aboutsummaryrefslogtreecommitdiff
path: root/src/wallet
diff options
context:
space:
mode:
Diffstat (limited to 'src/wallet')
-rw-r--r--src/wallet/bdb.cpp16
-rw-r--r--src/wallet/bdb.h3
-rw-r--r--src/wallet/db.cpp3
-rw-r--r--src/wallet/db.h8
-rw-r--r--src/wallet/dump.cpp10
-rw-r--r--src/wallet/init.cpp3
-rw-r--r--src/wallet/migrate.cpp784
-rw-r--r--src/wallet/migrate.h124
-rw-r--r--src/wallet/test/db_tests.cpp40
-rw-r--r--src/wallet/test/fuzz/wallet_bdb_parser.cpp133
-rw-r--r--src/wallet/test/util.cpp5
-rw-r--r--src/wallet/wallet.cpp5
-rw-r--r--src/wallet/walletdb.cpp10
-rw-r--r--src/wallet/wallettool.cpp5
14 files changed, 1135 insertions, 14 deletions
diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp
index 38cca32f80..d82d8d4513 100644
--- a/src/wallet/bdb.cpp
+++ b/src/wallet/bdb.cpp
@@ -65,6 +65,8 @@ RecursiveMutex cs_db;
std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment.
} // namespace
+static constexpr auto REVERSE_BYTE_ORDER{std::endian::native == std::endian::little ? 4321 : 1234};
+
bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const
{
return memcmp(value, &rhs.value, sizeof(value)) == 0;
@@ -300,7 +302,11 @@ static Span<const std::byte> SpanFromDbt(const SafeDbt& dbt)
}
BerkeleyDatabase::BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, fs::path filename, const DatabaseOptions& options) :
- WalletDatabase(), env(std::move(env)), m_filename(std::move(filename)), m_max_log_mb(options.max_log_mb)
+ WalletDatabase(),
+ env(std::move(env)),
+ m_byteswap(options.require_format == DatabaseFormat::BERKELEY_SWAP),
+ m_filename(std::move(filename)),
+ m_max_log_mb(options.max_log_mb)
{
auto inserted = this->env->m_databases.emplace(m_filename, std::ref(*this));
assert(inserted.second);
@@ -389,6 +395,10 @@ void BerkeleyDatabase::Open()
}
}
+ if (m_byteswap) {
+ pdb_temp->set_lorder(REVERSE_BYTE_ORDER);
+ }
+
ret = pdb_temp->open(nullptr, // Txn pointer
fMockDb ? nullptr : strFile.c_str(), // Filename
fMockDb ? strFile.c_str() : "main", // Logical db name
@@ -521,6 +531,10 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip)
BerkeleyBatch db(*this, true);
std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(env->dbenv.get(), 0);
+ if (m_byteswap) {
+ pdbCopy->set_lorder(REVERSE_BYTE_ORDER);
+ }
+
int ret = pdbCopy->open(nullptr, // Txn pointer
strFileRes.c_str(), // Filename
"main", // Logical db name
diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h
index 630630ebe0..af0c78f0d9 100644
--- a/src/wallet/bdb.h
+++ b/src/wallet/bdb.h
@@ -147,6 +147,9 @@ public:
/** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */
std::unique_ptr<Db> m_db;
+ // Whether to byteswap
+ bool m_byteswap;
+
fs::path m_filename;
int64_t m_max_log_mb;
diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp
index ea06767e9b..a5a5f8ec6f 100644
--- a/src/wallet/db.cpp
+++ b/src/wallet/db.cpp
@@ -16,6 +16,9 @@
#include <vector>
namespace wallet {
+bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); }
+bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; }
+
std::vector<fs::path> ListDatabases(const fs::path& wallet_dir)
{
std::vector<fs::path> paths;
diff --git a/src/wallet/db.h b/src/wallet/db.h
index 084fcadc24..b45076d10c 100644
--- a/src/wallet/db.h
+++ b/src/wallet/db.h
@@ -20,6 +20,12 @@ class ArgsManager;
struct bilingual_str;
namespace wallet {
+// BytePrefix compares equality with other byte spans that begin with the same prefix.
+struct BytePrefix {
+ Span<const std::byte> prefix;
+};
+bool operator<(BytePrefix a, Span<const std::byte> b);
+bool operator<(Span<const std::byte> a, BytePrefix b);
class DatabaseCursor
{
@@ -177,6 +183,8 @@ public:
enum class DatabaseFormat {
BERKELEY,
SQLITE,
+ BERKELEY_RO,
+ BERKELEY_SWAP,
};
struct DatabaseOptions {
diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp
index 7a36910dc1..db2756e0ca 100644
--- a/src/wallet/dump.cpp
+++ b/src/wallet/dump.cpp
@@ -60,7 +60,13 @@ bool DumpWallet(const ArgsManager& args, WalletDatabase& db, bilingual_str& erro
hasher << Span{line};
// Write out the file format
- line = strprintf("%s,%s\n", "format", db.Format());
+ std::string format = db.Format();
+ // BDB files that are opened using BerkeleyRODatabase have it's format as "bdb_ro"
+ // We want to override that format back to "bdb"
+ if (format == "bdb_ro") {
+ format = "bdb";
+ }
+ line = strprintf("%s,%s\n", "format", format);
dump_file.write(line.data(), line.size());
hasher << Span{line};
@@ -180,6 +186,8 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs::
data_format = DatabaseFormat::BERKELEY;
} else if (file_format == "sqlite") {
data_format = DatabaseFormat::SQLITE;
+ } else if (file_format == "bdb_swap") {
+ data_format = DatabaseFormat::BERKELEY_SWAP;
} else {
error = strprintf(_("Unknown wallet file format \"%s\" provided. Please provide one of \"bdb\" or \"sqlite\"."), file_format);
return false;
diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp
index 14e988ec1a..14d22bb54e 100644
--- a/src/wallet/init.cpp
+++ b/src/wallet/init.cpp
@@ -85,8 +85,9 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const
argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DatabaseOptions().max_log_mb), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", !DatabaseOptions().use_shared_memory), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
+ argsman.AddArg("-swapbdbendian", "Swaps the internal endianness of BDB wallet databases (default: false)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
#else
- argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb"});
+ argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb", "-swapbdbendian"});
#endif
#ifdef USE_SQLITE
diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp
new file mode 100644
index 0000000000..09254a76ad
--- /dev/null
+++ b/src/wallet/migrate.cpp
@@ -0,0 +1,784 @@
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <compat/byteswap.h>
+#include <crypto/common.h> // For ReadBE32
+#include <logging.h>
+#include <streams.h>
+#include <util/translation.h>
+#include <wallet/migrate.h>
+
+#include <optional>
+#include <variant>
+
+namespace wallet {
+// Magic bytes in both endianness's
+constexpr uint32_t BTREE_MAGIC = 0x00053162; // If the file endianness matches our system, we see this magic
+constexpr uint32_t BTREE_MAGIC_OE = 0x62310500; // If the file endianness is the other one, we will see this magic
+
+// Subdatabase name
+static const std::vector<std::byte> SUBDATABASE_NAME = {std::byte{'m'}, std::byte{'a'}, std::byte{'i'}, std::byte{'n'}};
+
+enum class PageType : uint8_t {
+ /*
+ * BDB has several page types, most of which we do not use
+ * They are listed here for completeness, but commented out
+ * to avoid opening something unintended.
+ INVALID = 0, // Invalid page type
+ DUPLICATE = 1, // Duplicate. Deprecated and no longer used
+ HASH_UNSORTED = 2, // Hash pages. Deprecated.
+ RECNO_INTERNAL = 4, // Recno internal
+ RECNO_LEAF = 6, // Recno leaf
+ HASH_META = 8, // Hash metadata
+ QUEUE_META = 10, // Queue Metadata
+ QUEUE_DATA = 11, // Queue Data
+ DUPLICATE_LEAF = 12, // Off-page duplicate leaf
+ HASH_SORTED = 13, // Sorted hash page
+ */
+ BTREE_INTERNAL = 3, // BTree internal
+ BTREE_LEAF = 5, // BTree leaf
+ OVERFLOW_DATA = 7, // Overflow
+ BTREE_META = 9, // BTree metadata
+};
+
+enum class RecordType : uint8_t {
+ KEYDATA = 1,
+ // DUPLICATE = 2, Unused as our databases do not support duplicate records
+ OVERFLOW_DATA = 3,
+ DELETE = 0x80, // Indicate this record is deleted. This is OR'd with the real type.
+};
+
+enum class BTreeFlags : uint32_t {
+ /*
+ * BTree databases have feature flags, but we do not use them except for
+ * subdatabases. The unused flags are included for completeness, but commented out
+ * to avoid accidental use.
+ DUP = 1, // Duplicates
+ RECNO = 2, // Recno tree
+ RECNUM = 4, // BTree: Maintain record counts
+ FIXEDLEN = 8, // Recno: fixed length records
+ RENUMBER = 0x10, // Recno: renumber on insert/delete
+ DUPSORT = 0x40, // Duplicates are sorted
+ COMPRESS = 0x80, // Compressed
+ */
+ SUBDB = 0x20, // Subdatabases
+};
+
+/** Berkeley DB BTree metadata page layout */
+class MetaPage
+{
+public:
+ uint32_t lsn_file; // Log Sequence Number file
+ uint32_t lsn_offset; // Log Sequence Number offset
+ uint32_t page_num; // Current page number
+ uint32_t magic; // Magic number
+ uint32_t version; // Version
+ uint32_t pagesize; // Page size
+ uint8_t encrypt_algo; // Encryption algorithm
+ PageType type; // Page type
+ uint8_t metaflags; // Meta-only flags
+ uint8_t unused1; // Unused
+ uint32_t free_list; // Free list page number
+ uint32_t last_page; // Page number of last page in db
+ uint32_t partitions; // Number of partitions
+ uint32_t key_count; // Cached key count
+ uint32_t record_count; // Cached record count
+ BTreeFlags flags; // Flags
+ std::array<std::byte, 20> uid; // 20 byte unique file ID
+ uint32_t unused2; // Unused
+ uint32_t minkey; // Minimum key
+ uint32_t re_len; // Recno: fixed length record length
+ uint32_t re_pad; // Recno: fixed length record pad
+ uint32_t root; // Root page number
+ char unused3[368]; // 92 * 4 bytes of unused space
+ uint32_t crypto_magic; // Crypto magic number
+ char trash[12]; // 3 * 4 bytes of trash space
+ unsigned char iv[20]; // Crypto IV
+ unsigned char chksum[16]; // Checksum
+
+ bool other_endian;
+ uint32_t expected_page_num;
+
+ MetaPage(uint32_t expected_page_num) : expected_page_num(expected_page_num) {}
+ MetaPage() = delete;
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ s >> lsn_file;
+ s >> lsn_offset;
+ s >> page_num;
+ s >> magic;
+ s >> version;
+ s >> pagesize;
+ s >> encrypt_algo;
+
+ other_endian = magic == BTREE_MAGIC_OE;
+
+ uint8_t uint8_type;
+ s >> uint8_type;
+ type = static_cast<PageType>(uint8_type);
+
+ s >> metaflags;
+ s >> unused1;
+ s >> free_list;
+ s >> last_page;
+ s >> partitions;
+ s >> key_count;
+ s >> record_count;
+
+ uint32_t uint32_flags;
+ s >> uint32_flags;
+ if (other_endian) {
+ uint32_flags = internal_bswap_32(uint32_flags);
+ }
+ flags = static_cast<BTreeFlags>(uint32_flags);
+
+ s >> uid;
+ s >> unused2;
+ s >> minkey;
+ s >> re_len;
+ s >> re_pad;
+ s >> root;
+ s >> unused3;
+ s >> crypto_magic;
+ s >> trash;
+ s >> iv;
+ s >> chksum;
+
+ if (other_endian) {
+ lsn_file = internal_bswap_32(lsn_file);
+ lsn_offset = internal_bswap_32(lsn_offset);
+ page_num = internal_bswap_32(page_num);
+ magic = internal_bswap_32(magic);
+ version = internal_bswap_32(version);
+ pagesize = internal_bswap_32(pagesize);
+ free_list = internal_bswap_32(free_list);
+ last_page = internal_bswap_32(last_page);
+ partitions = internal_bswap_32(partitions);
+ key_count = internal_bswap_32(key_count);
+ record_count = internal_bswap_32(record_count);
+ unused2 = internal_bswap_32(unused2);
+ minkey = internal_bswap_32(minkey);
+ re_len = internal_bswap_32(re_len);
+ re_pad = internal_bswap_32(re_pad);
+ root = internal_bswap_32(root);
+ crypto_magic = internal_bswap_32(crypto_magic);
+ }
+
+ // Page number must match
+ if (page_num != expected_page_num) {
+ throw std::runtime_error("Meta page number mismatch");
+ }
+
+ // Check magic
+ if (magic != BTREE_MAGIC) {
+ throw std::runtime_error("Not a BDB file");
+ }
+
+ // Only version 9 is supported
+ if (version != 9) {
+ throw std::runtime_error("Unsupported BDB data file version number");
+ }
+
+ // Page size must be 512 <= pagesize <= 64k, and be a power of 2
+ if (pagesize < 512 || pagesize > 65536 || (pagesize & (pagesize - 1)) != 0) {
+ throw std::runtime_error("Bad page size");
+ }
+
+ // Page type must be the btree type
+ if (type != PageType::BTREE_META) {
+ throw std::runtime_error("Unexpected page type, should be 9 (BTree Metadata)");
+ }
+
+ // Only supported meta-flag is subdatabase
+ if (flags != BTreeFlags::SUBDB) {
+ throw std::runtime_error("Unexpected database flags, should only be 0x20 (subdatabases)");
+ }
+ }
+};
+
+/** General class for records in a BDB BTree database. Contains common fields. */
+class RecordHeader
+{
+public:
+ uint16_t len; // Key/data item length
+ RecordType type; // Page type (BDB has this include a DELETE FLAG that we track separately)
+ bool deleted; // Whether the DELETE flag was set on type
+
+ static constexpr size_t SIZE = 3; // The record header is 3 bytes
+
+ bool other_endian;
+
+ RecordHeader(bool other_endian) : other_endian(other_endian) {}
+ RecordHeader() = delete;
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ s >> len;
+
+ uint8_t uint8_type;
+ s >> uint8_type;
+ type = static_cast<RecordType>(uint8_type & ~static_cast<uint8_t>(RecordType::DELETE));
+ deleted = uint8_type & static_cast<uint8_t>(RecordType::DELETE);
+
+ if (other_endian) {
+ len = internal_bswap_16(len);
+ }
+ }
+};
+
+/** Class for data in the record directly */
+class DataRecord
+{
+public:
+ DataRecord(const RecordHeader& header) : m_header(header) {}
+ DataRecord() = delete;
+
+ RecordHeader m_header;
+
+ std::vector<std::byte> data; // Variable length key/data item
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ data.resize(m_header.len);
+ s.read(AsWritableBytes(Span(data.data(), data.size())));
+ }
+};
+
+/** Class for records representing internal nodes of the BTree. */
+class InternalRecord
+{
+public:
+ InternalRecord(const RecordHeader& header) : m_header(header) {}
+ InternalRecord() = delete;
+
+ RecordHeader m_header;
+
+ uint8_t unused; // Padding, unused
+ uint32_t page_num; // Page number of referenced page
+ uint32_t records; // Subtree record count
+ std::vector<std::byte> data; // Variable length key item
+
+ static constexpr size_t FIXED_SIZE = 9; // Size of fixed data is 9 bytes
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ s >> unused;
+ s >> page_num;
+ s >> records;
+
+ data.resize(m_header.len);
+ s.read(AsWritableBytes(Span(data.data(), data.size())));
+
+ if (m_header.other_endian) {
+ page_num = internal_bswap_32(page_num);
+ records = internal_bswap_32(records);
+ }
+ }
+};
+
+/** Class for records representing overflow records of the BTree.
+ * Overflow records point to a page which contains the data in the record.
+ * Those pages may point to further pages with the rest of the data if it does not fit
+ * in one page */
+class OverflowRecord
+{
+public:
+ OverflowRecord(const RecordHeader& header) : m_header(header) {}
+ OverflowRecord() = delete;
+
+ RecordHeader m_header;
+
+ uint8_t unused2; // Padding, unused
+ uint32_t page_number; // Page number where data begins
+ uint32_t item_len; // Total length of item
+
+ static constexpr size_t SIZE = 9; // Overflow record is always 9 bytes
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ s >> unused2;
+ s >> page_number;
+ s >> item_len;
+
+ if (m_header.other_endian) {
+ page_number = internal_bswap_32(page_number);
+ item_len = internal_bswap_32(item_len);
+ }
+ }
+};
+
+/** A generic data page in the database. Contains fields common to all data pages. */
+class PageHeader
+{
+public:
+ uint32_t lsn_file; // Log Sequence Number file
+ uint32_t lsn_offset; // Log Sequence Number offset
+ uint32_t page_num; // Current page number
+ uint32_t prev_page; // Previous page number
+ uint32_t next_page; // Next page number
+ uint16_t entries; // Number of items on the page
+ uint16_t hf_offset; // High free byte page offset
+ uint8_t level; // Btree page level
+ PageType type; // Page type
+
+ static constexpr int64_t SIZE = 26; // The header is 26 bytes
+
+ uint32_t expected_page_num;
+ bool other_endian;
+
+ PageHeader(uint32_t page_num, bool other_endian) : expected_page_num(page_num), other_endian(other_endian) {}
+ PageHeader() = delete;
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ s >> lsn_file;
+ s >> lsn_offset;
+ s >> page_num;
+ s >> prev_page;
+ s >> next_page;
+ s >> entries;
+ s >> hf_offset;
+ s >> level;
+
+ uint8_t uint8_type;
+ s >> uint8_type;
+ type = static_cast<PageType>(uint8_type);
+
+ if (other_endian) {
+ lsn_file = internal_bswap_32(lsn_file);
+ lsn_offset = internal_bswap_32(lsn_offset);
+ page_num = internal_bswap_32(page_num);
+ prev_page = internal_bswap_32(prev_page);
+ next_page = internal_bswap_32(next_page);
+ entries = internal_bswap_16(entries);
+ hf_offset = internal_bswap_16(hf_offset);
+ }
+
+ if (expected_page_num != page_num) {
+ throw std::runtime_error("Page number mismatch");
+ }
+ if ((type != PageType::OVERFLOW_DATA && level < 1) || (type == PageType::OVERFLOW_DATA && level != 0)) {
+ throw std::runtime_error("Bad btree level");
+ }
+ }
+};
+
+/** A page of records in the database */
+class RecordsPage
+{
+public:
+ RecordsPage(const PageHeader& header) : m_header(header) {}
+ RecordsPage() = delete;
+
+ PageHeader m_header;
+
+ std::vector<uint16_t> indexes;
+ std::vector<std::variant<DataRecord, OverflowRecord>> records;
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ // Current position within the page
+ int64_t pos = PageHeader::SIZE;
+
+ // Get the items
+ for (uint32_t i = 0; i < m_header.entries; ++i) {
+ // Get the index
+ uint16_t index;
+ s >> index;
+ if (m_header.other_endian) {
+ index = internal_bswap_16(index);
+ }
+ indexes.push_back(index);
+ pos += sizeof(uint16_t);
+
+ // Go to the offset from the index
+ int64_t to_jump = index - pos;
+ if (to_jump < 0) {
+ throw std::runtime_error("Data record position not in page");
+ }
+ s.ignore(to_jump);
+
+ // Read the record
+ RecordHeader rec_hdr(m_header.other_endian);
+ s >> rec_hdr;
+ to_jump += RecordHeader::SIZE;
+
+ switch (rec_hdr.type) {
+ case RecordType::KEYDATA: {
+ DataRecord record(rec_hdr);
+ s >> record;
+ records.emplace_back(record);
+ to_jump += rec_hdr.len;
+ break;
+ }
+ case RecordType::OVERFLOW_DATA: {
+ OverflowRecord record(rec_hdr);
+ s >> record;
+ records.emplace_back(record);
+ to_jump += OverflowRecord::SIZE;
+ break;
+ }
+ default:
+ throw std::runtime_error("Unknown record type in records page");
+ }
+
+ // Go back to the indexes
+ s.seek(-to_jump, SEEK_CUR);
+ }
+ }
+};
+
+/** A page containing overflow data */
+class OverflowPage
+{
+public:
+ OverflowPage(const PageHeader& header) : m_header(header) {}
+ OverflowPage() = delete;
+
+ PageHeader m_header;
+
+ // BDB overloads some page fields to store overflow page data
+ // hf_offset contains the length of the overflow data stored on this page
+ // entries contains a reference count for references to this item
+
+ // The overflow data itself. Begins immediately following header
+ std::vector<std::byte> data;
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ data.resize(m_header.hf_offset);
+ s.read(AsWritableBytes(Span(data.data(), data.size())));
+ }
+};
+
+/** A page of records in the database */
+class InternalPage
+{
+public:
+ InternalPage(const PageHeader& header) : m_header(header) {}
+ InternalPage() = delete;
+
+ PageHeader m_header;
+
+ std::vector<uint16_t> indexes;
+ std::vector<InternalRecord> records;
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ // Current position within the page
+ int64_t pos = PageHeader::SIZE;
+
+ // Get the items
+ for (uint32_t i = 0; i < m_header.entries; ++i) {
+ // Get the index
+ uint16_t index;
+ s >> index;
+ if (m_header.other_endian) {
+ index = internal_bswap_16(index);
+ }
+ indexes.push_back(index);
+ pos += sizeof(uint16_t);
+
+ // Go to the offset from the index
+ int64_t to_jump = index - pos;
+ if (to_jump < 0) {
+ throw std::runtime_error("Internal record position not in page");
+ }
+ s.ignore(to_jump);
+
+ // Read the record
+ RecordHeader rec_hdr(m_header.other_endian);
+ s >> rec_hdr;
+ to_jump += RecordHeader::SIZE;
+
+ if (rec_hdr.type != RecordType::KEYDATA) {
+ throw std::runtime_error("Unknown record type in internal page");
+ }
+ InternalRecord record(rec_hdr);
+ s >> record;
+ records.emplace_back(record);
+ to_jump += InternalRecord::FIXED_SIZE + rec_hdr.len;
+
+ // Go back to the indexes
+ s.seek(-to_jump, SEEK_CUR);
+ }
+ }
+};
+
+static void SeekToPage(AutoFile& s, uint32_t page_num, uint32_t page_size)
+{
+ int64_t pos = int64_t{page_num} * page_size;
+ s.seek(pos, SEEK_SET);
+}
+
+void BerkeleyRODatabase::Open()
+{
+ // Open the file
+ FILE* file = fsbridge::fopen(m_filepath, "rb");
+ AutoFile db_file(file);
+ if (db_file.IsNull()) {
+ throw std::runtime_error("BerkeleyRODatabase: Failed to open database file");
+ }
+
+ uint32_t page_size = 4096; // Default page size
+
+ // Read the outer metapage
+ // Expected page number is 0
+ MetaPage outer_meta(0);
+ db_file >> outer_meta;
+ page_size = outer_meta.pagesize;
+
+ // Verify the size of the file is a multiple of the page size
+ db_file.seek(0, SEEK_END);
+ int64_t size = db_file.tell();
+
+ // Since BDB stores everything in a page, the file size should be a multiple of the page size;
+ // However, BDB doesn't actually check that this is the case, and enforcing this check results
+ // in us rejecting a database that BDB would not, so this check needs to be excluded.
+ // This is left commented out as a reminder to not accidentally implement this in the future.
+ // if (size % page_size != 0) {
+ // throw std::runtime_error("File size is not a multiple of page size");
+ // }
+
+ // Check the last page number
+ uint32_t expected_last_page = (size / page_size) - 1;
+ if (outer_meta.last_page != expected_last_page) {
+ throw std::runtime_error("Last page number could not fit in file");
+ }
+
+ // Make sure encryption is disabled
+ if (outer_meta.encrypt_algo != 0) {
+ throw std::runtime_error("BDB builtin encryption is not supported");
+ }
+
+ // Check all Log Sequence Numbers (LSN) point to file 0 and offset 1 which indicates that the LSNs were
+ // reset and that the log files are not necessary to get all of the data in the database.
+ for (uint32_t i = 0; i < outer_meta.last_page; ++i) {
+ // The LSN is composed of 2 32-bit ints, the first is a file id, the second an offset
+ // It will always be the first 8 bytes of a page, so we deserialize it directly for every page
+ uint32_t file;
+ uint32_t offset;
+ SeekToPage(db_file, i, page_size);
+ db_file >> file >> offset;
+ if (outer_meta.other_endian) {
+ file = internal_bswap_32(file);
+ offset = internal_bswap_32(offset);
+ }
+ if (file != 0 || offset != 1) {
+ throw std::runtime_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support");
+ }
+ }
+
+ // Read the root page
+ SeekToPage(db_file, outer_meta.root, page_size);
+ PageHeader header(outer_meta.root, outer_meta.other_endian);
+ db_file >> header;
+ if (header.type != PageType::BTREE_LEAF) {
+ throw std::runtime_error("Unexpected outer database root page type");
+ }
+ if (header.entries != 2) {
+ throw std::runtime_error("Unexpected number of entries in outer database root page");
+ }
+ RecordsPage page(header);
+ db_file >> page;
+
+ // First record should be the string "main"
+ if (!std::holds_alternative<DataRecord>(page.records.at(0)) || std::get<DataRecord>(page.records.at(0)).data != SUBDATABASE_NAME) {
+ throw std::runtime_error("Subdatabase has an unexpected name");
+ }
+ // Check length of page number for subdatabase location
+ if (!std::holds_alternative<DataRecord>(page.records.at(1)) || std::get<DataRecord>(page.records.at(1)).m_header.len != 4) {
+ throw std::runtime_error("Subdatabase page number has unexpected length");
+ }
+
+ // Read subdatabase page number
+ // It is written as a big endian 32 bit number
+ uint32_t main_db_page = ReadBE32(UCharCast(std::get<DataRecord>(page.records.at(1)).data.data()));
+
+ // The main database is in a page that doesn't exist
+ if (main_db_page > outer_meta.last_page) {
+ throw std::runtime_error("Page number is greater than database last page");
+ }
+
+ // Read the inner metapage
+ SeekToPage(db_file, main_db_page, page_size);
+ MetaPage inner_meta(main_db_page);
+ db_file >> inner_meta;
+
+ if (inner_meta.pagesize != page_size) {
+ throw std::runtime_error("Unexpected page size");
+ }
+
+ if (inner_meta.last_page > outer_meta.last_page) {
+ throw std::runtime_error("Subdatabase last page is greater than database last page");
+ }
+
+ // Make sure encryption is disabled
+ if (inner_meta.encrypt_algo != 0) {
+ throw std::runtime_error("BDB builtin encryption is not supported");
+ }
+
+ // Do a DFS through the BTree, starting at root
+ std::vector<uint32_t> pages{inner_meta.root};
+ while (pages.size() > 0) {
+ uint32_t curr_page = pages.back();
+ // It turns out BDB completely ignores this last_page field and doesn't actually update it to the correct
+ // last page. While we should be checking this, we can't.
+ // This is left commented out as a reminder to not accidentally implement this in the future.
+ // if (curr_page > inner_meta.last_page) {
+ // throw std::runtime_error("Page number is greater than subdatabase last page");
+ // }
+ pages.pop_back();
+ SeekToPage(db_file, curr_page, page_size);
+ PageHeader header(curr_page, inner_meta.other_endian);
+ db_file >> header;
+ switch (header.type) {
+ case PageType::BTREE_INTERNAL: {
+ InternalPage int_page(header);
+ db_file >> int_page;
+ for (const InternalRecord& rec : int_page.records) {
+ if (rec.m_header.deleted) continue;
+ pages.push_back(rec.page_num);
+ }
+ break;
+ }
+ case PageType::BTREE_LEAF: {
+ RecordsPage rec_page(header);
+ db_file >> rec_page;
+ if (rec_page.records.size() % 2 != 0) {
+ // BDB stores key value pairs in consecutive records, thus an odd number of records is unexpected
+ throw std::runtime_error("Records page has odd number of records");
+ }
+ bool is_key = true;
+ std::vector<std::byte> key;
+ for (const std::variant<DataRecord, OverflowRecord>& rec : rec_page.records) {
+ std::vector<std::byte> data;
+ if (const DataRecord* drec = std::get_if<DataRecord>(&rec)) {
+ if (drec->m_header.deleted) continue;
+ data = drec->data;
+ } else if (const OverflowRecord* orec = std::get_if<OverflowRecord>(&rec)) {
+ if (orec->m_header.deleted) continue;
+ uint32_t next_page = orec->page_number;
+ while (next_page != 0) {
+ SeekToPage(db_file, next_page, page_size);
+ PageHeader opage_header(next_page, inner_meta.other_endian);
+ db_file >> opage_header;
+ if (opage_header.type != PageType::OVERFLOW_DATA) {
+ throw std::runtime_error("Bad overflow record page type");
+ }
+ OverflowPage opage(opage_header);
+ db_file >> opage;
+ data.insert(data.end(), opage.data.begin(), opage.data.end());
+ next_page = opage_header.next_page;
+ }
+ }
+
+ if (is_key) {
+ key = data;
+ } else {
+ m_records.emplace(SerializeData{key.begin(), key.end()}, SerializeData{data.begin(), data.end()});
+ key.clear();
+ }
+ is_key = !is_key;
+ }
+ break;
+ }
+ default:
+ throw std::runtime_error("Unexpected page type");
+ }
+ }
+}
+
+std::unique_ptr<DatabaseBatch> BerkeleyRODatabase::MakeBatch(bool flush_on_close)
+{
+ return std::make_unique<BerkeleyROBatch>(*this);
+}
+
+bool BerkeleyRODatabase::Backup(const std::string& dest) const
+{
+ fs::path src(m_filepath);
+ fs::path dst(fs::PathFromString(dest));
+
+ if (fs::is_directory(dst)) {
+ dst = BDBDataFile(dst);
+ }
+ try {
+ if (fs::exists(dst) && fs::equivalent(src, dst)) {
+ LogPrintf("cannot backup to wallet source file %s\n", fs::PathToString(dst));
+ return false;
+ }
+
+ fs::copy_file(src, dst, fs::copy_options::overwrite_existing);
+ LogPrintf("copied %s to %s\n", fs::PathToString(m_filepath), fs::PathToString(dst));
+ return true;
+ } catch (const fs::filesystem_error& e) {
+ LogPrintf("error copying %s to %s - %s\n", fs::PathToString(m_filepath), fs::PathToString(dst), fsbridge::get_filesystem_error_message(e));
+ return false;
+ }
+}
+
+bool BerkeleyROBatch::ReadKey(DataStream&& key, DataStream& value)
+{
+ SerializeData key_data{key.begin(), key.end()};
+ const auto it{m_database.m_records.find(key_data)};
+ if (it == m_database.m_records.end()) {
+ return false;
+ }
+ auto val = it->second;
+ value.clear();
+ value.write(Span(val));
+ return true;
+}
+
+bool BerkeleyROBatch::HasKey(DataStream&& key)
+{
+ SerializeData key_data{key.begin(), key.end()};
+ return m_database.m_records.count(key_data) > 0;
+}
+
+BerkeleyROCursor::BerkeleyROCursor(const BerkeleyRODatabase& database, Span<const std::byte> prefix)
+ : m_database(database)
+{
+ std::tie(m_cursor, m_cursor_end) = m_database.m_records.equal_range(BytePrefix{prefix});
+}
+
+DatabaseCursor::Status BerkeleyROCursor::Next(DataStream& ssKey, DataStream& ssValue)
+{
+ if (m_cursor == m_cursor_end) {
+ return DatabaseCursor::Status::DONE;
+ }
+ ssKey.write(Span(m_cursor->first));
+ ssValue.write(Span(m_cursor->second));
+ m_cursor++;
+ return DatabaseCursor::Status::MORE;
+}
+
+std::unique_ptr<DatabaseCursor> BerkeleyROBatch::GetNewPrefixCursor(Span<const std::byte> prefix)
+{
+ return std::make_unique<BerkeleyROCursor>(m_database, prefix);
+}
+
+std::unique_ptr<BerkeleyRODatabase> MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error)
+{
+ fs::path data_file = BDBDataFile(path);
+ try {
+ std::unique_ptr<BerkeleyRODatabase> db = std::make_unique<BerkeleyRODatabase>(data_file);
+ status = DatabaseStatus::SUCCESS;
+ return db;
+ } catch (const std::runtime_error& e) {
+ error.original = e.what();
+ status = DatabaseStatus::FAILED_LOAD;
+ return nullptr;
+ }
+}
+} // namespace wallet
diff --git a/src/wallet/migrate.h b/src/wallet/migrate.h
new file mode 100644
index 0000000000..e4826450af
--- /dev/null
+++ b/src/wallet/migrate.h
@@ -0,0 +1,124 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_WALLET_MIGRATE_H
+#define BITCOIN_WALLET_MIGRATE_H
+
+#include <wallet/db.h>
+
+#include <optional>
+
+namespace wallet {
+
+using BerkeleyROData = std::map<SerializeData, SerializeData, std::less<>>;
+
+/**
+ * A class representing a BerkeleyDB file from which we can only read records.
+ * This is used only for migration of legacy to descriptor wallets
+ */
+class BerkeleyRODatabase : public WalletDatabase
+{
+private:
+ const fs::path m_filepath;
+
+public:
+ /** Create DB handle */
+ BerkeleyRODatabase(const fs::path& filepath, bool open = true) : WalletDatabase(), m_filepath(filepath)
+ {
+ if (open) Open();
+ }
+ ~BerkeleyRODatabase(){};
+
+ BerkeleyROData m_records;
+
+ /** Open the database if it is not already opened. */
+ void Open() override;
+
+ /** Indicate the a new database user has began using the database. Increments m_refcount */
+ void AddRef() override {}
+ /** Indicate that database user has stopped using the database and that it could be flushed or closed. Decrement m_refcount */
+ void RemoveRef() override {}
+
+ /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero
+ */
+ bool Rewrite(const char* pszSkip = nullptr) override { return false; }
+
+ /** Back up the entire database to a file.
+ */
+ bool Backup(const std::string& strDest) const override;
+
+ /** Make sure all changes are flushed to database file.
+ */
+ void Flush() override {}
+ /** Flush to the database file and close the database.
+ * Also close the environment if no other databases are open in it.
+ */
+ void Close() override {}
+ /* flush the wallet passively (TRY_LOCK)
+ ideal to be called periodically */
+ bool PeriodicFlush() override { return false; }
+
+ void IncrementUpdateCounter() override {}
+
+ void ReloadDbEnv() override {}
+
+ /** Return path to main database file for logs and error messages. */
+ std::string Filename() override { return fs::PathToString(m_filepath); }
+
+ std::string Format() override { return "bdb_ro"; }
+
+ /** Make a DatabaseBatch connected to this database */
+ std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override;
+};
+
+class BerkeleyROCursor : public DatabaseCursor
+{
+private:
+ const BerkeleyRODatabase& m_database;
+ BerkeleyROData::const_iterator m_cursor;
+ BerkeleyROData::const_iterator m_cursor_end;
+
+public:
+ explicit BerkeleyROCursor(const BerkeleyRODatabase& database, Span<const std::byte> prefix = {});
+ ~BerkeleyROCursor() {}
+
+ Status Next(DataStream& key, DataStream& value) override;
+};
+
+/** RAII class that provides access to a BerkeleyRODatabase */
+class BerkeleyROBatch : public DatabaseBatch
+{
+private:
+ const BerkeleyRODatabase& m_database;
+
+ bool ReadKey(DataStream&& key, DataStream& value) override;
+ // WriteKey returns true since various automatic upgrades for older wallets will expect writing to not fail.
+ // It is okay for this batch type to not actually write anything as those automatic upgrades will occur again after migration.
+ bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return true; }
+ bool EraseKey(DataStream&& key) override { return false; }
+ bool HasKey(DataStream&& key) override;
+ bool ErasePrefix(Span<const std::byte> prefix) override { return false; }
+
+public:
+ explicit BerkeleyROBatch(const BerkeleyRODatabase& database) : m_database(database) {}
+ ~BerkeleyROBatch() {}
+
+ BerkeleyROBatch(const BerkeleyROBatch&) = delete;
+ BerkeleyROBatch& operator=(const BerkeleyROBatch&) = delete;
+
+ void Flush() override {}
+ void Close() override {}
+
+ std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<BerkeleyROCursor>(m_database); }
+ std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override;
+ bool TxnBegin() override { return false; }
+ bool TxnCommit() override { return false; }
+ bool TxnAbort() override { return false; }
+};
+
+//! Return object giving access to Berkeley Read Only database at specified path.
+std::unique_ptr<BerkeleyRODatabase> MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error);
+} // namespace wallet
+
+#endif // BITCOIN_WALLET_MIGRATE_H
diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp
index 438dfceb7f..2fac356263 100644
--- a/src/wallet/test/db_tests.cpp
+++ b/src/wallet/test/db_tests.cpp
@@ -16,6 +16,7 @@
#ifdef USE_SQLITE
#include <wallet/sqlite.h>
#endif
+#include <wallet/migrate.h>
#include <wallet/test/util.h>
#include <wallet/walletutil.h> // for WALLET_FLAG_DESCRIPTORS
@@ -132,6 +133,8 @@ static std::vector<std::unique_ptr<WalletDatabase>> TestDatabases(const fs::path
bilingual_str error;
#ifdef USE_BDB
dbs.emplace_back(MakeBerkeleyDatabase(path_root / "bdb", options, status, error));
+ // Needs BDB to make the DB to read
+ dbs.emplace_back(std::make_unique<BerkeleyRODatabase>(BDBDataFile(path_root / "bdb"), /*open=*/false));
#endif
#ifdef USE_SQLITE
dbs.emplace_back(MakeSQLiteDatabase(path_root / "sqlite", options, status, error));
@@ -146,11 +149,16 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test)
for (const auto& database : TestDatabases(m_path_root)) {
std::vector<std::string> prefixes = {"", "FIRST", "SECOND", "P\xfe\xff", "P\xff\x01", "\xff\xff"};
- // Write elements to it
std::unique_ptr<DatabaseBatch> handler = Assert(database)->MakeBatch();
- for (unsigned int i = 0; i < 10; i++) {
- for (const auto& prefix : prefixes) {
- BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i));
+ if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
+ // For BerkeleyRO, open the file now. This must happen after BDB has written to the file
+ database->Open();
+ } else {
+ // Write elements to it if not berkeleyro
+ for (unsigned int i = 0; i < 10; i++) {
+ for (const auto& prefix : prefixes) {
+ BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i));
+ }
}
}
@@ -178,6 +186,8 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test)
// Let's now read it once more, it should return DONE
BOOST_CHECK(cursor->Next(key, value) == DatabaseCursor::Status::DONE);
}
+ handler.reset();
+ database->Close();
}
}
@@ -197,13 +207,23 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_byte_test)
ffs{StringData("\xff\xffsuffix"), StringData("ffs")};
for (const auto& database : TestDatabases(m_path_root)) {
std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
- for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) {
- batch->Write(Span{k}, Span{v});
+
+ if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
+ // For BerkeleyRO, open the file now. This must happen after BDB has written to the file
+ database->Open();
+ } else {
+ // Write elements to it if not berkeleyro
+ for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) {
+ batch->Write(Span{k}, Span{v});
+ }
}
+
CheckPrefix(*batch, StringBytes(""), {e, p, ps, f, fs, ff, ffs});
CheckPrefix(*batch, StringBytes("prefix"), {p, ps});
CheckPrefix(*batch, StringBytes("\xff"), {f, fs, ff, ffs});
CheckPrefix(*batch, StringBytes("\xff\xff"), {ff, ffs});
+ batch.reset();
+ database->Close();
}
}
@@ -213,6 +233,10 @@ BOOST_AUTO_TEST_CASE(db_availability_after_write_error)
// To simulate the behavior, record overwrites are disallowed, and the test verifies
// that the database remains active after failing to store an existing record.
for (const auto& database : TestDatabases(m_path_root)) {
+ if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
+ // Skip this test if BerkeleyRO
+ continue;
+ }
// Write original record
std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
std::string key = "key";
@@ -241,6 +265,10 @@ BOOST_AUTO_TEST_CASE(erase_prefix)
auto make_key = [](std::string type, std::string id) { return std::make_pair(type, id); };
for (const auto& database : TestDatabases(m_path_root)) {
+ if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
+ // Skip this test if BerkeleyRO
+ continue;
+ }
std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
// Write two entries with the same key type prefix, a third one with a different prefix
diff --git a/src/wallet/test/fuzz/wallet_bdb_parser.cpp b/src/wallet/test/fuzz/wallet_bdb_parser.cpp
new file mode 100644
index 0000000000..24ef75f791
--- /dev/null
+++ b/src/wallet/test/fuzz/wallet_bdb_parser.cpp
@@ -0,0 +1,133 @@
+// Copyright (c) 2023 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <config/bitcoin-config.h> // IWYU pragma: keep
+#include <test/fuzz/FuzzedDataProvider.h>
+#include <test/fuzz/fuzz.h>
+#include <test/fuzz/util.h>
+#include <test/util/setup_common.h>
+#include <util/fs.h>
+#include <util/time.h>
+#include <util/translation.h>
+#include <wallet/bdb.h>
+#include <wallet/db.h>
+#include <wallet/dump.h>
+#include <wallet/migrate.h>
+
+#include <fstream>
+#include <iostream>
+
+using wallet::DatabaseOptions;
+using wallet::DatabaseStatus;
+
+namespace {
+TestingSetup* g_setup;
+} // namespace
+
+void initialize_wallet_bdb_parser()
+{
+ static auto testing_setup = MakeNoLogFileContext<TestingSetup>();
+ g_setup = testing_setup.get();
+}
+
+FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser)
+{
+ const auto wallet_path = g_setup->m_args.GetDataDirNet() / "fuzzed_wallet.dat";
+
+ {
+ AutoFile outfile{fsbridge::fopen(wallet_path, "wb")};
+ outfile << Span{buffer};
+ }
+
+ const DatabaseOptions options{};
+ DatabaseStatus status;
+ bilingual_str error;
+
+ fs::path bdb_ro_dumpfile{g_setup->m_args.GetDataDirNet() / "fuzzed_dumpfile_bdb_ro.dump"};
+ if (fs::exists(bdb_ro_dumpfile)) { // Writing into an existing dump file will throw an exception
+ remove(bdb_ro_dumpfile);
+ }
+ g_setup->m_args.ForceSetArg("-dumpfile", fs::PathToString(bdb_ro_dumpfile));
+
+#ifdef USE_BDB
+ bool bdb_ro_err = false;
+ bool bdb_ro_pgno_err = false;
+#endif
+ auto db{MakeBerkeleyRODatabase(wallet_path, options, status, error)};
+ if (db) {
+ assert(DumpWallet(g_setup->m_args, *db, error));
+ } else {
+#ifdef USE_BDB
+ bdb_ro_err = true;
+#endif
+ if (error.original == "AutoFile::ignore: end of file: iostream error" ||
+ error.original == "AutoFile::read: end of file: iostream error" ||
+ error.original == "Not a BDB file" ||
+ error.original == "Unsupported BDB data file version number" ||
+ error.original == "Unexpected page type, should be 9 (BTree Metadata)" ||
+ error.original == "Unexpected database flags, should only be 0x20 (subdatabases)" ||
+ error.original == "Unexpected outer database root page type" ||
+ error.original == "Unexpected number of entries in outer database root page" ||
+ error.original == "Subdatabase has an unexpected name" ||
+ error.original == "Subdatabase page number has unexpected length" ||
+ error.original == "Unexpected inner database page type" ||
+ error.original == "Unknown record type in records page" ||
+ error.original == "Unknown record type in internal page" ||
+ error.original == "Unexpected page size" ||
+ error.original == "Unexpected page type" ||
+ error.original == "Page number mismatch" ||
+ error.original == "Bad btree level" ||
+ error.original == "Bad page size" ||
+ error.original == "File size is not a multiple of page size" ||
+ error.original == "Meta page number mismatch") {
+ // Do nothing
+ } else if (error.original == "Subdatabase last page is greater than database last page" ||
+ error.original == "Page number is greater than database last page" ||
+ error.original == "Page number is greater than subdatabase last page" ||
+ error.original == "Last page number could not fit in file") {
+#ifdef USE_BDB
+ bdb_ro_pgno_err = true;
+#endif
+ } else {
+ throw std::runtime_error(error.original);
+ }
+ }
+
+#ifdef USE_BDB
+ // Try opening with BDB
+ fs::path bdb_dumpfile{g_setup->m_args.GetDataDirNet() / "fuzzed_dumpfile_bdb.dump"};
+ if (fs::exists(bdb_dumpfile)) { // Writing into an existing dump file will throw an exception
+ remove(bdb_dumpfile);
+ }
+ g_setup->m_args.ForceSetArg("-dumpfile", fs::PathToString(bdb_dumpfile));
+
+ try {
+ auto db{MakeBerkeleyDatabase(wallet_path, options, status, error)};
+ if (bdb_ro_err && !db) {
+ return;
+ }
+ assert(db);
+ if (bdb_ro_pgno_err) {
+ // BerkeleyRO will throw on opening for errors involving bad page numbers, but BDB does not.
+ // Ignore those.
+ return;
+ }
+ assert(!bdb_ro_err);
+ assert(DumpWallet(g_setup->m_args, *db, error));
+ } catch (const std::runtime_error& e) {
+ if (bdb_ro_err) return;
+ throw e;
+ }
+
+ // Make sure the dumpfiles match
+ if (fs::exists(bdb_ro_dumpfile) && fs::exists(bdb_dumpfile)) {
+ std::ifstream bdb_ro_dump(bdb_ro_dumpfile, std::ios_base::binary | std::ios_base::in);
+ std::ifstream bdb_dump(bdb_dumpfile, std::ios_base::binary | std::ios_base::in);
+ assert(std::equal(
+ std::istreambuf_iterator<char>(bdb_ro_dump.rdbuf()),
+ std::istreambuf_iterator<char>(),
+ std::istreambuf_iterator<char>(bdb_dump.rdbuf())));
+ }
+#endif
+}
diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp
index 49d206f409..b21a9a601d 100644
--- a/src/wallet/test/util.cpp
+++ b/src/wallet/test/util.cpp
@@ -93,11 +93,6 @@ CTxDestination getNewDestination(CWallet& w, OutputType output_type)
return *Assert(w.GetNewDestination(output_type, ""));
}
-// BytePrefix compares equality with other byte spans that begin with the same prefix.
-struct BytePrefix { Span<const std::byte> prefix; };
-bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); }
-bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; }
-
MockableCursor::MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix)
{
m_pass = pass;
diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp
index 45f69f52d1..8a79cf730b 100644
--- a/src/wallet/wallet.cpp
+++ b/src/wallet/wallet.cpp
@@ -373,7 +373,12 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string&
uint64_t wallet_creation_flags = options.create_flags;
const SecureString& passphrase = options.create_passphrase;
+ ArgsManager& args = *Assert(context.args);
+
if (wallet_creation_flags & WALLET_FLAG_DESCRIPTORS) options.require_format = DatabaseFormat::SQLITE;
+ else if (args.GetBoolArg("-swapbdbendian", false)) {
+ options.require_format = DatabaseFormat::BERKELEY_SWAP;
+ }
// Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted
bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET);
diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp
index 3ba43cdb73..f34fcfc3fd 100644
--- a/src/wallet/walletdb.cpp
+++ b/src/wallet/walletdb.cpp
@@ -21,6 +21,7 @@
#ifdef USE_BDB
#include <wallet/bdb.h>
#endif
+#include <wallet/migrate.h>
#ifdef USE_SQLITE
#include <wallet/sqlite.h>
#endif
@@ -1387,6 +1388,11 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas
return nullptr;
}
+ // If BERKELEY was the format, then change the format from BERKELEY to BERKELEY_RO
+ if (format && options.require_format && format == DatabaseFormat::BERKELEY && options.require_format == DatabaseFormat::BERKELEY_RO) {
+ format = DatabaseFormat::BERKELEY_RO;
+ }
+
// A db already exists so format is set, but options also specifies the format, so make sure they agree
if (format && options.require_format && format != options.require_format) {
error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", fs::PathToString(path)));
@@ -1420,6 +1426,10 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas
}
}
+ if (format == DatabaseFormat::BERKELEY_RO) {
+ return MakeBerkeleyRODatabase(path, options, status, error);
+ }
+
#ifdef USE_BDB
if constexpr (true) {
return MakeBerkeleyDatabase(path, options, status, error);
diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp
index 7a1930fd31..10785ad354 100644
--- a/src/wallet/wallettool.cpp
+++ b/src/wallet/wallettool.cpp
@@ -192,6 +192,11 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command)
ReadDatabaseArgs(args, options);
options.require_existing = true;
DatabaseStatus status;
+
+ if (args.GetBoolArg("-withinternalbdb", false) && IsBDBFile(BDBDataFile(path))) {
+ options.require_format = DatabaseFormat::BERKELEY_RO;
+ }
+
bilingual_str error;
std::unique_ptr<WalletDatabase> database = MakeDatabase(path, options, status, error);
if (!database) {
generated by cgit v1.2.3 (git 2.26.2) at 2024-11-20 04:28:50 +0000