1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
#!/usr/bin/env python3
# Copyright (c) 2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""
Utilities for working directly with the wallet's BDB database file
This is specific to the configuration of BDB used in this project:
- pagesize: 4096 bytes
- Outer database contains single subdatabase named 'main'
- btree
- btree leaf pages
Each key-value pair is two entries in a btree leaf. The first is the key, the one that follows
is the value. And so on. Note that the entry data is itself not in the correct order. Instead
entry offsets are stored in the correct order and those offsets are needed to then retrieve
the data itself.
Page format can be found in BDB source code dbinc/db_page.h
This only implements the deserialization of btree metadata pages and normal btree pages. Overflow
pages are not implemented but may be needed in the future if dealing with wallets with large
transactions.
`db_dump -da wallet.dat` is useful to see the data in a wallet.dat BDB file
"""
import struct
# Important constants
PAGESIZE = 4096
OUTER_META_PAGE = 0
INNER_META_PAGE = 2
# Page type values
BTREE_INTERNAL = 3
BTREE_LEAF = 5
BTREE_META = 9
# Some magic numbers for sanity checking
BTREE_MAGIC = 0x053162
DB_VERSION = 9
# Deserializes a leaf page into a dict.
# Btree internal pages have the same header, for those, return None.
# For the btree leaf pages, deserialize them and put all the data into a dict
def dump_leaf_page(data):
page_info = {}
page_header = data[0:26]
_, pgno, prev_pgno, next_pgno, entries, hf_offset, level, pg_type = struct.unpack('QIIIHHBB', page_header)
page_info['pgno'] = pgno
page_info['prev_pgno'] = prev_pgno
page_info['next_pgno'] = next_pgno
page_info['hf_offset'] = hf_offset
page_info['level'] = level
page_info['pg_type'] = pg_type
page_info['entry_offsets'] = struct.unpack('{}H'.format(entries), data[26:26 + entries * 2])
page_info['entries'] = []
if pg_type == BTREE_INTERNAL:
# Skip internal pages. These are the internal nodes of the btree and don't contain anything relevant to us
return None
assert pg_type == BTREE_LEAF, 'A non-btree leaf page has been encountered while dumping leaves'
for i in range(0, entries):
offset = page_info['entry_offsets'][i]
entry = {'offset': offset}
page_data_header = data[offset:offset + 3]
e_len, pg_type = struct.unpack('HB', page_data_header)
entry['len'] = e_len
entry['pg_type'] = pg_type
entry['data'] = data[offset + 3:offset + 3 + e_len]
page_info['entries'].append(entry)
return page_info
# Deserializes a btree metadata page into a dict.
# Does a simple sanity check on the magic value, type, and version
def dump_meta_page(page):
# metadata page
# general metadata
metadata = {}
meta_page = page[0:72]
_, pgno, magic, version, pagesize, encrypt_alg, pg_type, metaflags, _, free, last_pgno, nparts, key_count, record_count, flags, uid = struct.unpack('QIIIIBBBBIIIIII20s', meta_page)
metadata['pgno'] = pgno
metadata['magic'] = magic
metadata['version'] = version
metadata['pagesize'] = pagesize
metadata['encrypt_alg'] = encrypt_alg
metadata['pg_type'] = pg_type
metadata['metaflags'] = metaflags
metadata['free'] = free
metadata['last_pgno'] = last_pgno
metadata['nparts'] = nparts
metadata['key_count'] = key_count
metadata['record_count'] = record_count
metadata['flags'] = flags
metadata['uid'] = uid.hex().encode()
assert magic == BTREE_MAGIC, 'bdb magic does not match bdb btree magic'
assert pg_type == BTREE_META, 'Metadata page is not a btree metadata page'
assert version == DB_VERSION, 'Database too new'
# btree metadata
btree_meta_page = page[72:512]
_, minkey, re_len, re_pad, root, _, crypto_magic, _, iv, chksum = struct.unpack('IIIII368sI12s16s20s', btree_meta_page)
metadata['minkey'] = minkey
metadata['re_len'] = re_len
metadata['re_pad'] = re_pad
metadata['root'] = root
metadata['crypto_magic'] = crypto_magic
metadata['iv'] = iv.hex().encode()
metadata['chksum'] = chksum.hex().encode()
return metadata
# Given the dict from dump_leaf_page, get the key-value pairs and put them into a dict
def extract_kv_pairs(page_data):
out = {}
last_key = None
for i, entry in enumerate(page_data['entries']):
# By virtue of these all being pairs, even number entries are keys, and odd are values
if i % 2 == 0:
out[entry['data']] = b''
last_key = entry['data']
else:
out[last_key] = entry['data']
return out
# Extract the key-value pairs of the BDB file given in filename
def dump_bdb_kv(filename):
# Read in the BDB file and start deserializing it
pages = []
with open(filename, 'rb') as f:
data = f.read(PAGESIZE)
while len(data) > 0:
pages.append(data)
data = f.read(PAGESIZE)
# Sanity check the meta pages
dump_meta_page(pages[OUTER_META_PAGE])
dump_meta_page(pages[INNER_META_PAGE])
# Fetch the kv pairs from the leaf pages
kv = {}
for i in range(3, len(pages)):
info = dump_leaf_page(pages[i])
if info is not None:
info_kv = extract_kv_pairs(info)
kv = {**kv, **info_kv}
return kv
|
