1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
|
#!/usr/bin/env python3
# Copyright (c) 2021 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test for assumeutxo, a means of quickly bootstrapping a node using
a serialized version of the UTXO set at a certain height, which corresponds
to a hash that has been compiled into bitcoind.
The assumeutxo value generated and used here is committed to in
`CRegTestParams::m_assumeutxo_data` in `src/chainparams.cpp`.
## Possible test improvements
- TODO: test submitting a transaction and verifying it appears in mempool
- TODO: test what happens with -reindex and -reindex-chainstate before the
snapshot is validated, and make sure it's deleted successfully.
Interesting test cases could be loading an assumeutxo snapshot file with:
- TODO: An invalid hash
- TODO: Valid hash but invalid snapshot file (bad coin height or truncated file or
bad other serialization)
- TODO: Valid snapshot file, but referencing an unknown block
- TODO: Valid snapshot file, but referencing a snapshot block that turns out to be
invalid, or has an invalid parent
- TODO: Valid snapshot file and snapshot block, but the block is not on the
most-work chain
Interesting starting states could be loading a snapshot when the current chain tip is:
- TODO: An ancestor of snapshot block
- TODO: Not an ancestor of the snapshot block but has less work
- TODO: The snapshot block
- TODO: A descendant of the snapshot block
- TODO: Not an ancestor or a descendant of the snapshot block and has more work
"""
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
assert_equal,
assert_raises_rpc_error,
)
import struct
START_HEIGHT = 199
SNAPSHOT_BASE_HEIGHT = 299
FINAL_HEIGHT = 399
COMPLETE_IDX = {'synced': True, 'best_block_height': FINAL_HEIGHT}
class AssumeutxoTest(BitcoinTestFramework):
def set_test_params(self):
"""Use the pregenerated, deterministic chain up to height 199."""
self.num_nodes = 3
self.rpc_timeout = 120
self.extra_args = [
[],
["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"],
["-txindex=1", "-blockfilterindex=1", "-coinstatsindex=1"],
]
def setup_network(self):
"""Start with the nodes disconnected so that one can generate a snapshot
including blocks the other hasn't yet seen."""
self.add_nodes(3)
self.start_nodes(extra_args=self.extra_args)
def test_invalid_snapshot_scenarios(self, valid_snapshot_path):
self.log.info("Test different scenarios of loading invalid snapshot files")
with open(valid_snapshot_path, 'rb') as f:
valid_snapshot_contents = f.read()
bad_snapshot_path = valid_snapshot_path + '.mod'
self.log.info(" - snapshot file refering to a block that is not in the assumeutxo parameters")
# we can only test this with a block that is already known, as otherwise the `loadtxoutset` RPC
# would time out (waiting to see the hash in the headers chain), rather than error immediately
bad_snapshot_height = SNAPSHOT_BASE_HEIGHT - 1
with open(bad_snapshot_path, 'wb') as f:
bad_snapshot_block_hash = self.nodes[0].getblockhash(bad_snapshot_height)
# block hash of the snapshot base is stored right at the start (first 32 bytes)
f.write(bytes.fromhex(bad_snapshot_block_hash)[::-1] + valid_snapshot_contents[32:])
expected_log = f"assumeutxo height in snapshot metadata not recognized ({bad_snapshot_height}) - refusing to load snapshot"
with self.nodes[1].assert_debug_log([expected_log]):
assert_raises_rpc_error(-32603, "Unable to load UTXO snapshot", self.nodes[1].loadtxoutset, bad_snapshot_path)
self.log.info(" - snapshot file with wrong number of coins")
valid_num_coins = struct.unpack("<I", valid_snapshot_contents[32:32 + 4])[0]
for off in [-1, +1]:
with open(bad_snapshot_path, 'wb') as f:
f.write(valid_snapshot_contents[:32])
f.write(struct.pack("<I", valid_num_coins + off))
f.write(valid_snapshot_contents[32 + 4:])
expected_log = f"bad snapshot - coins left over after deserializing 298 coins" if off == -1 else f"bad snapshot format or truncated snapshot after deserializing 299 coins"
with self.nodes[1].assert_debug_log([expected_log]):
assert_raises_rpc_error(-32603, "Unable to load UTXO snapshot", self.nodes[1].loadtxoutset, bad_snapshot_path)
def run_test(self):
"""
Bring up two (disconnected) nodes, mine some new blocks on the first,
and generate a UTXO snapshot.
Load the snapshot into the second, ensure it syncs to tip and completes
background validation when connected to the first.
"""
n0 = self.nodes[0]
n1 = self.nodes[1]
n2 = self.nodes[2]
# Mock time for a deterministic chain
for n in self.nodes:
n.setmocktime(n.getblockheader(n.getbestblockhash())['time'])
self.sync_blocks()
# Generate a series of blocks that `n0` will have in the snapshot,
# but that n1 doesn't yet see. In order for the snapshot to activate,
# though, we have to ferry over the new headers to n1 so that it
# isn't waiting forever to see the header of the snapshot's base block
# while disconnected from n0.
for i in range(100):
self.generate(n0, nblocks=1, sync_fun=self.no_op)
newblock = n0.getblock(n0.getbestblockhash(), 0)
# make n1 aware of the new header, but don't give it the block.
n1.submitheader(newblock)
n2.submitheader(newblock)
# Ensure everyone is seeing the same headers.
for n in self.nodes:
assert_equal(n.getblockchaininfo()["headers"], SNAPSHOT_BASE_HEIGHT)
self.log.info("-- Testing assumeutxo + some indexes + pruning")
assert_equal(n0.getblockcount(), SNAPSHOT_BASE_HEIGHT)
assert_equal(n1.getblockcount(), START_HEIGHT)
self.log.info(f"Creating a UTXO snapshot at height {SNAPSHOT_BASE_HEIGHT}")
dump_output = n0.dumptxoutset('utxos.dat')
assert_equal(
dump_output['txoutset_hash'],
'ef45ccdca5898b6c2145e4581d2b88c56564dd389e4bd75a1aaf6961d3edd3c0')
assert_equal(dump_output['nchaintx'], 300)
assert_equal(n0.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT)
# Mine more blocks on top of the snapshot that n1 hasn't yet seen. This
# will allow us to test n1's sync-to-tip on top of a snapshot.
self.generate(n0, nblocks=100, sync_fun=self.no_op)
assert_equal(n0.getblockcount(), FINAL_HEIGHT)
assert_equal(n1.getblockcount(), START_HEIGHT)
assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT)
self.test_invalid_snapshot_scenarios(dump_output['path'])
self.log.info(f"Loading snapshot into second node from {dump_output['path']}")
loaded = n1.loadtxoutset(dump_output['path'])
assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT)
assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT)
normal, snapshot = n1.getchainstates()["chainstates"]
assert_equal(normal['blocks'], START_HEIGHT)
assert_equal(normal.get('snapshot_blockhash'), None)
assert_equal(normal['validated'], True)
assert_equal(snapshot['blocks'], SNAPSHOT_BASE_HEIGHT)
assert_equal(snapshot['snapshot_blockhash'], dump_output['base_hash'])
assert_equal(snapshot['validated'], False)
assert_equal(n1.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT)
PAUSE_HEIGHT = FINAL_HEIGHT - 40
self.log.info("Restarting node to stop at height %d", PAUSE_HEIGHT)
self.restart_node(1, extra_args=[
f"-stopatheight={PAUSE_HEIGHT}", *self.extra_args[1]])
# Finally connect the nodes and let them sync.
#
# Set `wait_for_connect=False` to avoid a race between performing connection
# assertions and the -stopatheight tripping.
self.connect_nodes(0, 1, wait_for_connect=False)
n1.wait_until_stopped(timeout=5)
self.log.info("Checking that blocks are segmented on disk")
assert self.has_blockfile(n1, "00000"), "normal blockfile missing"
assert self.has_blockfile(n1, "00001"), "assumed blockfile missing"
assert not self.has_blockfile(n1, "00002"), "too many blockfiles"
self.log.info("Restarted node before snapshot validation completed, reloading...")
self.restart_node(1, extra_args=self.extra_args[1])
self.connect_nodes(0, 1)
self.log.info(f"Ensuring snapshot chain syncs to tip. ({FINAL_HEIGHT})")
self.wait_until(lambda: n1.getchainstates()['chainstates'][-1]['blocks'] == FINAL_HEIGHT)
self.sync_blocks(nodes=(n0, n1))
self.log.info("Ensuring background validation completes")
self.wait_until(lambda: len(n1.getchainstates()['chainstates']) == 1)
# Ensure indexes have synced.
completed_idx_state = {
'basic block filter index': COMPLETE_IDX,
'coinstatsindex': COMPLETE_IDX,
}
self.wait_until(lambda: n1.getindexinfo() == completed_idx_state)
for i in (0, 1):
n = self.nodes[i]
self.log.info(f"Restarting node {i} to ensure (Check|Load)BlockIndex passes")
self.restart_node(i, extra_args=self.extra_args[i])
assert_equal(n.getblockchaininfo()["blocks"], FINAL_HEIGHT)
chainstate, = n.getchainstates()['chainstates']
assert_equal(chainstate['blocks'], FINAL_HEIGHT)
if i != 0:
# Ensure indexes have synced for the assumeutxo node
self.wait_until(lambda: n.getindexinfo() == completed_idx_state)
# Node 2: all indexes + reindex
# -----------------------------
self.log.info("-- Testing all indexes + reindex")
assert_equal(n2.getblockcount(), START_HEIGHT)
self.log.info(f"Loading snapshot into third node from {dump_output['path']}")
loaded = n2.loadtxoutset(dump_output['path'])
assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT)
assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT)
normal, snapshot = n2.getchainstates()['chainstates']
assert_equal(normal['blocks'], START_HEIGHT)
assert_equal(normal.get('snapshot_blockhash'), None)
assert_equal(normal['validated'], True)
assert_equal(snapshot['blocks'], SNAPSHOT_BASE_HEIGHT)
assert_equal(snapshot['snapshot_blockhash'], dump_output['base_hash'])
assert_equal(snapshot['validated'], False)
self.connect_nodes(0, 2)
self.wait_until(lambda: n2.getchainstates()['chainstates'][-1]['blocks'] == FINAL_HEIGHT)
self.sync_blocks()
self.log.info("Ensuring background validation completes")
self.wait_until(lambda: len(n2.getchainstates()['chainstates']) == 1)
completed_idx_state = {
'basic block filter index': COMPLETE_IDX,
'coinstatsindex': COMPLETE_IDX,
'txindex': COMPLETE_IDX,
}
self.wait_until(lambda: n2.getindexinfo() == completed_idx_state)
for i in (0, 2):
n = self.nodes[i]
self.log.info(f"Restarting node {i} to ensure (Check|Load)BlockIndex passes")
self.restart_node(i, extra_args=self.extra_args[i])
assert_equal(n.getblockchaininfo()["blocks"], FINAL_HEIGHT)
chainstate, = n.getchainstates()['chainstates']
assert_equal(chainstate['blocks'], FINAL_HEIGHT)
if i != 0:
# Ensure indexes have synced for the assumeutxo node
self.wait_until(lambda: n.getindexinfo() == completed_idx_state)
self.log.info("Test -reindex-chainstate of an assumeutxo-synced node")
self.restart_node(2, extra_args=[
'-reindex-chainstate=1', *self.extra_args[2]])
assert_equal(n2.getblockchaininfo()["blocks"], FINAL_HEIGHT)
self.wait_until(lambda: n2.getblockcount() == FINAL_HEIGHT)
self.log.info("Test -reindex of an assumeutxo-synced node")
self.restart_node(2, extra_args=['-reindex=1', *self.extra_args[2]])
self.connect_nodes(0, 2)
self.wait_until(lambda: n2.getblockcount() == FINAL_HEIGHT)
if __name__ == '__main__':
AssumeutxoTest().main()
|
