diff options
| -rw-r--r-- | contrib/linearize/README.md | 13 | ||||
| -rwxr-xr-x | contrib/linearize/linearize-data.py | 28 | ||||
| -rwxr-xr-x | contrib/linearize/linearize-hashes.py | 32 |
3 files changed, 47 insertions, 26 deletions
diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 07597481e0..adc9a559cc 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -1,5 +1,6 @@ # Linearize -Construct a linear, no-fork, best version of the Bitcoin blockchain. +Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts +run using Python 3 but are compatible with Python 2. ## Step 1: Download hash list @@ -9,7 +10,8 @@ Required configuration file settings for linearize-hashes: * RPC: `rpcuser`, `rpcpassword` Optional config file setting for linearize-hashes: -* RPC: `host`, `port` (Default: `127.0.0.1:8332`) +* RPC: `host` (Default: `127.0.0.1`) +* RPC: `port` (Default: `8332`) * Blockchain: `min_height`, `max_height` * `rev_hash_bytes`: If true, the written block hash list will be byte-reversed. (In other words, the hash returned by getblockhash will have its @@ -17,6 +19,9 @@ bytes reversed.) False by default. Intended for generation of standalone hash lists but safe to use with linearize-data.py, which will output the same data no matter which byte format is chosen. +The `linearize-hashes` script requires a connection, local or remote, to a +JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient. + ## Step 2: Copy local block data $ ./linearize-data.py linearize.cfg @@ -24,13 +29,13 @@ the same data no matter which byte format is chosen. Required configuration file settings: * `output_file`: The file that will contain the final blockchain. or -* `output`: Output directory for linearized blocks/blkNNNNN.dat output. +* `output`: Output directory for linearized `blocks/blkNNNNN.dat` output. Optional config file setting for linearize-data: * `file_timestamp`: Set each file's last-modified time to that of the most recent block in that file. * `genesis`: The hash of the genesis block in the blockchain. -* `input: bitcoind blocks/ directory containing blkNNNNN.dat +* `input`: bitcoind blocks/ directory containing blkNNNNN.dat * `hashlist`: text file containing list of block hashes created by linearize-hashes.py. * `max_out_sz`: Maximum size for files created by the `output_file` option. diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index adea35dbd5..043bf2e814 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # linearize-data.py: Construct a linear, no-fork version of the chain. # @@ -8,29 +8,33 @@ # from __future__ import print_function, division +try: # Python 3 + import http.client as httplib +except ImportError: # Python 2 + import httplib import json import struct import re import os import os.path import base64 -import httplib import sys import hashlib import datetime import time from collections import namedtuple +from binascii import hexlify, unhexlify settings = {} ##### Switch endian-ness ##### def hex_switchEndian(s): """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i]+s[i+1] for i in range(0,len(s),2)] - return ''.join(pairList[::-1]) + pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] + return b''.join(pairList[::-1]).decode() def uint32(x): - return x & 0xffffffffL + return x & 0xffffffff def bytereverse(x): return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | @@ -41,14 +45,14 @@ def bufreverse(in_buf): for i in range(0, len(in_buf), 4): word = struct.unpack('@I', in_buf[i:i+4])[0] out_words.append(struct.pack('@I', bytereverse(word))) - return ''.join(out_words) + return b''.join(out_words) def wordreverse(in_buf): out_words = [] for i in range(0, len(in_buf), 4): out_words.append(in_buf[i:i+4]) out_words.reverse() - return ''.join(out_words) + return b''.join(out_words) def calc_hdr_hash(blk_hdr): hash1 = hashlib.sha256() @@ -65,7 +69,7 @@ def calc_hash_str(blk_hdr): hash = calc_hdr_hash(blk_hdr) hash = bufreverse(hash) hash = wordreverse(hash) - hash_str = hash.encode('hex') + hash_str = hexlify(hash).decode('utf-8') return hash_str def get_blk_dt(blk_hdr): @@ -217,7 +221,7 @@ class BlockDataCopier: inMagic = inhdr[:4] if (inMagic != self.settings['netmagic']): - print("Invalid magic: " + inMagic.encode('hex')) + print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) return inLenLE = inhdr[4:] su = struct.unpack("<I", inLenLE) @@ -294,14 +298,14 @@ if __name__ == '__main__': if 'split_timestamp' not in settings: settings['split_timestamp'] = 0 if 'max_out_sz' not in settings: - settings['max_out_sz'] = 1000L * 1000 * 1000 + settings['max_out_sz'] = 1000 * 1000 * 1000 if 'out_of_order_cache_sz' not in settings: settings['out_of_order_cache_sz'] = 100 * 1000 * 1000 - settings['max_out_sz'] = long(settings['max_out_sz']) + settings['max_out_sz'] = int(settings['max_out_sz']) settings['split_timestamp'] = int(settings['split_timestamp']) settings['file_timestamp'] = int(settings['file_timestamp']) - settings['netmagic'] = settings['netmagic'].decode('hex') + settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8')) settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) if 'output_file' not in settings and 'output' not in settings: diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 9efdf13255..f749da5396 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # @@ -8,11 +8,14 @@ # from __future__ import print_function +try: # Python 3 + import http.client as httplib +except ImportError: # Python 2 + import httplib import json import struct import re import base64 -import httplib import sys settings = {} @@ -20,26 +23,32 @@ settings = {} ##### Switch endian-ness ##### def hex_switchEndian(s): """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i]+s[i+1] for i in range(0,len(s),2)] - return ''.join(pairList[::-1]) + pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] + return b''.join(pairList[::-1]).decode() class BitcoinRPC: def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) - self.authhdr = "Basic %s" % (base64.b64encode(authpair)) - self.conn = httplib.HTTPConnection(host, port, False, 30) + authpair = authpair.encode('utf-8') + self.authhdr = b"Basic " + base64.b64encode(authpair) + self.conn = httplib.HTTPConnection(host, port=port, timeout=30) def execute(self, obj): - self.conn.request('POST', '/', json.dumps(obj), - { 'Authorization' : self.authhdr, - 'Content-type' : 'application/json' }) + try: + self.conn.request('POST', '/', json.dumps(obj), + { 'Authorization' : self.authhdr, + 'Content-type' : 'application/json' }) + except ConnectionRefusedError: + print('RPC connection refused. Check RPC settings and the server status.', + file=sys.stderr) + return None resp = self.conn.getresponse() if resp is None: print("JSON-RPC: no response", file=sys.stderr) return None - body = resp.read() + body = resp.read().decode('utf-8') resp_obj = json.loads(body) return resp_obj @@ -70,6 +79,9 @@ def get_block_hashes(settings, max_blocks_per_call=10000): batch.append(rpc.build_request(x, 'getblockhash', [height + x])) reply = rpc.execute(batch) + if reply is None: + print('Cannot continue. Program will halt.') + return None for x,resp_obj in enumerate(reply): if rpc.response_is_error(resp_obj): |