diff options
Diffstat (limited to 'contrib/linearize/linearize-data.py')
| -rwxr-xr-x | contrib/linearize/linearize-data.py | 44 |
1 files changed, 35 insertions, 9 deletions
diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 77bae6e3c6..ea94f25fae 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -14,8 +14,7 @@ import base64 import httplib import sys import hashlib - -MAX_OUT_SZ = 128 * 1024 * 1024 +import datetime settings = {} @@ -41,9 +40,7 @@ def wordreverse(in_buf): out_words.reverse() return ''.join(out_words) -def calc_hdr_hash(rawblock): - blk_hdr = rawblock[:80] - +def calc_hdr_hash(blk_hdr): hash1 = hashlib.sha256() hash1.update(blk_hdr) hash1_o = hash1.digest() @@ -54,13 +51,18 @@ def calc_hdr_hash(rawblock): return hash2_o -def calc_hash_str(rawblock): - hash = calc_hdr_hash(rawblock) +def calc_hash_str(blk_hdr): + hash = calc_hdr_hash(blk_hdr) hash = bufreverse(hash) hash = wordreverse(hash) hash_str = hash.encode('hex') return hash_str +def get_blk_year(blk_hdr): + members = struct.unpack("<I", blk_hdr[68:68+4]) + dt = datetime.datetime.fromtimestamp(members[0]) + return dt.year + def get_block_hashes(settings): blkindex = [] f = open(settings['hashlist'], "r") @@ -86,9 +88,14 @@ def copydata(settings, blkindex, blkset): outF = None blkCount = 0 + lastYear = 0 + splitYear = False fileOutput = True + maxOutSz = settings['max_out_sz'] if 'output' in settings: fileOutput = False + if settings['split_year'] != 0: + splitYear = True while True: if not inF: @@ -111,17 +118,30 @@ def copydata(settings, blkindex, blkset): su = struct.unpack("<I", inLenLE) inLen = su[0] rawblock = inF.read(inLen) + blk_hdr = rawblock[:80] - hash_str = calc_hash_str(rawblock) + hash_str = calc_hash_str(blk_hdr) if not hash_str in blkset: print("Skipping unknown block " + hash_str) continue - if not fileOutput and ((outsz + inLen) > MAX_OUT_SZ): + if not fileOutput and ((outsz + inLen) > maxOutSz): outF.close() outF = None outFn = outFn + 1 outsz = 0 + + if splitYear: + blkYear = get_blk_year(blk_hdr) + if blkYear > lastYear: + print("New year " + str(blkYear) + " @ " + hash_str) + lastYear = blkYear + if outF: + outF.close() + outF = None + outFn = outFn + 1 + outsz = 0 + if not outF: if fileOutput: fname = settings['output_file'] @@ -164,7 +184,13 @@ if __name__ == '__main__': settings['input'] = 'input' if 'hashlist' not in settings: settings['hashlist'] = 'hashlist.txt' + if 'split_year' not in settings: + settings['split_year'] = 0 + if 'max_out_sz' not in settings: + settings['max_out_sz'] = 1000L * 1000 * 1000 + settings['max_out_sz'] = long(settings['max_out_sz']) + settings['split_year'] = int(settings['split_year']) settings['netmagic'] = settings['netmagic'].decode('hex') if 'output_file' not in settings and 'output' not in settings: |