diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/linearize/README.md | 5 | ||||
| -rwxr-xr-x | contrib/linearize/linearize-data.py | 73 |
2 files changed, 52 insertions, 26 deletions
diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index b5c6e7824e..157586e4d4 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -27,6 +27,7 @@ output. Optional config file setting for linearize-data: * "netmagic": network magic number * "max_out_sz": maximum output file size (default 1000*1000*1000) -* "split_year": Split files when a new year is first seen, in addition to +* "split_timestamp": Split files when a new month is first seen, in addition to reaching a maximum file size. - +* "file_timestamp": Set each file's last-modified time to that of the +most recent block in that file. diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index ea94f25fae..383bb38198 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -10,11 +10,13 @@ import json import struct import re +import os import base64 import httplib import sys import hashlib import datetime +import time settings = {} @@ -58,10 +60,12 @@ def calc_hash_str(blk_hdr): hash_str = hash.encode('hex') return hash_str -def get_blk_year(blk_hdr): +def get_blk_dt(blk_hdr): members = struct.unpack("<I", blk_hdr[68:68+4]) - dt = datetime.datetime.fromtimestamp(members[0]) - return dt.year + nTime = members[0] + dt = datetime.datetime.fromtimestamp(nTime) + dt_ym = datetime.datetime(dt.year, dt.month, 1) + return (dt_ym, nTime) def get_block_hashes(settings): blkindex = [] @@ -86,16 +90,21 @@ def copydata(settings, blkindex, blkset): outFn = 0 outsz = 0 outF = None + outFname = None blkCount = 0 - lastYear = 0 - splitYear = False + lastDate = datetime.datetime(2000, 1, 1) + highTS = 1408893517 - 315360000 + timestampSplit = False fileOutput = True + setFileTime = False maxOutSz = settings['max_out_sz'] if 'output' in settings: fileOutput = False - if settings['split_year'] != 0: - splitYear = True + if settings['file_timestamp'] != 0: + setFileTime = True + if settings['split_timestamp'] != 0: + timestampSplit = True while True: if not inF: @@ -125,36 +134,49 @@ def copydata(settings, blkindex, blkset): print("Skipping unknown block " + hash_str) continue + if blkindex[blkCount] != hash_str: + print("Out of order block.") + print("Expected " + blkindex[blkCount]) + print("Got " + hash_str) + sys.exit(1) + if not fileOutput and ((outsz + inLen) > maxOutSz): outF.close() + if setFileTime: + os.utime(outFname, (int(time.time()), highTS)) outF = None + outFname = None outFn = outFn + 1 outsz = 0 - if splitYear: - blkYear = get_blk_year(blk_hdr) - if blkYear > lastYear: - print("New year " + str(blkYear) + " @ " + hash_str) - lastYear = blkYear - if outF: - outF.close() - outF = None - outFn = outFn + 1 - outsz = 0 + (blkDate, blkTS) = get_blk_dt(blk_hdr) + if timestampSplit and (blkDate > lastDate): + print("New month " + blkDate.strftime("%Y-%m") + " @ " + hash_str) + lastDate = blkDate + if outF: + outF.close() + if setFileTime: + os.utime(outFname, (int(time.time()), highTS)) + outF = None + outFname = None + outFn = outFn + 1 + outsz = 0 if not outF: if fileOutput: - fname = settings['output_file'] + outFname = settings['output_file'] else: - fname = "%s/blk%05d.dat" % (settings['output'], outFn) - print("Output file" + fname) - outF = open(fname, "wb") + outFname = "%s/blk%05d.dat" % (settings['output'], outFn) + print("Output file" + outFname) + outF = open(outFname, "wb") outF.write(inhdr) outF.write(rawblock) outsz = outsz + inLen + 8 blkCount = blkCount + 1 + if blkTS > highTS: + highTS = blkTS if (blkCount % 1000) == 0: print("Wrote " + str(blkCount) + " blocks") @@ -184,13 +206,16 @@ if __name__ == '__main__': settings['input'] = 'input' if 'hashlist' not in settings: settings['hashlist'] = 'hashlist.txt' - if 'split_year' not in settings: - settings['split_year'] = 0 + if 'file_timestamp' not in settings: + settings['file_timestamp'] = 0 + if 'split_timestamp' not in settings: + settings['split_timestamp'] = 0 if 'max_out_sz' not in settings: settings['max_out_sz'] = 1000L * 1000 * 1000 settings['max_out_sz'] = long(settings['max_out_sz']) - settings['split_year'] = int(settings['split_year']) + settings['split_timestamp'] = int(settings['split_timestamp']) + settings['file_timestamp'] = int(settings['file_timestamp']) settings['netmagic'] = settings['netmagic'].decode('hex') if 'output_file' not in settings and 'output' not in settings: |