contrib: Improvements to hardcoded seeds scripts

- Moved all seed related scripts to contrib/seeds for consistency
- Updated `makeseeds.py` to handle IPv6 and onions, fix regular
  expression for recent Bitcoin Core versions
- Fixed a bug in `generate-seeds.py` with regard to IPv6 parsing

5 files changed, 735 insertions, 14 deletions

diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md
index bc88201f0f..63647fa11a 100644
--- a/contrib/seeds/README.md
+++ b/contrib/seeds/README.md
@@ -1,7 +1,7 @@
 ### Seeds ###
 
 Utility to generate the seeds.txt list that is compiled into the client
-(see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and [share/seeds](/share/seeds)).
+(see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and other utilities in [contrib/seeds](/contrib/seeds)).
 
 The 512 seeds compiled into the 0.10 release were created from sipa's DNS seed data, like this:
 
diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py
new file mode 100755
index 0000000000..167c219c6e
--- /dev/null
+++ b/contrib/seeds/generate-seeds.py
@@ -0,0 +1,138 @@
+#!/usr/bin/python
+# Copyright (c) 2014 Wladmir J. van der Laan
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+'''
+Script to generate list of seed nodes for chainparams.cpp.
+
+This script expects two text files in the directory that is passed as an
+argument:
+
+    nodes_main.txt
+    nodes_test.txt
+
+These files must consist of lines in the format 
+
+    <ip>
+    <ip>:<port>
+    [<ipv6>]
+    [<ipv6>]:<port>
+    <onion>.onion
+    0xDDBBCCAA (IPv4 little-endian old pnSeeds format)
+
+The output will be two data structures with the peers in binary format:
+
+   static SeedSpec6 pnSeed6_main[]={
+   ...
+   }
+   static SeedSpec6 pnSeed6_test[]={
+   ...
+   }
+
+These should be pasted into `src/chainparamsseeds.h`.
+'''
+from __future__ import print_function, division
+from base64 import b32decode
+from binascii import a2b_hex
+import sys, os
+import re
+
+# ipv4 in ipv6 prefix
+pchIPv4 = bytearray([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff])
+# tor-specific ipv6 prefix
+pchOnionCat = bytearray([0xFD,0x87,0xD8,0x7E,0xEB,0x43])
+
+def name_to_ipv6(addr):
+    if len(addr)>6 and addr.endswith('.onion'):
+        vchAddr = b32decode(addr[0:-6], True)
+        if len(vchAddr) != 16-len(pchOnionCat):
+            raise ValueError('Invalid onion %s' % s)
+        return pchOnionCat + vchAddr
+    elif '.' in addr: # IPv4
+        return pchIPv4 + bytearray((int(x) for x in addr.split('.')))
+    elif ':' in addr: # IPv6
+        sub = [[], []] # prefix, suffix
+        x = 0
+        addr = addr.split(':')
+        for i,comp in enumerate(addr):
+            if comp == '':
+                if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end
+                    continue
+                x += 1 # :: skips to suffix
+                assert(x < 2)
+            else: # two bytes per component
+                val = int(comp, 16)
+                sub[x].append(val >> 8)
+                sub[x].append(val & 0xff)
+        nullbytes = 16 - len(sub[0]) - len(sub[1])
+        assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0))
+        return bytearray(sub[0] + ([0] * nullbytes) + sub[1])
+    elif addr.startswith('0x'): # IPv4-in-little-endian
+        return pchIPv4 + bytearray(reversed(a2b_hex(addr[2:])))
+    else:
+        raise ValueError('Could not parse address %s' % addr)
+
+def parse_spec(s, defaultport):
+    match = re.match('\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s)
+    if match: # ipv6
+        host = match.group(1)
+        port = match.group(2)
+    elif s.count(':') > 1: # ipv6, no port
+        host = s
+        port = ''
+    else:
+        (host,_,port) = s.partition(':')
+
+    if not port:
+        port = defaultport
+    else:
+        port = int(port)
+
+    host = name_to_ipv6(host)
+
+    return (host,port)
+
+def process_nodes(g, f, structname, defaultport):
+    g.write('static SeedSpec6 %s[] = {\n' % structname)
+    first = True
+    for line in f:
+        comment = line.find('#')
+        if comment != -1:
+            line = line[0:comment]
+        line = line.strip()
+        if not line:
+            continue
+        if not first:
+            g.write(',\n')
+        first = False
+
+        (host,port) = parse_spec(line, defaultport)
+        hoststr = ','.join(('0x%02x' % b) for b in host)
+        g.write('    {{%s}, %i}' % (hoststr, port))
+    g.write('\n};\n')
+
+def main():
+    if len(sys.argv)<2:
+        print(('Usage: %s <path_to_nodes_txt>' % sys.argv[0]), file=sys.stderr)
+        exit(1)
+    g = sys.stdout
+    indir = sys.argv[1]
+    g.write('#ifndef BITCOIN_CHAINPARAMSSEEDS_H\n')
+    g.write('#define BITCOIN_CHAINPARAMSSEEDS_H\n')
+    g.write('/**\n')
+    g.write(' * List of fixed seed nodes for the bitcoin network\n')
+    g.write(' * AUTOGENERATED by contrib/seeds/generate-seeds.py\n')
+    g.write(' *\n')
+    g.write(' * Each line contains a 16-byte IPv6 address and a port.\n')
+    g.write(' * IPv4 as well as onion addresses are wrapped inside a IPv6 address accordingly.\n')
+    g.write(' */\n')
+    with open(os.path.join(indir,'nodes_main.txt'),'r') as f:
+        process_nodes(g, f, 'pnSeed6_main', 8333)
+    g.write('\n')
+    with open(os.path.join(indir,'nodes_test.txt'),'r') as f:
+        process_nodes(g, f, 'pnSeed6_test', 18333)
+    g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n')
+            
+if __name__ == '__main__':
+    main()
+
diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py
index b831395f2c..0bd91b8a7e 100755
--- a/contrib/seeds/makeseeds.py
+++ b/contrib/seeds/makeseeds.py
@@ -24,24 +24,42 @@ import sys
 import dns.resolver
 
 PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):8333$")
-PATTERN_AGENT = re.compile(r"^(\/Satoshi:0.8.6\/|\/Satoshi:0.9.(2|3)\/|\/Satoshi:0.10.\d{1,2}\/)$")
+PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:8333$")
+PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):8333$")
+PATTERN_AGENT = re.compile(r"^(\/Satoshi:0\.8\.6\/|\/Satoshi:0\.9\.(2|3|4|5)\/|\/Satoshi:0\.10\.\d{1,2}\/|\/Satoshi:0\.11\.\d{1,2}\/)$")
 
 def parseline(line):
     sline = line.split()
     if len(sline) < 11:
        return None
-    # Match only IPv4
     m = PATTERN_IPV4.match(sline[0])
+    sortkey = None
+    ip = None
     if m is None:
-        return None
-    # Do IPv4 sanity check
-    ip = 0
-    for i in range(0,4):
-        if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
+        m = PATTERN_IPV6.match(sline[0])
+        if m is None:
+            m = PATTERN_ONION.match(sline[0])
+            if m is None:
+                return None
+            else:
+                net = 'onion'
+                sortkey = m.group(1)
+        else:
+            net = 'ipv6'
+            if m.group(1) in ['::']: # Not interested in localhost
+                return None
+            sortkey = m.group(1) # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds
+    else:
+        # Do IPv4 sanity check
+        ip = 0
+        for i in range(0,4):
+            if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
+                return None
+            ip = ip + (int(m.group(i+2)) << (8*(3-i)))
+        if ip == 0:
             return None
-        ip = ip + (int(m.group(i+2)) << (8*(3-i)))
-    if ip == 0:
-        return None
+        net = 'ipv4'
+        sortkey = ip
     # Skip bad results.
     if sline[1] == 0:
         return None
@@ -59,6 +77,7 @@ def parseline(line):
     blocks = int(sline[8])
     # Construct result.
     return {
+        'net': net,
         'ip': m.group(1),
         'ipnum': ip,
         'uptime': uptime30,
@@ -67,13 +86,20 @@ def parseline(line):
         'agent': agent,
         'service': service,
         'blocks': blocks,
+        'sortkey': sortkey,
     }
 
 # Based on Greg Maxwell's seed_filter.py
 def filterbyasn(ips, max_per_asn, max_total):
+    # Sift out ips by type
+    ips_ipv4 = [ip for ip in ips if ip['net'] == 'ipv4']
+    ips_ipv6 = [ip for ip in ips if ip['net'] == 'ipv6']
+    ips_onion = [ip for ip in ips if ip['net'] == 'onion']
+
+    # Filter IPv4 by ASN
     result = []
     asn_count = {}
-    for ip in ips:
+    for ip in ips_ipv4:
         if len(result) == max_total:
             break
         try:
@@ -86,13 +112,19 @@ def filterbyasn(ips, max_per_asn, max_total):
             result.append(ip)
         except:
             sys.stderr.write('ERR: Could not resolve ASN for "' + ip['ip'] + '"\n')
+
+    # TODO: filter IPv6 by ASN
+
+    # Add back non-IPv4
+    result.extend(ips_ipv6)
+    result.extend(ips_onion)
     return result
 
 def main():
     lines = sys.stdin.readlines()
     ips = [parseline(line) for line in lines]
 
-    # Skip entries with valid IPv4 address.
+    # Skip entries with valid address.
     ips = [ip for ip in ips if ip is not None]
     # Skip entries from suspicious hosts.
     ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
@@ -109,7 +141,7 @@ def main():
     # Look up ASNs and limit results, both per ASN and globally.
     ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
     # Sort the results by IP address (for deterministic output).
-    ips.sort(key=lambda x: (x['ipnum']))
+    ips.sort(key=lambda x: (x['net'], x['sortkey']))
 
     for ip in ips:
         print ip['ip']
diff --git a/contrib/seeds/nodes_main.txt b/contrib/seeds/nodes_main.txt
new file mode 100644
index 0000000000..3dba6d8a64
--- /dev/null
+++ b/contrib/seeds/nodes_main.txt
@@ -0,0 +1,540 @@
+# List of fixed seed nodes for main network
+
+# IPv4 nodes (generated using contrib/seeds/makeseeds.py)
+1.33.197.110
+1.34.180.245
+1.202.128.218
+2.35.195.25
+5.100.123.19
+5.175.145.169
+5.199.133.193
+5.199.151.10
+5.228.1.230
+14.200.200.145
+18.228.0.188
+18.228.0.200
+23.30.243.153
+23.88.232.49
+23.99.105.9
+23.226.137.208
+23.227.177.161
+23.227.191.50
+23.229.45.32
+23.236.144.69
+23.253.148.113
+23.253.241.22
+23.255.227.231
+24.20.205.222
+24.23.120.252
+24.94.98.96
+24.98.95.201
+24.111.90.55
+24.119.119.105
+24.138.25.149
+31.3.214.45
+31.186.87.46
+31.186.101.98
+31.186.250.186
+31.204.153.107
+37.44.16.231
+37.44.44.11
+37.120.168.204
+37.143.86.26
+37.187.75.24
+37.188.68.169
+37.192.95.150
+37.201.246.116
+37.205.10.140
+46.10.210.17
+46.19.138.154
+46.28.204.123
+46.28.205.67
+46.38.235.229
+46.163.76.230
+46.166.162.91
+46.173.190.50
+46.227.66.132
+46.229.238.187
+46.236.116.209
+47.55.14.65
+50.7.252.229
+50.46.159.91
+50.78.49.181
+50.78.231.57
+50.79.153.65
+50.116.34.44
+50.126.86.253
+50.142.41.23
+50.199.113.193
+50.200.78.107
+50.206.138.177
+50.252.52.49
+54.165.25.75
+54.169.107.40
+54.179.190.56
+54.187.82.121
+54.246.85.246
+58.74.7.205
+58.96.183.121
+61.62.58.38
+61.63.91.72
+61.63.91.112
+61.72.211.228
+62.43.40.154
+62.43.130.178
+62.80.185.213
+62.109.49.26
+62.173.139.58
+62.181.238.186
+62.210.114.127
+63.141.228.138
+63.153.213.78
+63.223.84.145
+63.251.88.112
+64.31.110.50
+64.34.121.45
+64.114.6.42
+64.140.125.98
+64.156.193.100
+65.30.47.116
+65.35.132.177
+65.96.193.165
+65.111.189.26
+66.68.10.30
+66.114.33.250
+66.130.46.63
+66.175.215.135
+66.190.253.165
+66.194.38.254
+66.244.98.111
+67.162.238.30
+67.169.255.17
+67.183.173.25
+67.219.233.140
+67.227.240.115
+67.247.222.71
+68.43.114.66
+68.52.33.36
+68.198.245.241
+69.12.226.165
+69.13.198.188
+69.15.179.62
+69.39.239.47
+69.47.45.87
+69.62.217.206
+69.64.42.31
+69.64.81.61
+69.67.219.200
+69.90.132.157
+69.94.30.177
+69.136.175.241
+70.61.97.228
+70.123.118.132
+71.59.152.182
+71.198.248.151
+71.200.242.89
+71.225.179.157
+72.14.187.51
+72.38.34.180
+72.52.72.187
+72.91.144.182
+72.167.49.217
+72.201.243.55
+72.223.60.249
+72.228.153.102
+73.26.101.228
+73.50.158.200
+73.181.204.170
+74.57.199.180
+74.63.222.226
+74.81.231.21
+74.193.126.82
+74.207.235.164
+75.83.197.114
+75.144.114.9
+76.112.5.247
+76.174.20.247
+77.37.240.142
+77.57.202.107
+77.172.123.53
+77.221.91.253
+77.235.48.48
+77.245.78.2
+78.8.58.249
+78.27.191.182
+78.129.236.141
+78.131.88.47
+78.157.205.6
+79.132.230.144
+79.143.188.155
+79.160.221.140
+79.161.111.114
+80.100.189.3
+80.147.140.121
+80.203.75.133
+80.220.99.227
+80.222.20.169
+80.241.1.7
+81.23.191.243
+81.38.11.202
+81.80.9.71
+81.110.213.165
+81.133.155.237
+81.171.34.37
+81.181.155.180
+82.39.156.137
+82.73.161.95
+82.130.45.40
+82.165.153.47
+82.168.128.133
+82.179.225.118
+82.194.245.158
+82.199.102.10
+82.211.30.243
+82.217.133.145
+82.221.128.35
+82.221.131.177
+82.233.225.205
+83.0.249.146
+83.89.31.249
+83.128.29.231
+83.128.253.142
+83.143.130.56
+83.150.2.99
+83.150.9.196
+83.161.64.45
+83.212.103.212
+83.212.111.114
+83.246.75.8
+83.254.81.31
+83.254.150.54
+84.2.34.104
+84.15.61.60
+84.17.25.135
+84.42.144.19
+84.212.210.135
+84.215.165.231
+84.238.140.176
+84.240.31.184
+85.25.214.137
+85.139.163.132
+85.199.4.228
+85.214.61.209
+85.214.108.77
+86.123.16.17
+87.48.42.199
+87.104.168.104
+87.229.73.171
+87.236.196.77
+88.97.56.98
+88.134.178.89
+88.150.233.19
+88.168.133.3
+88.208.18.246
+88.208.33.202
+89.18.28.21
+89.85.220.84
+89.163.227.28
+89.184.83.60
+89.231.96.83
+89.236.49.117
+91.90.66.209
+91.106.194.97
+91.134.75.115
+91.152.193.36
+91.152.219.35
+91.197.10.234
+91.209.77.101
+91.210.106.147
+91.214.200.205
+91.223.115.38
+91.234.48.232
+91.250.86.18
+92.27.7.209
+92.255.207.73
+93.74.163.234
+93.84.114.106
+93.152.166.29
+93.171.216.221
+93.185.177.71
+94.19.12.244
+94.42.115.50
+94.79.177.206
+94.136.147.119
+94.143.245.5
+94.188.50.39
+94.190.227.112
+94.198.135.29
+94.226.107.86
+94.242.219.90
+94.242.229.168
+94.244.160.84
+95.31.10.209
+95.85.25.41
+95.105.161.136
+95.154.165.45
+95.154.200.216
+95.167.109.125
+95.211.125.231
+95.211.216.235
+96.33.25.17
+96.43.130.178
+97.118.8.236
+98.102.6.125
+98.202.20.45
+98.217.125.225
+98.234.210.111
+98.237.20.123
+98.255.144.176
+99.113.64.43
+99.229.22.8
+103.1.212.19
+103.30.42.189
+103.224.165.48
+103.243.94.140
+104.131.107.107
+104.131.116.184
+104.143.0.156
+104.219.184.9
+106.185.38.174
+107.6.4.145
+107.150.8.27
+107.150.33.20
+107.170.228.129
+107.170.240.173
+108.51.20.86
+108.61.149.222
+108.61.151.172
+108.161.129.247
+108.170.140.21
+109.60.211.216
+109.73.42.36
+109.73.172.138
+109.163.235.239
+109.190.196.220
+109.201.135.216
+109.228.152.2
+109.228.154.81
+109.230.220.125
+109.234.156.218
+109.235.49.27
+109.235.69.84
+112.124.71.0
+113.146.68.251
+115.29.17.82
+115.70.176.17
+117.41.162.184
+118.27.8.170
+119.230.7.211
+119.246.71.52
+121.172.8.100
+122.128.109.148
+123.231.224.63
+128.175.195.31
+128.199.164.96
+128.199.254.244
+129.97.69.76
+129.123.7.7
+129.123.7.39
+129.186.17.17
+131.247.169.190
+133.242.209.63
+134.102.94.38
+134.119.17.145
+137.116.160.176
+137.226.34.42
+138.210.217.170
+141.255.166.194
+143.215.129.126
+144.76.244.19
+146.148.52.162
+146.148.80.57
+146.185.19.30
+146.185.142.86
+146.185.253.51
+148.251.6.214
+149.154.155.235
+149.210.133.244
+151.224.248.252
+153.121.75.229
+153.127.251.67
+154.20.2.139
+157.13.61.5
+158.58.173.48
+159.253.23.132
+162.209.110.218
+162.213.254.205
+162.239.254.100
+162.242.150.39
+162.243.81.138
+162.243.235.56
+162.244.79.16
+162.245.217.119
+162.248.102.117
+162.251.108.53
+162.254.149.139
+162.255.116.78
+166.70.94.106
+167.88.45.124
+167.88.120.210
+173.26.49.43
+173.30.14.6
+173.80.114.197
+173.167.214.243
+173.208.219.108
+173.220.67.156
+173.236.101.34
+173.246.107.34
+173.255.237.241
+174.2.213.209
+174.51.23.224
+174.51.123.159
+174.57.212.121
+174.109.33.28
+175.126.124.91
+175.126.124.92
+176.10.116.242
+176.36.35.126
+176.36.99.222
+176.124.110.47
+176.194.33.44
+176.223.201.198
+178.62.26.83
+178.62.36.48
+178.62.212.141
+178.62.254.59
+178.78.250.3
+178.155.86.226
+178.175.134.35
+178.248.111.4
+178.254.1.170
+178.254.34.161
+179.43.114.14
+182.213.208.28
+184.68.2.46
+184.72.238.42
+184.94.226.34
+184.94.227.58
+184.107.139.58
+184.107.206.45
+185.10.48.117
+185.21.216.156
+185.38.47.224
+185.45.192.129
+185.53.129.230
+185.53.131.114
+185.55.53.61
+185.55.53.63
+185.61.119.2
+185.61.148.203
+186.2.167.23
+188.92.75.178
+188.122.92.134
+188.138.9.208
+188.165.209.148
+188.226.206.239
+190.10.8.124
+190.10.10.147
+192.0.130.142
+192.3.89.159
+192.73.234.138
+192.75.95.107
+192.95.100.102
+192.155.84.181
+192.169.233.206
+192.198.93.86
+192.227.135.216
+193.0.109.3
+193.77.50.208
+193.109.68.62
+193.150.121.37
+193.224.69.98
+194.79.8.37
+194.141.86.10
+195.12.180.94
+195.56.63.10
+195.116.93.93
+195.154.174.226
+195.159.111.98
+195.169.138.2
+195.189.126.35
+195.197.175.190
+197.242.93.82
+198.11.214.147
+198.49.41.21
+199.33.124.186
+199.204.186.146
+199.233.238.115
+199.241.189.66
+202.60.68.242
+202.60.69.232
+203.183.151.39
+203.219.14.204
+204.44.123.109
+204.44.123.162
+204.45.120.178
+206.190.134.44
+206.248.184.127
+207.244.73.8
+208.66.30.27
+209.81.9.223
+209.105.243.229
+209.126.70.159
+209.140.30.169
+209.165.128.235
+209.190.2.242
+210.66.254.236
+210.73.27.33
+211.72.66.229
+212.25.37.124
+212.71.235.114
+212.71.252.109
+212.114.48.31
+212.174.151.118
+213.66.205.194
+213.129.248.139
+213.136.87.34
+213.165.82.133
+213.167.17.6
+213.179.158.253
+213.189.53.125
+213.222.208.93
+216.49.158.161
+216.55.143.154
+216.131.91.100
+216.245.206.181
+216.250.138.230
+217.11.225.189
+217.23.6.133
+217.75.88.178
+217.172.143.140
+217.195.169.209
+217.196.248.106
+219.138.161.162
+222.167.248.90
+223.18.254.55
+
+# Onion nodes
+bitcoinostk4e4re.onion:8333
+5k4vwyy5stro33fb.onion:8333
+zy3kdqowmrb7xm7h.onion:8333
+e3tn727fywnioxrc.onion:8333
+kjy2eqzk4zwi5zd3.onion:8333
+pt2awtcs2ulm75ig.onion:8333
+td7tgof3imei3fm6.onion:8333
+czsbwh4pq4mh3izl.onion:8333
+xdnigz4qn5dbbw2t.onion:8333
+ymnsfdejmc74vcfb.onion:7033
+jxrvw5iqizppbgml.onion:8333
+bk5ejfe56xakvtkk.onion:8333
+szsm43ou7otwwyfv.onion:8333
+5ghqw4wj6hpgfvdg.onion:8333
+evolynhit7shzeet.onion:8333
+4crhf372poejlc44.onion:8333
+tfu4kqfhsw5slqp2.onion:8333
+i2r5tbaizb75h26f.onion:8333
+btcnet3utgzyz2bf.onion:8333
+vso3r6cmjoomhhgg.onion:8333
+pqosrh6wfaucet32.onion:8333
+zy3kdqowmrb7xm7h.onion:8333
+r4de4zf4lyniu4mx.onion:8444
diff --git a/contrib/seeds/nodes_test.txt b/contrib/seeds/nodes_test.txt
new file mode 100644
index 0000000000..98365ee505
--- /dev/null
+++ b/contrib/seeds/nodes_test.txt
@@ -0,0 +1,11 @@
+# List of fixed seed nodes for testnet
+
+# Onion nodes
+thfsmmn2jbitcoin.onion
+it2pj4f7657g3rhi.onion
+nkf5e6b7pl4jfd4a.onion
+4zhkir2ofl7orfom.onion
+t6xj6wilh4ytvcs7.onion
+i6y6ivorwakd7nw3.onion
+ubqj4rsu3nqtxmtp.onion
+