Merge #14954: build: Require python 3.5

fa2797808e test: Remove python3.4 workaround in feature_dbcrash (MarcoFalke)
dddd1d05d3 .python-version: Specify full version 3.5.6 (MarcoFalke)
faa7cdf764 scripted-diff: Update copyright in ./test (MarcoFalke)
fa0e65b772 scripted-diff: test: Remove brackets after assert (MarcoFalke)
fab5a1e0f4 build: Require python 3.5 (MarcoFalke)
fa6bf21f5e scripted-diff: test: Use py3.5 bytes::hex() method (MarcoFalke)

Pull request description:

  Python 3.4 is EOL after March 2019, so switch to 3.5. See https://devguide.python.org/#status-of-python-branches

  This pull does the following in a bunch of commits:
  * scripted diff to use the `bytes::hex()` method in place of previous wrappers (`b2x`, `bytes_to_hex_str`, `hexlify`, ...)
  * Update the build system (gitian and travis) to remove python2.7 and replace it with python3.5
  * Another scripted-diff to remove brackets after `assert`. This is unrelated to the python3.5 switch, but a stylistic commit, so probably not worth to split up. The motivation behind it is to avoid asserting on data structures (such as tuples of length one), which never fails:
  ```py
  >>> assert(False,)   # with brackets
  >>> assert False,    # without brackets
  SyntaxError: invalid syntax
  >>> assert False     # proper assertion
  AssertionError
  ```
  * And then a final scripted diff to update the copyright headers in the `test` subfolder, since I touched most of the files anyway and it wouldn't make sense to split this commit out into a separate pull.

  For reference (contributed by luke-jr):

  Ubuntu LTS (bionic): 3.6.5
  Debian stable (stretch): 3.5.3
  RHEL 8 (expected before v0.19): 3.6.x
  Gentoo stable: 3.6.5
  Arch: 3.7.1

Tree-SHA512: 643c28cd2d5b9543ce4bf8ad2a8b282bc79b37dc5b25c9c8358e6ce201e2a67a546463e5f3430b16652eb2489d7c3ed4b0772cd2e2bf790fe68a5e3cc8a25029

10 files changed, 37 insertions, 52 deletions

diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py
index 456d43aa2e..f36cffe957 100644
--- a/test/functional/test_framework/address.py
+++ b/test/functional/test_framework/address.py
@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
-# Copyright (c) 2016-2018 The Bitcoin Core developers
+# Copyright (c) 2016-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Encode and decode BASE58, P2PKH and P2SH addresses."""
 
 from .script import hash256, hash160, sha256, CScript, OP_0
-from .util import bytes_to_hex_str, hex_str_to_bytes
+from .util import hex_str_to_bytes
 
 from . import segwit_addr
 
@@ -16,9 +16,9 @@ chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
 
 def byte_to_base58(b, version):
     result = ''
-    str = bytes_to_hex_str(b)
-    str = bytes_to_hex_str(chr(version).encode('latin-1')) + str
-    checksum = bytes_to_hex_str(hash256(hex_str_to_bytes(str)))
+    str = b.hex()
+    str = chr(version).encode('latin-1').hex() + str
+    checksum = hash256(hex_str_to_bytes(str)).hex()
     str += checksum[:8]
     value = int('0x'+str,0)
     while value > 0:
@@ -32,12 +32,12 @@ def byte_to_base58(b, version):
 # TODO: def base58_decode
 
 def keyhash_to_p2pkh(hash, main = False):
-    assert (len(hash) == 20)
+    assert len(hash) == 20
     version = 0 if main else 111
     return byte_to_base58(hash, version)
 
 def scripthash_to_p2sh(hash, main = False):
-    assert (len(hash) == 20)
+    assert len(hash) == 20
     version = 5 if main else 196
     return byte_to_base58(hash, version)
 
@@ -80,11 +80,11 @@ def check_key(key):
         key = hex_str_to_bytes(key) # Assuming this is hex string
     if (type(key) is bytes and (len(key) == 33 or len(key) == 65)):
         return key
-    assert(False)
+    assert False
 
 def check_script(script):
     if (type(script) is str):
         script = hex_str_to_bytes(script) # Assuming this is hex string
     if (type(script) is bytes or type(script) is CScript):
         return script
-    assert(False)
+    assert False
diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py
index 15f4502994..7ac044d0d0 100644
--- a/test/functional/test_framework/blocktools.py
+++ b/test/functional/test_framework/blocktools.py
@@ -20,7 +20,6 @@ from .messages import (
     CTxOut,
     FromHex,
     ToHex,
-    bytes_to_hex_str,
     hash256,
     hex_str_to_bytes,
     ser_string,
@@ -132,7 +131,7 @@ def create_tx_with_script(prevtx, n, script_sig=b"", *, amount, script_pub_key=C
        Can optionally pass scriptPubKey and scriptSig, default is anyone-can-spend output.
     """
     tx = CTransaction()
-    assert(n < len(prevtx.vout))
+    assert n < len(prevtx.vout)
     tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), script_sig, 0xffffffff))
     tx.vout.append(CTxOut(amount, script_pub_key))
     tx.calc_sha256()
@@ -190,7 +189,7 @@ def witness_script(use_p2wsh, pubkey):
         witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG])
         scripthash = sha256(witness_program)
         pkscript = CScript([OP_0, scripthash])
-    return bytes_to_hex_str(pkscript)
+    return pkscript.hex()
 
 def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount):
     """Return a transaction (in hex) that spends the given utxo to a segwit output.
@@ -215,7 +214,7 @@ def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=Tru
     tx_to_witness = create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount)
     if (sign):
         signed = node.signrawtransactionwithwallet(tx_to_witness)
-        assert("errors" not in signed or len(["errors"]) == 0)
+        assert "errors" not in signed or len(["errors"]) == 0
         return node.sendrawtransaction(signed["hex"])
     else:
         if (insert_redeem_script):
diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py
index 4bd58519c5..7cf51d9223 100755
--- a/test/functional/test_framework/messages.py
+++ b/test/functional/test_framework/messages.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # Copyright (c) 2010 ArtForz -- public domain half-a-node
 # Copyright (c) 2012 Jeff Garzik
-# Copyright (c) 2010-2018 The Bitcoin Core developers
+# Copyright (c) 2010-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Bitcoin test framework primitive and message structures
@@ -28,7 +28,7 @@ import struct
 import time
 
 from test_framework.siphash import siphash256
-from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, assert_equal
+from test_framework.util import hex_str_to_bytes, assert_equal
 
 MIN_VERSION_SUPPORTED = 60001
 MY_VERSION = 70014  # past bip-31 for ping/pong
@@ -181,7 +181,7 @@ def FromHex(obj, hex_string):
 
 # Convert a binary-serializable object to hex (eg for submission via RPC)
 def ToHex(obj):
-    return bytes_to_hex_str(obj.serialize())
+    return obj.serialize().hex()
 
 # Objects that map to bitcoind objects, which can be serialized/deserialized
 
@@ -319,7 +319,7 @@ class CTxIn:
 
     def __repr__(self):
         return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
-            % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
+            % (repr(self.prevout), self.scriptSig.hex(),
                self.nSequence)
 
 
@@ -343,7 +343,7 @@ class CTxOut:
     def __repr__(self):
         return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
             % (self.nValue // COIN, self.nValue % COIN,
-               bytes_to_hex_str(self.scriptPubKey))
+               self.scriptPubKey.hex())
 
 
 class CScriptWitness:
@@ -355,7 +355,7 @@ class CScriptWitness:
 
     def __repr__(self):
         return "CScriptWitness(%s)" % \
-               (",".join([bytes_to_hex_str(x) for x in self.stack]))
+               (",".join([x.hex() for x in self.stack]))
 
     def is_null(self):
         if self.stack:
diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py
index ac7cc068bd..52a840941b 100755
--- a/test/functional/test_framework/mininode.py
+++ b/test/functional/test_framework/mininode.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # Copyright (c) 2010 ArtForz -- public domain half-a-node
 # Copyright (c) 2012 Jeff Garzik
-# Copyright (c) 2010-2018 The Bitcoin Core developers
+# Copyright (c) 2010-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Bitcoin P2P network half-a-node.
@@ -218,10 +218,7 @@ class P2PConnection(asyncio.Protocol):
         def maybe_write():
             if not self._transport:
                 return
-            # Python <3.4.4 does not have is_closing, so we have to check for
-            # its existence explicitly as long as Bitcoin Core supports all
-            # Python 3.4 versions.
-            if hasattr(self._transport, 'is_closing') and self._transport.is_closing():
+            if self._transport.is_closing():
                 return
             self._transport.write(raw_message_bytes)
         NetworkThread.network_event_loop.call_soon_threadsafe(maybe_write)
diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py
index 45c182f630..c98424e8e2 100644
--- a/test/functional/test_framework/netutil.py
+++ b/test/functional/test_framework/netutil.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2014-2018 The Bitcoin Core developers
+# Copyright (c) 2014-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Linux network utilities.
@@ -12,7 +12,7 @@ import socket
 import struct
 import array
 import os
-from binascii import unhexlify, hexlify
+from binascii import unhexlify
 
 # STATE_ESTABLISHED = '01'
 # STATE_SYN_SENT  = '02'
@@ -129,17 +129,17 @@ def addr_to_hex(addr):
                 if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end
                     continue
                 x += 1 # :: skips to suffix
-                assert(x < 2)
+                assert x < 2
             else: # two bytes per component
                 val = int(comp, 16)
                 sub[x].append(val >> 8)
                 sub[x].append(val & 0xff)
         nullbytes = 16 - len(sub[0]) - len(sub[1])
-        assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0))
+        assert (x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)
         addr = sub[0] + ([0] * nullbytes) + sub[1]
     else:
         raise ValueError('Could not parse address %s' % addr)
-    return hexlify(bytearray(addr)).decode('ascii')
+    return bytearray(addr).hex()
 
 def test_ipv6_local():
     '''
diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py
index 012c80a1be..384062b808 100644
--- a/test/functional/test_framework/script.py
+++ b/test/functional/test_framework/script.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Copyright (c) 2015-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Functionality to build scripts, as well as SignatureHash().
@@ -9,7 +9,6 @@ This file is modified from python-bitcoinlib.
 
 from .messages import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string
 
-from binascii import hexlify
 import hashlib
 import struct
 
@@ -450,10 +449,6 @@ class CScript(bytes):
         # join makes no sense for a CScript()
         raise NotImplementedError
 
-    # Python 3.4 compatibility
-    def hex(self):
-        return hexlify(self).decode('ascii')
-
     def __new__(cls, value=b''):
         if isinstance(value, bytes) or isinstance(value, bytearray):
             return super(CScript, cls).__new__(cls, value)
@@ -545,7 +540,7 @@ class CScript(bytes):
     def __repr__(self):
         def _repr(o):
             if isinstance(o, bytes):
-                return "x('%s')" % hexlify(o).decode('ascii')
+                return "x('%s')" % o.hex()
             else:
                 return repr(o)
 
diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py
index a21c864e75..799b1c74b8 100644
--- a/test/functional/test_framework/socks5.py
+++ b/test/functional/test_framework/socks5.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Copyright (c) 2015-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Dummy Socks5 server for testing."""
@@ -144,7 +144,7 @@ class Socks5Server():
                 thread.start()
 
     def start(self):
-        assert(not self.running)
+        assert not self.running
         self.running = True
         self.thread = threading.Thread(None, self.run)
         self.thread.daemon = True
diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py
index 09d7d877a7..15d2e08c93 100755
--- a/test/functional/test_framework/test_framework.py
+++ b/test/functional/test_framework/test_framework.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2014-2018 The Bitcoin Core developers
+# Copyright (c) 2014-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Base class for RPC testing."""
diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index 37fd2a8744..ec5d3b267e 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2017-2018 The Bitcoin Core developers
+# Copyright (c) 2017-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Class for bitcoind node under test"""
@@ -31,9 +31,6 @@ from .util import (
     p2p_port,
 )
 
-# For Python 3.4 compatibility
-JSONDecodeError = getattr(json, "JSONDecodeError", ValueError)
-
 BITCOIND_PROC_WAIT_TIMEOUT = 60
 
 
@@ -565,5 +562,5 @@ class TestNodeCLI():
             raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr)
         try:
             return json.loads(cli_stdout, parse_float=decimal.Decimal)
-        except JSONDecodeError:
+        except json.JSONDecodeError:
             return cli_stdout.rstrip("\n")
diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py
index fef9982412..034ed893f4 100644
--- a/test/functional/test_framework/util.py
+++ b/test/functional/test_framework/util.py
@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
-# Copyright (c) 2014-2018 The Bitcoin Core developers
+# Copyright (c) 2014-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Helpful routines for regression testing."""
 
 from base64 import b64encode
-from binascii import hexlify, unhexlify
+from binascii import unhexlify
 from decimal import Decimal, ROUND_DOWN
 import hashlib
 import inspect
@@ -182,9 +182,6 @@ def check_json_precision():
 def count_bytes(hex_string):
     return len(bytearray.fromhex(hex_string))
 
-def bytes_to_hex_str(byte_str):
-    return hexlify(byte_str).decode('ascii')
-
 def hash256(byte_str):
     sha256 = hashlib.sha256()
     sha256.update(byte_str)
@@ -267,7 +264,7 @@ def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None):
     return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile)
 
 def p2p_port(n):
-    assert(n <= MAX_NODES)
+    assert n <= MAX_NODES
     return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES)
 
 def rpc_port(n):
@@ -425,7 +422,7 @@ def gather_inputs(from_node, amount_needed, confirmations_required=1):
     """
     Return a random set of unspent txouts that are enough to pay amount_needed
     """
-    assert(confirmations_required >= 0)
+    assert confirmations_required >= 0
     utxo = from_node.listunspent(confirmations_required)
     random.shuffle(utxo)
     inputs = []
@@ -503,7 +500,7 @@ def create_confirmed_utxos(fee, node, count):
         node.generate(1)
 
     utxos = node.listunspent()
-    assert(len(utxos) >= count)
+    assert len(utxos) >= count
     return utxos
 
 # Create large OP_RETURN txouts that can be appended to a transaction