Merge pull request #6703

45bfa13 PARTIAL: typofixes (found by misspell_fixer) (Veres Lajos)
21c406e add support for miniupnpc api version 14 (Pavel Vasin)
13bd5a7 rpc-tests: re-enable rpc-tests for Windows (Cory Fields)
ccc4ad6 net: Set SO_REUSEADDR for Windows too (Cory Fields)
1f6772e add unit test for CNetAddr::GetGroup. (Alex Morcos)
13642a5 Fix masking of irrelevant bits in address groups. (Alex Morcos)
6b51b9b Replace boost::reverse_lock with our own. (Casey Rodarmor)
626c5e6 Make sure we re-acquire lock if a task throws (Casey Rodarmor)
4877053 Add missing files to files.md (fanquake)
f171fee Handle leveldb::DestroyDB() errors on wipe failure (Adam Weiss)
c5b89fe Fix race condition on test node shutdown (Casey Rodarmor)
4a37410 Handle no chain tip available in InvalidChainFound() (Ross Nicoll)
f6d29a6 Use unique name for AlertNotify tempfile (Casey Rodarmor)
e6adac7 Delay initial pruning until after wallet init (Adam Weiss)
e0020d4 Make sure LogPrint strings are line-terminated (J Ross Nicoll)
7ff9d12 Make sure LogPrintf strings are line-terminated (Wladimir J. van der Laan)
5a39133 build: fix libressl detection (Cory Fields)
f6355e6 Avoid leaking file descriptors in RegisterLoad (Casey Rodarmor)
60457d3 locking: fix a few small issues uncovered by -Wthread-safety (Cory Fields)
a496e11 Remove bash test note from rpc-tests readme (fanquake)
49c6a64 tests: Remove old sh-based test framework (Wladimir J. van der Laan)
a37567d Add autogen.sh to source tarball. (randy-waterhouse)
1f4d7cf travis: for travis generating an extra build (Cory Fields)

7 files changed, 50 insertions, 338 deletions

diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh
index 9b318650e9..97aecc5e3c 100755
--- a/qa/pull-tester/rpc-tests.sh
+++ b/qa/pull-tester/rpc-tests.sh
@@ -8,11 +8,6 @@ CURDIR=$(cd $(dirname "$0"); pwd)
 export BITCOINCLI=${BUILDDIR}/qa/pull-tester/run-bitcoin-cli
 export BITCOIND=${REAL_BITCOIND}
 
-if [ "x${EXEEXT}" = "x.exe" ]; then
-  echo "Win tests currently disabled"
-  exit 0
-fi
-
 #Run the tests
 
 testScripts=(
diff --git a/qa/rpc-tests/README.md b/qa/rpc-tests/README.md
index 6221c93d8b..08c67c4ef1 100644
--- a/qa/rpc-tests/README.md
+++ b/qa/rpc-tests/README.md
@@ -12,10 +12,6 @@ Base class for new regression tests.
 ### [test_framework/util.py](test_framework/util.py)
 Generally useful functions.
 
-Bash-based tests, to be ported to Python:
------------------------------------------
-- conflictedbalance.sh : More testing of malleable transaction handling
-
 Notes
 =====
 
diff --git a/qa/rpc-tests/conflictedbalance.sh b/qa/rpc-tests/conflictedbalance.sh
deleted file mode 100755
index 7e44097374..0000000000
--- a/qa/rpc-tests/conflictedbalance.sh
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) 2014 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-# Test marking of spent outputs
-
-# Create a transaction graph with four transactions,
-# A/B/C/D
-# C spends A
-# D spends B and C
-
-# Then simulate C being mutated, to create C'
-#  that is mined.
-# A is still (correctly) considered spent.
-# B should be treated as unspent
-
-if [ $# -lt 1 ]; then
-        echo "Usage: $0 path_to_binaries"
-        echo "e.g. $0 ../../src"
-        echo "Env vars BITCOIND and BITCOINCLI may be used to specify the exact binaries used"
-        exit 1
-fi
-
-set -f
-
-BITCOIND=${BITCOIND:-${1}/bitcoind}
-CLI=${BITCOINCLI:-${1}/bitcoin-cli}
-
-DIR="${BASH_SOURCE%/*}"
-SENDANDWAIT="${DIR}/send.sh"
-if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
-. "$DIR/util.sh"
-
-D=$(mktemp -d test.XXXXX)
-
-# Two nodes; one will play the part of merchant, the
-# other an evil transaction-mutating miner.
-
-D1=${D}/node1
-CreateDataDir $D1 port=11000 rpcport=11001
-B1ARGS="-datadir=$D1 -debug=mempool"
-$BITCOIND $B1ARGS &
-B1PID=$!
-
-D2=${D}/node2
-CreateDataDir $D2 port=11010 rpcport=11011
-B2ARGS="-datadir=$D2 -debug=mempool"
-$BITCOIND $B2ARGS &
-B2PID=$!
-
-# Wait until both nodes are at the same block number
-function WaitBlocks {
-    while :
-    do
-        sleep 1
-        declare -i BLOCKS1=$( GetBlocks $B1ARGS )
-        declare -i BLOCKS2=$( GetBlocks $B2ARGS )
-        if (( BLOCKS1 == BLOCKS2 ))
-        then
-            break
-        fi
-    done
-}
-
-# Wait until node has $N peers
-function WaitPeers {
-    while :
-    do
-        declare -i PEERS=$( $CLI $1 getconnectioncount )
-        if (( PEERS == "$2" ))
-        then
-            break
-        fi
-        sleep 1
-    done
-}
-
-echo "Generating test blockchain..."
-
-# Start with B2 connected to B1:
-$CLI $B2ARGS addnode 127.0.0.1:11000 onetry
-WaitPeers "$B1ARGS" 1
-
-# 2 block, 50 XBT each == 100 XBT
-# These will be transactions "A" and "B"
-$CLI $B1ARGS generate 2
-
-WaitBlocks
-# 100 blocks, 0 mature == 0 XBT
-$CLI $B2ARGS generate 100
-WaitBlocks
-
-CheckBalance "$B1ARGS" 100
-CheckBalance "$B2ARGS" 0
-
-# restart B2 with no connection
-$CLI $B2ARGS stop > /dev/null 2>&1
-wait $B2PID
-$BITCOIND $B2ARGS &
-B2PID=$!
-
-B1ADDRESS=$( $CLI $B1ARGS getnewaddress )
-B2ADDRESS=$( $CLI $B2ARGS getnewaddress )
-
-# Transaction C: send-to-self, spend A
-TXID_C=$( $CLI $B1ARGS sendtoaddress $B1ADDRESS 50.0)
-
-# Transaction D: spends B and C
-TXID_D=$( $CLI $B1ARGS sendtoaddress $B2ADDRESS 100.0)
-
-CheckBalance "$B1ARGS" 0
-
-# Mutate TXID_C and add it to B2's memory pool:
-RAWTX_C=$( $CLI $B1ARGS getrawtransaction $TXID_C )
-
-# ... mutate C to create C'
-L=${RAWTX_C:82:2}
-NEWLEN=$( printf "%x" $(( 16#$L + 1 )) )
-MUTATEDTX_C=${RAWTX_C:0:82}${NEWLEN}4c${RAWTX_C:84}
-# ... give mutated tx1 to B2:
-MUTATEDTXID=$( $CLI $B2ARGS sendrawtransaction $MUTATEDTX_C )
-
-echo "TXID_C: " $TXID_C
-echo "Mutated: " $MUTATEDTXID
-
-# Re-connect nodes, and have both nodes mine some blocks:
-$CLI $B2ARGS addnode 127.0.0.1:11000 onetry
-WaitPeers "$B1ARGS" 1
-
-# Having B2 mine the next block puts the mutated
-# transaction C in the chain:
-$CLI $B2ARGS generate 1
-WaitBlocks
-
-# B1 should still be able to spend 100, because D is conflicted
-# so does not count as a spend of B
-CheckBalance "$B1ARGS" 100
-
-$CLI $B2ARGS stop > /dev/null 2>&1
-wait $B2PID
-$CLI $B1ARGS stop > /dev/null 2>&1
-wait $B1PID
-
-echo "Tests successful, cleaning up"
-rm -rf $D
-exit 0
diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py
index 6c51b2fcd9..15de037360 100755
--- a/qa/rpc-tests/rest.py
+++ b/qa/rpc-tests/rest.py
@@ -199,7 +199,7 @@ class RESTTest (BitcoinTestFramework):
         response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True)
         assert_equal(response.status, 200) #must be a 500 because we exceeding the limits
 
-        self.nodes[0].generate(1) #generate block to not affect upcomming tests
+        self.nodes[0].generate(1) #generate block to not affect upcoming tests
         self.sync_all()
 
         ################
diff --git a/qa/rpc-tests/send.sh b/qa/rpc-tests/send.sh
deleted file mode 100755
index 2d54cc6ded..0000000000
--- a/qa/rpc-tests/send.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2014 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-TIMEOUT=10
-SIGNAL=HUP
-PIDFILE=.send.pid
-if [ $# -eq 0 ]; then
-  echo -e "Usage:\t$0 <cmd>"
-  echo -e "\tRuns <cmd> and wait ${TIMEOUT} seconds or until SIG${SIGNAL} is received."
-  echo -e "\tReturns: 0 if SIG${SIGNAL} is received, 1 otherwise."
-  echo -e "Or:\t$0 -STOP"
-  echo -e "\tsends SIG${SIGNAL} to running send.sh"
-  exit 0
-fi
-
-if [ $1 = "-STOP" ]; then
-  if [ -s ${PIDFILE} ]; then
-      kill -s ${SIGNAL} $(<$PIDFILE 2>/dev/null) 2>/dev/null
-  fi
-  exit 0
-fi
-
-trap '[[ ${PID} ]] && kill ${PID}' ${SIGNAL}
-trap 'rm -f ${PIDFILE}' EXIT
-echo $$ > ${PIDFILE}
-"$@"
-sleep ${TIMEOUT} & PID=$!
-wait ${PID} && exit 1
-
-exit 0
diff --git a/qa/rpc-tests/test_framework/comptool.py b/qa/rpc-tests/test_framework/comptool.py
index 23a979250c..24ae05807a 100755
--- a/qa/rpc-tests/test_framework/comptool.py
+++ b/qa/rpc-tests/test_framework/comptool.py
@@ -27,6 +27,20 @@ generator that returns TestInstance objects.  See below for definition.
 
 global mininode_lock
 
+def wait_until(predicate, attempts=float('inf'), timeout=float('inf')):
+    attempt = 0
+    elapsed = 0
+
+    while attempt < attempts and elapsed < timeout:
+        with mininode_lock:
+            if predicate():
+                return True
+        attempt += 1
+        elapsed += 0.05
+        time.sleep(0.05)
+
+    return False
+
 class TestNode(NodeConnCB):
 
     def __init__(self, block_store, tx_store):
@@ -43,6 +57,10 @@ class TestNode(NodeConnCB):
         # a response
         self.pingMap = {} 
         self.lastInv = []
+        self.closed = False
+
+    def on_close(self, conn):
+        self.closed = True
 
     def add_connection(self, conn):
         self.conn = conn
@@ -116,7 +134,7 @@ class TestNode(NodeConnCB):
 #    is reached) and then sent out in one inv message.  Then the final block
 #    will be synced across all connections, and the outcome of the final 
 #    block will be tested.
-# sync_every_tx: analagous to behavior for sync_every_block, except if outcome
+# sync_every_tx: analogous to behavior for sync_every_block, except if outcome
 #    on the final tx is None, then contents of entire mempool are compared
 #    across all connections.  (If outcome of final tx is specified as true
 #    or false, then only the last tx is tested against outcome.)
@@ -132,6 +150,7 @@ class TestManager(object):
     def __init__(self, testgen, datadir):
         self.test_generator = testgen
         self.connections    = []
+        self.test_nodes     = []
         self.block_store    = BlockStore(datadir)
         self.tx_store       = TxStore(datadir)
         self.ping_counter   = 1
@@ -139,54 +158,40 @@ class TestManager(object):
     def add_all_connections(self, nodes):
         for i in range(len(nodes)):
             # Create a p2p connection to each node
-            self.connections.append(NodeConn('127.0.0.1', p2p_port(i), 
-                        nodes[i], TestNode(self.block_store, self.tx_store)))
+            test_node = TestNode(self.block_store, self.tx_store)
+            self.test_nodes.append(test_node)
+            self.connections.append(NodeConn('127.0.0.1', p2p_port(i), nodes[i], test_node))
             # Make sure the TestNode (callback class) has a reference to its
             # associated NodeConn
-            self.connections[-1].cb.add_connection(self.connections[-1])
+            test_node.add_connection(self.connections[-1])
+
+    def wait_for_disconnections(self):
+        def disconnected():
+            return all(node.closed for node in self.test_nodes)
+        return wait_until(disconnected, timeout=10)
 
     def wait_for_verack(self):
-        sleep_time = 0.05
-        max_tries = 10 / sleep_time  # Wait at most 10 seconds
-        while max_tries > 0:
-            done = True
-            with mininode_lock:
-                for c in self.connections:
-                    if c.cb.verack_received is False:
-                        done = False
-                        break
-            if done:
-                break
-            time.sleep(sleep_time)
+        def veracked():
+            return all(node.verack_received for node in self.test_nodes)
+        return wait_until(veracked, timeout=10)
 
     def wait_for_pings(self, counter):
-        received_pongs = False
-        while received_pongs is not True:
-            time.sleep(0.05)
-            received_pongs = True
-            with mininode_lock:
-                for c in self.connections:
-                    if c.cb.received_ping_response(counter) is not True:
-                        received_pongs = False
-                        break
+        def received_pongs():
+            return all(node.received_ping_response(counter) for node in self.test_nodes)
+        return wait_until(received_pongs)
 
     # sync_blocks: Wait for all connections to request the blockhash given
     # then send get_headers to find out the tip of each node, and synchronize
     # the response by using a ping (and waiting for pong with same nonce).
     def sync_blocks(self, blockhash, num_blocks):
-        # Wait for nodes to request block (50ms sleep * 20 tries * num_blocks)
-        max_tries = 20*num_blocks
-        while max_tries > 0:
-            with mininode_lock:
-                results = [ blockhash in c.cb.block_request_map and
-                            c.cb.block_request_map[blockhash] for c in self.connections ]
-            if False not in results:
-                break
-            time.sleep(0.05)
-            max_tries -= 1
+        def blocks_requested():
+            return all(
+                blockhash in node.block_request_map and node.block_request_map[blockhash]
+                for node in self.test_nodes
+            )
 
         # --> error if not requested
-        if max_tries == 0:
+        if not wait_until(blocks_requested, attempts=20*num_blocks):
             # print [ c.cb.block_request_map for c in self.connections ]
             raise AssertionError("Not all nodes requested block")
         # --> Answer request (we did this inline!)
@@ -202,18 +207,14 @@ class TestManager(object):
     # Analogous to sync_block (see above)
     def sync_transaction(self, txhash, num_events):
         # Wait for nodes to request transaction (50ms sleep * 20 tries * num_events)
-        max_tries = 20*num_events
-        while max_tries > 0:
-            with mininode_lock:
-                results = [ txhash in c.cb.tx_request_map and
-                            c.cb.tx_request_map[txhash] for c in self.connections ]
-            if False not in results:
-                break
-            time.sleep(0.05)
-            max_tries -= 1
+        def transaction_requested():
+            return all(
+                txhash in node.tx_request_map and node.tx_request_map[txhash]
+                for node in self.test_nodes
+            )
 
         # --> error if not requested
-        if max_tries == 0:
+        if not wait_until(transaction_requested, attempts=20*num_events):
             # print [ c.cb.tx_request_map for c in self.connections ]
             raise AssertionError("Not all nodes requested transaction")
         # --> Answer request (we did this inline!)
@@ -336,6 +337,7 @@ class TestManager(object):
             print "Test %d: PASS" % test_number, [ c.rpc.getblockcount() for c in self.connections ]
             test_number += 1
 
+        [ c.disconnect_node() for c in self.connections ]
+        self.wait_for_disconnections()
         self.block_store.close()
         self.tx_store.close()
-        [ c.disconnect_node() for c in self.connections ]
diff --git a/qa/rpc-tests/util.sh b/qa/rpc-tests/util.sh
deleted file mode 100644
index c2b7004308..0000000000
--- a/qa/rpc-tests/util.sh
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) 2014 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-# Functions used by more than one test
-
-function echoerr {
-  echo "$@" 1>&2;
-}
-
-# Usage: ExtractKey <key> "<json_object_string>"
-# Warning: this will only work for the very-well-behaved
-# JSON produced by bitcoind, do NOT use it to try to
-# parse arbitrary/nested/etc JSON.
-function ExtractKey {
-    echo $2 | tr -d ' "{}\n' | awk -v RS=',' -F: "\$1 ~ /$1/ { print \$2}"
-}
-
-function CreateDataDir {
-  DIR=$1
-  mkdir -p $DIR
-  CONF=$DIR/bitcoin.conf
-  echo "regtest=1" >> $CONF
-  echo "keypool=2" >> $CONF
-  echo "rpcuser=rt" >> $CONF
-  echo "rpcpassword=rt" >> $CONF
-  echo "rpcwait=1" >> $CONF
-  echo "walletnotify=${SENDANDWAIT} -STOP" >> $CONF
-  shift
-  while (( "$#" )); do
-      echo $1 >> $CONF
-      shift
-  done
-}
-
-function AssertEqual {
-  if (( $( echo "$1 == $2" | bc ) == 0 ))
-  then
-    echoerr "AssertEqual: $1 != $2"
-    declare -f CleanUp > /dev/null 2>&1
-    if [[ $? -eq 0 ]] ; then
-        CleanUp
-    fi
-    exit 1
-  fi
-}
-
-# CheckBalance -datadir=... amount account minconf
-function CheckBalance {
-  declare -i EXPECT="$2"
-  B=$( $CLI $1 getbalance $3 $4 )
-  if (( $( echo "$B == $EXPECT" | bc ) == 0 ))
-  then
-    echoerr "bad balance: $B (expected $2)"
-    declare -f CleanUp > /dev/null 2>&1
-    if [[ $? -eq 0 ]] ; then
-        CleanUp
-    fi
-    exit 1
-  fi
-}
-
-# Use: Address <datadir> [account]
-function Address {
-  $CLI $1 getnewaddress $2
-}
-
-# Send from to amount
-function Send {
-  from=$1
-  to=$2
-  amount=$3
-  address=$(Address $to)
-  txid=$( ${SENDANDWAIT} $CLI $from sendtoaddress $address $amount )
-}
-
-# Use: Unspent <datadir> <n'th-last-unspent> <var>
-function Unspent {
-  local r=$( $CLI $1 listunspent | awk -F'[ |:,"]+' "\$2 ~ /$3/ { print \$3 }" | tail -n $2 | head -n 1)
-  echo $r
-}
-
-# Use: CreateTxn1 <datadir> <n'th-last-unspent> <destaddress>
-# produces hex from signrawtransaction
-function CreateTxn1 {
-  TXID=$(Unspent $1 $2 txid)
-  AMOUNT=$(Unspent $1 $2 amount)
-  VOUT=$(Unspent $1 $2 vout)
-  RAWTXN=$( $CLI $1 createrawtransaction "[{\"txid\":\"$TXID\",\"vout\":$VOUT}]" "{\"$3\":$AMOUNT}")
-  ExtractKey hex "$( $CLI $1 signrawtransaction $RAWTXN )"
-}
-
-# Use: SendRawTxn <datadir> <hex_txn_data>
-function SendRawTxn {
-  ${SENDANDWAIT} $CLI $1 sendrawtransaction $2
-}
-
-# Use: GetBlocks <datadir>
-# returns number of blocks from getinfo
-function GetBlocks {
-    $CLI $1 getblockcount
-}