17 files changed, 118 insertions, 54 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index fd1a912c4f..06fcd8dd81 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -178,7 +178,9 @@ language is used within pull-request comments:
 
   - ACK means "I have tested the code and I agree it should be merged";
   - NACK means "I disagree this should be merged", and must be accompanied by
-    sound technical justification. NACKs without accompanying reasoning may be disregarded;
+    sound technical justification (or in certain cases of copyright/patent/licensing
+    issues, legal justification). NACKs without accompanying reasoning may be
+    disregarded;
   - utACK means "I have not tested the code, but I have reviewed it and it looks
     OK, I agree it can be merged";
   - Concept ACK means "I agree in the general principle of this pull request";
diff --git a/doc/build-unix.md b/doc/build-unix.md
index 62e3e793e9..5202072f8b 100644
--- a/doc/build-unix.md
+++ b/doc/build-unix.md
@@ -308,3 +308,35 @@ To build executables for ARM:
 
 
 For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory.
+
+Building on FreeBSD
+--------------------
+
+(Updated as of FreeBSD 10.3)
+
+Clang is installed by default as `cc` compiler, this makes it easier to get
+started than on [OpenBSD](build-openbsd.md). Installing dependencies:
+
+    pkg install autoconf automake libtool pkgconf
+    pkg install boost-libs openssl libevent2
+
+(`libressl` instead of `openssl` will also work)
+
+For the wallet (optional):
+
+    pkg install db5
+
+This will give a warning "configure: WARNING: Found Berkeley DB other
+than 4.8; wallets opened by this build will not be portable!", but as FreeBSD never
+had a binary release, this may not matter. If backwards compatibility
+with 4.8-built Bitcoin Core is needed follow the steps under "Berkeley DB" above.
+
+Then build using:
+
+    ./autogen.sh
+    ./configure --with-incompatible-bdb CPPFLAGS=-I/usr/local/include/db5 LDFLAGS=-L/usr/local/lib/db5
+    make
+
+*Note on debugging*: The version of `gdb` installed by default is [ancient and considered harmful](https://wiki.freebsd.org/GdbRetirement).
+It is not suitable for debugging a multi-threaded C++ program, not even for getting backtraces. Please install the package `gdb` and
+use the versioned gdb command e.g. `gdb7111`.
diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py
index d91e10d77c..e7f5a1c9c6 100755
--- a/qa/rpc-tests/p2p-compactblocks.py
+++ b/qa/rpc-tests/p2p-compactblocks.py
@@ -30,6 +30,10 @@ class TestNode(SingleNodeConnCB):
         self.last_getblocktxn = None
         self.last_block = None
         self.last_blocktxn = None
+        # Store the hashes of blocks we've seen announced.
+        # This is for synchronizing the p2p message traffic,
+        # so we can eg wait until a particular block is announced.
+        self.set_announced_blockhashes = set()
 
     def on_sendcmpct(self, conn, message):
         self.last_sendcmpct.append(message)
@@ -40,14 +44,22 @@ class TestNode(SingleNodeConnCB):
     def on_cmpctblock(self, conn, message):
         self.last_cmpctblock = message
         self.block_announced = True
+        self.last_cmpctblock.header_and_shortids.header.calc_sha256()
+        self.set_announced_blockhashes.add(self.last_cmpctblock.header_and_shortids.header.sha256)
 
     def on_headers(self, conn, message):
         self.last_headers = message
         self.block_announced = True
+        for x in self.last_headers.headers:
+            x.calc_sha256()
+            self.set_announced_blockhashes.add(x.sha256)
 
     def on_inv(self, conn, message):
         self.last_inv = message
-        self.block_announced = True
+        for x in self.last_inv.inv:
+            if x.type == 2:
+                self.block_announced = True
+                self.set_announced_blockhashes.add(x.hash)
 
     def on_getdata(self, conn, message):
         self.last_getdata = message
@@ -87,6 +99,12 @@ class TestNode(SingleNodeConnCB):
         assert(self.received_block_announcement())
         self.clear_block_announcement()
 
+    # Block until a block announcement for a particular block hash is
+    # received.
+    def wait_for_block_announcement(self, block_hash, timeout=30):
+        def received_hash():
+            return (block_hash in self.set_announced_blockhashes)
+        return wait_until(received_hash, timeout=timeout)
 
 class CompactBlocksTest(BitcoinTestFramework):
     def __init__(self):
@@ -278,7 +296,9 @@ class CompactBlocksTest(BitcoinTestFramework):
         if use_witness_address:
             assert(segwit_tx_generated) # check that our test is not broken
 
-        self.test_node.sync_with_ping()
+        # Wait until we've seen the block announcement for the resulting tip
+        tip = int(self.nodes[0].getbestblockhash(), 16)
+        assert(self.test_node.wait_for_block_announcement(tip))
 
         # Now mine a block, and look at the resulting compact block.
         test_node.clear_block_announcement()
@@ -382,7 +402,9 @@ class CompactBlocksTest(BitcoinTestFramework):
             comp_block = HeaderAndShortIDs()
             comp_block.header = CBlockHeader(block)
             comp_block.nonce = 0
-            comp_block.shortids = [1]  # this is useless, and wrong
+            [k0, k1] = comp_block.get_siphash_keys()
+            comp_block.shortids = [
+                    calculate_shortid(k0, k1, block.vtx[0].sha256) ]
             test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
             assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
             # Expect a getblocktxn message.
diff --git a/qa/rpc-tests/sendheaders.py b/qa/rpc-tests/sendheaders.py
index c3f3180b6b..81b2442e6a 100755
--- a/qa/rpc-tests/sendheaders.py
+++ b/qa/rpc-tests/sendheaders.py
@@ -80,20 +80,19 @@ e. Announce one more that doesn't connect.
    Expect: disconnect.
 '''
 
-class BaseNode(NodeConnCB):
+direct_fetch_response_time = 0.05
+
+class BaseNode(SingleNodeConnCB):
     def __init__(self):
-        NodeConnCB.__init__(self)
-        self.connection = None
+        SingleNodeConnCB.__init__(self)
         self.last_inv = None
         self.last_headers = None
         self.last_block = None
-        self.ping_counter = 1
-        self.last_pong = msg_pong(0)
         self.last_getdata = None
-        self.sleep_time = 0.05
         self.block_announced = False
         self.last_getheaders = None
         self.disconnected = False
+        self.last_blockhash_announced = None
 
     def clear_last_announcement(self):
         with mininode_lock:
@@ -101,9 +100,6 @@ class BaseNode(NodeConnCB):
             self.last_inv = None
             self.last_headers = None
 
-    def add_connection(self, conn):
-        self.connection = conn
-
     # Request data for a list of block hashes
     def get_data(self, block_hashes):
         msg = msg_getdata()
@@ -122,17 +118,17 @@ class BaseNode(NodeConnCB):
         msg.inv = [CInv(2, blockhash)]
         self.connection.send_message(msg)
 
-    # Wrapper for the NodeConn's send_message function
-    def send_message(self, message):
-        self.connection.send_message(message)
-
     def on_inv(self, conn, message):
         self.last_inv = message
         self.block_announced = True
+        self.last_blockhash_announced = message.inv[-1].hash
 
     def on_headers(self, conn, message):
         self.last_headers = message
-        self.block_announced = True
+        if len(message.headers):
+            self.block_announced = True
+            message.headers[-1].calc_sha256()
+            self.last_blockhash_announced = message.headers[-1].sha256
 
     def on_block(self, conn, message):
         self.last_block = message.block
@@ -141,9 +137,6 @@ class BaseNode(NodeConnCB):
     def on_getdata(self, conn, message):
         self.last_getdata = message
 
-    def on_pong(self, conn, message):
-        self.last_pong = message
-
     def on_getheaders(self, conn, message):
         self.last_getheaders = message
 
@@ -157,7 +150,7 @@ class BaseNode(NodeConnCB):
         expect_headers = headers if headers != None else []
         expect_inv = inv if inv != None else []
         test_function = lambda: self.block_announced
-        self.sync(test_function)
+        assert(wait_until(test_function, timeout=60))
         with mininode_lock:
             self.block_announced = False
 
@@ -180,30 +173,14 @@ class BaseNode(NodeConnCB):
         return success
 
     # Syncing helpers
-    def sync(self, test_function, timeout=60):
-        while timeout > 0:
-            with mininode_lock:
-                if test_function():
-                    return
-            time.sleep(self.sleep_time)
-            timeout -= self.sleep_time
-        raise AssertionError("Sync failed to complete")
-        
-    def sync_with_ping(self, timeout=60):
-        self.send_message(msg_ping(nonce=self.ping_counter))
-        test_function = lambda: self.last_pong.nonce == self.ping_counter
-        self.sync(test_function, timeout)
-        self.ping_counter += 1
-        return
-
     def wait_for_block(self, blockhash, timeout=60):
         test_function = lambda: self.last_block != None and self.last_block.sha256 == blockhash
-        self.sync(test_function, timeout)
+        assert(wait_until(test_function, timeout=timeout))
         return
 
     def wait_for_getheaders(self, timeout=60):
         test_function = lambda: self.last_getheaders != None
-        self.sync(test_function, timeout)
+        assert(wait_until(test_function, timeout=timeout))
         return
 
     def wait_for_getdata(self, hash_list, timeout=60):
@@ -211,12 +188,17 @@ class BaseNode(NodeConnCB):
             return
 
         test_function = lambda: self.last_getdata != None and [x.hash for x in self.last_getdata.inv] == hash_list
-        self.sync(test_function, timeout)
+        assert(wait_until(test_function, timeout=timeout))
         return
 
     def wait_for_disconnect(self, timeout=60):
         test_function = lambda: self.disconnected
-        self.sync(test_function, timeout)
+        assert(wait_until(test_function, timeout=timeout))
+        return
+
+    def wait_for_block_announcement(self, block_hash, timeout=60):
+        test_function = lambda: self.last_blockhash_announced == block_hash
+        assert(wait_until(test_function, timeout=timeout))
         return
 
     def send_header_for_blocks(self, new_blocks):
@@ -266,7 +248,9 @@ class SendHeadersTest(BitcoinTestFramework):
     def mine_reorg(self, length):
         self.nodes[0].generate(length) # make sure all invalidated blocks are node0's
         sync_blocks(self.nodes, wait=0.1)
-        [x.clear_last_announcement() for x in self.p2p_connections]
+        for x in self.p2p_connections:
+            x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
+            x.clear_last_announcement()
 
         tip_height = self.nodes[1].getblockcount()
         hash_to_invalidate = self.nodes[1].getblockhash(tip_height-(length-1))
@@ -495,7 +479,7 @@ class SendHeadersTest(BitcoinTestFramework):
 
         test_node.send_header_for_blocks(blocks)
         test_node.sync_with_ping()
-        test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=test_node.sleep_time)
+        test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=direct_fetch_response_time)
 
         [ test_node.send_message(msg_block(x)) for x in blocks ]
 
@@ -526,13 +510,13 @@ class SendHeadersTest(BitcoinTestFramework):
         # both blocks (same work as tip)
         test_node.send_header_for_blocks(blocks[1:2])
         test_node.sync_with_ping()
-        test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=test_node.sleep_time)
+        test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=direct_fetch_response_time)
 
         # Announcing 16 more headers should trigger direct fetch for 14 more
         # blocks
         test_node.send_header_for_blocks(blocks[2:18])
         test_node.sync_with_ping()
-        test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=test_node.sleep_time)
+        test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=direct_fetch_response_time)
 
         # Announcing 1 more header should not trigger any response
         test_node.last_getdata = None
diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include
index a2ed4f47d8..1f9a901d75 100644
--- a/src/Makefile.qt.include
+++ b/src/Makefile.qt.include
@@ -289,7 +289,7 @@ RES_ICONS = \
   qt/res/icons/warning.png \
   qt/res/icons/verify.png
 
-BITCOIN_QT_CPP = \
+BITCOIN_QT_BASE_CPP = \
   qt/bantablemodel.cpp \
   qt/bitcoinaddressvalidator.cpp \
   qt/bitcoinamountfield.cpp \
@@ -313,12 +313,9 @@ BITCOIN_QT_CPP = \
   qt/trafficgraphwidget.cpp \
   qt/utilitydialog.cpp
 
-if TARGET_WINDOWS
-BITCOIN_QT_CPP += qt/winshutdownmonitor.cpp
-endif
+BITCOIN_QT_WINDOWS_CPP = qt/winshutdownmonitor.cpp
 
-if ENABLE_WALLET
-BITCOIN_QT_CPP += \
+BITCOIN_QT_WALLET_CPP = \
   qt/addressbookpage.cpp \
   qt/addresstablemodel.cpp \
   qt/askpassphrasedialog.cpp \
@@ -345,6 +342,13 @@ BITCOIN_QT_CPP += \
   qt/walletmodel.cpp \
   qt/walletmodeltransaction.cpp \
   qt/walletview.cpp
+
+BITCOIN_QT_CPP = $(BITCOIN_QT_BASE_CPP)
+if TARGET_WINDOWS
+BITCOIN_QT_CPP += $(BITCOIN_QT_WINDOWS_CPP)
+endif
+if ENABLE_WALLET
+BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP)
 endif
 
 RES_IMAGES = 
@@ -413,7 +417,7 @@ $(srcdir)/qt/bitcoinstrings.cpp: $(libbitcoin_server_a_SOURCES) $(libbitcoin_wal
 	@test -n $(XGETTEXT) || echo "xgettext is required for updating translations"
 	$(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) PACKAGE_NAME="$(PACKAGE_NAME)" COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" COPYRIGHT_HOLDERS_SUBSTITUTION="$(COPYRIGHT_HOLDERS_SUBSTITUTION)" $(PYTHON) ../share/qt/extract_strings_qt.py $^
 
-translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(BITCOIN_MM)
+translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCOIN_QT_BASE_CPP) $(BITCOIN_QT_WINDOWS_CPP) $(BITCOIN_QT_WALLET_CPP) $(BITCOIN_QT_H) $(BITCOIN_MM)
 	@test -n $(LUPDATE) || echo "lupdate is required for updating translations"
 	$(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LUPDATE) $^ -locations relative -no-obsolete -ts $(srcdir)/qt/locale/bitcoin_en.ts
 
diff --git a/src/core_write.cpp b/src/core_write.cpp
index 6f9e2266a3..ea01ddc10d 100644
--- a/src/core_write.cpp
+++ b/src/core_write.cpp
@@ -151,11 +151,13 @@ void ScriptPubKeyToUniv(const CScript& scriptPubKey,
 void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry)
 {
     entry.pushKV("txid", tx.GetHash().GetHex());
+    entry.pushKV("hash", tx.GetWitnessHash().GetHex());
     entry.pushKV("version", tx.nVersion);
     entry.pushKV("locktime", (int64_t)tx.nLockTime);
 
     UniValue vin(UniValue::VARR);
-    BOOST_FOREACH(const CTxIn& txin, tx.vin) {
+    for (unsigned int i = 0; i < tx.vin.size(); i++) {
+        const CTxIn& txin = tx.vin[i];
         UniValue in(UniValue::VOBJ);
         if (tx.IsCoinBase())
             in.pushKV("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()));
@@ -166,6 +168,13 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry)
             o.pushKV("asm", ScriptToAsmStr(txin.scriptSig, true));
             o.pushKV("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()));
             in.pushKV("scriptSig", o);
+            if (!tx.wit.IsNull() && i < tx.wit.vtxinwit.size() && !tx.wit.vtxinwit[i].IsNull()) {
+                UniValue txinwitness(UniValue::VARR);
+                for (const auto& item : tx.wit.vtxinwit[i].scriptWitness.stack) {
+                    txinwitness.push_back(HexStr(item.begin(), item.end()));
+                }
+                in.pushKV("txinwitness", txinwitness);
+            }
         }
         in.pushKV("sequence", (int64_t)txin.nSequence);
         vin.push_back(in);
diff --git a/src/test/data/blanktx.json b/src/test/data/blanktx.json
index f6d6ab5884..51c25a5a98 100644
--- a/src/test/data/blanktx.json
+++ b/src/test/data/blanktx.json
@@ -1,5 +1,6 @@
 {
     "txid": "d21633ba23f70118185227be58a63527675641ad37967e2aa461559f577aec43",
+    "hash": "d21633ba23f70118185227be58a63527675641ad37967e2aa461559f577aec43",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/tt-delin1-out.json b/src/test/data/tt-delin1-out.json
index 2c7a68636a..712a2c27f8 100644
--- a/src/test/data/tt-delin1-out.json
+++ b/src/test/data/tt-delin1-out.json
@@ -1,5 +1,6 @@
 {
     "txid": "81b2035be1da1abe745c6141174a73d151009ec17b3d5ebffa2e177408c50dfd",
+    "hash": "81b2035be1da1abe745c6141174a73d151009ec17b3d5ebffa2e177408c50dfd",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/tt-delout1-out.json b/src/test/data/tt-delout1-out.json
index 9cf8cbb16c..afc4e95762 100644
--- a/src/test/data/tt-delout1-out.json
+++ b/src/test/data/tt-delout1-out.json
@@ -1,5 +1,6 @@
 {
     "txid": "c46ccd75b5050e942b2e86a3648f843f525fe6fc000bf0534ba5973063354493",
+    "hash": "c46ccd75b5050e942b2e86a3648f843f525fe6fc000bf0534ba5973063354493",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/tt-locktime317000-out.json b/src/test/data/tt-locktime317000-out.json
index 65b6a4451b..2b9075f8ac 100644
--- a/src/test/data/tt-locktime317000-out.json
+++ b/src/test/data/tt-locktime317000-out.json
@@ -1,5 +1,6 @@
 {
     "txid": "aded538f642c17e15f4d3306b8be7e1a4d1ae0c4616d641ab51ea09ba65e5cb5",
+    "hash": "aded538f642c17e15f4d3306b8be7e1a4d1ae0c4616d641ab51ea09ba65e5cb5",
     "version": 1,
     "locktime": 317000,
     "vin": [
diff --git a/src/test/data/txcreate1.json b/src/test/data/txcreate1.json
index 3890dbaf6e..567e8026a3 100644
--- a/src/test/data/txcreate1.json
+++ b/src/test/data/txcreate1.json
@@ -1,5 +1,6 @@
 {
     "txid": "f70f0d6c71416ed538e37549f430ab3665fee2437a42f10238c1bd490e782231",
+    "hash": "f70f0d6c71416ed538e37549f430ab3665fee2437a42f10238c1bd490e782231",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/txcreate2.json b/src/test/data/txcreate2.json
index c56293eaf2..a70c1d302a 100644
--- a/src/test/data/txcreate2.json
+++ b/src/test/data/txcreate2.json
@@ -1,5 +1,6 @@
 {
     "txid": "cf90229625e9eb10f6be8156bf6aa5ec2eca19a42b1e05c11f3029b560a32e13",
+    "hash": "cf90229625e9eb10f6be8156bf6aa5ec2eca19a42b1e05c11f3029b560a32e13",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/txcreatedata1.json b/src/test/data/txcreatedata1.json
index 2fed228108..760518d30a 100644
--- a/src/test/data/txcreatedata1.json
+++ b/src/test/data/txcreatedata1.json
@@ -1,5 +1,6 @@
 {
     "txid": "07894b4d12fe7853dd911402db1620920d261b9627c447f931417d330c25f06e",
+    "hash": "07894b4d12fe7853dd911402db1620920d261b9627c447f931417d330c25f06e",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/txcreatedata2.json b/src/test/data/txcreatedata2.json
index 3d4d367f37..56dfe4a1b0 100644
--- a/src/test/data/txcreatedata2.json
+++ b/src/test/data/txcreatedata2.json
@@ -1,5 +1,6 @@
 {
     "txid": "4ed17118f5e932ba8c75c461787d171bc02a016d8557cb5bcf34cd416c27bb8b",
+    "hash": "4ed17118f5e932ba8c75c461787d171bc02a016d8557cb5bcf34cd416c27bb8b",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/txcreatedata_seq0.json b/src/test/data/txcreatedata_seq0.json
index f25aa43c2b..9bc0ed4593 100644
--- a/src/test/data/txcreatedata_seq0.json
+++ b/src/test/data/txcreatedata_seq0.json
@@ -1,5 +1,6 @@
 {
     "txid": "71603ccb1cd76d73d76eb6cfd5f0b9df6d65d90d76860ee52cb461c4be7032e8",
+    "hash": "71603ccb1cd76d73d76eb6cfd5f0b9df6d65d90d76860ee52cb461c4be7032e8",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/txcreatedata_seq1.json b/src/test/data/txcreatedata_seq1.json
index 33585d6dfa..d323255418 100644
--- a/src/test/data/txcreatedata_seq1.json
+++ b/src/test/data/txcreatedata_seq1.json
@@ -1,5 +1,6 @@
 {
     "txid": "c4dea671b0d7b48f8ab10bc46650e8329d3c5766931f548f513847a19f5ba75b",
+    "hash": "c4dea671b0d7b48f8ab10bc46650e8329d3c5766931f548f513847a19f5ba75b",
     "version": 1,
     "locktime": 0,
     "vin": [
diff --git a/src/test/data/txcreatesign.json b/src/test/data/txcreatesign.json
index 057fe9b010..ff39e71b40 100644
--- a/src/test/data/txcreatesign.json
+++ b/src/test/data/txcreatesign.json
@@ -1,5 +1,6 @@
 {
     "txid": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af",
+    "hash": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af",
     "version": 1,
     "locktime": 0,
     "vin": [