Merge #13756: wallet: "avoid_reuse" wallet flag for improved privacy

5ebc6b0eb267e0552c66fffc5e5afe7df8becf80 bitcoind: update -avoidpartialspends description to account for auto-enable for avoid_reuse wallets (Karl-Johan Alm)
ada258f8c8f92d44d893cf9f22d15acdeca40b1a doc: release notes for avoid_reuse (Karl-Johan Alm)
27669551da52099e4a6a401acd7aa32b32832423 wallet: enable avoid_partial_spends by default if avoid_reuse is set (Karl-Johan Alm)
8f2e208f7c0468f9ba92bc789a698281b1c81284 test: add test for avoidreuse feature (Karl-Johan Alm)
0bdfbd34cf4015de87741ff549db35e5064f4e16 wallet/rpc: add 'avoid_reuse' option to RPC commands (Karl-Johan Alm)
f904723e0d5883309cb0dd14b826bc45c5e776fb wallet/rpc: add setwalletflag RPC and MUTABLE_WALLET_FLAGS (Karl-Johan Alm)
8247a0da3a46d7c38943ee0304343ab7465305bd wallet: enable avoid_reuse feature (Karl-Johan Alm)
eec15662fad917b169f5e3b8baaf4301dcf00a7b wallet: avoid reuse flags (Karl-Johan Alm)
58928098c299efdc7c5ddf2dc20716ca5272f21b wallet: make IsWalletFlagSet() const (Karl-Johan Alm)
129a5bafd9a3efa2fa16d780885048a06566d262 wallet: rename g_known_wallet_flags constant to KNOWN_WALLET_FLAGS (Karl-Johan Alm)

Pull request description:

  Add a new wallet flag called `avoid_reuse` which, when enabled, will keep track of when a specific destination has been spent from, and will actively "blacklist" any new UTXOs which send to an already-spent-from destination.

  This improves privacy, as a payer could otherwise begin tracking a payee's wallet by regularly peppering a known UTXO with dust outputs, which would then be scooped up and used in payments by the payee, allowing the payer to map out (1) the inputs owned by the payee and (2) the destinations to which the payee is making payments.

  This replaces #10386 and together with the (now merged) #12257 it addresses #10065 in full. The concerns raised in https://github.com/bitcoin/bitcoin/pull/10386#issuecomment-302361381 are also addressed due to #12257.

  ~~Note: this builds on top of #15780.~~ (merged)

ACKs for commit 5ebc6b:
  jnewbery:
    ACK 5ebc6b0eb
  laanwj:
    Concept and code-review ACK 5ebc6b0eb267e0552c66fffc5e5afe7df8becf80
  meshcollider:
    Code review ACK https://github.com/bitcoin/bitcoin/pull/13756/commits/5ebc6b0eb267e0552c66fffc5e5afe7df8becf80
  achow101:
    ACK 5ebc6b0eb267e0552c66fffc5e5afe7df8becf80 modulo above nits

Tree-SHA512: fdef45826af544cbbb45634ac367852cc467ec87081d86d08b53ca849e588617e9a0a255b7e7bb28692d15332de58d6c3d274ac003355220e4213d7d9070742e

2 files changed, 218 insertions, 0 deletions

diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 80dced733e..098dbd647b 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -120,6 +120,7 @@ BASE_SCRIPTS = [
     'rpc_misc.py',
     'interface_rest.py',
     'mempool_spend_coinbase.py',
+    'wallet_avoidreuse.py',
     'mempool_reorg.py',
     'mempool_persist.py',
     'wallet_multiwallet.py',
diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py
new file mode 100755
index 0000000000..1dec040f68
--- /dev/null
+++ b/test/functional/wallet_avoidreuse.py
@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the avoid_reuse and setwalletflag features."""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_raises_rpc_error,
+    connect_nodes_bi,
+)
+
+# TODO: Copied from wallet_groups.py -- should perhaps move into util.py
+def assert_approx(v, vexp, vspan=0.00001):
+    if v < vexp - vspan:
+        raise AssertionError("%s < [%s..%s]" % (str(v), str(vexp - vspan), str(vexp + vspan)))
+    if v > vexp + vspan:
+        raise AssertionError("%s > [%s..%s]" % (str(v), str(vexp - vspan), str(vexp + vspan)))
+
+def reset_balance(node, discardaddr):
+    '''Throw away all owned coins by the node so it gets a balance of 0.'''
+    balance = node.getbalance(avoid_reuse=False)
+    if balance > 0.5:
+        node.sendtoaddress(address=discardaddr, amount=balance, subtractfeefromamount=True, avoid_reuse=False)
+
+def count_unspent(node):
+    '''Count the unspent outputs for the given node and return various statistics'''
+    r = {
+        "total": {
+            "count": 0,
+            "sum": 0,
+        },
+        "reused": {
+            "count": 0,
+            "sum": 0,
+        },
+    }
+    supports_reused = True
+    for utxo in node.listunspent(minconf=0):
+        r["total"]["count"] += 1
+        r["total"]["sum"] += utxo["amount"]
+        if supports_reused and "reused" in utxo:
+            if utxo["reused"]:
+                r["reused"]["count"] += 1
+                r["reused"]["sum"] += utxo["amount"]
+        else:
+            supports_reused = False
+    r["reused"]["supported"] = supports_reused
+    return r
+
+def assert_unspent(node, total_count=None, total_sum=None, reused_supported=None, reused_count=None, reused_sum=None):
+    '''Make assertions about a node's unspent output statistics'''
+    stats = count_unspent(node)
+    if total_count is not None:
+        assert_equal(stats["total"]["count"], total_count)
+    if total_sum is not None:
+        assert_approx(stats["total"]["sum"], total_sum, 0.001)
+    if reused_supported is not None:
+        assert_equal(stats["reused"]["supported"], reused_supported)
+    if reused_count is not None:
+        assert_equal(stats["reused"]["count"], reused_count)
+    if reused_sum is not None:
+        assert_approx(stats["reused"]["sum"], reused_sum, 0.001)
+
+class AvoidReuseTest(BitcoinTestFramework):
+
+    def set_test_params(self):
+        self.setup_clean_chain = False
+        self.num_nodes = 2
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def run_test(self):
+        '''Set up initial chain and run tests defined below'''
+
+        self.test_persistence()
+        self.test_immutable()
+
+        self.nodes[0].generate(110)
+        self.sync_all()
+        reset_balance(self.nodes[1], self.nodes[0].getnewaddress())
+        self.test_fund_send_fund_senddirty()
+        reset_balance(self.nodes[1], self.nodes[0].getnewaddress())
+        self.test_fund_send_fund_send()
+
+    def test_persistence(self):
+        '''Test that wallet files persist the avoid_reuse flag.'''
+        # Configure node 1 to use avoid_reuse
+        self.nodes[1].setwalletflag('avoid_reuse')
+
+        # Flags should be node1.avoid_reuse=false, node2.avoid_reuse=true
+        assert_equal(self.nodes[0].getwalletinfo()["avoid_reuse"], False)
+        assert_equal(self.nodes[1].getwalletinfo()["avoid_reuse"], True)
+
+        # Stop and restart node 1
+        self.stop_node(1)
+        self.start_node(1)
+        connect_nodes_bi(self.nodes, 0, 1)
+
+        # Flags should still be node1.avoid_reuse=false, node2.avoid_reuse=true
+        assert_equal(self.nodes[0].getwalletinfo()["avoid_reuse"], False)
+        assert_equal(self.nodes[1].getwalletinfo()["avoid_reuse"], True)
+
+        # Attempting to set flag to its current state should throw
+        assert_raises_rpc_error(-8, "Wallet flag is already set to false", self.nodes[0].setwalletflag, 'avoid_reuse', False)
+        assert_raises_rpc_error(-8, "Wallet flag is already set to true", self.nodes[1].setwalletflag, 'avoid_reuse', True)
+
+    def test_immutable(self):
+        '''Test immutable wallet flags'''
+        # Attempt to set the disable_private_keys flag; this should not work
+        assert_raises_rpc_error(-8, "Wallet flag is immutable", self.nodes[1].setwalletflag, 'disable_private_keys')
+
+        tempwallet = ".wallet_avoidreuse.py_test_immutable_wallet.dat"
+
+        # Create a wallet with disable_private_keys set; this should work
+        self.nodes[1].createwallet(tempwallet, True)
+        w = self.nodes[1].get_wallet_rpc(tempwallet)
+
+        # Attempt to unset the disable_private_keys flag; this should not work
+        assert_raises_rpc_error(-8, "Wallet flag is immutable", w.setwalletflag, 'disable_private_keys', False)
+
+        # Unload temp wallet
+        self.nodes[1].unloadwallet(tempwallet)
+
+    def test_fund_send_fund_senddirty(self):
+        '''
+        Test the same as test_fund_send_fund_send, except send the 10 BTC with
+        the avoid_reuse flag set to false. This means the 10 BTC send should succeed,
+        where it fails in test_fund_send_fund_send.
+        '''
+
+        fundaddr = self.nodes[1].getnewaddress()
+        retaddr = self.nodes[0].getnewaddress()
+
+        self.nodes[0].sendtoaddress(fundaddr, 10)
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        # listunspent should show 1 single, unused 10 btc output
+        assert_unspent(self.nodes[1], total_count=1, total_sum=10, reused_supported=True, reused_count=0)
+
+        self.nodes[1].sendtoaddress(retaddr, 5)
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        # listunspent should show 1 single, unused 5 btc output
+        assert_unspent(self.nodes[1], total_count=1, total_sum=5, reused_supported=True, reused_count=0)
+
+        self.nodes[0].sendtoaddress(fundaddr, 10)
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10)
+        assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10)
+
+        self.nodes[1].sendtoaddress(address=retaddr, amount=10, avoid_reuse=False)
+
+        # listunspent should show 1 total outputs (5 btc), unused
+        assert_unspent(self.nodes[1], total_count=1, total_sum=5, reused_count=0)
+
+        # node 1 should now have about 5 btc left (for both cases)
+        assert_approx(self.nodes[1].getbalance(), 5, 0.001)
+        assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 5, 0.001)
+
+    def test_fund_send_fund_send(self):
+        '''
+        Test the simple case where [1] generates a new address A, then
+        [0] sends 10 BTC to A.
+        [1] spends 5 BTC from A. (leaving roughly 5 BTC useable)
+        [0] sends 10 BTC to A again.
+        [1] tries to spend 10 BTC (fails; dirty).
+        [1] tries to spend 4 BTC (succeeds; change address sufficient)
+        '''
+
+        fundaddr = self.nodes[1].getnewaddress()
+        retaddr = self.nodes[0].getnewaddress()
+
+        self.nodes[0].sendtoaddress(fundaddr, 10)
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        # listunspent should show 1 single, unused 10 btc output
+        assert_unspent(self.nodes[1], total_count=1, total_sum=10, reused_supported=True, reused_count=0)
+
+        self.nodes[1].sendtoaddress(retaddr, 5)
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        # listunspent should show 1 single, unused 5 btc output
+        assert_unspent(self.nodes[1], total_count=1, total_sum=5, reused_supported=True, reused_count=0)
+
+        self.nodes[0].sendtoaddress(fundaddr, 10)
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10)
+        assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10)
+
+        # node 1 should now have a balance of 5 (no dirty) or 15 (including dirty)
+        assert_approx(self.nodes[1].getbalance(), 5, 0.001)
+        assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 15, 0.001)
+
+        assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[1].sendtoaddress, retaddr, 10)
+
+        self.nodes[1].sendtoaddress(retaddr, 4)
+
+        # listunspent should show 2 total outputs (1, 10 btc), one unused (1), one reused (10)
+        assert_unspent(self.nodes[1], total_count=2, total_sum=11, reused_count=1, reused_sum=10)
+
+        # node 1 should now have about 1 btc left (no dirty) and 11 (including dirty)
+        assert_approx(self.nodes[1].getbalance(), 1, 0.001)
+        assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001)
+
+if __name__ == '__main__':
+    AvoidReuseTest().main()