Merge #19953: Implement BIP 340-342 validation (Schnorr/taproot/tapscript)

0e2a5e448f426219a6464b9aaadcc715534114e6 tests: dumping and minimizing of script assets data (Pieter Wuille)
4567ba034c5ae6e6cc161360f7425c9e844738f0 tests: add generic qa-asset-based script verification unit test (Pieter Wuille)
f06e6d03452cf5e0b1a0863afb08c9e6d3ef452e tests: functional tests for Schnorr/Taproot/Tapscript (Pieter Wuille)
3c226639eb134314a0640d34e4ccb6148dbde22f tests: add BIP340 Schnorr signature support to test framework (Pieter Wuille)
206fb180ec6ee5f916afc6f574000d716daf79b7 --- [TAPROOT] Tests --- (Pieter Wuille)
d7ff237f2996a4c11fdf9399187c2d2b26bf9809 Activate Taproot/Tapscript on regtest (BIP 341, BIP 342) (Pieter Wuille)
e9a021d7e6a454d610a45cb9b3995f0d96a5fbb6 Make Taproot spends standard + policy limits (Pieter Wuille)
865d2c37e2e44678498b7f425b65e01b1e231cde --- [TAPROOT] Regtest activation and policy --- (Pieter Wuille)
72422ce396b8eba7b1a72c171c2f07dae691d1b5 Implement Tapscript script validation rules (BIP 342) (Johnson Lau)
330de894a9a48515d9a473448b6c67adc3d188be Use ScriptExecutionData to pass through annex hash (Pieter Wuille)
8bbed4b7acf4c76eaea8c0e10f3cbf6ba4e53809 Implement Taproot validation (BIP 341) (Pieter Wuille)
0664f5fe1f77f08d235aa3750b59428257b0b91d Support for Schnorr signatures and integration in SignatureCheckers (BIP 340) (Pieter Wuille)
5de246ca8159dcffaa4c136a60c8bfed2028e2ee Implement Taproot signature hashing (BIP 341) (Johnson Lau)
9eb590894f15ff40806039bfd32972fbc260e30d Add TaggedHash function (BIP 340) (Pieter Wuille)
450d2b23710ad296eede81339195376021ab5500 --- [TAPROOT] BIP340/341/342 consensus rules --- (Pieter Wuille)
5d62e3a68b6ea9bb03556ee1fbf5678f20be01a2 refactor: keep spent outputs in PrecomputedTransactionData (Pieter Wuille)
8bd2b4e78452ff69c08c37acf164a6b80e503f13 refactor: rename scriptPubKey in VerifyWitnessProgram to exec_script (Pieter Wuille)
107b57df9fa8b2d625d2b342dc77722282a6ae4c scripted-diff: put ECDSA in name of signature functions (Pieter Wuille)
f8c099e2207c90d758e7a659d6a55fa7ccb7ceaa --- [TAPROOT] Refactors --- (Pieter Wuille)

Pull request description:

  This is an implementation of the Schnorr/taproot consensus rules proposed by BIPs [340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki), [341](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki), and [342](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki).

  See the list of commits [below](https://github.com/bitcoin/bitcoin/pull/19953#issuecomment-691815830). No signing or wallet support of any kind is included, as testing is done entirely through the Python test framework.

  This is a successor to https://github.com/bitcoin/bitcoin/pull/17977 (see discussion following [this comment](https://github.com/bitcoin/bitcoin/pull/17977#issuecomment-682285983)), and will have further changes squashed/rebased. The history of this PR can be found in #19997.

ACKs for top commit:
  instagibbs:
    reACK https://github.com/bitcoin/bitcoin/pull/19953/commits/0e2a5e448f426219a6464b9aaadcc715534114e6
  benthecarman:
    reACK 0e2a5e4
  kallewoof:
    reACK 0e2a5e448f426219a6464b9aaadcc715534114e6
  jonasnick:
    ACK 0e2a5e448f426219a6464b9aaadcc715534114e6 almost only looked at bip340/libsecp related code
  jonatack:
    ACK 0e2a5e448f426219a6464b9aaadcc715534114e6 modulo the last four commits (tests) that I plan to finish reviewing tomorrow
  fjahr:
    reACK 0e2a5e448f426219a6464b9aaadcc715534114e6
  achow101:
    ACK 0e2a5e448f426219a6464b9aaadcc715534114e6

Tree-SHA512: 1b00314450a2938a22bccbb4e177230cf08bd365d72055f9d526891f334b364c997e260c10bc19ca78440b6767712c9feea7faad9a1045dd51a5b96f7ca8146e

1 files changed, 200 insertions, 0 deletions

diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp
new file mode 100644
index 0000000000..d20fa43d68
--- /dev/null
+++ b/src/test/fuzz/script_assets_test_minimizer.cpp
@@ -0,0 +1,200 @@
+// Copyright (c) 2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <test/fuzz/fuzz.h>
+
+#include <primitives/transaction.h>
+#include <pubkey.h>
+#include <script/interpreter.h>
+#include <serialize.h>
+#include <streams.h>
+#include <univalue.h>
+#include <util/strencodings.h>
+
+#include <boost/algorithm/string.hpp>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+// This fuzz "test" can be used to minimize test cases for script_assets_test in
+// src/test/script_tests.cpp. While it written as a fuzz test, and can be used as such,
+// fuzzing the inputs is unlikely to construct useful test cases.
+//
+// Instead, it is primarily intended to be run on a test set that was generated
+// externally, for example using test/functional/feature_taproot.py's --dumptests mode.
+// The minimized set can then be concatenated together, surrounded by '[' and ']',
+// and used as the script_assets_test.json input to the script_assets_test unit test:
+//
+// (normal build)
+// $ mkdir dump
+// $ for N in $(seq 1 10); do TEST_DUMP_DIR=dump test/functional/feature_taproot --dumptests; done
+// $ ...
+//
+// (fuzz test build)
+// $ mkdir dump-min
+// $ ./src/test/fuzz/script_assets_test_minimizer -merge=1 dump-min/ dump/
+// $ (echo -en '[\n'; cat dump-min/* | head -c -2; echo -en '\n]') >script_assets_test.json
+
+namespace {
+
+std::vector<unsigned char> CheckedParseHex(const std::string& str)
+{
+    if (str.size() && !IsHex(str)) throw std::runtime_error("Non-hex input '" + str + "'");
+    return ParseHex(str);
+}
+
+CScript ScriptFromHex(const std::string& str)
+{
+    std::vector<unsigned char> data = CheckedParseHex(str);
+    return CScript(data.begin(), data.end());
+}
+
+CMutableTransaction TxFromHex(const std::string& str)
+{
+    CMutableTransaction tx;
+    try {
+        VectorReader(SER_DISK, SERIALIZE_TRANSACTION_NO_WITNESS, CheckedParseHex(str), 0) >> tx;
+    } catch (const std::ios_base::failure&) {
+        throw std::runtime_error("Tx deserialization failure");
+    }
+    return tx;
+}
+
+std::vector<CTxOut> TxOutsFromJSON(const UniValue& univalue)
+{
+    if (!univalue.isArray()) throw std::runtime_error("Prevouts must be array");
+    std::vector<CTxOut> prevouts;
+    for (size_t i = 0; i < univalue.size(); ++i) {
+        CTxOut txout;
+        try {
+            VectorReader(SER_DISK, 0, CheckedParseHex(univalue[i].get_str()), 0) >> txout;
+        } catch (const std::ios_base::failure&) {
+            throw std::runtime_error("Prevout invalid format");
+        }
+        prevouts.push_back(std::move(txout));
+    }
+    return prevouts;
+}
+
+CScriptWitness ScriptWitnessFromJSON(const UniValue& univalue)
+{
+    if (!univalue.isArray()) throw std::runtime_error("Script witness is not array");
+    CScriptWitness scriptwitness;
+    for (size_t i = 0; i < univalue.size(); ++i) {
+        auto bytes = CheckedParseHex(univalue[i].get_str());
+        scriptwitness.stack.push_back(std::move(bytes));
+    }
+    return scriptwitness;
+}
+
+const std::map<std::string, unsigned int> FLAG_NAMES = {
+    {std::string("P2SH"), (unsigned int)SCRIPT_VERIFY_P2SH},
+    {std::string("DERSIG"), (unsigned int)SCRIPT_VERIFY_DERSIG},
+    {std::string("NULLDUMMY"), (unsigned int)SCRIPT_VERIFY_NULLDUMMY},
+    {std::string("CHECKLOCKTIMEVERIFY"), (unsigned int)SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY},
+    {std::string("CHECKSEQUENCEVERIFY"), (unsigned int)SCRIPT_VERIFY_CHECKSEQUENCEVERIFY},
+    {std::string("WITNESS"), (unsigned int)SCRIPT_VERIFY_WITNESS},
+    {std::string("TAPROOT"), (unsigned int)SCRIPT_VERIFY_TAPROOT},
+};
+
+std::vector<unsigned int> AllFlags()
+{
+    std::vector<unsigned int> ret;
+
+    for (unsigned int i = 0; i < 128; ++i) {
+        unsigned int flag = 0;
+        if (i & 1) flag |= SCRIPT_VERIFY_P2SH;
+        if (i & 2) flag |= SCRIPT_VERIFY_DERSIG;
+        if (i & 4) flag |= SCRIPT_VERIFY_NULLDUMMY;
+        if (i & 8) flag |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY;
+        if (i & 16) flag |= SCRIPT_VERIFY_CHECKSEQUENCEVERIFY;
+        if (i & 32) flag |= SCRIPT_VERIFY_WITNESS;
+        if (i & 64) flag |= SCRIPT_VERIFY_TAPROOT;
+
+        // SCRIPT_VERIFY_WITNESS requires SCRIPT_VERIFY_P2SH
+        if (flag & SCRIPT_VERIFY_WITNESS && !(flag & SCRIPT_VERIFY_P2SH)) continue;
+        // SCRIPT_VERIFY_TAPROOT requires SCRIPT_VERIFY_WITNESS
+        if (flag & SCRIPT_VERIFY_TAPROOT && !(flag & SCRIPT_VERIFY_WITNESS)) continue;
+
+        ret.push_back(flag);
+    }
+
+    return ret;
+}
+
+const std::vector<unsigned int> ALL_FLAGS = AllFlags();
+
+unsigned int ParseScriptFlags(const std::string& str)
+{
+    if (str.empty()) return 0;
+
+    unsigned int flags = 0;
+    std::vector<std::string> words;
+    boost::algorithm::split(words, str, boost::algorithm::is_any_of(","));
+
+    for (const std::string& word : words)
+    {
+        auto it = FLAG_NAMES.find(word);
+        if (it == FLAG_NAMES.end()) throw std::runtime_error("Unknown verification flag " + word);
+        flags |= it->second;
+    }
+
+    return flags;
+}
+
+void Test(const std::string& str)
+{
+    UniValue test;
+    if (!test.read(str) || !test.isObject()) throw std::runtime_error("Non-object test input");
+
+    CMutableTransaction tx = TxFromHex(test["tx"].get_str());
+    const std::vector<CTxOut> prevouts = TxOutsFromJSON(test["prevouts"]);
+    if (prevouts.size() != tx.vin.size()) throw std::runtime_error("Incorrect number of prevouts");
+    size_t idx = test["index"].get_int64();
+    if (idx >= tx.vin.size()) throw std::runtime_error("Invalid index");
+    unsigned int test_flags = ParseScriptFlags(test["flags"].get_str());
+    bool final = test.exists("final") && test["final"].get_bool();
+
+    if (test.exists("success")) {
+        tx.vin[idx].scriptSig = ScriptFromHex(test["success"]["scriptSig"].get_str());
+        tx.vin[idx].scriptWitness = ScriptWitnessFromJSON(test["success"]["witness"]);
+        PrecomputedTransactionData txdata;
+        txdata.Init(tx, std::vector<CTxOut>(prevouts));
+        MutableTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, txdata);
+        for (const auto flags : ALL_FLAGS) {
+            // "final": true tests are valid for all flags. Others are only valid with flags that are
+            // a subset of test_flags.
+            if (final || ((flags & test_flags) == flags)) {
+                (void)VerifyScript(tx.vin[idx].scriptSig, prevouts[idx].scriptPubKey, &tx.vin[idx].scriptWitness, flags, txcheck, nullptr);
+            }
+        }
+    }
+
+    if (test.exists("failure")) {
+        tx.vin[idx].scriptSig = ScriptFromHex(test["failure"]["scriptSig"].get_str());
+        tx.vin[idx].scriptWitness = ScriptWitnessFromJSON(test["failure"]["witness"]);
+        PrecomputedTransactionData txdata;
+        txdata.Init(tx, std::vector<CTxOut>(prevouts));
+        MutableTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, txdata);
+        for (const auto flags : ALL_FLAGS) {
+            // If a test is supposed to fail with test_flags, it should also fail with any superset thereof.
+            if ((flags & test_flags) == test_flags) {
+                (void)VerifyScript(tx.vin[idx].scriptSig, prevouts[idx].scriptPubKey, &tx.vin[idx].scriptWitness, flags, txcheck, nullptr);
+            }
+        }
+    }
+}
+
+ECCVerifyHandle handle;
+
+}
+
+void test_one_input(const std::vector<uint8_t>& buffer)
+{
+    if (buffer.size() < 2 || buffer.back() != '\n' || buffer[buffer.size() - 2] != ',') return;
+    const std::string str((const char*)buffer.data(), buffer.size() - 2);
+    try {
+        Test(str);
+    } catch (const std::runtime_error&) {}
+}