Make STRICTENC invalid pubkeys fail the script rather than the opcode.

This turns STRICTENC turn into a softforking-safe change (even though it
is not intended as a consensus rule), and as a result guarantee that using
it for mempool validation only results in consensus-valid transactions in
the mempool.

7 files changed, 53 insertions, 23 deletions

diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp
index 760086eaba..a2a2edce6d 100644
--- a/src/script/interpreter.cpp
+++ b/src/script/interpreter.cpp
@@ -207,9 +207,9 @@ bool static CheckSignatureEncoding(const valtype &vchSig, unsigned int flags, Sc
     return true;
 }
 
-bool static CheckPubKeyEncoding(const valtype &vchSig, unsigned int flags) {
+bool static CheckPubKeyEncoding(const valtype &vchSig, unsigned int flags, ScriptError* serror) {
     if ((flags & SCRIPT_VERIFY_STRICTENC) != 0 && !IsCompressedOrUncompressedPubKey(vchSig)) {
-        return false;
+        return set_error(serror, SCRIPT_ERR_PUBKEYTYPE);
     }
     return true;
 }
@@ -792,11 +792,11 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, un
                     // Drop the signature, since there's no way for a signature to sign itself
                     scriptCode.FindAndDelete(CScript(vchSig));
 
-                    if (!CheckSignatureEncoding(vchSig, flags, serror)) {
+                    if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, serror)) {
                         //serror is set
                         return false;
                     }
-                    bool fSuccess = CheckPubKeyEncoding(vchPubKey, flags) && checker.CheckSig(vchSig, vchPubKey, scriptCode);
+                    bool fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode);
 
                     popstack(stack);
                     popstack(stack);
@@ -855,13 +855,13 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, un
                         valtype& vchSig    = stacktop(-isig);
                         valtype& vchPubKey = stacktop(-ikey);
 
-                        if (!CheckSignatureEncoding(vchSig, flags, serror)) {
+                        if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, serror)) {
                             // serror is set
                             return false;
                         }
 
                         // Check signature
-                        bool fOk = CheckPubKeyEncoding(vchPubKey, flags) && checker.CheckSig(vchSig, vchPubKey, scriptCode);
+                        bool fOk = checker.CheckSig(vchSig, vchPubKey, scriptCode);
 
                         if (fOk) {
                             isig++;
diff --git a/src/script/interpreter.h b/src/script/interpreter.h
index 12b2719414..35b2f6c65a 100644
--- a/src/script/interpreter.h
+++ b/src/script/interpreter.h
@@ -35,8 +35,8 @@ enum
     SCRIPT_VERIFY_P2SH      = (1U << 0),
 
     // Passing a non-strict-DER signature or one with undefined hashtype to a checksig operation causes script failure.
-    // Passing a pubkey that is not (0x04 + 64 bytes) or (0x02 or 0x03 + 32 bytes) to checksig causes that pubkey to be
-    // skipped (not softfork safe: this flag can widen the validity of OP_CHECKSIG OP_NOT).
+    // Evaluating a pubkey that is not (0x04 + 64 bytes) or (0x02 or 0x03 + 32 bytes) by checksig causes script failure.
+    // (softfork safe, but not used or intended as a consensus rule).
     SCRIPT_VERIFY_STRICTENC = (1U << 1),
 
     // Passing a non-strict-DER signature to a checksig operation causes script failure (softfork safe, BIP62 rule 1)
diff --git a/src/script/script_error.cpp b/src/script/script_error.cpp
index 793fc0da41..5d24ed98ba 100644
--- a/src/script/script_error.cpp
+++ b/src/script/script_error.cpp
@@ -61,6 +61,8 @@ const char* ScriptErrorString(const ScriptError serror)
             return "Dummy CHECKMULTISIG argument must be zero";
         case SCRIPT_ERR_DISCOURAGE_UPGRADABLE_NOPS:
             return "NOPx reserved for soft-fork upgrades";
+        case SCRIPT_ERR_PUBKEYTYPE:
+            return "Public key is neither compressed or uncompressed";
         case SCRIPT_ERR_UNKNOWN_ERROR:
         case SCRIPT_ERR_ERROR_COUNT:
         default: break;
diff --git a/src/script/script_error.h b/src/script/script_error.h
index 21153f1bd1..ac1f2deae5 100644
--- a/src/script/script_error.h
+++ b/src/script/script_error.h
@@ -42,6 +42,7 @@ typedef enum ScriptError_t
     SCRIPT_ERR_SIG_PUSHONLY,
     SCRIPT_ERR_SIG_HIGH_S,
     SCRIPT_ERR_SIG_NULLDUMMY,
+    SCRIPT_ERR_PUBKEYTYPE,
 
     /* softfork safeness */
     SCRIPT_ERR_DISCOURAGE_UPGRADABLE_NOPS,
diff --git a/src/test/data/script_invalid.json b/src/test/data/script_invalid.json
index 0356d0be1c..96ff0a2911 100644
--- a/src/test/data/script_invalid.json
+++ b/src/test/data/script_invalid.json
@@ -593,6 +593,24 @@ nSequences are max.
     "P2PK NOT with hybrid pubkey but no STRICTENC"
 ],
 [
+    "0x47 0x3044022078033e4227aa05ded69d8da579966578e230d8a7fb44d5f1a0620c3853c24f78022006a2e3f4d872ac8dfdc529110aa37301d65a76255a4b6cce2992adacd4d2c4e201",
+    "0x41 0x0679be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 CHECKSIG NOT",
+    "STRICTENC",
+    "P2PK NOT with hybrid pubkey"
+],
+[
+    "0x47 0x304402207592427de20e315d644839754f2a5cca5b978b983a15e6da82109ede01722baa022032ceaf78590faa3f7743821e1b47b897ed1a57f6ee1c8a7519d23774d8de3c4401",
+    "0x41 0x0679be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 CHECKSIG NOT",
+    "STRICTENC",
+    "P2PK NOT with invalid hybrid pubkey"
+],
+[
+    "0 0x47 0x304402206797289d3dc81692edae58430276d04641ea5d86967be557163f8494da32fd78022006fc6ab77aaed4ac11ea69cd878ab26e3e24290f47a43e9adf34075d52b7142c01",
+    "1 0x21 0x038282263212c609d9ea2a6e3e172de238d8c39cabd5ac1ca10646e23fd5f51508 0x41 0x0679be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 2 CHECKMULTISIG",
+    "STRICTENC",
+    "1-of-2 with the first 1 hybrid pubkey"
+],
+[
     "0x47 0x304402201f82b99a813c9c48c8dee8d2c43b8f637b72353fe9bdcc084537bc17e2ab770402200c43b96a5f7e115f0114eabda32e068145965cb6c7b5ef64833bb4fcf9fc1b3b05",
     "0x41 0x048282263212c609d9ea2a6e3e172de238d8c39cabd5ac1ca10646e23fd5f5150811f8a8098557dfe45e8256e830b60ace62d613ac2f7b17bed31b6eaff6e26caf CHECKSIG",
     "STRICTENC",
diff --git a/src/test/data/script_valid.json b/src/test/data/script_valid.json
index 0cf1563163..94e1abfaae 100644
--- a/src/test/data/script_valid.json
+++ b/src/test/data/script_valid.json
@@ -744,22 +744,22 @@ nSequences are max.
     "P2PK with hybrid pubkey but no STRICTENC"
 ],
 [
-    "0x47 0x3044022078033e4227aa05ded69d8da579966578e230d8a7fb44d5f1a0620c3853c24f78022006a2e3f4d872ac8dfdc529110aa37301d65a76255a4b6cce2992adacd4d2c4e201",
-    "0x41 0x0679be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 CHECKSIG NOT",
-    "STRICTENC",
-    "P2PK NOT with hybrid pubkey"
-],
-[
     "0x47 0x3044022078d6c447887e88dcbe1bc5b613645280df6f4e5935648bc226e9d91da71b3216022047d6b7ef0949b228fc1b359afb8d50500268711354298217b983c26970790c7601",
     "0x41 0x0679be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 CHECKSIG NOT",
     "",
     "P2PK NOT with invalid hybrid pubkey but no STRICTENC"
 ],
 [
-    "0x47 0x304402207592427de20e315d644839754f2a5cca5b978b983a15e6da82109ede01722baa022032ceaf78590faa3f7743821e1b47b897ed1a57f6ee1c8a7519d23774d8de3c4401",
-    "0x41 0x0679be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 CHECKSIG NOT",
+    "0 0x47 0x304402203b269b9fbc0936877bf855b5fb41757218d9548b246370d991442a5f5bd1c3440220235268a4eaa8c67e543c6e37da81dd36d3b1be2de6b4fef04113389ca6ddc04501",
+    "1 0x41 0x0679be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 0x21 0x038282263212c609d9ea2a6e3e172de238d8c39cabd5ac1ca10646e23fd5f51508 2 CHECKMULTISIG",
+    "",
+    "1-of-2 with the second 1 hybrid pubkey and no STRICTENC"
+],
+[
+    "0 0x47 0x3044022044dc17b0887c161bb67ba9635bf758735bdde503e4b0a0987f587f14a4e1143d022009a215772d49a85dae40d8ca03955af26ad3978a0ff965faa12915e9586249a501",
+    "1 0x41 0x0679be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 0x21 0x038282263212c609d9ea2a6e3e172de238d8c39cabd5ac1ca10646e23fd5f51508 2 CHECKMULTISIG",
     "STRICTENC",
-    "P2PK NOT with invalid hybrid pubkey"
+    "1-of-2 with the second 1 hybrid pubkey"
 ],
 [
     "0x47 0x304402204649e9517ef0377a8f8270bd423053fd98ddff62d74ea553e9579558abbb75e4022044a2b2344469c12e35ed898987711272b634733dd0f5e051288eceb04bd4669e05",
diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp
index 36aaa6903f..6952f4c584 100644
--- a/src/test/script_tests.cpp
+++ b/src/test/script_tests.cpp
@@ -428,15 +428,24 @@ BOOST_AUTO_TEST_CASE(script_build)
     bad.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey0H) << OP_CHECKSIG << OP_NOT,
                               "P2PK NOT with hybrid pubkey but no STRICTENC", 0
                              ).PushSig(keys.key0, SIGHASH_ALL));
-    good.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey0H) << OP_CHECKSIG << OP_NOT,
-                               "P2PK NOT with hybrid pubkey", SCRIPT_VERIFY_STRICTENC
-                              ).PushSig(keys.key0, SIGHASH_ALL));
+    bad.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey0H) << OP_CHECKSIG << OP_NOT,
+                              "P2PK NOT with hybrid pubkey", SCRIPT_VERIFY_STRICTENC
+                             ).PushSig(keys.key0, SIGHASH_ALL));
     good.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey0H) << OP_CHECKSIG << OP_NOT,
                                "P2PK NOT with invalid hybrid pubkey but no STRICTENC", 0
                               ).PushSig(keys.key0, SIGHASH_ALL).DamagePush(10));
-    good.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey0H) << OP_CHECKSIG << OP_NOT,
-                               "P2PK NOT with invalid hybrid pubkey", SCRIPT_VERIFY_STRICTENC
-                              ).PushSig(keys.key0, SIGHASH_ALL).DamagePush(10));
+    bad.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey0H) << OP_CHECKSIG << OP_NOT,
+                              "P2PK NOT with invalid hybrid pubkey", SCRIPT_VERIFY_STRICTENC
+                             ).PushSig(keys.key0, SIGHASH_ALL).DamagePush(10));
+    good.push_back(TestBuilder(CScript() << OP_1 << ToByteVector(keys.pubkey0H) << ToByteVector(keys.pubkey1C) << OP_2 << OP_CHECKMULTISIG,
+                               "1-of-2 with the second 1 hybrid pubkey and no STRICTENC", 0
+                              ).Num(0).PushSig(keys.key1, SIGHASH_ALL));
+    good.push_back(TestBuilder(CScript() << OP_1 << ToByteVector(keys.pubkey0H) << ToByteVector(keys.pubkey1C) << OP_2 << OP_CHECKMULTISIG,
+                               "1-of-2 with the second 1 hybrid pubkey", SCRIPT_VERIFY_STRICTENC
+                              ).Num(0).PushSig(keys.key1, SIGHASH_ALL));
+    bad.push_back(TestBuilder(CScript() << OP_1 << ToByteVector(keys.pubkey1C) << ToByteVector(keys.pubkey0H) << OP_2 << OP_CHECKMULTISIG,
+                              "1-of-2 with the first 1 hybrid pubkey", SCRIPT_VERIFY_STRICTENC
+                             ).Num(0).PushSig(keys.key1, SIGHASH_ALL));
 
     good.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey1) << OP_CHECKSIG,
                                "P2PK with undefined hashtype but no STRICTENC", 0