consensus: guard against openssl's new strict DER checks

New versions of OpenSSL will reject non-canonical DER signatures. However,
it'll happily decode them. Decode then re-encode before verification in order
to ensure that it is properly consumed.

Github-Pull: #5634
Rebased-From: 488ed32f2ada1d1dd108fc245d025c4d5f252783

1 files changed, 13 insertions, 3 deletions

diff --git a/src/key.cpp b/src/key.cpp
index 5b261bb285..e5943af79a 100644
--- a/src/key.cpp
+++ b/src/key.cpp
@@ -227,10 +227,20 @@ public:
     }
 
     bool Verify(const uint256 &hash, const std::vector<unsigned char>& vchSig) {
-        // -1 = error, 0 = bad sig, 1 = good
-        if (ECDSA_verify(0, (unsigned char*)&hash, sizeof(hash), &vchSig[0], vchSig.size(), pkey) != 1)
+        // New versions of OpenSSL will reject non-canonical DER signatures. de/re-serialize first.
+        unsigned char *norm_der = NULL;
+        ECDSA_SIG *norm_sig = ECDSA_SIG_new();
+        const unsigned char* sigptr = &vchSig[0];
+        d2i_ECDSA_SIG(&norm_sig, &sigptr, vchSig.size());
+        int derlen = i2d_ECDSA_SIG(norm_sig, &norm_der);
+        ECDSA_SIG_free(norm_sig);
+        if (derlen <= 0)
             return false;
-        return true;
+
+        // -1 = error, 0 = bad sig, 1 = good
+        bool ret = ECDSA_verify(0, (unsigned char*)&hash, sizeof(hash), norm_der, derlen, pkey) == 1;
+        OPENSSL_free(norm_der);
+        return ret;
     }
 
     bool SignCompact(const uint256 &hash, unsigned char *p64, int &rec) {