1 files changed, 87 insertions, 12 deletions

diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp
index 100d0d1fac..fbdbef0bc6 100644
--- a/src/crypto/sha256.cpp
+++ b/src/crypto/sha256.cpp
@@ -9,7 +9,7 @@
 #include <string.h>
 #include <atomic>
 
-#if defined(__x86_64__) || defined(__amd64__)
+#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__)
 #if defined(USE_ASM)
 #include <cpuid.h>
 namespace sha256_sse4
@@ -29,6 +29,16 @@ namespace sha256d64_avx2
 void Transform_8way(unsigned char* out, const unsigned char* in);
 }
 
+namespace sha256d64_shani
+{
+void Transform_2way(unsigned char* out, const unsigned char* in);
+}
+
+namespace sha256_shani
+{
+void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks);
+}
+
 // Internal implementation code.
 namespace
 {
@@ -448,6 +458,7 @@ void TransformD64Wrapper(unsigned char* out, const unsigned char* in)
 
 TransformType Transform = sha256::Transform;
 TransformD64Type TransformD64 = sha256::TransformD64;
+TransformD64Type TransformD64_2way = nullptr;
 TransformD64Type TransformD64_4way = nullptr;
 TransformD64Type TransformD64_8way = nullptr;
 
@@ -512,6 +523,13 @@ bool SelfTest() {
     TransformD64(out, data + 1);
     if (!std::equal(out, out + 32, result_d64)) return false;
 
+    // Test TransformD64_2way, if available.
+    if (TransformD64_2way) {
+        unsigned char out[64];
+        TransformD64_2way(out, data + 1);
+        if (!std::equal(out, out + 64, result_d64)) return false;
+    }
+
     // Test TransformD64_4way, if available.
     if (TransformD64_4way) {
         unsigned char out[128];
@@ -534,7 +552,19 @@ bool SelfTest() {
 // We can't use cpuid.h's __get_cpuid as it does not support subleafs.
 void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d)
 {
+#ifdef __GNUC__
+    __cpuid_count(leaf, subleaf, a, b, c, d);
+#else
   __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf));
+#endif
+}
+
+/** Check whether the OS has enabled AVX registers. */
+bool AVXEnabled()
+{
+    uint32_t a, d;
+    __asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(0));
+    return (a & 6) == 6;
 }
 #endif
 } // namespace
@@ -544,27 +574,64 @@ std::string SHA256AutoDetect()
 {
     std::string ret = "standard";
 #if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__))
+    bool have_sse4 = false;
+    bool have_xsave = false;
+    bool have_avx = false;
+    bool have_avx2 = false;
+    bool have_shani = false;
+    bool enabled_avx = false;
+
+    (void)AVXEnabled;
+    (void)have_sse4;
+    (void)have_avx;
+    (void)have_xsave;
+    (void)have_avx2;
+    (void)have_shani;
+    (void)enabled_avx;
+
     uint32_t eax, ebx, ecx, edx;
     cpuid(1, 0, eax, ebx, ecx, edx);
-    if ((ecx >> 19) & 1) {
+    have_sse4 = (ecx >> 19) & 1;
+    have_xsave = (ecx >> 27) & 1;
+    have_avx = (ecx >> 28) & 1;
+    if (have_xsave && have_avx) {
+        enabled_avx = AVXEnabled();
+    }
+    if (have_sse4) {
+        cpuid(7, 0, eax, ebx, ecx, edx);
+        have_avx2 = (ebx >> 5) & 1;
+        have_shani = (ebx >> 29) & 1;
+    }
+
+#if defined(ENABLE_SHANI) && !defined(BUILD_BITCOIN_INTERNAL)
+    if (have_shani) {
+        Transform = sha256_shani::Transform;
+        TransformD64 = TransformD64Wrapper<sha256_shani::Transform>;
+        TransformD64_2way = sha256d64_shani::Transform_2way;
+        ret = "shani(1way,2way)";
+        have_sse4 = false; // Disable SSE4/AVX2;
+        have_avx2 = false;
+    }
+#endif
+
+    if (have_sse4) {
 #if defined(__x86_64__) || defined(__amd64__)
         Transform = sha256_sse4::Transform;
         TransformD64 = TransformD64Wrapper<sha256_sse4::Transform>;
+        ret = "sse4(1way)";
 #endif
 #if defined(ENABLE_SSE41) && !defined(BUILD_BITCOIN_INTERNAL)
         TransformD64_4way = sha256d64_sse41::Transform_4way;
-        ret = "sse4(1way+4way)";
-#if defined(ENABLE_AVX2) && !defined(BUILD_BITCOIN_INTERNAL)
-        cpuid(7, 0, eax, ebx, ecx, edx);
-        if ((ebx >> 5) & 1) {
-            TransformD64_8way = sha256d64_avx2::Transform_8way;
-            ret += ",avx2(8way)";
-        }
-#endif
-#else
-        ret = "sse4";
+        ret += ",sse41(4way)";
 #endif
     }
+
+#if defined(ENABLE_AVX2) && !defined(BUILD_BITCOIN_INTERNAL)
+    if (have_avx2 && have_avx && enabled_avx) {
+        TransformD64_8way = sha256d64_avx2::Transform_8way;
+        ret += ",avx2(8way)";
+    }
+#endif
 #endif
 
     assert(SelfTest());
@@ -646,6 +713,14 @@ void SHA256D64(unsigned char* out, const unsigned char* in, size_t blocks)
             blocks -= 4;
         }
     }
+    if (TransformD64_2way) {
+        while (blocks >= 2) {
+            TransformD64_2way(out, in);
+            out += 64;
+            in += 128;
+            blocks -= 2;
+        }
+    }
     while (blocks) {
         TransformD64(out, in);
         out += 32;