diff options
| -rw-r--r-- | configure.ac | 37 | ||||
| -rw-r--r-- | src/Makefile.am | 26 | ||||
| -rw-r--r-- | src/crypto/sha256.cpp | 74 | ||||
| -rw-r--r-- | src/crypto/sha256_arm_shani.cpp | 899 | ||||
| -rw-r--r-- | src/crypto/sha256_x86_shani.cpp (renamed from src/crypto/sha256_shani.cpp) | 6 |
5 files changed, 1012 insertions, 30 deletions
diff --git a/configure.ac b/configure.ac index 11a01f310c..452b7deec7 100644 --- a/configure.ac +++ b/configure.ac @@ -469,7 +469,7 @@ AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CXXFLAGS="$CXXFLAGS -fno-ext enable_sse42=no enable_sse41=no enable_avx2=no -enable_shani=no +enable_x86_shani=no if test "$use_asm" = "yes"; then @@ -481,7 +481,7 @@ dnl x86 AX_CHECK_COMPILE_FLAG([-msse4.2], [SSE42_CXXFLAGS="-msse4.2"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-msse4.1], [SSE41_CXXFLAGS="-msse4.1"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-mavx -mavx2], [AVX2_CXXFLAGS="-mavx -mavx2"], [], [$CXXFLAG_WERROR]) -AX_CHECK_COMPILE_FLAG([-msse4 -msha], [SHANI_CXXFLAGS="-msse4 -msha"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-msse4 -msha], [X86_SHANI_CXXFLAGS="-msse4 -msha"], [], [$CXXFLAG_WERROR]) enable_clmul= AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([ @@ -554,8 +554,8 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $SHANI_CXXFLAGS" -AC_MSG_CHECKING([for SHA-NI intrinsics]) +CXXFLAGS="$CXXFLAGS $X86_SHANI_CXXFLAGS" +AC_MSG_CHECKING([for x86 SHA-NI intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <stdint.h> #include <immintrin.h> @@ -565,17 +565,18 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m128i k = _mm_set1_epi32(2); return _mm_extract_epi32(_mm_sha256rnds2_epu32(i, i, k), 0); ]])], - [ AC_MSG_RESULT([yes]); enable_shani=yes; AC_DEFINE([ENABLE_SHANI], [1], [Define this symbol to build code that uses SHA-NI intrinsics]) ], + [ AC_MSG_RESULT([yes]); enable_x86_shani=yes; AC_DEFINE([ENABLE_X86_SHANI], [1], [Define this symbol to build code that uses x86 SHA-NI intrinsics]) ], [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" # ARM AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS" -AC_MSG_CHECKING([for AArch64 CRC32 intrinsics]) +AC_MSG_CHECKING([for ARMv8 CRC32 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <arm_acle.h> #include <arm_neon.h> @@ -592,6 +593,24 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ) CXXFLAGS="$TEMP_CXXFLAGS" +TEMP_CXXFLAGS="$CXXFLAGS" +CXXFLAGS="$CXXFLAGS $ARM_SHANI_CXXFLAGS" +AC_MSG_CHECKING([for ARMv8 SHA-NI intrinsics]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include <arm_acle.h> + #include <arm_neon.h> + ]],[[ + uint32x4_t a, b, c; + vsha256h2q_u32(a, b, c); + vsha256hq_u32(a, b, c); + vsha256su0q_u32(a, b); + vsha256su1q_u32(a, b, c); + ]])], + [ AC_MSG_RESULT([yes]); enable_arm_shani=yes; AC_DEFINE([ENABLE_ARM_SHANI], [1], [Define this symbol to build code that uses ARMv8 SHA-NI intrinsics]) ], + [ AC_MSG_RESULT([no])] +) +CXXFLAGS="$TEMP_CXXFLAGS" + fi CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO" @@ -1774,8 +1793,9 @@ AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) AM_CONDITIONAL([ENABLE_SSE42], [test "$enable_sse42" = "yes"]) AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) AM_CONDITIONAL([ENABLE_AVX2], [test "$enable_avx2" = "yes"]) -AM_CONDITIONAL([ENABLE_SHANI], [test "$enable_shani" = "yes"]) +AM_CONDITIONAL([ENABLE_X86_SHANI], [test "$enable_x86_shani" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_CRC], [test "$enable_arm_crc" = "yes"]) +AM_CONDITIONAL([ENABLE_ARM_SHANI], [test "$enable_arm_shani" = "yes"]) AM_CONDITIONAL([USE_ASM], [test "$use_asm" = "yes"]) AM_CONDITIONAL([WORDS_BIGENDIAN], [test "$ac_cv_c_bigendian" = "yes"]) AM_CONDITIONAL([USE_NATPMP], [test "$use_natpmp" = "yes"]) @@ -1832,8 +1852,9 @@ AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS) AC_SUBST(CLMUL_CXXFLAGS) AC_SUBST(AVX2_CXXFLAGS) -AC_SUBST(SHANI_CXXFLAGS) +AC_SUBST(X86_SHANI_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS) +AC_SUBST(ARM_SHANI_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_SQLITE) AC_SUBST(USE_BDB) diff --git a/src/Makefile.am b/src/Makefile.am index efc32da55d..417a611181 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -46,9 +46,13 @@ if ENABLE_AVX2 LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.a LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) endif -if ENABLE_SHANI -LIBBITCOIN_CRYPTO_SHANI = crypto/libbitcoin_crypto_shani.a -LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SHANI) +if ENABLE_X86_SHANI +LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.a +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_SHANI) +endif +if ENABLE_ARM_SHANI +LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.a +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI) endif $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*) @@ -498,11 +502,17 @@ crypto_libbitcoin_crypto_avx2_a_CXXFLAGS += $(AVX2_CXXFLAGS) crypto_libbitcoin_crypto_avx2_a_CPPFLAGS += -DENABLE_AVX2 crypto_libbitcoin_crypto_avx2_a_SOURCES = crypto/sha256_avx2.cpp -crypto_libbitcoin_crypto_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_shani_a_CXXFLAGS += $(SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_shani_a_CPPFLAGS += -DENABLE_SHANI -crypto_libbitcoin_crypto_shani_a_SOURCES = crypto/sha256_shani.cpp +crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS += $(X86_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS += -DENABLE_X86_SHANI +crypto_libbitcoin_crypto_x86_shani_a_SOURCES = crypto/sha256_x86_shani.cpp + +crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS += -DENABLE_ARM_SHANI +crypto_libbitcoin_crypto_arm_shani_a_SOURCES = crypto/sha256_arm_shani.cpp # consensus: shared between all executables that validate any consensus rules. libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index e35d526d35..cde543e68c 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -10,6 +10,16 @@ #include <compat/cpuid.h> +#if defined(__linux__) && defined(ENABLE_ARM_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) +#include <sys/auxv.h> +#include <asm/hwcap.h> +#endif + +#if defined(MAC_OSX) && defined(ENABLE_ARM_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) +#include <sys/types.h> +#include <sys/sysctl.h> +#endif + #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) #if defined(USE_ASM) namespace sha256_sse4 @@ -29,16 +39,26 @@ namespace sha256d64_avx2 void Transform_8way(unsigned char* out, const unsigned char* in); } -namespace sha256d64_shani +namespace sha256d64_x86_shani { void Transform_2way(unsigned char* out, const unsigned char* in); } -namespace sha256_shani +namespace sha256_x86_shani { void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); } +namespace sha256_arm_shani +{ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); +} + +namespace sha256d64_arm_shani +{ +void Transform_2way(unsigned char* out, const unsigned char* in); +} + // Internal implementation code. namespace { @@ -567,7 +587,7 @@ std::string SHA256AutoDetect() bool have_xsave = false; bool have_avx = false; bool have_avx2 = false; - bool have_shani = false; + bool have_x86_shani = false; bool enabled_avx = false; (void)AVXEnabled; @@ -575,7 +595,7 @@ std::string SHA256AutoDetect() (void)have_avx; (void)have_xsave; (void)have_avx2; - (void)have_shani; + (void)have_x86_shani; (void)enabled_avx; uint32_t eax, ebx, ecx, edx; @@ -589,15 +609,15 @@ std::string SHA256AutoDetect() if (have_sse4) { GetCPUID(7, 0, eax, ebx, ecx, edx); have_avx2 = (ebx >> 5) & 1; - have_shani = (ebx >> 29) & 1; + have_x86_shani = (ebx >> 29) & 1; } -#if defined(ENABLE_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) - if (have_shani) { - Transform = sha256_shani::Transform; - TransformD64 = TransformD64Wrapper<sha256_shani::Transform>; - TransformD64_2way = sha256d64_shani::Transform_2way; - ret = "shani(1way,2way)"; +#if defined(ENABLE_X86_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) + if (have_x86_shani) { + Transform = sha256_x86_shani::Transform; + TransformD64 = TransformD64Wrapper<sha256_x86_shani::Transform>; + TransformD64_2way = sha256d64_x86_shani::Transform_2way; + ret = "x86_shani(1way,2way)"; have_sse4 = false; // Disable SSE4/AVX2; have_avx2 = false; } @@ -623,6 +643,38 @@ std::string SHA256AutoDetect() #endif #endif +#if defined(ENABLE_ARM_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) + bool have_arm_shani = false; + +#if defined(__linux__) +#if defined(__arm__) // 32-bit + if (getauxval(AT_HWCAP2) & HWCAP2_SHA2) { + have_arm_shani = true; + } +#endif +#if defined(__aarch64__) // 64-bit + if (getauxval(AT_HWCAP) & HWCAP_SHA2) { + have_arm_shani = true; + } +#endif +#endif + +#if defined(MAC_OSX) + int val = 0; + size_t len = sizeof(val); + if (sysctlbyname("hw.optional.arm.FEAT_SHA256", &val, &len, nullptr, 0) == 0) { + have_arm_shani = val != 0; + } +#endif + + if (have_arm_shani) { + Transform = sha256_arm_shani::Transform; + TransformD64 = TransformD64Wrapper<sha256_arm_shani::Transform>; + TransformD64_2way = sha256d64_arm_shani::Transform_2way; + ret = "arm_shani(1way,2way)"; + } +#endif + assert(SelfTest()); return ret; } diff --git a/src/crypto/sha256_arm_shani.cpp b/src/crypto/sha256_arm_shani.cpp new file mode 100644 index 0000000000..7ffa56be70 --- /dev/null +++ b/src/crypto/sha256_arm_shani.cpp @@ -0,0 +1,899 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// Based on https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-arm.c, +// Written and placed in public domain by Jeffrey Walton. +// Based on code from ARM, and by Johannes Schneiders, Skip Hovsmith and +// Barry O'Rourke for the mbedTLS project. +// Variant specialized for 64-byte inputs added by Pieter Wuille. + +#ifdef ENABLE_ARM_SHANI + +#include <array> +#include <cstdint> +#include <cstddef> +#include <arm_acle.h> +#include <arm_neon.h> + +namespace { +alignas(uint32x4_t) static constexpr std::array<uint32_t, 64> K = +{ + 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, + 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5, + 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, + 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174, + 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, + 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA, + 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, + 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967, + 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, + 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85, + 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, + 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070, + 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, + 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3, + 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, + 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2, +}; +} + +namespace sha256_arm_shani { +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) +{ + uint32x4_t STATE0, STATE1, ABEF_SAVE, CDGH_SAVE; + uint32x4_t MSG0, MSG1, MSG2, MSG3; + uint32x4_t TMP0, TMP2; + + // Load state + STATE0 = vld1q_u32(&s[0]); + STATE1 = vld1q_u32(&s[4]); + + while (blocks--) + { + // Save state + ABEF_SAVE = STATE0; + CDGH_SAVE = STATE1; + + // Load and convert input chunk to Big Endian + MSG0 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 0))); + MSG1 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 16))); + MSG2 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 32))); + MSG3 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 48))); + chunk += 64; + + // Original implemenation preloaded message and constant addition which was 1-3% slower. + // Now included as first step in quad round code saving one Q Neon register + // "TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[0]));" + + // Rounds 1-4 + TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[0])); + TMP2 = STATE0; + MSG0 = vsha256su0q_u32(MSG0, MSG1); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3); + + // Rounds 5-8 + TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[4])); + TMP2 = STATE0; + MSG1 = vsha256su0q_u32(MSG1, MSG2); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0); + + // Rounds 9-12 + TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[8])); + TMP2 = STATE0; + MSG2 = vsha256su0q_u32(MSG2, MSG3); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1); + + // Rounds 13-16 + TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[12])); + TMP2 = STATE0; + MSG3 = vsha256su0q_u32(MSG3, MSG0); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2); + + // Rounds 17-20 + TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[16])); + TMP2 = STATE0; + MSG0 = vsha256su0q_u32(MSG0, MSG1); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3); + + // Rounds 21-24 + TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[20])); + TMP2 = STATE0; + MSG1 = vsha256su0q_u32(MSG1, MSG2); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0); + + // Rounds 25-28 + TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[24])); + TMP2 = STATE0; + MSG2 = vsha256su0q_u32(MSG2, MSG3); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1); + + // Rounds 29-32 + TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[28])); + TMP2 = STATE0; + MSG3 = vsha256su0q_u32(MSG3, MSG0); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2); + + // Rounds 33-36 + TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[32])); + TMP2 = STATE0; + MSG0 = vsha256su0q_u32(MSG0, MSG1); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3); + + // Rounds 37-40 + TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[36])); + TMP2 = STATE0; + MSG1 = vsha256su0q_u32(MSG1, MSG2); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0); + + // Rounds 41-44 + TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[40])); + TMP2 = STATE0; + MSG2 = vsha256su0q_u32(MSG2, MSG3); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1); + + // Rounds 45-48 + TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[44])); + TMP2 = STATE0; + MSG3 = vsha256su0q_u32(MSG3, MSG0); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2); + + // Rounds 49-52 + TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[48])); + TMP2 = STATE0; + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + + // Rounds 53-56 + TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[52])); + TMP2 = STATE0; + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + + // Rounds 57-60 + TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[56])); + TMP2 = STATE0; + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + + // Rounds 61-64 + TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[60])); + TMP2 = STATE0; + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + + // Update state + STATE0 = vaddq_u32(STATE0, ABEF_SAVE); + STATE1 = vaddq_u32(STATE1, CDGH_SAVE); + } + + // Save final state + vst1q_u32(&s[0], STATE0); + vst1q_u32(&s[4], STATE1); +} +} + +namespace sha256d64_arm_shani { +void Transform_2way(unsigned char* output, const unsigned char* input) +{ + /* Initial state. */ + alignas(uint32x4_t) static constexpr std::array<uint32_t, 8> INIT = { + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 + }; + + /* Precomputed message schedule for the 2nd transform. */ + alignas(uint32x4_t) static constexpr std::array<uint32_t, 64> MIDS = { + 0xc28a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf374, + 0x649b69c1, 0xf0fe4786, 0x0fe1edc6, 0x240cf254, + 0x4fe9346f, 0x6cc984be, 0x61b9411e, 0x16f988fa, + 0xf2c65152, 0xa88e5a6d, 0xb019fc65, 0xb9d99ec7, + 0x9a1231c3, 0xe70eeaa0, 0xfdb1232b, 0xc7353eb0, + 0x3069bad5, 0xcb976d5f, 0x5a0f118f, 0xdc1eeefd, + 0x0a35b689, 0xde0b7a04, 0x58f4ca9d, 0xe15d5b16, + 0x007f3e86, 0x37088980, 0xa507ea32, 0x6fab9537, + 0x17406110, 0x0d8cd6f1, 0xcdaa3b6d, 0xc0bbbe37, + 0x83613bda, 0xdb48a363, 0x0b02e931, 0x6fd15ca7, + 0x521afaca, 0x31338431, 0x6ed41a95, 0x6d437890, + 0xc39c91f2, 0x9eccabbd, 0xb5c9a0e6, 0x532fb63c, + 0xd2c741c6, 0x07237ea3, 0xa4954b68, 0x4c191d76 + }; + + /* A few precomputed message schedule values for the 3rd transform. */ + alignas(uint32x4_t) static constexpr std::array<uint32_t, 12> FINS = { + 0x5807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x80000000, 0x00000000, 0x00000000, 0x00000000, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf274 + }; + + /* Padding processed in the 3rd transform (byteswapped). */ + alignas(uint32x4_t) static constexpr std::array<uint32_t, 8> FINAL = {0x80000000, 0, 0, 0, 0, 0, 0, 0x100}; + + uint32x4_t STATE0A, STATE0B, STATE1A, STATE1B, ABEF_SAVEA, ABEF_SAVEB, CDGH_SAVEA, CDGH_SAVEB; + uint32x4_t MSG0A, MSG0B, MSG1A, MSG1B, MSG2A, MSG2B, MSG3A, MSG3B; + uint32x4_t TMP0A, TMP0B, TMP2A, TMP2B, TMP; + + // Transform 1: Load state + STATE0A = vld1q_u32(&INIT[0]); + STATE0B = STATE0A; + STATE1A = vld1q_u32(&INIT[4]); + STATE1B = STATE1A; + + // Transform 1: Load and convert input chunk to Big Endian + MSG0A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 0))); + MSG1A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 16))); + MSG2A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 32))); + MSG3A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 48))); + MSG0B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 64))); + MSG1B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 80))); + MSG2B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 96))); + MSG3B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 112))); + + // Transform 1: Rounds 1-4 + TMP = vld1q_u32(&K[0]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 1: Rounds 5-8 + TMP = vld1q_u32(&K[4]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 1: Rounds 9-12 + TMP = vld1q_u32(&K[8]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 1: Rounds 13-16 + TMP = vld1q_u32(&K[12]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 1: Rounds 17-20 + TMP = vld1q_u32(&K[16]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 1: Rounds 21-24 + TMP = vld1q_u32(&K[20]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 1: Rounds 25-28 + TMP = vld1q_u32(&K[24]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 1: Rounds 29-32 + TMP = vld1q_u32(&K[28]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 1: Rounds 33-36 + TMP = vld1q_u32(&K[32]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 1: Rounds 37-40 + TMP = vld1q_u32(&K[36]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 1: Rounds 41-44 + TMP = vld1q_u32(&K[40]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 1: Rounds 45-48 + TMP = vld1q_u32(&K[44]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 1: Rounds 49-52 + TMP = vld1q_u32(&K[48]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 1: Rounds 53-56 + TMP = vld1q_u32(&K[52]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 1: Rounds 57-60 + TMP = vld1q_u32(&K[56]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 1: Rounds 61-64 + TMP = vld1q_u32(&K[60]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 1: Update state + TMP = vld1q_u32(&INIT[0]); + STATE0A = vaddq_u32(STATE0A, TMP); + STATE0B = vaddq_u32(STATE0B, TMP); + TMP = vld1q_u32(&INIT[4]); + STATE1A = vaddq_u32(STATE1A, TMP); + STATE1B = vaddq_u32(STATE1B, TMP); + + // Transform 2: Save state + ABEF_SAVEA = STATE0A; + ABEF_SAVEB = STATE0B; + CDGH_SAVEA = STATE1A; + CDGH_SAVEB = STATE1B; + + // Transform 2: Rounds 1-4 + TMP = vld1q_u32(&MIDS[0]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 5-8 + TMP = vld1q_u32(&MIDS[4]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 9-12 + TMP = vld1q_u32(&MIDS[8]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 13-16 + TMP = vld1q_u32(&MIDS[12]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 17-20 + TMP = vld1q_u32(&MIDS[16]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 21-24 + TMP = vld1q_u32(&MIDS[20]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 25-28 + TMP = vld1q_u32(&MIDS[24]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 29-32 + TMP = vld1q_u32(&MIDS[28]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 33-36 + TMP = vld1q_u32(&MIDS[32]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 37-40 + TMP = vld1q_u32(&MIDS[36]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 41-44 + TMP = vld1q_u32(&MIDS[40]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 45-48 + TMP = vld1q_u32(&MIDS[44]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 49-52 + TMP = vld1q_u32(&MIDS[48]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 53-56 + TMP = vld1q_u32(&MIDS[52]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 57-60 + TMP = vld1q_u32(&MIDS[56]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 61-64 + TMP = vld1q_u32(&MIDS[60]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Update state + STATE0A = vaddq_u32(STATE0A, ABEF_SAVEA); + STATE0B = vaddq_u32(STATE0B, ABEF_SAVEB); + STATE1A = vaddq_u32(STATE1A, CDGH_SAVEA); + STATE1B = vaddq_u32(STATE1B, CDGH_SAVEB); + + // Transform 3: Pad previous output + MSG0A = STATE0A; + MSG0B = STATE0B; + MSG1A = STATE1A; + MSG1B = STATE1B; + MSG2A = vld1q_u32(&FINAL[0]); + MSG2B = MSG2A; + MSG3A = vld1q_u32(&FINAL[4]); + MSG3B = MSG3A; + + // Transform 3: Load state + STATE0A = vld1q_u32(&INIT[0]); + STATE0B = STATE0A; + STATE1A = vld1q_u32(&INIT[4]); + STATE1B = STATE1A; + + // Transform 3: Rounds 1-4 + TMP = vld1q_u32(&K[0]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 3: Rounds 5-8 + TMP = vld1q_u32(&K[4]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 3: Rounds 9-12 + TMP = vld1q_u32(&FINS[0]); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vld1q_u32(&FINS[4]); + MSG2B = MSG2A; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 3: Rounds 13-16 + TMP = vld1q_u32(&FINS[8]); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 3: Rounds 17-20 + TMP = vld1q_u32(&K[16]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 3: Rounds 21-24 + TMP = vld1q_u32(&K[20]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 3: Rounds 25-28 + TMP = vld1q_u32(&K[24]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 3: Rounds 29-32 + TMP = vld1q_u32(&K[28]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 3: Rounds 33-36 + TMP = vld1q_u32(&K[32]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 3: Rounds 37-40 + TMP = vld1q_u32(&K[36]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 3: Rounds 41-44 + TMP = vld1q_u32(&K[40]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 3: Rounds 45-48 + TMP = vld1q_u32(&K[44]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 3: Rounds 49-52 + TMP = vld1q_u32(&K[48]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 3: Rounds 53-56 + TMP = vld1q_u32(&K[52]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 3: Rounds 57-60 + TMP = vld1q_u32(&K[56]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 3: Rounds 61-64 + TMP = vld1q_u32(&K[60]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 3: Update state + TMP = vld1q_u32(&INIT[0]); + STATE0A = vaddq_u32(STATE0A, TMP); + STATE0B = vaddq_u32(STATE0B, TMP); + TMP = vld1q_u32(&INIT[4]); + STATE1A = vaddq_u32(STATE1A, TMP); + STATE1B = vaddq_u32(STATE1B, TMP); + + // Store result + vst1q_u8(output, vrev32q_u8(vreinterpretq_u8_u32(STATE0A))); + vst1q_u8(output + 16, vrev32q_u8(vreinterpretq_u8_u32(STATE1A))); + vst1q_u8(output + 32, vrev32q_u8(vreinterpretq_u8_u32(STATE0B))); + vst1q_u8(output + 48, vrev32q_u8(vreinterpretq_u8_u32(STATE1B))); +} +} + +#endif diff --git a/src/crypto/sha256_shani.cpp b/src/crypto/sha256_x86_shani.cpp index 4f4d5b5837..a82802199f 100644 --- a/src/crypto/sha256_shani.cpp +++ b/src/crypto/sha256_x86_shani.cpp @@ -6,7 +6,7 @@ // Written and placed in public domain by Jeffrey Walton. // Based on code from Intel, and by Sean Gulley for the miTLS project. -#ifdef ENABLE_SHANI +#ifdef ENABLE_X86_SHANI #include <stdint.h> #include <immintrin.h> @@ -74,7 +74,7 @@ void inline __attribute__((always_inline)) Save(unsigned char* out, __m128i s) } } -namespace sha256_shani { +namespace sha256_x86_shani { void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) { __m128i m0, m1, m2, m3, s0, s1, so0, so1; @@ -139,7 +139,7 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) } } -namespace sha256d64_shani { +namespace sha256d64_x86_shani { void Transform_2way(unsigned char* out, const unsigned char* in) { |