/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifdef FREEBL_NO_DEPEND #include "stubs.h" #endif #include "blapii.h" #include "mpi.h" #include "secerr.h" #include "prtypes.h" #include "prinit.h" #include "prenv.h" #if defined(_MSC_VER) && !defined(_M_IX86) #include /* for _xgetbv() */ #endif #if defined(_WIN64) && defined(__aarch64__) #include #endif #if defined(DARWIN) #include #endif static PRCallOnceType coFreeblInit; /* State variables. */ static PRBool aesni_support_ = PR_FALSE; static PRBool clmul_support_ = PR_FALSE; static PRBool sha_support_ = PR_FALSE; static PRBool avx_support_ = PR_FALSE; static PRBool avx2_support_ = PR_FALSE; static PRBool adx_support_ = PR_FALSE; static PRBool ssse3_support_ = PR_FALSE; static PRBool sse4_1_support_ = PR_FALSE; static PRBool sse4_2_support_ = PR_FALSE; static PRBool arm_neon_support_ = PR_FALSE; static PRBool arm_aes_support_ = PR_FALSE; static PRBool arm_sha1_support_ = PR_FALSE; static PRBool arm_sha2_support_ = PR_FALSE; static PRBool arm_pmull_support_ = PR_FALSE; static PRBool ppc_crypto_support_ = PR_FALSE; #ifdef NSS_X86_OR_X64 /* * Adapted from the example code in "How to detect New Instruction support in * the 4th generation Intel Core processor family" by Max Locktyukhin. * https://www.intel.com/content/dam/develop/external/us/en/documents/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family.pdf * * XGETBV: * Reads an extended control register (XCR) specified by ECX into EDX:EAX. */ static PRBool check_xcr0_ymm() { PRUint32 xcr0; #if defined(_MSC_VER) #if defined(_M_IX86) __asm { mov ecx, 0 xgetbv mov xcr0, eax } #else xcr0 = (PRUint32)_xgetbv(0); /* Requires VS2010 SP1 or later. */ #endif /* _M_IX86 */ #else /* _MSC_VER */ /* Old OSX compilers don't support xgetbv. Use byte form. */ __asm__(".byte 0x0F, 0x01, 0xd0" : "=a"(xcr0) : "c"(0) : "%edx"); #endif /* _MSC_VER */ /* Check if xmm and ymm state are enabled in XCR0. */ return (xcr0 & 6) == 6; } #define ECX_AESNI (1 << 25) #define ECX_CLMUL (1 << 1) #define ECX_XSAVE (1 << 26) #define ECX_OSXSAVE (1 << 27) #define ECX_AVX (1 << 28) #define EBX_AVX2 (1 << 5) #define EBX_ADX (1 << 19) #define EBX_BMI1 (1 << 3) #define EBX_BMI2 (1 << 8) #define EBX_SHA (1 << 29) #define ECX_FMA (1 << 12) #define ECX_MOVBE (1 << 22) #define ECX_SSSE3 (1 << 9) #define ECX_SSE4_1 (1 << 19) #define ECX_SSE4_2 (1 << 20) #define AVX_BITS (ECX_XSAVE | ECX_OSXSAVE | ECX_AVX) #define AVX2_EBX_BITS (EBX_AVX2 | EBX_BMI1 | EBX_BMI2) #define AVX2_ECX_BITS (ECX_FMA | ECX_MOVBE) void CheckX86CPUSupport() { unsigned long eax, ebx, ecx, edx; unsigned long eax7, ebx7, ecx7, edx7; char *disable_hw_aes = PR_GetEnvSecure("NSS_DISABLE_HW_AES"); char *disable_pclmul = PR_GetEnvSecure("NSS_DISABLE_PCLMUL"); char *disable_hw_sha = PR_GetEnvSecure("NSS_DISABLE_HW_SHA"); char *disable_avx = PR_GetEnvSecure("NSS_DISABLE_AVX"); char *disable_avx2 = PR_GetEnvSecure("NSS_DISABLE_AVX2"); char *disable_adx = PR_GetEnvSecure("NSS_DISABLE_ADX"); char *disable_ssse3 = PR_GetEnvSecure("NSS_DISABLE_SSSE3"); char *disable_sse4_1 = PR_GetEnvSecure("NSS_DISABLE_SSE4_1"); char *disable_sse4_2 = PR_GetEnvSecure("NSS_DISABLE_SSE4_2"); freebl_cpuid(1, &eax, &ebx, &ecx, &edx); freebl_cpuid(7, &eax7, &ebx7, &ecx7, &edx7); aesni_support_ = (PRBool)((ecx & ECX_AESNI) != 0 && disable_hw_aes == NULL); clmul_support_ = (PRBool)((ecx & ECX_CLMUL) != 0 && disable_pclmul == NULL); sha_support_ = (PRBool)((ebx7 & EBX_SHA) != 0 && disable_hw_sha == NULL); /* For AVX we ensure that: * - The AVX, OSXSAVE, and XSAVE bits of ECX from CPUID(EAX=1) are set, and * - the SSE and AVX state bits of XCR0 are set (check_xcr0_ymm). */ avx_support_ = (PRBool)((ecx & AVX_BITS) == AVX_BITS) && check_xcr0_ymm() && disable_avx == NULL; /* For AVX2 we ensure that: * - AVX is supported, * - the AVX2, BMI1, and BMI2 bits of EBX from CPUID(EAX=7) are set, and * - the FMA, and MOVBE bits of ECX from CPUID(EAX=1) are set. * We do not check for LZCNT support. */ avx2_support_ = (PRBool)(avx_support_ == PR_TRUE && (ebx7 & AVX2_EBX_BITS) == AVX2_EBX_BITS && (ecx & AVX2_ECX_BITS) == AVX2_ECX_BITS && disable_avx2 == NULL); /* CPUID.(EAX=07H, ECX=0H):EBX.ADX[bit 19]=1 indicates the processor supports ADCX and ADOX instructions.*/ adx_support_ = (PRBool)((ebx7 & EBX_ADX) != 0 && disable_adx == NULL); ssse3_support_ = (PRBool)((ecx & ECX_SSSE3) != 0 && disable_ssse3 == NULL); sse4_1_support_ = (PRBool)((ecx & ECX_SSE4_1) != 0 && disable_sse4_1 == NULL); sse4_2_support_ = (PRBool)((ecx & ECX_SSE4_2) != 0 && disable_sse4_2 == NULL); } #endif /* NSS_X86_OR_X64 */ /* clang-format off */ #if (defined(__aarch64__) || defined(__arm__)) && !defined(TARGET_OS_IPHONE) #ifndef __has_include #define __has_include(x) 0 #endif #if (__has_include() || defined(__linux__)) && \ defined(__GNUC__) && __GNUC__ >= 2 && defined(__ELF__) /* This might be conflict with host compiler */ #if !defined(__ANDROID__) #include #endif extern unsigned long getauxval(unsigned long type) __attribute__((weak)); #elif defined(__arm__) || (!defined(__OpenBSD__) && !defined(_WIN64)) static unsigned long (*getauxval)(unsigned long) = NULL; #endif /* defined(__GNUC__) && __GNUC__ >= 2 && defined(__ELF__)*/ #if defined(__FreeBSD__) && !defined(__aarch64__) && __has_include() /* Avoid conflict with static declaration above */ #define getauxval freebl_getauxval static unsigned long getauxval(unsigned long type) { /* Only AT_HWCAP* return unsigned long */ if (type != AT_HWCAP && type != AT_HWCAP2) { return 0; } unsigned long ret = 0; elf_aux_info(type, &ret, sizeof(ret)); return ret; } #endif #ifndef AT_HWCAP2 #define AT_HWCAP2 26 #endif #ifndef AT_HWCAP #define AT_HWCAP 16 #endif #endif /* defined(__aarch64__) || defined(__arm__) */ /* clang-format on */ #if defined(__aarch64__) #if defined(__linux__) // Defines from hwcap.h in Linux kernel - ARM64 #ifndef HWCAP_AES #define HWCAP_AES (1 << 3) #endif #ifndef HWCAP_PMULL #define HWCAP_PMULL (1 << 4) #endif #ifndef HWCAP_SHA1 #define HWCAP_SHA1 (1 << 5) #endif #ifndef HWCAP_SHA2 #define HWCAP_SHA2 (1 << 6) #endif #endif /* defined(__linux__) */ #if defined(__FreeBSD__) #include #include // Support for older version of armreg.h #ifndef ID_AA64ISAR0_AES_VAL #define ID_AA64ISAR0_AES_VAL ID_AA64ISAR0_AES #endif #ifndef ID_AA64ISAR0_SHA1_VAL #define ID_AA64ISAR0_SHA1_VAL ID_AA64ISAR0_SHA1 #endif #ifndef ID_AA64ISAR0_SHA2_VAL #define ID_AA64ISAR0_SHA2_VAL ID_AA64ISAR0_SHA2 #endif #endif /* defined(__FreeBSD__) */ #if defined(__OpenBSD__) #include #include #include #endif /* defined(__OpenBSD__) */ void CheckARMSupport() { #if defined(_WIN64) BOOL arm_crypto_support = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); arm_aes_support_ = arm_crypto_support; arm_pmull_support_ = arm_crypto_support; arm_sha1_support_ = arm_crypto_support; arm_sha2_support_ = arm_crypto_support; #elif defined(__linux__) if (getauxval) { long hwcaps = getauxval(AT_HWCAP); arm_aes_support_ = (hwcaps & HWCAP_AES) == HWCAP_AES; arm_pmull_support_ = (hwcaps & HWCAP_PMULL) == HWCAP_PMULL; arm_sha1_support_ = (hwcaps & HWCAP_SHA1) == HWCAP_SHA1; arm_sha2_support_ = (hwcaps & HWCAP_SHA2) == HWCAP_SHA2; } #elif defined(__FreeBSD__) /* qemu-user does not support register access from userspace */ if (PR_GetEnvSecure("QEMU_EMULATING") == NULL) { uint64_t isar0 = READ_SPECIALREG(id_aa64isar0_el1); arm_aes_support_ = ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_BASE; arm_pmull_support_ = ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_PMULL; arm_sha1_support_ = ID_AA64ISAR0_SHA1_VAL(isar0) >= ID_AA64ISAR0_SHA1_BASE; arm_sha2_support_ = ID_AA64ISAR0_SHA2_VAL(isar0) >= ID_AA64ISAR0_SHA2_BASE; } #elif defined(__OpenBSD__) const int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 }; uint64_t isar0; size_t len = sizeof(isar0); if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) < 0) return; arm_aes_support_ = ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_BASE; arm_pmull_support_ = ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_PMULL; arm_sha1_support_ = ID_AA64ISAR0_SHA1(isar0) >= ID_AA64ISAR0_SHA1_BASE; arm_sha2_support_ = ID_AA64ISAR0_SHA2(isar0) >= ID_AA64ISAR0_SHA2_BASE; #elif defined(__ARM_FEATURE_CRYPTO) /* * Although no feature detection, default compiler option allows ARM * Crypto Extension. */ arm_aes_support_ = PR_TRUE; arm_pmull_support_ = PR_TRUE; arm_sha1_support_ = PR_TRUE; arm_sha2_support_ = PR_TRUE; #endif /* aarch64 must support NEON. */ arm_neon_support_ = PR_GetEnvSecure("NSS_DISABLE_ARM_NEON") == NULL; arm_aes_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_AES") == NULL; arm_pmull_support_ &= PR_GetEnvSecure("NSS_DISABLE_PMULL") == NULL; arm_sha1_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_SHA1") == NULL; arm_sha2_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_SHA2") == NULL; } #endif /* defined(__aarch64__) */ #if defined(__arm__) // Defines from hwcap.h in Linux kernel - ARM /* * HWCAP flags - for elf_hwcap (in kernel) and AT_HWCAP */ #ifndef HWCAP_NEON #define HWCAP_NEON (1 << 12) #endif /* * HWCAP2 flags - for elf_hwcap2 (in kernel) and AT_HWCAP2 */ #ifndef HWCAP2_AES #define HWCAP2_AES (1 << 0) #endif #ifndef HWCAP2_PMULL #define HWCAP2_PMULL (1 << 1) #endif #ifndef HWCAP2_SHA1 #define HWCAP2_SHA1 (1 << 2) #endif #ifndef HWCAP2_SHA2 #define HWCAP2_SHA2 (1 << 3) #endif PRBool GetNeonSupport() { char *disable_arm_neon = PR_GetEnvSecure("NSS_DISABLE_ARM_NEON"); if (disable_arm_neon) { return PR_FALSE; } #if defined(__ARM_NEON) || defined(__ARM_NEON__) // Compiler generates NEON instruction as default option. // If no getauxval, compiler generate NEON instruction by default, // we should allow NOEN support. return PR_TRUE; #elif !defined(__ANDROID__) // Android's cpu-features.c detects features by the following logic // // - Call getauxval(AT_HWCAP) // - Parse /proc/self/auxv if getauxval is nothing or returns 0 // - Parse /proc/cpuinfo if both cannot detect features // // But we don't use it for Android since Android document // (https://developer.android.com/ndk/guides/cpu-features) says // one problem with AT_HWCAP sometimes devices (Nexus 4 and emulator) // are mistaken for IDIV. if (getauxval) { return (getauxval(AT_HWCAP) & HWCAP_NEON); } #endif /* defined(__ARM_NEON) || defined(__ARM_NEON__) */ return PR_FALSE; } #ifdef __linux__ static long ReadCPUInfoForHWCAP2() { FILE *cpuinfo; char buf[512]; char *p; long hwcap2 = 0; cpuinfo = fopen("/proc/cpuinfo", "r"); if (!cpuinfo) { return 0; } while (fgets(buf, 511, cpuinfo)) { if (!memcmp(buf, "Features", 8)) { p = strstr(buf, " aes"); if (p && (p[4] == ' ' || p[4] == '\n')) { hwcap2 |= HWCAP2_AES; } p = strstr(buf, " sha1"); if (p && (p[5] == ' ' || p[5] == '\n')) { hwcap2 |= HWCAP2_SHA1; } p = strstr(buf, " sha2"); if (p && (p[5] == ' ' || p[5] == '\n')) { hwcap2 |= HWCAP2_SHA2; } p = strstr(buf, " pmull"); if (p && (p[6] == ' ' || p[6] == '\n')) { hwcap2 |= HWCAP2_PMULL; } break; } } fclose(cpuinfo); return hwcap2; } #endif /* __linux__ */ void CheckARMSupport() { char *disable_hw_aes = PR_GetEnvSecure("NSS_DISABLE_HW_AES"); if (getauxval) { // Android's cpu-features.c uses AT_HWCAP2 for newer features. // AT_HWCAP2 is implemented on newer devices / kernel, so we can trust // it since cpu-features.c doesn't have workaround / fallback. // Also, AT_HWCAP2 is supported by glibc 2.18+ on Linux/arm, If // AT_HWCAP2 isn't supported by glibc or Linux kernel, getauxval will // returns 0. long hwcaps = getauxval(AT_HWCAP2); #ifdef __linux__ if (!hwcaps) { // Some ARMv8 devices may not implement AT_HWCAP2. So we also // read /proc/cpuinfo if AT_HWCAP2 is 0. hwcaps = ReadCPUInfoForHWCAP2(); } #endif arm_aes_support_ = hwcaps & HWCAP2_AES && disable_hw_aes == NULL; arm_pmull_support_ = hwcaps & HWCAP2_PMULL; arm_sha1_support_ = hwcaps & HWCAP2_SHA1; arm_sha2_support_ = hwcaps & HWCAP2_SHA2; } arm_neon_support_ = GetNeonSupport(); arm_sha1_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_SHA1") == NULL; arm_sha2_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_SHA2") == NULL; } #endif /* defined(__arm__) */ // Enable when Firefox can use it for Android API 16 and 17. // #if defined(__ANDROID__) && (defined(__arm__) || defined(__aarch64__)) // #include // void // CheckARMSupport() // { // char *disable_arm_neon = PR_GetEnvSecure("NSS_DISABLE_ARM_NEON"); // char *disable_hw_aes = PR_GetEnvSecure("NSS_DISABLE_HW_AES"); // AndroidCpuFamily family = android_getCpuFamily(); // uint64_t features = android_getCpuFeatures(); // if (family == ANDROID_CPU_FAMILY_ARM64) { // arm_aes_support_ = features & ANDROID_CPU_ARM64_FEATURE_AES && // disable_hw_aes == NULL; // arm_pmull_support_ = features & ANDROID_CPU_ARM64_FEATURE_PMULL; // arm_sha1_support_ = features & ANDROID_CPU_ARM64_FEATURE_SHA1; // arm_sha2_support_ = features & ANDROID_CPU_ARM64_FEATURE_SHA2; // arm_neon_support_ = disable_arm_neon == NULL; // } // if (family == ANDROID_CPU_FAMILY_ARM) { // arm_aes_support_ = features & ANDROID_CPU_ARM_FEATURE_AES && // disable_hw_aes == NULL; // arm_pmull_support_ = features & ANDROID_CPU_ARM_FEATURE_PMULL; // arm_sha1_support_ = features & ANDROID_CPU_ARM_FEATURE_SHA1; // arm_sha2_support_ = features & ANDROID_CPU_ARM_FEATURE_SHA2; // arm_neon_support_ = hwcaps & ANDROID_CPU_ARM_FEATURE_NEON && // disable_arm_neon == NULL; // } // } // #endif /* defined(__ANDROID__) && (defined(__arm__) || defined(__aarch64__)) */ PRBool aesni_support() { return aesni_support_; } PRBool clmul_support() { return clmul_support_; } PRBool sha_support() { return sha_support_; } PRBool avx_support() { return avx_support_; } PRBool avx2_support() { return avx2_support_; } PRBool adx_support() { return adx_support_; } PRBool ssse3_support() { return ssse3_support_; } PRBool sse4_1_support() { return sse4_1_support_; } PRBool sse4_2_support() { return sse4_2_support_; } PRBool arm_neon_support() { return arm_neon_support_; } PRBool arm_aes_support() { return arm_aes_support_; } PRBool arm_pmull_support() { return arm_pmull_support_; } PRBool arm_sha1_support() { return arm_sha1_support_; } PRBool arm_sha2_support() { return arm_sha2_support_; } PRBool ppc_crypto_support() { return ppc_crypto_support_; } #if defined(__powerpc__) #ifndef __has_include #define __has_include(x) 0 #endif /* clang-format off */ #if defined(__linux__) || (defined(__FreeBSD__) && __FreeBSD__ >= 12) #if __has_include() #include #endif #elif (defined(__FreeBSD__) && __FreeBSD__ < 12) #include #endif // Defines from cputable.h in Linux kernel - PPC, letting us build on older kernels #ifndef PPC_FEATURE2_VEC_CRYPTO #define PPC_FEATURE2_VEC_CRYPTO 0x02000000 #endif static void CheckPPCSupport() { char *disable_hw_crypto = PR_GetEnvSecure("NSS_DISABLE_PPC_GHASH"); unsigned long hwcaps = 0; #if defined(__linux__) #if __has_include() hwcaps = getauxval(AT_HWCAP2); #endif #elif defined(__FreeBSD__) #if __FreeBSD__ >= 12 #if __has_include() elf_aux_info(AT_HWCAP2, &hwcaps, sizeof(hwcaps)); #endif #else size_t len = sizeof(hwcaps); sysctlbyname("hw.cpu_features2", &hwcaps, &len, NULL, 0); #endif #endif ppc_crypto_support_ = hwcaps & PPC_FEATURE2_VEC_CRYPTO && disable_hw_crypto == NULL; } /* clang-format on */ #endif /* __powerpc__ */ static PRStatus FreeblInit(void) { #ifdef NSS_X86_OR_X64 CheckX86CPUSupport(); #elif (defined(__aarch64__) || defined(__arm__)) CheckARMSupport(); #elif (defined(__powerpc__)) CheckPPCSupport(); #endif return PR_SUCCESS; } SECStatus BL_Init() { if (PR_CallOnce(&coFreeblInit, FreeblInit) != PR_SUCCESS) { PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); return SECFailure; } RSA_Init(); return SECSuccess; }