/********************************************************************** Copyright(c) 2020 Arm Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Arm Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ #ifndef __AARCH64_MULTIBINARY_H__ #define __AARCH64_MULTIBINARY_H__ #ifndef __aarch64__ #error "This file is for aarch64 only" #endif #include #ifdef __ASSEMBLY__ /** * # mbin_interface : the wrapper layer for isal-l api * * ## references: * * https://sourceware.org/git/gitweb.cgi?p=glibc.git;a=blob;f=sysdeps/aarch64/dl-trampoline.S * * http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf * * https://static.docs.arm.com/ihi0057/b/IHI0057B_aadwarf64.pdf?_ga=2.80574487.1870739014.1564969896-1634778941.1548729310 * * ## Usage: * 1. Define dispather function * 2. name must be \name\()_dispatcher * 3. Prototype should be *"void * \name\()_dispatcher"* * 4. The dispather should return the right function pointer , revision and a string information . **/ .macro mbin_interface name:req .extern \name\()_dispatcher .section .data .balign 8 .global \name\()_dispatcher_info .type \name\()_dispatcher_info,%object \name\()_dispatcher_info: .quad \name\()_mbinit //func_entry .size \name\()_dispatcher_info,. - \name\()_dispatcher_info .balign 8 .text \name\()_mbinit: //save lp fp, sub sp .cfi_startproc stp x29, x30, [sp, -224]! //add cfi directive to avoid GDB bt cmds error //set cfi(Call Frame Information) .cfi_def_cfa_offset 224 .cfi_offset 29, -224 .cfi_offset 30, -216 //save parameter/result/indirect result registers stp x8, x9, [sp, 16] .cfi_offset 8, -208 .cfi_offset 9, -200 stp x0, x1, [sp, 32] .cfi_offset 0, -192 .cfi_offset 1, -184 stp x2, x3, [sp, 48] .cfi_offset 2, -176 .cfi_offset 3, -168 stp x4, x5, [sp, 64] .cfi_offset 4, -160 .cfi_offset 5, -152 stp x6, x7, [sp, 80] .cfi_offset 6, -144 .cfi_offset 7, -136 stp q0, q1, [sp, 96] .cfi_offset 64, -128 .cfi_offset 65, -112 stp q2, q3, [sp, 128] .cfi_offset 66, -96 .cfi_offset 67, -80 stp q4, q5, [sp, 160] .cfi_offset 68, -64 .cfi_offset 69, -48 stp q6, q7, [sp, 192] .cfi_offset 70, -32 .cfi_offset 71, -16 /** * The dispatcher functions have the following prototype: * void * function_dispatcher(void) * As the dispatcher is returning a struct, by the AAPCS, */ bl \name\()_dispatcher //restore temp/indirect result registers ldp x8, x9, [sp, 16] .cfi_restore 8 .cfi_restore 9 // save function entry str x0, [x9] //restore parameter/result registers ldp x0, x1, [sp, 32] .cfi_restore 0 .cfi_restore 1 ldp x2, x3, [sp, 48] .cfi_restore 2 .cfi_restore 3 ldp x4, x5, [sp, 64] .cfi_restore 4 .cfi_restore 5 ldp x6, x7, [sp, 80] .cfi_restore 6 .cfi_restore 7 ldp q0, q1, [sp, 96] .cfi_restore 64 .cfi_restore 65 ldp q2, q3, [sp, 128] .cfi_restore 66 .cfi_restore 67 ldp q4, q5, [sp, 160] .cfi_restore 68 .cfi_restore 69 ldp q6, q7, [sp, 192] .cfi_restore 70 .cfi_restore 71 //save lp fp and sp ldp x29, x30, [sp], 224 //restore cfi setting .cfi_restore 30 .cfi_restore 29 .cfi_def_cfa_offset 0 .cfi_endproc .global \name .type \name,%function .align 2 \name\(): adrp x9, :got:\name\()_dispatcher_info ldr x9, [x9, #:got_lo12:\name\()_dispatcher_info] ldr x10,[x9] br x10 .size \name,. - \name .endm /** * mbin_interface_base is used for the interfaces which have only * noarch implementation */ .macro mbin_interface_base name:req, base:req .extern \base .section .data .balign 8 .global \name\()_dispatcher_info .type \name\()_dispatcher_info,%object \name\()_dispatcher_info: .quad \base //func_entry .size \name\()_dispatcher_info,. - \name\()_dispatcher_info .balign 8 .text .global \name .type \name,%function .align 2 \name\(): adrp x9, :got:\name\()_dispatcher_info ldr x9, [x9, #:got_lo12:\name\()_dispatcher_info] ldr x10,[x9] br x10 .size \name,. - \name .endm #else /* __ASSEMBLY__ */ #include #define DEFINE_INTERFACE_DISPATCHER(name) \ void * name##_dispatcher(void) #define PROVIDER_BASIC(name) \ PROVIDER_INFO(name##_base) #define DO_DIGNOSTIC(x) _Pragma GCC diagnostic ignored "-W"#x #define DO_PRAGMA(x) _Pragma (#x) #define DIGNOSTIC_IGNORE(x) DO_PRAGMA(GCC diagnostic ignored #x) #define DIGNOSTIC_PUSH() DO_PRAGMA(GCC diagnostic push) #define DIGNOSTIC_POP() DO_PRAGMA(GCC diagnostic pop) #define PROVIDER_INFO(_func_entry) \ ({ DIGNOSTIC_PUSH() \ DIGNOSTIC_IGNORE(-Wnested-externs) \ extern void _func_entry(void); \ DIGNOSTIC_POP() \ _func_entry; \ }) /** * Micro-Architector definitions * Reference: https://developer.arm.com/docs/ddi0595/f/aarch64-system-registers/midr_el1 */ #define CPU_IMPLEMENTER_RESERVE 0x00 #define CPU_IMPLEMENTER_ARM 0x41 #define CPU_PART_CORTEX_A57 0xD07 #define CPU_PART_CORTEX_A72 0xD08 #define CPU_PART_NEOVERSE_N1 0xD0C #define MICRO_ARCH_ID(imp,part) \ (((CPU_IMPLEMENTER_##imp&0xff)<<24)|((CPU_PART_##part&0xfff)<<4)) #ifndef HWCAP_CPUID #define HWCAP_CPUID (1<<11) #endif /** * @brief get_micro_arch_id * * read micro-architector register instruction if possible.This function * provides microarchitecture information and make microarchitecture optimization * possible. * * Read system registers(MRS) is forbidden in userspace. If executed, it * will raise illegal instruction error. Kernel provides a solution for * this issue. The solution depends on HWCAP_CPUID flags. Reference(1) * describes how to use it. It provides a "illegal insstruction" handler * in kernel space, the handler will execute MRS and return the correct * value to userspace. * * To avoid too many kernel trap, this function MUST be only called in * dispatcher. And HWCAP must be match,That will make sure there are no * illegal instruction errors. HWCAP_CPUID should be available to get the * best performance. * * NOTICE: * - HWCAP_CPUID should be available. Otherwise it returns reserve value * - It MUST be called inside dispather. * - It MUST meet the HWCAP requirements * * Example: * DEFINE_INTERFACE_DISPATCHER(crc32_iscsi) * { * unsigned long auxval = getauxval(AT_HWCAP); * // MUST do the judgement is MUST. * if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) { * switch (get_micro_arch_id()) { * case MICRO_ARCH_ID(ARM, CORTEX_A57): * return PROVIDER_INFO(crc32_pmull_crc_for_a57); * case MICRO_ARCH_ID(ARM, CORTEX_A72): * return PROVIDER_INFO(crc32_pmull_crc_for_a72); * case MICRO_ARCH_ID(ARM, NEOVERSE_N1): * return PROVIDER_INFO(crc32_pmull_crc_for_n1); * case default: * return PROVIDER_INFO(crc32_pmull_crc_for_others); * } * } * return PROVIDER_BASIC(crc32_iscsi); * } * KNOWN ISSUE: * On a heterogeneous system (big.LITTLE), it will work but the performance * might not be the best one as expected. * * If this function is called on the big core, it will return the function * optimized for the big core. * * If execution is then scheduled to the little core. It will still work (1), * but the function won't be optimized for the little core, thus the performance * won't be as expected. * * References: * - [CPU Feature detection](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/arm64/cpu-feature-registers.rst?h=v5.5) * */ static inline uint32_t get_micro_arch_id(void) { uint32_t id=CPU_IMPLEMENTER_RESERVE; if ((getauxval(AT_HWCAP) & HWCAP_CPUID)) { /** Here will trap into kernel space */ asm("mrs %0, MIDR_EL1 " : "=r" (id)); } return id&0xff00fff0; } #endif /* __ASSEMBLY__ */ #endif