diff options
Diffstat (limited to '')
276 files changed, 59357 insertions, 0 deletions
diff --git a/lib/aarch32/arm32_aeabi_divmod.c b/lib/aarch32/arm32_aeabi_divmod.c new file mode 100644 index 0000000..89a9033 --- /dev/null +++ b/lib/aarch32/arm32_aeabi_divmod.c @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2017-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Form ABI specifications: + * int __aeabi_idiv(int numerator, int denominator); + * unsigned __aeabi_uidiv(unsigned numerator, unsigned denominator); + * + * typedef struct { int quot; int rem; } idiv_return; + * typedef struct { unsigned quot; unsigned rem; } uidiv_return; + * + * __value_in_regs idiv_return __aeabi_idivmod(int numerator, + * int *denominator); + * __value_in_regs uidiv_return __aeabi_uidivmod(unsigned *numerator, + * unsigned denominator); + */ + +/* struct qr - stores quotient/remainder to handle divmod EABI interfaces. */ +struct qr { + unsigned int q; /* computed quotient */ + unsigned int r; /* computed remainder */ + unsigned int q_n; /* specifies if quotient shall be negative */ + unsigned int r_n; /* specifies if remainder shall be negative */ +}; + +static void uint_div_qr(unsigned int numerator, unsigned int denominator, + struct qr *qr); + +/* returns in R0 and R1 by tail calling an asm function */ +unsigned int __aeabi_uidivmod(unsigned int numerator, unsigned int denominator); + +unsigned int __aeabi_uidiv(unsigned int numerator, unsigned int denominator); + +/* returns in R0 and R1 by tail calling an asm function */ +signed int __aeabi_idivmod(signed int numerator, signed int denominator); + +signed int __aeabi_idiv(signed int numerator, signed int denominator); + +/* + * __ste_idivmod_ret_t __aeabi_idivmod(signed numerator, signed denominator) + * Numerator and Denominator are received in R0 and R1. + * Where __ste_idivmod_ret_t is returned in R0 and R1. + * + * __ste_uidivmod_ret_t __aeabi_uidivmod(unsigned numerator, + * unsigned denominator) + * Numerator and Denominator are received in R0 and R1. + * Where __ste_uidivmod_ret_t is returned in R0 and R1. + */ +#ifdef __GNUC__ +signed int ret_idivmod_values(signed int quotient, signed int remainder); +unsigned int ret_uidivmod_values(unsigned int quotient, unsigned int remainder); +#else +#error "Compiler not supported" +#endif + +static void division_qr(unsigned int n, unsigned int p, struct qr *qr) +{ + unsigned int i = 1, q = 0; + + if (p == 0) { + qr->r = 0xFFFFFFFF; /* division by 0 */ + return; + } + + while ((p >> 31) == 0) { + i = i << 1; /* count the max division steps */ + p = p << 1; /* increase p until it has maximum size*/ + } + + while (i > 0) { + q = q << 1; /* write bit in q at index (size-1) */ + if (n >= p) { + n -= p; + q++; + } + p = p >> 1; /* decrease p */ + i = i >> 1; /* decrease remaining size in q */ + } + qr->r = n; + qr->q = q; +} + +static void uint_div_qr(unsigned int numerator, unsigned int denominator, + struct qr *qr) +{ + division_qr(numerator, denominator, qr); + + /* negate quotient and/or remainder according to requester */ + if (qr->q_n) + qr->q = -qr->q; + if (qr->r_n) + qr->r = -qr->r; +} + +unsigned int __aeabi_uidiv(unsigned int numerator, unsigned int denominator) +{ + struct qr qr = { .q_n = 0, .r_n = 0 }; + + uint_div_qr(numerator, denominator, &qr); + + return qr.q; +} + +unsigned int __aeabi_uidivmod(unsigned int numerator, unsigned int denominator) +{ + struct qr qr = { .q_n = 0, .r_n = 0 }; + + uint_div_qr(numerator, denominator, &qr); + + return ret_uidivmod_values(qr.q, qr.r); +} + +signed int __aeabi_idiv(signed int numerator, signed int denominator) +{ + struct qr qr = { .q_n = 0, .r_n = 0 }; + + if (((numerator < 0) && (denominator > 0)) || + ((numerator > 0) && (denominator < 0))) + qr.q_n = 1; /* quotient shall be negate */ + + if (numerator < 0) { + numerator = -numerator; + qr.r_n = 1; /* remainder shall be negate */ + } + + if (denominator < 0) + denominator = -denominator; + + uint_div_qr(numerator, denominator, &qr); + + return qr.q; +} + +signed int __aeabi_idivmod(signed int numerator, signed int denominator) +{ + struct qr qr = { .q_n = 0, .r_n = 0 }; + + if (((numerator < 0) && (denominator > 0)) || + ((numerator > 0) && (denominator < 0))) + qr.q_n = 1; /* quotient shall be negate */ + + if (numerator < 0) { + numerator = -numerator; + qr.r_n = 1; /* remainder shall be negate */ + } + + if (denominator < 0) + denominator = -denominator; + + uint_div_qr(numerator, denominator, &qr); + + return ret_idivmod_values(qr.q, qr.r); +} diff --git a/lib/aarch32/arm32_aeabi_divmod_a32.S b/lib/aarch32/arm32_aeabi_divmod_a32.S new file mode 100644 index 0000000..9155100 --- /dev/null +++ b/lib/aarch32/arm32_aeabi_divmod_a32.S @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + +/* + * EABI wrappers from the udivmod and idivmod functions + */ + + .globl ret_uidivmod_values + .globl ret_idivmod_values + +/* + * signed ret_idivmod_values(signed quot, signed rem); + * return quotient and remaining the EABI way (regs r0,r1) + */ +func ret_idivmod_values + bx lr +endfunc ret_idivmod_values + +/* + * unsigned ret_uidivmod_values(unsigned quot, unsigned rem); + * return quotient and remaining the EABI way (regs r0,r1) + */ +func ret_uidivmod_values + bx lr +endfunc ret_uidivmod_values diff --git a/lib/aarch32/armclang_printf.S b/lib/aarch32/armclang_printf.S new file mode 100644 index 0000000..e736a11 --- /dev/null +++ b/lib/aarch32/armclang_printf.S @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + +/* Symbols needed by armclang */ + + .globl __0printf + .globl __1printf + .globl __2printf + +func __0printf +__1printf: +__2printf: + b printf +endfunc __0printf diff --git a/lib/aarch32/cache_helpers.S b/lib/aarch32/cache_helpers.S new file mode 100644 index 0000000..13d1872 --- /dev/null +++ b/lib/aarch32/cache_helpers.S @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> + + .globl flush_dcache_range + .globl clean_dcache_range + .globl inv_dcache_range + .globl dcsw_op_louis + .globl dcsw_op_all + .globl dcsw_op_level1 + .globl dcsw_op_level2 + .globl dcsw_op_level3 + +/* + * This macro can be used for implementing various data cache operations `op` + */ +.macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2 + /* Exit early if size is zero */ + cmp r1, #0 + beq exit_loop_\op + dcache_line_size r2, r3 + add r1, r0, r1 + sub r3, r2, #1 + bic r0, r0, r3 +loop_\op: + stcopr r0, \coproc, \opc1, \CRn, \CRm, \opc2 + add r0, r0, r2 + cmp r0, r1 + blo loop_\op + dsb sy +exit_loop_\op: + bx lr +.endm + + /* ------------------------------------------ + * Clean+Invalidate from base address till + * size. 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +func flush_dcache_range + do_dcache_maintenance_by_mva cimvac, DCCIMVAC +endfunc flush_dcache_range + + /* ------------------------------------------ + * Clean from base address till size. + * 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +func clean_dcache_range + do_dcache_maintenance_by_mva cmvac, DCCMVAC +endfunc clean_dcache_range + + /* ------------------------------------------ + * Invalidate from base address till + * size. 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +func inv_dcache_range + do_dcache_maintenance_by_mva imvac, DCIMVAC +endfunc inv_dcache_range + + /* ---------------------------------------------------------------- + * Data cache operations by set/way to the level specified + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * r1: The cache level to begin operation from + * r2: clidr_el1 + * r3: The last cache level to operate on + * and will carry out the operation on each data cache from level 0 + * to the level in r3 in sequence + * + * The dcsw_op macro sets up the r2 and r3 parameters based on + * clidr_el1 cache information before invoking the main function + * ---------------------------------------------------------------- + */ + + .macro dcsw_op shift, fw, ls + ldcopr r2, CLIDR + ubfx r3, r2, \shift, \fw + lsl r3, r3, \ls + mov r1, #0 + b do_dcsw_op + .endm + +func do_dcsw_op + push {r4-r12, lr} + ldcopr r8, ID_MMFR4 // stash FEAT_CCIDX identifier in r8 + ubfx r8, r8, #ID_MMFR4_CCIDX_SHIFT, #ID_MMFR4_CCIDX_LENGTH + adr r11, dcsw_loop_table // compute cache op based on the operation type + add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions +loop1: + add r10, r1, r1, LSR #1 // Work out 3x current cache level + mov r12, r2, LSR r10 // extract cache type bits from clidr + and r12, r12, #7 // mask the bits for current cache only + cmp r12, #2 // see what cache we have at this level + blo level_done // no cache or only instruction cache at this level + + stcopr r1, CSSELR // select current cache level in csselr + isb // isb to sych the new cssr&csidr + ldcopr r12, CCSIDR // read the new ccsidr + and r10, r12, #7 // extract the length of the cache lines + add r10, r10, #4 // add 4 (r10 = line length offset) + + cmp r8, #0 // check for FEAT_CCIDX for Associativity + beq 1f + ubfx r4, r12, #3, #21 // r4 = associativity CCSIDR[23:3] + b 2f +1: + ubfx r4, r12, #3, #10 // r4 = associativity CCSIDR[12:3] +2: + clz r5, r4 // r5 = the bit position of the way size increment + mov r9, r4 // r9 working copy of the aligned max way number + +loop2: + cmp r8, #0 // check for FEAT_CCIDX for NumSets + beq 3f + ldcopr r12, CCSIDR2 // FEAT_CCIDX numsets is in CCSIDR2 + ubfx r7, r12, #0, #24 // r7 = numsets CCSIDR2[23:0] + b loop3 +3: + ubfx r7, r12, #13, #15 // r7 = numsets CCSIDR[27:13] +loop3: + orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0 + orr r0, r0, r7, LSL r10 // factor in the set number + + blx r6 + subs r7, r7, #1 // decrement the set number + bhs loop3 + subs r9, r9, #1 // decrement the way number + bhs loop2 +level_done: + add r1, r1, #2 // increment the cache number + cmp r3, r1 + // Ensure completion of previous cache maintenance instruction. Note + // this also mitigates erratum 814220 on Cortex-A7 + dsb sy + bhi loop1 + + mov r6, #0 + stcopr r6, CSSELR //select cache level 0 in csselr + dsb sy + isb + pop {r4-r12, pc} + +dcsw_loop_table: + stcopr r0, DCISW + bx lr + stcopr r0, DCCISW + bx lr + stcopr r0, DCCSW + bx lr + +endfunc do_dcsw_op + + /* --------------------------------------------------------------- + * Data cache operations by set/way till PoU. + * + * The function requires : + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_louis + dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT +endfunc dcsw_op_louis + + /* --------------------------------------------------------------- + * Data cache operations by set/way till PoC. + * + * The function requires : + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_all + dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT +endfunc dcsw_op_all + + + /* --------------------------------------------------------------- + * Helper macro for data cache operations by set/way for the + * level specified + * --------------------------------------------------------------- + */ + .macro dcsw_op_level level + ldcopr r2, CLIDR + mov r3, \level + sub r1, r3, #2 + b do_dcsw_op + .endm + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 1 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level1 + dcsw_op_level #(1 << LEVEL_SHIFT) +endfunc dcsw_op_level1 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 2 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level2 + dcsw_op_level #(2 << LEVEL_SHIFT) +endfunc dcsw_op_level2 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 3 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level3 + dcsw_op_level #(3 << LEVEL_SHIFT) +endfunc dcsw_op_level3 diff --git a/lib/aarch32/misc_helpers.S b/lib/aarch32/misc_helpers.S new file mode 100644 index 0000000..c010cbc --- /dev/null +++ b/lib/aarch32/misc_helpers.S @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <common/bl_common.h> +#include <lib/xlat_tables/xlat_tables_defs.h> + + .globl smc + .globl zeromem + .globl zero_normalmem + .globl memcpy4 + .globl disable_mmu_icache_secure + .globl disable_mmu_secure + .globl fixup_gdt_reloc + +#define PAGE_START_MASK ~(PAGE_SIZE_MASK) + +func smc + /* + * For AArch32 only r0-r3 will be in the registers; + * rest r4-r6 will be pushed on to the stack. So here, we'll + * have to load them from the stack to registers r4-r6 explicitly. + * Clobbers: r4-r6 + */ + ldm sp, {r4, r5, r6} + smc #0 +endfunc smc + +/* ----------------------------------------------------------------------- + * void zeromem(void *mem, unsigned int length) + * + * Initialise a region in normal memory to 0. This functions complies with the + * AAPCS and can be called from C code. + * + * ----------------------------------------------------------------------- + */ +func zeromem + /* + * Readable names for registers + * + * Registers r0, r1 and r2 are also set by zeromem which + * branches into the fallback path directly, so cursor, length and + * stop_address should not be retargeted to other registers. + */ + cursor .req r0 /* Start address and then current address */ + length .req r1 /* Length in bytes of the region to zero out */ + /* + * Reusing the r1 register as length is only used at the beginning of + * the function. + */ + stop_address .req r1 /* Address past the last zeroed byte */ + zeroreg1 .req r2 /* Source register filled with 0 */ + zeroreg2 .req r3 /* Source register filled with 0 */ + tmp .req r12 /* Temporary scratch register */ + + mov zeroreg1, #0 + + /* stop_address is the address past the last to zero */ + add stop_address, cursor, length + + /* + * Length cannot be used anymore as it shares the same register with + * stop_address. + */ + .unreq length + + /* + * If the start address is already aligned to 8 bytes, skip this loop. + */ + tst cursor, #(8-1) + beq .Lzeromem_8bytes_aligned + + /* Calculate the next address aligned to 8 bytes */ + orr tmp, cursor, #(8-1) + adds tmp, tmp, #1 + /* If it overflows, fallback to byte per byte zeroing */ + beq .Lzeromem_1byte_aligned + /* If the next aligned address is after the stop address, fall back */ + cmp tmp, stop_address + bhs .Lzeromem_1byte_aligned + + /* zero byte per byte */ +1: + strb zeroreg1, [cursor], #1 + cmp cursor, tmp + bne 1b + + /* zero 8 bytes at a time */ +.Lzeromem_8bytes_aligned: + + /* Calculate the last 8 bytes aligned address. */ + bic tmp, stop_address, #(8-1) + + cmp cursor, tmp + bhs 2f + + mov zeroreg2, #0 +1: + stmia cursor!, {zeroreg1, zeroreg2} + cmp cursor, tmp + blo 1b +2: + + /* zero byte per byte */ +.Lzeromem_1byte_aligned: + cmp cursor, stop_address + beq 2f +1: + strb zeroreg1, [cursor], #1 + cmp cursor, stop_address + bne 1b +2: + bx lr + + .unreq cursor + /* + * length is already unreq'ed to reuse the register for another + * variable. + */ + .unreq stop_address + .unreq zeroreg1 + .unreq zeroreg2 + .unreq tmp +endfunc zeromem + +/* + * AArch32 does not have special ways of zeroing normal memory as AArch64 does + * using the DC ZVA instruction, so we just alias zero_normalmem to zeromem. + */ +.equ zero_normalmem, zeromem + +/* -------------------------------------------------------------------------- + * void memcpy4(void *dest, const void *src, unsigned int length) + * + * Copy length bytes from memory area src to memory area dest. + * The memory areas should not overlap. + * Destination and source addresses must be 4-byte aligned. + * -------------------------------------------------------------------------- + */ +func memcpy4 +#if ENABLE_ASSERTIONS + orr r3, r0, r1 + tst r3, #0x3 + ASM_ASSERT(eq) +#endif +/* copy 4 bytes at a time */ +m_loop4: + cmp r2, #4 + blo m_loop1 + ldr r3, [r1], #4 + str r3, [r0], #4 + subs r2, r2, #4 + bne m_loop4 + bx lr + +/* copy byte per byte */ +m_loop1: + ldrb r3, [r1], #1 + strb r3, [r0], #1 + subs r2, r2, #1 + bne m_loop1 + bx lr +endfunc memcpy4 + +/* --------------------------------------------------------------------------- + * Disable the MMU in Secure State + * --------------------------------------------------------------------------- + */ + +func disable_mmu_secure + mov r1, #(SCTLR_M_BIT | SCTLR_C_BIT) +do_disable_mmu: +#if ERRATA_A9_794073 + stcopr r0, BPIALL + dsb +#endif + ldcopr r0, SCTLR + bic r0, r0, r1 + stcopr r0, SCTLR + isb // ensure MMU is off + dsb sy + bx lr +endfunc disable_mmu_secure + + +func disable_mmu_icache_secure + ldr r1, =(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) + b do_disable_mmu +endfunc disable_mmu_icache_secure + +/* --------------------------------------------------------------------------- + * Helper to fixup Global Descriptor table (GDT) and dynamic relocations + * (.rel.dyn) at runtime. + * + * This function is meant to be used when the firmware is compiled with -fpie + * and linked with -pie options. We rely on the linker script exporting + * appropriate markers for start and end of the section. For GOT, we + * expect __GOT_START__ and __GOT_END__. Similarly for .rela.dyn, we expect + * __RELA_START__ and __RELA_END__. + * + * The function takes the limits of the memory to apply fixups to as + * arguments (which is usually the limits of the relocable BL image). + * r0 - the start of the fixup region + * r1 - the limit of the fixup region + * These addresses have to be 4KB page aligned. + * --------------------------------------------------------------------------- + */ + +/* Relocation codes */ +#define R_ARM_RELATIVE 23 + +func fixup_gdt_reloc + mov r6, r0 + mov r7, r1 + +#if ENABLE_ASSERTIONS + /* Test if the limits are 4K aligned */ + orr r0, r0, r1 + mov r1, #(PAGE_SIZE_MASK) + tst r0, r1 + ASM_ASSERT(eq) +#endif + /* + * Calculate the offset based on return address in lr. + * Assume that this function is called within a page at the start of + * fixup region. + */ + ldr r1, =PAGE_START_MASK + and r2, lr, r1 + subs r0, r2, r6 /* Diff(S) = Current Address - Compiled Address */ + beq 3f /* Diff(S) = 0. No relocation needed */ + + ldr r1, =__GOT_START__ + add r1, r1, r0 + ldr r2, =__GOT_END__ + add r2, r2, r0 + + /* + * GOT is an array of 32_bit addresses which must be fixed up as + * new_addr = old_addr + Diff(S). + * The new_addr is the address currently the binary is executing from + * and old_addr is the address at compile time. + */ +1: ldr r3, [r1] + + /* Skip adding offset if address is < lower limit */ + cmp r3, r6 + blo 2f + + /* Skip adding offset if address is > upper limit */ + cmp r3, r7 + bhi 2f + add r3, r3, r0 + str r3, [r1] + +2: add r1, r1, #4 + cmp r1, r2 + blo 1b + + /* Starting dynamic relocations. Use ldr to get RELA_START and END */ +3: ldr r1, =__RELA_START__ + add r1, r1, r0 + ldr r2, =__RELA_END__ + add r2, r2, r0 + + /* + * According to ELF-32 specification, the RELA data structure is as + * follows: + * typedef struct { + * Elf32_Addr r_offset; + * Elf32_Xword r_info; + * } Elf32_Rela; + * + * r_offset is address of reference + * r_info is symbol index and type of relocation (in this case + * code 23 which corresponds to R_ARM_RELATIVE). + * + * Size of Elf32_Rela structure is 8 bytes. + */ + + /* Skip R_ARM_NONE entry with code 0 */ +1: ldr r3, [r1, #4] + ands r3, r3, #0xff + beq 2f + +#if ENABLE_ASSERTIONS + /* Assert that the relocation type is R_ARM_RELATIVE */ + cmp r3, #R_ARM_RELATIVE + ASM_ASSERT(eq) +#endif + ldr r3, [r1] /* r_offset */ + add r3, r0, r3 /* Diff(S) + r_offset */ + ldr r4, [r3] + + /* Skip adding offset if address is < lower limit */ + cmp r4, r6 + blo 2f + + /* Skip adding offset if address is > upper limit */ + cmp r4, r7 + bhi 2f + + add r4, r0, r4 + str r4, [r3] + +2: add r1, r1, #8 + cmp r1, r2 + blo 1b + bx lr +endfunc fixup_gdt_reloc diff --git a/lib/aarch64/armclang_printf.S b/lib/aarch64/armclang_printf.S new file mode 100644 index 0000000..f9326fd --- /dev/null +++ b/lib/aarch64/armclang_printf.S @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + +/* Symbols needed by armclang */ + + .globl __0printf + .globl __1printf + .globl __2printf + +func __0printf + b printf +endfunc __0printf + +func __1printf + b printf +endfunc __1printf + +func __2printf + b printf +endfunc __2printf diff --git a/lib/aarch64/cache_helpers.S b/lib/aarch64/cache_helpers.S new file mode 100644 index 0000000..314ed6e --- /dev/null +++ b/lib/aarch64/cache_helpers.S @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> + + .globl flush_dcache_range + .globl flush_dcache_to_popa_range + .globl clean_dcache_range + .globl inv_dcache_range + .globl dcsw_op_louis + .globl dcsw_op_all + .globl dcsw_op_level1 + .globl dcsw_op_level2 + .globl dcsw_op_level3 + +/* + * This macro can be used for implementing various data cache operations `op` + */ +.macro do_dcache_maintenance_by_mva op + /* Exit early if size is zero */ + cbz x1, exit_loop_\op + dcache_line_size x2, x3 + add x1, x0, x1 + sub x3, x2, #1 + bic x0, x0, x3 +loop_\op: + dc \op, x0 + add x0, x0, x2 + cmp x0, x1 + b.lo loop_\op + dsb sy +exit_loop_\op: + ret +.endm + +.macro check_plat_can_cmo +#if CONDITIONAL_CMO + mov x3, x30 + mov x2, x0 + bl plat_can_cmo + mov x30, x3 + cbnz x0, 1f + ret +1: + mov x0, x2 +#endif +.endm + /* ------------------------------------------ + * Clean+Invalidate from base address till + * size. 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +func flush_dcache_range + check_plat_can_cmo + do_dcache_maintenance_by_mva civac +endfunc flush_dcache_range + + /* ------------------------------------------ + * Clean from base address till size. + * 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +func clean_dcache_range + check_plat_can_cmo + do_dcache_maintenance_by_mva cvac +endfunc clean_dcache_range + + /* ------------------------------------------ + * Invalidate from base address till + * size. 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +func inv_dcache_range + check_plat_can_cmo + do_dcache_maintenance_by_mva ivac +endfunc inv_dcache_range + + + /* + * On implementations with FEAT_MTE2, + * Root firmware must issue DC_CIGDPAPA instead of DC_CIPAPA , + * in order to additionally clean and invalidate Allocation Tags + * associated with the affected locations. + * + * ------------------------------------------ + * Clean+Invalidate by PA to POPA + * from base address till size. + * 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +func flush_dcache_to_popa_range + /* Exit early if size is zero */ + cbz x1, exit_loop_dc_cipapa + check_plat_can_cmo + dcache_line_size x2, x3 + sub x3, x2, #1 + bic x0, x0, x3 + add x1, x1, x0 +loop_dc_cipapa: + sys #6, c7, c14, #1, x0 /* DC CIPAPA,<Xt> */ + add x0, x0, x2 + cmp x0, x1 + b.lo loop_dc_cipapa + dsb osh +exit_loop_dc_cipapa: + ret +endfunc flush_dcache_to_popa_range + + /* --------------------------------------------------------------- + * Data cache operations by set/way to the level specified + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in arch.h + * x3: The last cache level to operate on + * x9: clidr_el1 + * x10: The cache level to begin operation from + * and will carry out the operation on each data cache from level 0 + * to the level in x3 in sequence + * + * The dcsw_op macro sets up the x3 and x9 parameters based on + * clidr_el1 cache information before invoking the main function + * --------------------------------------------------------------- + */ + + .macro dcsw_op shift, fw, ls + mrs x9, clidr_el1 + ubfx x3, x9, \shift, \fw + lsl x3, x3, \ls + mov x10, xzr + b do_dcsw_op + .endm + +func do_dcsw_op + cbz x3, exit + mrs x12, ID_AA64MMFR2_EL1 // stash FEAT_CCIDX identifier in x12 + ubfx x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH + adr x14, dcsw_loop_table // compute inner loop address + add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions +#if ENABLE_BTI + add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction +#endif + mov x0, x9 + mov w8, #1 +loop1: + add x2, x10, x10, lsr #1 // work out 3x current cache level + lsr x1, x0, x2 // extract cache type bits from clidr + and x1, x1, #7 // mask the bits for current cache only + cmp x1, #2 // see what cache we have at this level + b.lo level_done // nothing to do if no cache or icache + + msr csselr_el1, x10 // select current cache level in csselr + isb // isb to sych the new cssr&csidr + mrs x1, ccsidr_el1 // read the new ccsidr + and x2, x1, #7 // extract the length of the cache lines + add x2, x2, #4 // add 4 (line length offset) + + cbz x12, 1f // check for FEAT_CCIDX for Associativity + ubfx x4, x1, #3, #21 // x4 = associativity CCSIDR_EL1[23:3] + b 2f +1: + ubfx x4, x1, #3, #10 // x4 = associativity CCSIDR_EL1[12:3] +2: + clz w5, w4 // bit position of way size increment + lsl w9, w4, w5 // w9 = aligned max way number + lsl w16, w8, w5 // w16 = way number loop decrement + orr w9, w10, w9 // w9 = combine way and cache number + + cbz x12, 3f // check for FEAT_CCIDX for NumSets + ubfx x6, x1, #32, #24 // x6 (w6) = numsets CCSIDR_EL1[55:32] + // ISA will not allow x->w ubfx + b 4f +3: + ubfx w6, w1, #13, #15 // w6 = numsets CCSIDR_EL1[27:13] +4: + lsl w17, w8, w2 // w17 = set number loop decrement + dsb sy // barrier before we start this level + br x14 // jump to DC operation specific loop + + .macro dcsw_loop _op +#if ENABLE_BTI + bti j +#endif +loop2_\_op: + lsl w7, w6, w2 // w7 = aligned max set number + +loop3_\_op: + orr w11, w9, w7 // combine cache, way and set number + dc \_op, x11 + subs w7, w7, w17 // decrement set number + b.hs loop3_\_op + + subs x9, x9, x16 // decrement way number + b.hs loop2_\_op + + b level_done + .endm + +level_done: + add x10, x10, #2 // increment cache number + cmp x3, x10 + b.hi loop1 + msr csselr_el1, xzr // select cache level 0 in csselr + dsb sy // barrier to complete final cache operation + isb +exit: + ret +endfunc do_dcsw_op + +dcsw_loop_table: + dcsw_loop isw + dcsw_loop cisw + dcsw_loop csw + + +func dcsw_op_louis + check_plat_can_cmo + dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT +endfunc dcsw_op_louis + + +func dcsw_op_all + check_plat_can_cmo + dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT +endfunc dcsw_op_all + + /* --------------------------------------------------------------- + * Helper macro for data cache operations by set/way for the + * level specified + * --------------------------------------------------------------- + */ + .macro dcsw_op_level level + mrs x9, clidr_el1 + mov x3, \level + sub x10, x3, #2 + b do_dcsw_op + .endm + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 1 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level1 + check_plat_can_cmo + dcsw_op_level #(1 << LEVEL_SHIFT) +endfunc dcsw_op_level1 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 2 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level2 + check_plat_can_cmo + dcsw_op_level #(2 << LEVEL_SHIFT) +endfunc dcsw_op_level2 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 3 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level3 + check_plat_can_cmo + dcsw_op_level #(3 << LEVEL_SHIFT) +endfunc dcsw_op_level3 diff --git a/lib/aarch64/misc_helpers.S b/lib/aarch64/misc_helpers.S new file mode 100644 index 0000000..f9c4baf --- /dev/null +++ b/lib/aarch64/misc_helpers.S @@ -0,0 +1,613 @@ +/* + * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <common/bl_common.h> +#include <lib/xlat_tables/xlat_tables_defs.h> + + .globl smc + + .globl zero_normalmem + .globl zeromem + .globl memcpy16 + .globl gpt_tlbi_by_pa_ll + + .globl disable_mmu_el1 + .globl disable_mmu_el3 + .globl disable_mmu_icache_el1 + .globl disable_mmu_icache_el3 + .globl fixup_gdt_reloc +#if SUPPORT_VFP + .globl enable_vfp +#endif + +func smc + smc #0 +endfunc smc + +/* ----------------------------------------------------------------------- + * void zero_normalmem(void *mem, unsigned int length); + * + * Initialise a region in normal memory to 0. This functions complies with the + * AAPCS and can be called from C code. + * + * NOTE: MMU must be enabled when using this function as it can only operate on + * normal memory. It is intended to be mainly used from C code when MMU + * is usually enabled. + * ----------------------------------------------------------------------- + */ +.equ zero_normalmem, zeromem_dczva + +/* ----------------------------------------------------------------------- + * void zeromem(void *mem, unsigned int length); + * + * Initialise a region of device memory to 0. This functions complies with the + * AAPCS and can be called from C code. + * + * NOTE: When data caches and MMU are enabled, zero_normalmem can usually be + * used instead for faster zeroing. + * + * ----------------------------------------------------------------------- + */ +func zeromem + /* x2 is the address past the last zeroed address */ + add x2, x0, x1 + /* + * Uses the fallback path that does not use DC ZVA instruction and + * therefore does not need enabled MMU + */ + b .Lzeromem_dczva_fallback_entry +endfunc zeromem + +/* ----------------------------------------------------------------------- + * void zeromem_dczva(void *mem, unsigned int length); + * + * Fill a region of normal memory of size "length" in bytes with null bytes. + * MMU must be enabled and the memory be of + * normal type. This is because this function internally uses the DC ZVA + * instruction, which generates an Alignment fault if used on any type of + * Device memory (see section D3.4.9 of the ARMv8 ARM, issue k). When the MMU + * is disabled, all memory behaves like Device-nGnRnE memory (see section + * D4.2.8), hence the requirement on the MMU being enabled. + * NOTE: The code assumes that the block size as defined in DCZID_EL0 + * register is at least 16 bytes. + * + * ----------------------------------------------------------------------- + */ +func zeromem_dczva + + /* + * The function consists of a series of loops that zero memory one byte + * at a time, 16 bytes at a time or using the DC ZVA instruction to + * zero aligned block of bytes, which is assumed to be more than 16. + * In the case where the DC ZVA instruction cannot be used or if the + * first 16 bytes loop would overflow, there is fallback path that does + * not use DC ZVA. + * Note: The fallback path is also used by the zeromem function that + * branches to it directly. + * + * +---------+ zeromem_dczva + * | entry | + * +----+----+ + * | + * v + * +---------+ + * | checks |>o-------+ (If any check fails, fallback) + * +----+----+ | + * | |---------------+ + * v | Fallback path | + * +------+------+ |---------------+ + * | 1 byte loop | | + * +------+------+ .Lzeromem_dczva_initial_1byte_aligned_end + * | | + * v | + * +-------+-------+ | + * | 16 bytes loop | | + * +-------+-------+ | + * | | + * v | + * +------+------+ .Lzeromem_dczva_blocksize_aligned + * | DC ZVA loop | | + * +------+------+ | + * +--------+ | | + * | | | | + * | v v | + * | +-------+-------+ .Lzeromem_dczva_final_16bytes_aligned + * | | 16 bytes loop | | + * | +-------+-------+ | + * | | | + * | v | + * | +------+------+ .Lzeromem_dczva_final_1byte_aligned + * | | 1 byte loop | | + * | +-------------+ | + * | | | + * | v | + * | +---+--+ | + * | | exit | | + * | +------+ | + * | | + * | +--------------+ +------------------+ zeromem + * | | +----------------| zeromem function | + * | | | +------------------+ + * | v v + * | +-------------+ .Lzeromem_dczva_fallback_entry + * | | 1 byte loop | + * | +------+------+ + * | | + * +-----------+ + */ + + /* + * Readable names for registers + * + * Registers x0, x1 and x2 are also set by zeromem which + * branches into the fallback path directly, so cursor, length and + * stop_address should not be retargeted to other registers. + */ + cursor .req x0 /* Start address and then current address */ + length .req x1 /* Length in bytes of the region to zero out */ + /* Reusing x1 as length is never used after block_mask is set */ + block_mask .req x1 /* Bitmask of the block size read in DCZID_EL0 */ + stop_address .req x2 /* Address past the last zeroed byte */ + block_size .req x3 /* Size of a block in bytes as read in DCZID_EL0 */ + tmp1 .req x4 + tmp2 .req x5 + +#if ENABLE_ASSERTIONS + /* + * Check for M bit (MMU enabled) of the current SCTLR_EL(1|3) + * register value and panic if the MMU is disabled. + */ +#if defined(IMAGE_BL1) || defined(IMAGE_BL31) || (defined(IMAGE_BL2) && \ + BL2_RUNS_AT_EL3) + mrs tmp1, sctlr_el3 +#else + mrs tmp1, sctlr_el1 +#endif + + tst tmp1, #SCTLR_M_BIT + ASM_ASSERT(ne) +#endif /* ENABLE_ASSERTIONS */ + + /* stop_address is the address past the last to zero */ + add stop_address, cursor, length + + /* + * Get block_size = (log2(<block size>) >> 2) (see encoding of + * dczid_el0 reg) + */ + mrs block_size, dczid_el0 + + /* + * Select the 4 lowest bits and convert the extracted log2(<block size + * in words>) to <block size in bytes> + */ + ubfx block_size, block_size, #0, #4 + mov tmp2, #(1 << 2) + lsl block_size, tmp2, block_size + +#if ENABLE_ASSERTIONS + /* + * Assumes block size is at least 16 bytes to avoid manual realignment + * of the cursor at the end of the DCZVA loop. + */ + cmp block_size, #16 + ASM_ASSERT(hs) +#endif + /* + * Not worth doing all the setup for a region less than a block and + * protects against zeroing a whole block when the area to zero is + * smaller than that. Also, as it is assumed that the block size is at + * least 16 bytes, this also protects the initial aligning loops from + * trying to zero 16 bytes when length is less than 16. + */ + cmp length, block_size + b.lo .Lzeromem_dczva_fallback_entry + + /* + * Calculate the bitmask of the block alignment. It will never + * underflow as the block size is between 4 bytes and 2kB. + * block_mask = block_size - 1 + */ + sub block_mask, block_size, #1 + + /* + * length alias should not be used after this point unless it is + * defined as a register other than block_mask's. + */ + .unreq length + + /* + * If the start address is already aligned to zero block size, go + * straight to the cache zeroing loop. This is safe because at this + * point, the length cannot be smaller than a block size. + */ + tst cursor, block_mask + b.eq .Lzeromem_dczva_blocksize_aligned + + /* + * Calculate the first block-size-aligned address. It is assumed that + * the zero block size is at least 16 bytes. This address is the last + * address of this initial loop. + */ + orr tmp1, cursor, block_mask + add tmp1, tmp1, #1 + + /* + * If the addition overflows, skip the cache zeroing loops. This is + * quite unlikely however. + */ + cbz tmp1, .Lzeromem_dczva_fallback_entry + + /* + * If the first block-size-aligned address is past the last address, + * fallback to the simpler code. + */ + cmp tmp1, stop_address + b.hi .Lzeromem_dczva_fallback_entry + + /* + * If the start address is already aligned to 16 bytes, skip this loop. + * It is safe to do this because tmp1 (the stop address of the initial + * 16 bytes loop) will never be greater than the final stop address. + */ + tst cursor, #0xf + b.eq .Lzeromem_dczva_initial_1byte_aligned_end + + /* Calculate the next address aligned to 16 bytes */ + orr tmp2, cursor, #0xf + add tmp2, tmp2, #1 + /* If it overflows, fallback to the simple path (unlikely) */ + cbz tmp2, .Lzeromem_dczva_fallback_entry + /* + * Next aligned address cannot be after the stop address because the + * length cannot be smaller than 16 at this point. + */ + + /* First loop: zero byte per byte */ +1: + strb wzr, [cursor], #1 + cmp cursor, tmp2 + b.ne 1b +.Lzeromem_dczva_initial_1byte_aligned_end: + + /* + * Second loop: we need to zero 16 bytes at a time from cursor to tmp1 + * before being able to use the code that deals with block-size-aligned + * addresses. + */ + cmp cursor, tmp1 + b.hs 2f +1: + stp xzr, xzr, [cursor], #16 + cmp cursor, tmp1 + b.lo 1b +2: + + /* + * Third loop: zero a block at a time using DC ZVA cache block zeroing + * instruction. + */ +.Lzeromem_dczva_blocksize_aligned: + /* + * Calculate the last block-size-aligned address. If the result equals + * to the start address, the loop will exit immediately. + */ + bic tmp1, stop_address, block_mask + + cmp cursor, tmp1 + b.hs 2f +1: + /* Zero the block containing the cursor */ + dc zva, cursor + /* Increment the cursor by the size of a block */ + add cursor, cursor, block_size + cmp cursor, tmp1 + b.lo 1b +2: + + /* + * Fourth loop: zero 16 bytes at a time and then byte per byte the + * remaining area + */ +.Lzeromem_dczva_final_16bytes_aligned: + /* + * Calculate the last 16 bytes aligned address. It is assumed that the + * block size will never be smaller than 16 bytes so that the current + * cursor is aligned to at least 16 bytes boundary. + */ + bic tmp1, stop_address, #15 + + cmp cursor, tmp1 + b.hs 2f +1: + stp xzr, xzr, [cursor], #16 + cmp cursor, tmp1 + b.lo 1b +2: + + /* Fifth and final loop: zero byte per byte */ +.Lzeromem_dczva_final_1byte_aligned: + cmp cursor, stop_address + b.eq 2f +1: + strb wzr, [cursor], #1 + cmp cursor, stop_address + b.ne 1b +2: + ret + + /* Fallback for unaligned start addresses */ +.Lzeromem_dczva_fallback_entry: + /* + * If the start address is already aligned to 16 bytes, skip this loop. + */ + tst cursor, #0xf + b.eq .Lzeromem_dczva_final_16bytes_aligned + + /* Calculate the next address aligned to 16 bytes */ + orr tmp1, cursor, #15 + add tmp1, tmp1, #1 + /* If it overflows, fallback to byte per byte zeroing */ + cbz tmp1, .Lzeromem_dczva_final_1byte_aligned + /* If the next aligned address is after the stop address, fall back */ + cmp tmp1, stop_address + b.hs .Lzeromem_dczva_final_1byte_aligned + + /* Fallback entry loop: zero byte per byte */ +1: + strb wzr, [cursor], #1 + cmp cursor, tmp1 + b.ne 1b + + b .Lzeromem_dczva_final_16bytes_aligned + + .unreq cursor + /* + * length is already unreq'ed to reuse the register for another + * variable. + */ + .unreq stop_address + .unreq block_size + .unreq block_mask + .unreq tmp1 + .unreq tmp2 +endfunc zeromem_dczva + +/* -------------------------------------------------------------------------- + * void memcpy16(void *dest, const void *src, unsigned int length) + * + * Copy length bytes from memory area src to memory area dest. + * The memory areas should not overlap. + * Destination and source addresses must be 16-byte aligned. + * -------------------------------------------------------------------------- + */ +func memcpy16 +#if ENABLE_ASSERTIONS + orr x3, x0, x1 + tst x3, #0xf + ASM_ASSERT(eq) +#endif +/* copy 16 bytes at a time */ +m_loop16: + cmp x2, #16 + b.lo m_loop1 + ldp x3, x4, [x1], #16 + stp x3, x4, [x0], #16 + sub x2, x2, #16 + b m_loop16 +/* copy byte per byte */ +m_loop1: + cbz x2, m_end + ldrb w3, [x1], #1 + strb w3, [x0], #1 + subs x2, x2, #1 + b.ne m_loop1 +m_end: + ret +endfunc memcpy16 + +/* --------------------------------------------------------------------------- + * Disable the MMU at EL3 + * --------------------------------------------------------------------------- + */ + +func disable_mmu_el3 + mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) +do_disable_mmu_el3: + mrs x0, sctlr_el3 + bic x0, x0, x1 + msr sctlr_el3, x0 + isb /* ensure MMU is off */ + dsb sy + ret +endfunc disable_mmu_el3 + + +func disable_mmu_icache_el3 + mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) + b do_disable_mmu_el3 +endfunc disable_mmu_icache_el3 + +/* --------------------------------------------------------------------------- + * Disable the MMU at EL1 + * --------------------------------------------------------------------------- + */ + +func disable_mmu_el1 + mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) +do_disable_mmu_el1: + mrs x0, sctlr_el1 + bic x0, x0, x1 + msr sctlr_el1, x0 + isb /* ensure MMU is off */ + dsb sy + ret +endfunc disable_mmu_el1 + + +func disable_mmu_icache_el1 + mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) + b do_disable_mmu_el1 +endfunc disable_mmu_icache_el1 + +/* --------------------------------------------------------------------------- + * Enable the use of VFP at EL3 + * --------------------------------------------------------------------------- + */ +#if SUPPORT_VFP +func enable_vfp + mrs x0, cpacr_el1 + orr x0, x0, #CPACR_VFP_BITS + msr cpacr_el1, x0 + mrs x0, cptr_el3 + mov x1, #AARCH64_CPTR_TFP + bic x0, x0, x1 + msr cptr_el3, x0 + isb + ret +endfunc enable_vfp +#endif + +/* --------------------------------------------------------------------------- + * Helper to fixup Global Descriptor table (GDT) and dynamic relocations + * (.rela.dyn) at runtime. + * + * This function is meant to be used when the firmware is compiled with -fpie + * and linked with -pie options. We rely on the linker script exporting + * appropriate markers for start and end of the section. For GOT, we + * expect __GOT_START__ and __GOT_END__. Similarly for .rela.dyn, we expect + * __RELA_START__ and __RELA_END__. + * + * The function takes the limits of the memory to apply fixups to as + * arguments (which is usually the limits of the relocable BL image). + * x0 - the start of the fixup region + * x1 - the limit of the fixup region + * These addresses have to be 4KB page aligned. + * --------------------------------------------------------------------------- + */ + +/* Relocation codes */ +#define R_AARCH64_NONE 0 +#define R_AARCH64_RELATIVE 1027 + +func fixup_gdt_reloc + mov x6, x0 + mov x7, x1 + +#if ENABLE_ASSERTIONS + /* Test if the limits are 4KB aligned */ + orr x0, x0, x1 + tst x0, #(PAGE_SIZE_MASK) + ASM_ASSERT(eq) +#endif + /* + * Calculate the offset based on return address in x30. + * Assume that this function is called within a page at the start of + * fixup region. + */ + and x2, x30, #~(PAGE_SIZE_MASK) + subs x0, x2, x6 /* Diff(S) = Current Address - Compiled Address */ + b.eq 3f /* Diff(S) = 0. No relocation needed */ + + adrp x1, __GOT_START__ + add x1, x1, :lo12:__GOT_START__ + adrp x2, __GOT_END__ + add x2, x2, :lo12:__GOT_END__ + + /* + * GOT is an array of 64_bit addresses which must be fixed up as + * new_addr = old_addr + Diff(S). + * The new_addr is the address currently the binary is executing from + * and old_addr is the address at compile time. + */ +1: ldr x3, [x1] + + /* Skip adding offset if address is < lower limit */ + cmp x3, x6 + b.lo 2f + + /* Skip adding offset if address is > upper limit */ + cmp x3, x7 + b.hi 2f + add x3, x3, x0 + str x3, [x1] + +2: add x1, x1, #8 + cmp x1, x2 + b.lo 1b + + /* Starting dynamic relocations. Use adrp/adr to get RELA_START and END */ +3: adrp x1, __RELA_START__ + add x1, x1, :lo12:__RELA_START__ + adrp x2, __RELA_END__ + add x2, x2, :lo12:__RELA_END__ + + /* + * According to ELF-64 specification, the RELA data structure is as + * follows: + * typedef struct { + * Elf64_Addr r_offset; + * Elf64_Xword r_info; + * Elf64_Sxword r_addend; + * } Elf64_Rela; + * + * r_offset is address of reference + * r_info is symbol index and type of relocation (in this case + * code 1027 which corresponds to R_AARCH64_RELATIVE). + * r_addend is constant part of expression. + * + * Size of Elf64_Rela structure is 24 bytes. + */ + + /* Skip R_AARCH64_NONE entry with code 0 */ +1: ldr x3, [x1, #8] + cbz x3, 2f + +#if ENABLE_ASSERTIONS + /* Assert that the relocation type is R_AARCH64_RELATIVE */ + cmp x3, #R_AARCH64_RELATIVE + ASM_ASSERT(eq) +#endif + ldr x3, [x1] /* r_offset */ + add x3, x0, x3 + ldr x4, [x1, #16] /* r_addend */ + + /* Skip adding offset if r_addend is < lower limit */ + cmp x4, x6 + b.lo 2f + + /* Skip adding offset if r_addend entry is > upper limit */ + cmp x4, x7 + b.hi 2f + + add x4, x0, x4 /* Diff(S) + r_addend */ + str x4, [x3] + +2: add x1, x1, #24 + cmp x1, x2 + b.lo 1b + ret +endfunc fixup_gdt_reloc + +/* + * TODO: Currently only supports size of 4KB, + * support other sizes as well. + */ +func gpt_tlbi_by_pa_ll +#if ENABLE_ASSERTIONS + cmp x1, #PAGE_SIZE_4KB + ASM_ASSERT(eq) + tst x0, #(PAGE_SIZE_MASK) + ASM_ASSERT(eq) +#endif + lsr x0, x0, #FOUR_KB_SHIFT /* 4KB size encoding is zero */ + sys #6, c8, c4, #7, x0 /* TLBI RPALOS, <Xt> */ + dsb sy + ret +endfunc gpt_tlbi_by_pa_ll diff --git a/lib/bl_aux_params/bl_aux_params.c b/lib/bl_aux_params/bl_aux_params.c new file mode 100644 index 0000000..6e30ecf --- /dev/null +++ b/lib/bl_aux_params/bl_aux_params.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <inttypes.h> +#include <stdint.h> + +#include <common/debug.h> +#include <lib/coreboot.h> +#include <lib/bl_aux_params/bl_aux_params.h> + +void bl_aux_params_parse(u_register_t head, + bl_aux_param_handler_t handler) +{ + struct bl_aux_param_header *p; + + for (p = (void *)head; p; p = (void *)(uintptr_t)p->next) { + if (handler && handler(p)) + continue; + + switch (p->type) { +#if COREBOOT + case BL_AUX_PARAM_COREBOOT_TABLE: + coreboot_table_setup((void *)(uintptr_t) + ((struct bl_aux_param_uint64 *)p)->value); + break; +#endif + default: + ERROR("Ignoring unknown BL aux parameter: 0x%" PRIx64, + p->type); + break; + } + } +} diff --git a/lib/compiler-rt/LICENSE.TXT b/lib/compiler-rt/LICENSE.TXT new file mode 100644 index 0000000..a17dc12 --- /dev/null +++ b/lib/compiler-rt/LICENSE.TXT @@ -0,0 +1,91 @@ +============================================================================== +compiler_rt License +============================================================================== + +The compiler_rt library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2016 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +============================================================================== +Copyrights and Licenses for Third Party Software Distributed with LLVM: +============================================================================== +The LLVM software contains code written by third parties. Such software will +have its own individual LICENSE.TXT file in the directory in which it appears. +This file will describe the copyrights, license, and restrictions which apply +to that code. + +The disclaimer of warranty in the University of Illinois Open Source License +applies to all code in the LLVM Distribution, and nothing in any of the +other licenses gives permission to use the names of the LLVM Team or the +University of Illinois to endorse or promote products derived from this +Software. + diff --git a/lib/compiler-rt/builtins/arm/aeabi_ldivmod.S b/lib/compiler-rt/builtins/arm/aeabi_ldivmod.S new file mode 100644 index 0000000..d0d06be --- /dev/null +++ b/lib/compiler-rt/builtins/arm/aeabi_ldivmod.S @@ -0,0 +1,45 @@ +//===-- aeabi_ldivmod.S - EABI ldivmod implementation ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "../assembly.h" + +// struct { int64_t quot, int64_t rem} +// __aeabi_ldivmod(int64_t numerator, int64_t denominator) { +// int64_t rem, quot; +// quot = __divmoddi4(numerator, denominator, &rem); +// return {quot, rem}; +// } + +#if defined(__MINGW32__) +#define __aeabi_ldivmod __rt_sdiv64 +#endif + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_ldivmod) + push {r6, lr} + sub sp, sp, #16 + add r6, sp, #8 + str r6, [sp] +#if defined(__MINGW32__) + movs r6, r0 + movs r0, r2 + movs r2, r6 + movs r6, r1 + movs r1, r3 + movs r3, r6 +#endif + bl SYMBOL_NAME(__divmoddi4) + ldr r2, [sp, #8] + ldr r3, [sp, #12] + add sp, sp, #16 + pop {r6, pc} +END_COMPILERRT_FUNCTION(__aeabi_ldivmod) + +NO_EXEC_STACK_DIRECTIVE + diff --git a/lib/compiler-rt/builtins/arm/aeabi_memcpy.S b/lib/compiler-rt/builtins/arm/aeabi_memcpy.S new file mode 100644 index 0000000..93e1b05 --- /dev/null +++ b/lib/compiler-rt/builtins/arm/aeabi_memcpy.S @@ -0,0 +1,29 @@ +//===-- aeabi_memcpy.S - EABI memcpy implementation -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "../assembly.h" + +// void __aeabi_memcpy(void *dest, void *src, size_t n) { memcpy(dest, src, n); } + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_memcpy) +#ifdef USE_THUMB_1 + push {r7, lr} + bl memcpy + pop {r7, pc} +#else + b memcpy +#endif +END_COMPILERRT_FUNCTION(__aeabi_memcpy) + +DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memcpy4, __aeabi_memcpy) +DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memcpy8, __aeabi_memcpy) + +NO_EXEC_STACK_DIRECTIVE + diff --git a/lib/compiler-rt/builtins/arm/aeabi_memset.S b/lib/compiler-rt/builtins/arm/aeabi_memset.S new file mode 100644 index 0000000..2aa8ec0 --- /dev/null +++ b/lib/compiler-rt/builtins/arm/aeabi_memset.S @@ -0,0 +1,49 @@ +//===-- aeabi_memset.S - EABI memset implementation -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "../assembly.h" + +// void __aeabi_memset(void *dest, size_t n, int c) { memset(dest, c, n); } +// void __aeabi_memclr(void *dest, size_t n) { __aeabi_memset(dest, n, 0); } + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_memset) + mov r3, r1 + mov r1, r2 + mov r2, r3 +#ifdef USE_THUMB_1 + push {r7, lr} + bl memset + pop {r7, pc} +#else + b memset +#endif +END_COMPILERRT_FUNCTION(__aeabi_memset) + +DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset4, __aeabi_memset) +DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset8, __aeabi_memset) + + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_memclr) + mov r2, r1 + movs r1, #0 +#ifdef USE_THUMB_1 + push {r7, lr} + bl memset + pop {r7, pc} +#else + b memset +#endif +END_COMPILERRT_FUNCTION(__aeabi_memclr) + +DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memclr4, __aeabi_memclr) +DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memclr8, __aeabi_memclr) + +NO_EXEC_STACK_DIRECTIVE + diff --git a/lib/compiler-rt/builtins/arm/aeabi_uldivmod.S b/lib/compiler-rt/builtins/arm/aeabi_uldivmod.S new file mode 100644 index 0000000..4fc9770 --- /dev/null +++ b/lib/compiler-rt/builtins/arm/aeabi_uldivmod.S @@ -0,0 +1,45 @@ +//===-- aeabi_uldivmod.S - EABI uldivmod implementation -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "../assembly.h" + +// struct { uint64_t quot, uint64_t rem} +// __aeabi_uldivmod(uint64_t numerator, uint64_t denominator) { +// uint64_t rem, quot; +// quot = __udivmoddi4(numerator, denominator, &rem); +// return {quot, rem}; +// } + +#if defined(__MINGW32__) +#define __aeabi_uldivmod __rt_udiv64 +#endif + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_uldivmod) + push {r6, lr} + sub sp, sp, #16 + add r6, sp, #8 + str r6, [sp] +#if defined(__MINGW32__) + movs r6, r0 + movs r0, r2 + movs r2, r6 + movs r6, r1 + movs r1, r3 + movs r3, r6 +#endif + bl SYMBOL_NAME(__udivmoddi4) + ldr r2, [sp, #8] + ldr r3, [sp, #12] + add sp, sp, #16 + pop {r6, pc} +END_COMPILERRT_FUNCTION(__aeabi_uldivmod) + +NO_EXEC_STACK_DIRECTIVE + diff --git a/lib/compiler-rt/builtins/assembly.h b/lib/compiler-rt/builtins/assembly.h new file mode 100644 index 0000000..169d496 --- /dev/null +++ b/lib/compiler-rt/builtins/assembly.h @@ -0,0 +1,292 @@ +//===-- assembly.h - compiler-rt assembler support macros -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines macros for use in compiler-rt assembler source. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// + +#ifndef COMPILERRT_ASSEMBLY_H +#define COMPILERRT_ASSEMBLY_H + +#if defined(__linux__) && defined(__CET__) +#if __has_include(<cet.h>) +#include <cet.h> +#endif +#endif + +#if defined(__APPLE__) && defined(__aarch64__) +#define SEPARATOR %% +#else +#define SEPARATOR ; +#endif + +#if defined(__APPLE__) +#define HIDDEN(name) .private_extern name +#define LOCAL_LABEL(name) L_##name +// tell linker it can break up file at label boundaries +#define FILE_LEVEL_DIRECTIVE .subsections_via_symbols +#define SYMBOL_IS_FUNC(name) +#define CONST_SECTION .const + +#define NO_EXEC_STACK_DIRECTIVE + +#elif defined(__ELF__) + +#define HIDDEN(name) .hidden name +#define LOCAL_LABEL(name) .L_##name +#define FILE_LEVEL_DIRECTIVE +#if defined(__arm__) || defined(__aarch64__) +#define SYMBOL_IS_FUNC(name) .type name,%function +#else +#define SYMBOL_IS_FUNC(name) .type name,@function +#endif +#define CONST_SECTION .section .rodata + +#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ + defined(__linux__) +#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits +#else +#define NO_EXEC_STACK_DIRECTIVE +#endif + +#else // !__APPLE__ && !__ELF__ + +#define HIDDEN(name) +#define LOCAL_LABEL(name) .L ## name +#define FILE_LEVEL_DIRECTIVE +#define SYMBOL_IS_FUNC(name) \ + .def name SEPARATOR \ + .scl 2 SEPARATOR \ + .type 32 SEPARATOR \ + .endef +#define CONST_SECTION .section .rdata,"rd" + +#define NO_EXEC_STACK_DIRECTIVE + +#endif + +#if defined(__arm__) || defined(__aarch64__) +#define FUNC_ALIGN \ + .text SEPARATOR \ + .balign 16 SEPARATOR +#else +#define FUNC_ALIGN +#endif + +// BTI and PAC gnu property note +#define NT_GNU_PROPERTY_TYPE_0 5 +#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 +#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1 +#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) +#define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI +#else +#define BTI_FLAG 0 +#endif + +#if __ARM_FEATURE_PAC_DEFAULT & 3 +#define PAC_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_PAC +#else +#define PAC_FLAG 0 +#endif + +#define GNU_PROPERTY(type, value) \ + .pushsection .note.gnu.property, "a" SEPARATOR \ + .p2align 3 SEPARATOR \ + .word 4 SEPARATOR \ + .word 16 SEPARATOR \ + .word NT_GNU_PROPERTY_TYPE_0 SEPARATOR \ + .asciz "GNU" SEPARATOR \ + .word type SEPARATOR \ + .word 4 SEPARATOR \ + .word value SEPARATOR \ + .word 0 SEPARATOR \ + .popsection + +#if BTI_FLAG != 0 +#define BTI_C hint #34 +#define BTI_J hint #36 +#else +#define BTI_C +#define BTI_J +#endif + +#if (BTI_FLAG | PAC_FLAG) != 0 +#define GNU_PROPERTY_BTI_PAC \ + GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG | PAC_FLAG) +#else +#define GNU_PROPERTY_BTI_PAC +#endif + +#if defined(__clang__) || defined(__GCC_HAVE_DWARF2_CFI_ASM) +#define CFI_START .cfi_startproc +#define CFI_END .cfi_endproc +#else +#define CFI_START +#define CFI_END +#endif + +#if defined(__arm__) + +// Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: +// - for '-mthumb -march=armv6' compiler defines '__thumb__' +// - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' +#if defined(__thumb2__) || defined(__thumb__) +#define DEFINE_CODE_STATE .thumb SEPARATOR +#define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR +#if defined(__thumb2__) +#define USE_THUMB_2 +#define IT(cond) it cond +#define ITT(cond) itt cond +#define ITE(cond) ite cond +#else +#define USE_THUMB_1 +#define IT(cond) +#define ITT(cond) +#define ITE(cond) +#endif // defined(__thumb__2) +#else // !defined(__thumb2__) && !defined(__thumb__) +#define DEFINE_CODE_STATE .arm SEPARATOR +#define DECLARE_FUNC_ENCODING +#define IT(cond) +#define ITT(cond) +#define ITE(cond) +#endif + +#if defined(USE_THUMB_1) && defined(USE_THUMB_2) +#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together." +#endif + +#if defined(__ARM_ARCH_4T__) || __ARM_ARCH >= 5 +#define ARM_HAS_BX +#endif +#if !defined(__ARM_FEATURE_CLZ) && !defined(USE_THUMB_1) && \ + (__ARM_ARCH >= 6 || (__ARM_ARCH == 5 && !defined(__ARM_ARCH_5__))) +#define __ARM_FEATURE_CLZ +#endif + +#ifdef ARM_HAS_BX +#define JMP(r) bx r +#define JMPc(r, c) bx##c r +#else +#define JMP(r) mov pc, r +#define JMPc(r, c) mov##c pc, r +#endif + +// pop {pc} can't switch Thumb mode on ARMv4T +#if __ARM_ARCH >= 5 +#define POP_PC() pop {pc} +#else +#define POP_PC() \ + pop {ip}; \ + JMP(ip) +#endif + +#if defined(USE_THUMB_2) +#define WIDE(op) op.w +#else +#define WIDE(op) op +#endif +#else // !defined(__arm) +#define DECLARE_FUNC_ENCODING +#define DEFINE_CODE_STATE +#endif + +#define GLUE2_(a, b) a##b +#define GLUE(a, b) GLUE2_(a, b) +#define GLUE2(a, b) GLUE2_(a, b) +#define GLUE3_(a, b, c) a##b##c +#define GLUE3(a, b, c) GLUE3_(a, b, c) +#define GLUE4_(a, b, c, d) a##b##c##d +#define GLUE4(a, b, c, d) GLUE4_(a, b, c, d) + +#define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name) + +#ifdef VISIBILITY_HIDDEN +#define DECLARE_SYMBOL_VISIBILITY(name) \ + HIDDEN(SYMBOL_NAME(name)) SEPARATOR +#define DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) \ + HIDDEN(name) SEPARATOR +#else +#define DECLARE_SYMBOL_VISIBILITY(name) +#define DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) +#endif + +#define DEFINE_COMPILERRT_FUNCTION(name) \ + DEFINE_CODE_STATE \ + FILE_LEVEL_DIRECTIVE SEPARATOR \ + .globl SYMBOL_NAME(name) SEPARATOR \ + SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_SYMBOL_VISIBILITY(name) \ + DECLARE_FUNC_ENCODING \ + SYMBOL_NAME(name): + +#define DEFINE_COMPILERRT_THUMB_FUNCTION(name) \ + DEFINE_CODE_STATE \ + FILE_LEVEL_DIRECTIVE SEPARATOR \ + .globl SYMBOL_NAME(name) SEPARATOR \ + SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \ + .thumb_func SEPARATOR \ + SYMBOL_NAME(name): + +#define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \ + DEFINE_CODE_STATE \ + FILE_LEVEL_DIRECTIVE SEPARATOR \ + .globl SYMBOL_NAME(name) SEPARATOR \ + SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ + HIDDEN(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_FUNC_ENCODING \ + SYMBOL_NAME(name): + +#define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \ + DEFINE_CODE_STATE \ + .globl name SEPARATOR \ + SYMBOL_IS_FUNC(name) SEPARATOR \ + HIDDEN(name) SEPARATOR \ + DECLARE_FUNC_ENCODING \ + name: + +#define DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(name) \ + DEFINE_CODE_STATE \ + FUNC_ALIGN \ + .globl name SEPARATOR \ + SYMBOL_IS_FUNC(name) SEPARATOR \ + DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) SEPARATOR \ + CFI_START SEPARATOR \ + DECLARE_FUNC_ENCODING \ + name: SEPARATOR BTI_C + +#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \ + .globl SYMBOL_NAME(name) SEPARATOR \ + SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \ + .set SYMBOL_NAME(name), SYMBOL_NAME(target) SEPARATOR + +#if defined(__ARM_EABI__) +#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name) \ + DEFINE_COMPILERRT_FUNCTION_ALIAS(aeabi_name, name) +#else +#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name) +#endif + +#ifdef __ELF__ +#define END_COMPILERRT_FUNCTION(name) \ + .size SYMBOL_NAME(name), . - SYMBOL_NAME(name) +#define END_COMPILERRT_OUTLINE_FUNCTION(name) \ + CFI_END SEPARATOR \ + .size SYMBOL_NAME(name), . - SYMBOL_NAME(name) +#else +#define END_COMPILERRT_FUNCTION(name) +#define END_COMPILERRT_OUTLINE_FUNCTION(name) \ + CFI_END +#endif + +#endif // COMPILERRT_ASSEMBLY_H diff --git a/lib/compiler-rt/builtins/ctzdi2.c b/lib/compiler-rt/builtins/ctzdi2.c new file mode 100644 index 0000000..26c908d --- /dev/null +++ b/lib/compiler-rt/builtins/ctzdi2.c @@ -0,0 +1,35 @@ +//===-- ctzdi2.c - Implement __ctzdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ctzdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// + +#include "int_lib.h" + +// Returns: the number of trailing 0-bits + +#if !defined(__clang__) && \ + ((defined(__sparc__) && defined(__arch64__)) || defined(__mips64) || \ + (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) +// On 64-bit architectures with neither a native clz instruction nor a native +// ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than +// __ctzsi2, leading to infinite recursion. +#define __builtin_ctz(a) __ctzsi2(a) +extern int __ctzsi2(si_int); +#endif + +// Precondition: a != 0 + +COMPILER_RT_ABI int __ctzdi2(di_int a) { + dwords x; + x.all = a; + const si_int f = -(x.s.low == 0); + return ctzsi((x.s.high & f) | (x.s.low & ~f)) + + (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); +} diff --git a/lib/compiler-rt/builtins/divdi3.c b/lib/compiler-rt/builtins/divdi3.c new file mode 100644 index 0000000..d71e138 --- /dev/null +++ b/lib/compiler-rt/builtins/divdi3.c @@ -0,0 +1,22 @@ +//===-- divdi3.c - Implement __divdi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// + +#include "int_lib.h" + +// Returns: a / b + +#define fixint_t di_int +#define fixuint_t du_int +#define COMPUTE_UDIV(a, b) __udivmoddi4((a), (b), (du_int *)0) +#include "int_div_impl.inc" + +COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b) { return __divXi3(a, b); } diff --git a/lib/compiler-rt/builtins/divmoddi4.c b/lib/compiler-rt/builtins/divmoddi4.c new file mode 100644 index 0000000..64bbb69 --- /dev/null +++ b/lib/compiler-rt/builtins/divmoddi4.c @@ -0,0 +1,28 @@ +//===-- divmoddi4.c - Implement __divmoddi4 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divmoddi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// + +#include "int_lib.h" + +// Returns: a / b, *rem = a % b + +COMPILER_RT_ABI di_int __divmoddi4(di_int a, di_int b, di_int *rem) { + const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; + di_int s_a = a >> bits_in_dword_m1; // s_a = a < 0 ? -1 : 0 + di_int s_b = b >> bits_in_dword_m1; // s_b = b < 0 ? -1 : 0 + a = (du_int)(a ^ s_a) - s_a; // negate if s_a == -1 + b = (du_int)(b ^ s_b) - s_b; // negate if s_b == -1 + s_b ^= s_a; // sign of quotient + du_int r; + di_int q = (__udivmoddi4(a, b, &r) ^ s_b) - s_b; // negate if s_b == -1 + *rem = (r ^ s_a) - s_a; // negate if s_a == -1 + return q; +} diff --git a/lib/compiler-rt/builtins/int_div_impl.inc b/lib/compiler-rt/builtins/int_div_impl.inc new file mode 100644 index 0000000..dc1f97c --- /dev/null +++ b/lib/compiler-rt/builtins/int_div_impl.inc @@ -0,0 +1,95 @@ +//===-- int_div_impl.inc - Integer division ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Helpers used by __udivsi3, __umodsi3, __udivdi3, and __umodsi3. +// +//===----------------------------------------------------------------------===// + +#define clz(a) (sizeof(a) == sizeof(unsigned long long) ? __builtin_clzll(a) : clzsi(a)) + +// Adapted from Figure 3-40 of The PowerPC Compiler Writer's Guide +static __inline fixuint_t __udivXi3(fixuint_t n, fixuint_t d) { + const unsigned N = sizeof(fixuint_t) * CHAR_BIT; + // d == 0 cases are unspecified. + unsigned sr = (d ? clz(d) : N) - (n ? clz(n) : N); + // 0 <= sr <= N - 1 or sr is very large. + if (sr > N - 1) // n < d + return 0; + if (sr == N - 1) // d == 1 + return n; + ++sr; + // 1 <= sr <= N - 1. Shifts do not trigger UB. + fixuint_t r = n >> sr; + n <<= N - sr; + fixuint_t carry = 0; + for (; sr > 0; --sr) { + r = (r << 1) | (n >> (N - 1)); + n = (n << 1) | carry; + // Branch-less version of: + // carry = 0; + // if (r >= d) r -= d, carry = 1; + const fixint_t s = (fixint_t)(d - r - 1) >> (N - 1); + carry = s & 1; + r -= d & s; + } + n = (n << 1) | carry; + return n; +} + +// Mostly identical to __udivXi3 but the return values are different. +static __inline fixuint_t __umodXi3(fixuint_t n, fixuint_t d) { + const unsigned N = sizeof(fixuint_t) * CHAR_BIT; + // d == 0 cases are unspecified. + unsigned sr = (d ? clz(d) : N) - (n ? clz(n) : N); + // 0 <= sr <= N - 1 or sr is very large. + if (sr > N - 1) // n < d + return n; + if (sr == N - 1) // d == 1 + return 0; + ++sr; + // 1 <= sr <= N - 1. Shifts do not trigger UB. + fixuint_t r = n >> sr; + n <<= N - sr; + fixuint_t carry = 0; + for (; sr > 0; --sr) { + r = (r << 1) | (n >> (N - 1)); + n = (n << 1) | carry; + // Branch-less version of: + // carry = 0; + // if (r >= d) r -= d, carry = 1; + const fixint_t s = (fixint_t)(d - r - 1) >> (N - 1); + carry = s & 1; + r -= d & s; + } + return r; +} + +#ifdef COMPUTE_UDIV +static __inline fixint_t __divXi3(fixint_t a, fixint_t b) { + const int N = (int)(sizeof(fixint_t) * CHAR_BIT) - 1; + fixint_t s_a = a >> N; // s_a = a < 0 ? -1 : 0 + fixint_t s_b = b >> N; // s_b = b < 0 ? -1 : 0 + fixuint_t a_u = (fixuint_t)(a ^ s_a) + (-s_a); // negate if s_a == -1 + fixuint_t b_u = (fixuint_t)(b ^ s_b) + (-s_b); // negate if s_b == -1 + s_a ^= s_b; // sign of quotient + return (COMPUTE_UDIV(a_u, b_u) ^ s_a) + (-s_a); // negate if s_a == -1 +} +#endif // COMPUTE_UDIV + +#ifdef ASSIGN_UMOD +static __inline fixint_t __modXi3(fixint_t a, fixint_t b) { + const int N = (int)(sizeof(fixint_t) * CHAR_BIT) - 1; + fixint_t s = b >> N; // s = b < 0 ? -1 : 0 + fixuint_t b_u = (fixuint_t)(b ^ s) + (-s); // negate if s == -1 + s = a >> N; // s = a < 0 ? -1 : 0 + fixuint_t a_u = (fixuint_t)(a ^ s) + (-s); // negate if s == -1 + fixuint_t res; + ASSIGN_UMOD(res, a_u, b_u); + return (res ^ s) + (-s); // negate if s == -1 +} +#endif // ASSIGN_UMOD diff --git a/lib/compiler-rt/builtins/int_endianness.h b/lib/compiler-rt/builtins/int_endianness.h new file mode 100644 index 0000000..291c6b5 --- /dev/null +++ b/lib/compiler-rt/builtins/int_endianness.h @@ -0,0 +1,114 @@ +//===-- int_endianness.h - configuration header for compiler-rt -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a configuration header for compiler-rt. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// + +#ifndef INT_ENDIANNESS_H +#define INT_ENDIANNESS_H + +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ + defined(__ORDER_LITTLE_ENDIAN__) + +// Clang and GCC provide built-in endianness definitions. +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define _YUGA_LITTLE_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 1 +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define _YUGA_LITTLE_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 0 +#endif // __BYTE_ORDER__ + +#else // Compilers other than Clang or GCC. + +#if defined(__SVR4) && defined(__sun) +#include <sys/byteorder.h> + +#if defined(_BIG_ENDIAN) +#define _YUGA_LITTLE_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 1 +#elif defined(_LITTLE_ENDIAN) +#define _YUGA_LITTLE_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 0 +#else // !_LITTLE_ENDIAN +#error "unknown endianness" +#endif // !_LITTLE_ENDIAN + +#endif // Solaris + +// .. + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || \ + defined(__minix) +#include <sys/endian.h> + +#if _BYTE_ORDER == _BIG_ENDIAN +#define _YUGA_LITTLE_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 1 +#elif _BYTE_ORDER == _LITTLE_ENDIAN +#define _YUGA_LITTLE_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 0 +#endif // _BYTE_ORDER + +#endif // *BSD + +#if defined(__OpenBSD__) +#include <machine/endian.h> + +#if _BYTE_ORDER == _BIG_ENDIAN +#define _YUGA_LITTLE_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 1 +#elif _BYTE_ORDER == _LITTLE_ENDIAN +#define _YUGA_LITTLE_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 0 +#endif // _BYTE_ORDER + +#endif // OpenBSD + +// .. + +// Mac OSX has __BIG_ENDIAN__ or __LITTLE_ENDIAN__ automatically set by the +// compiler (at least with GCC) +#if defined(__APPLE__) || defined(__ellcc__) + +#ifdef __BIG_ENDIAN__ +#if __BIG_ENDIAN__ +#define _YUGA_LITTLE_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 1 +#endif +#endif // __BIG_ENDIAN__ + +#ifdef __LITTLE_ENDIAN__ +#if __LITTLE_ENDIAN__ +#define _YUGA_LITTLE_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 0 +#endif +#endif // __LITTLE_ENDIAN__ + +#endif // Mac OSX + +// .. + +#if defined(_WIN32) + +#define _YUGA_LITTLE_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 0 + +#endif // Windows + +#endif // Clang or GCC. + +// . + +#if !defined(_YUGA_LITTLE_ENDIAN) || !defined(_YUGA_BIG_ENDIAN) +#error Unable to determine endian +#endif // Check we found an endianness correctly. + +#endif // INT_ENDIANNESS_H diff --git a/lib/compiler-rt/builtins/int_lib.h b/lib/compiler-rt/builtins/int_lib.h new file mode 100644 index 0000000..04ea2d9 --- /dev/null +++ b/lib/compiler-rt/builtins/int_lib.h @@ -0,0 +1,171 @@ +//===-- int_lib.h - configuration header for compiler-rt -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a configuration header for compiler-rt. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// + +#ifndef INT_LIB_H +#define INT_LIB_H + +// Assumption: Signed integral is 2's complement. +// Assumption: Right shift of signed negative is arithmetic shift. +// Assumption: Endianness is little or big (not mixed). + +// ABI macro definitions + +#if __ARM_EABI__ +#ifdef COMPILER_RT_ARMHF_TARGET +#define COMPILER_RT_ABI +#else +#define COMPILER_RT_ABI __attribute__((__pcs__("aapcs"))) +#endif +#else +#define COMPILER_RT_ABI +#endif + +#define AEABI_RTABI __attribute__((__pcs__("aapcs"))) + +#if defined(_MSC_VER) && !defined(__clang__) +#define ALWAYS_INLINE __forceinline +#define NOINLINE __declspec(noinline) +#define NORETURN __declspec(noreturn) +#define UNUSED +#else +#define ALWAYS_INLINE __attribute__((always_inline)) +#define NOINLINE __attribute__((noinline)) +#define NORETURN __attribute__((noreturn)) +#define UNUSED __attribute__((unused)) +#endif + +#define STR(a) #a +#define XSTR(a) STR(a) +#define SYMBOL_NAME(name) XSTR(__USER_LABEL_PREFIX__) #name + +#if defined(__ELF__) || defined(__MINGW32__) || defined(__wasm__) || \ + defined(_AIX) || defined(__CYGWIN__) +#define COMPILER_RT_ALIAS(name, aliasname) \ + COMPILER_RT_ABI __typeof(name) aliasname __attribute__((__alias__(#name))); +#elif defined(__APPLE__) +#if defined(VISIBILITY_HIDDEN) +#define COMPILER_RT_ALIAS_VISIBILITY(name) \ + __asm__(".private_extern " SYMBOL_NAME(name)); +#else +#define COMPILER_RT_ALIAS_VISIBILITY(name) +#endif +#define COMPILER_RT_ALIAS(name, aliasname) \ + __asm__(".globl " SYMBOL_NAME(aliasname)); \ + COMPILER_RT_ALIAS_VISIBILITY(aliasname) \ + __asm__(SYMBOL_NAME(aliasname) " = " SYMBOL_NAME(name)); \ + COMPILER_RT_ABI __typeof(name) aliasname; +#elif defined(_WIN32) +#define COMPILER_RT_ALIAS(name, aliasname) +#else +#error Unsupported target +#endif + +#if (defined(__FreeBSD__) || defined(__NetBSD__)) && \ + (defined(_KERNEL) || defined(_STANDALONE)) +// +// Kernel and boot environment can't use normal headers, +// so use the equivalent system headers. +// NB: FreeBSD (and OpenBSD) deprecate machine/limits.h in +// favour of sys/limits.h, so prefer the former, but fall +// back on the latter if not available since NetBSD only has +// the latter. +// +#if defined(__has_include) && __has_include(<sys/limits.h>) +#include <sys/limits.h> +#else +#include <machine/limits.h> +#endif +#include <sys/stdint.h> +#include <sys/types.h> +#else +// Include the standard compiler builtin headers we use functionality from. +#include <float.h> +#include <limits.h> +#include <stdbool.h> +#include <stdint.h> +#endif + +// Include the commonly used internal type definitions. +#include "int_types.h" + +// Include internal utility function declarations. +#include "int_util.h" + +COMPILER_RT_ABI int __paritysi2(si_int a); +COMPILER_RT_ABI int __paritydi2(di_int a); + +COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b); +COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b); +COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d); + +COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int *rem); +COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem); +#ifdef CRT_HAS_128BIT +COMPILER_RT_ABI int __clzti2(ti_int a); +COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem); +#endif + +// Definitions for builtins unavailable on MSVC +#if defined(_MSC_VER) && !defined(__clang__) +#include <intrin.h> + +int __inline __builtin_ctz(uint32_t value) { + unsigned long trailing_zero = 0; + if (_BitScanForward(&trailing_zero, value)) + return trailing_zero; + return 32; +} + +int __inline __builtin_clz(uint32_t value) { + unsigned long leading_zero = 0; + if (_BitScanReverse(&leading_zero, value)) + return 31 - leading_zero; + return 32; +} + +#if defined(_M_ARM) || defined(_M_X64) +int __inline __builtin_clzll(uint64_t value) { + unsigned long leading_zero = 0; + if (_BitScanReverse64(&leading_zero, value)) + return 63 - leading_zero; + return 64; +} +#else +int __inline __builtin_clzll(uint64_t value) { + if (value == 0) + return 64; + uint32_t msh = (uint32_t)(value >> 32); + uint32_t lsh = (uint32_t)(value & 0xFFFFFFFF); + if (msh != 0) + return __builtin_clz(msh); + return 32 + __builtin_clz(lsh); +} +#endif + +#define __builtin_clzl __builtin_clzll + +bool __inline __builtin_sadd_overflow(int x, int y, int *result) { + if ((x < 0) != (y < 0)) { + *result = x + y; + return false; + } + int tmp = (unsigned int)x + (unsigned int)y; + if ((tmp < 0) != (x < 0)) + return true; + *result = tmp; + return false; +} + +#endif // defined(_MSC_VER) && !defined(__clang__) + +#endif // INT_LIB_H diff --git a/lib/compiler-rt/builtins/int_math.h b/lib/compiler-rt/builtins/int_math.h new file mode 100644 index 0000000..74d3e31 --- /dev/null +++ b/lib/compiler-rt/builtins/int_math.h @@ -0,0 +1,108 @@ +//===-- int_math.h - internal math inlines --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines substitutes for the libm functions used in some of the +// compiler-rt implementations, defined in such a way that there is not a direct +// dependency on libm or math.h. Instead, we use the compiler builtin versions +// where available. This reduces our dependencies on the system SDK by foisting +// the responsibility onto the compiler. +// +//===----------------------------------------------------------------------===// + +#ifndef INT_MATH_H +#define INT_MATH_H + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#include <math.h> +#include <stdlib.h> +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define CRT_INFINITY INFINITY +#else +#define CRT_INFINITY __builtin_huge_valf() +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_isfinite(x) _finite((x)) +#define crt_isinf(x) !_finite((x)) +#define crt_isnan(x) _isnan((x)) +#else +// Define crt_isfinite in terms of the builtin if available, otherwise provide +// an alternate version in terms of our other functions. This supports some +// versions of GCC which didn't have __builtin_isfinite. +#if __has_builtin(__builtin_isfinite) +#define crt_isfinite(x) __builtin_isfinite((x)) +#elif defined(__GNUC__) +#define crt_isfinite(x) \ + __extension__(({ \ + __typeof((x)) x_ = (x); \ + !crt_isinf(x_) && !crt_isnan(x_); \ + })) +#else +#error "Do not know how to check for infinity" +#endif // __has_builtin(__builtin_isfinite) +#define crt_isinf(x) __builtin_isinf((x)) +#define crt_isnan(x) __builtin_isnan((x)) +#endif // _MSC_VER + +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_copysign(x, y) copysign((x), (y)) +#define crt_copysignf(x, y) copysignf((x), (y)) +#define crt_copysignl(x, y) copysignl((x), (y)) +#else +#define crt_copysign(x, y) __builtin_copysign((x), (y)) +#define crt_copysignf(x, y) __builtin_copysignf((x), (y)) +#define crt_copysignl(x, y) __builtin_copysignl((x), (y)) +#if __has_builtin(__builtin_copysignf128) +#define crt_copysignf128(x, y) __builtin_copysignf128((x), (y)) +#elif __has_builtin(__builtin_copysignq) || (defined(__GNUC__) && __GNUC__ >= 7) +#define crt_copysignf128(x, y) __builtin_copysignq((x), (y)) +#endif +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_fabs(x) fabs((x)) +#define crt_fabsf(x) fabsf((x)) +#define crt_fabsl(x) fabs((x)) +#else +#define crt_fabs(x) __builtin_fabs((x)) +#define crt_fabsf(x) __builtin_fabsf((x)) +#define crt_fabsl(x) __builtin_fabsl((x)) +#if __has_builtin(__builtin_fabsf128) +#define crt_fabsf128(x) __builtin_fabsf128((x)) +#elif __has_builtin(__builtin_fabsq) || (defined(__GNUC__) && __GNUC__ >= 7) +#define crt_fabsf128(x) __builtin_fabsq((x)) +#endif +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_fmaxl(x, y) __max((x), (y)) +#else +#define crt_fmaxl(x, y) __builtin_fmaxl((x), (y)) +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_logbl(x) logbl((x)) +#else +#define crt_logbl(x) __builtin_logbl((x)) +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_scalbnl(x, y) scalbnl((x), (y)) +#else +#define crt_scalbnl(x, y) __builtin_scalbnl((x), (y)) +#endif + +#endif // INT_MATH_H diff --git a/lib/compiler-rt/builtins/int_types.h b/lib/compiler-rt/builtins/int_types.h new file mode 100644 index 0000000..18bf0a7 --- /dev/null +++ b/lib/compiler-rt/builtins/int_types.h @@ -0,0 +1,274 @@ +//===-- int_lib.h - configuration header for compiler-rt -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines various standard types, most importantly a number of unions +// used to access parts of larger types. +// +//===----------------------------------------------------------------------===// + +#ifndef INT_TYPES_H +#define INT_TYPES_H + +#include "int_endianness.h" + +// si_int is defined in Linux sysroot's asm-generic/siginfo.h +#ifdef si_int +#undef si_int +#endif +typedef int32_t si_int; +typedef uint32_t su_int; +#if UINT_MAX == 0xFFFFFFFF +#define clzsi __builtin_clz +#define ctzsi __builtin_ctz +#elif ULONG_MAX == 0xFFFFFFFF +#define clzsi __builtin_clzl +#define ctzsi __builtin_ctzl +#else +#error could not determine appropriate clzsi macro for this system +#endif + +typedef int64_t di_int; +typedef uint64_t du_int; + +typedef union { + di_int all; + struct { +#if _YUGA_LITTLE_ENDIAN + su_int low; + si_int high; +#else + si_int high; + su_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; +} dwords; + +typedef union { + du_int all; + struct { +#if _YUGA_LITTLE_ENDIAN + su_int low; + su_int high; +#else + su_int high; + su_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; +} udwords; + +#if defined(__LP64__) || defined(__wasm__) || defined(__mips64) || \ + defined(__SIZEOF_INT128__) || defined(_WIN64) +#define CRT_HAS_128BIT +#endif + +// MSVC doesn't have a working 128bit integer type. Users should really compile +// compiler-rt with clang, but if they happen to be doing a standalone build for +// asan or something else, disable the 128 bit parts so things sort of work. +#if defined(_MSC_VER) && !defined(__clang__) +#undef CRT_HAS_128BIT +#endif + +#ifdef CRT_HAS_128BIT +typedef int ti_int __attribute__((mode(TI))); +typedef unsigned tu_int __attribute__((mode(TI))); + +typedef union { + ti_int all; + struct { +#if _YUGA_LITTLE_ENDIAN + du_int low; + di_int high; +#else + di_int high; + du_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; +} twords; + +typedef union { + tu_int all; + struct { +#if _YUGA_LITTLE_ENDIAN + du_int low; + du_int high; +#else + du_int high; + du_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; +} utwords; + +static __inline ti_int make_ti(di_int h, di_int l) { + twords r; + r.s.high = h; + r.s.low = l; + return r.all; +} + +static __inline tu_int make_tu(du_int h, du_int l) { + utwords r; + r.s.high = h; + r.s.low = l; + return r.all; +} + +#endif // CRT_HAS_128BIT + +// FreeBSD's boot environment does not support using floating-point and poisons +// the float and double keywords. +#if defined(__FreeBSD__) && defined(_STANDALONE) +#define CRT_HAS_FLOATING_POINT 0 +#else +#define CRT_HAS_FLOATING_POINT 1 +#endif + +#if CRT_HAS_FLOATING_POINT +typedef union { + su_int u; + float f; +} float_bits; + +typedef union { + udwords u; + double f; +} double_bits; +#endif + +typedef struct { +#if _YUGA_LITTLE_ENDIAN + udwords low; + udwords high; +#else + udwords high; + udwords low; +#endif // _YUGA_LITTLE_ENDIAN +} uqwords; + +// Check if the target supports 80 bit extended precision long doubles. +// Notably, on x86 Windows, MSVC only provides a 64-bit long double, but GCC +// still makes it 80 bits. Clang will match whatever compiler it is trying to +// be compatible with. On 32-bit x86 Android, long double is 64 bits, while on +// x86_64 Android, long double is 128 bits. +#if (defined(__i386__) || defined(__x86_64__)) && \ + !(defined(_MSC_VER) || defined(__ANDROID__)) +#define HAS_80_BIT_LONG_DOUBLE 1 +#elif defined(__m68k__) || defined(__ia64__) +#define HAS_80_BIT_LONG_DOUBLE 1 +#else +#define HAS_80_BIT_LONG_DOUBLE 0 +#endif + +#if HAS_80_BIT_LONG_DOUBLE +typedef long double xf_float; +typedef union { + uqwords u; + xf_float f; +} xf_bits; +#endif + +#ifdef __powerpc64__ +// From https://gcc.gnu.org/wiki/Ieee128PowerPC: +// PowerPC64 uses the following suffixes: +// IFmode: IBM extended double +// KFmode: IEEE 128-bit floating point +// TFmode: Matches the default for long double. With -mabi=ieeelongdouble, +// it is IEEE 128-bit, with -mabi=ibmlongdouble IBM extended double +// Since compiler-rt only implements the tf set of libcalls, we use long double +// for the tf_float typedef. +typedef long double tf_float; +#define CRT_LDBL_128BIT +#define CRT_HAS_F128 +#if __LDBL_MANT_DIG__ == 113 && !defined(__LONG_DOUBLE_IBM128__) +#define CRT_HAS_IEEE_TF +#define CRT_LDBL_IEEE_F128 +#endif +#define TF_C(x) x##L +#elif __LDBL_MANT_DIG__ == 113 +// Use long double instead of __float128 if it matches the IEEE 128-bit format. +#define CRT_LDBL_128BIT +#define CRT_HAS_F128 +#define CRT_HAS_IEEE_TF +#define CRT_LDBL_IEEE_F128 +typedef long double tf_float; +#define TF_C(x) x##L +#elif defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__) +#define CRT_HAS___FLOAT128_KEYWORD +#define CRT_HAS_F128 +// NB: we assume the __float128 type uses IEEE representation. +#define CRT_HAS_IEEE_TF +typedef __float128 tf_float; +#define TF_C(x) x##Q +#endif + +#ifdef CRT_HAS_F128 +typedef union { + uqwords u; + tf_float f; +} tf_bits; +#endif + +// __(u)int128_t is currently needed to compile the *tf builtins as we would +// otherwise need to manually expand the bit manipulation on two 64-bit value. +#if defined(CRT_HAS_128BIT) && defined(CRT_HAS_F128) +#define CRT_HAS_TF_MODE +#endif + +#if CRT_HAS_FLOATING_POINT +#if __STDC_VERSION__ >= 199901L +typedef float _Complex Fcomplex; +typedef double _Complex Dcomplex; +typedef long double _Complex Lcomplex; +#if defined(CRT_LDBL_128BIT) +typedef Lcomplex Qcomplex; +#define CRT_HAS_NATIVE_COMPLEX_F128 +#elif defined(CRT_HAS___FLOAT128_KEYWORD) +#if defined(__clang_major__) && __clang_major__ > 10 +// Clang prior to 11 did not support __float128 _Complex. +typedef __float128 _Complex Qcomplex; +#define CRT_HAS_NATIVE_COMPLEX_F128 +#elif defined(__GNUC__) && __GNUC__ >= 7 +// GCC does not allow __float128 _Complex, but accepts _Float128 _Complex. +typedef _Float128 _Complex Qcomplex; +#define CRT_HAS_NATIVE_COMPLEX_F128 +#endif +#endif + +#define COMPLEX_REAL(x) __real__(x) +#define COMPLEX_IMAGINARY(x) __imag__(x) +#else +typedef struct { + float real, imaginary; +} Fcomplex; + +typedef struct { + double real, imaginary; +} Dcomplex; + +typedef struct { + long double real, imaginary; +} Lcomplex; + +#define COMPLEX_REAL(x) (x).real +#define COMPLEX_IMAGINARY(x) (x).imaginary +#endif + +#ifdef CRT_HAS_NATIVE_COMPLEX_F128 +#define COMPLEXTF_REAL(x) __real__(x) +#define COMPLEXTF_IMAGINARY(x) __imag__(x) +#elif defined(CRT_HAS_F128) +typedef struct { + tf_float real, imaginary; +} Qcomplex; +#define COMPLEXTF_REAL(x) (x).real +#define COMPLEXTF_IMAGINARY(x) (x).imaginary +#endif + +#endif +#endif // INT_TYPES_H diff --git a/lib/compiler-rt/builtins/int_util.h b/lib/compiler-rt/builtins/int_util.h new file mode 100644 index 0000000..c372c2e --- /dev/null +++ b/lib/compiler-rt/builtins/int_util.h @@ -0,0 +1,47 @@ +//===-- int_util.h - internal utility functions ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines non-inline utilities which are available for use in the +// library. The function definitions themselves are all contained in int_util.c +// which will always be compiled into any compiler-rt library. +// +//===----------------------------------------------------------------------===// + +#ifndef INT_UTIL_H +#define INT_UTIL_H + +/// \brief Trigger a program abort (or panic for kernel code). +#define compilerrt_abort() __compilerrt_abort_impl(__FILE__, __LINE__, __func__) + +NORETURN void __compilerrt_abort_impl(const char *file, int line, + const char *function); + +#define COMPILE_TIME_ASSERT(expr) COMPILE_TIME_ASSERT1(expr, __COUNTER__) +#define COMPILE_TIME_ASSERT1(expr, cnt) COMPILE_TIME_ASSERT2(expr, cnt) +#define COMPILE_TIME_ASSERT2(expr, cnt) \ + typedef char ct_assert_##cnt[(expr) ? 1 : -1] UNUSED + +// Force unrolling the code specified to be repeated N times. +#define REPEAT_0_TIMES(code_to_repeat) /* do nothing */ +#define REPEAT_1_TIMES(code_to_repeat) code_to_repeat +#define REPEAT_2_TIMES(code_to_repeat) \ + REPEAT_1_TIMES(code_to_repeat) \ + code_to_repeat +#define REPEAT_3_TIMES(code_to_repeat) \ + REPEAT_2_TIMES(code_to_repeat) \ + code_to_repeat +#define REPEAT_4_TIMES(code_to_repeat) \ + REPEAT_3_TIMES(code_to_repeat) \ + code_to_repeat + +#define REPEAT_N_TIMES_(N, code_to_repeat) REPEAT_##N##_TIMES(code_to_repeat) +#define REPEAT_N_TIMES(N, code_to_repeat) REPEAT_N_TIMES_(N, code_to_repeat) + +#endif // INT_UTIL_H diff --git a/lib/compiler-rt/builtins/lshrdi3.c b/lib/compiler-rt/builtins/lshrdi3.c new file mode 100644 index 0000000..6072152 --- /dev/null +++ b/lib/compiler-rt/builtins/lshrdi3.c @@ -0,0 +1,38 @@ +//===-- lshrdi3.c - Implement __lshrdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __lshrdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// + +#include "int_lib.h" + +// Returns: logical a >> b + +// Precondition: 0 <= b < bits_in_dword + +COMPILER_RT_ABI di_int __lshrdi3(di_int a, int b) { + const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); + udwords input; + udwords result; + input.all = a; + if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ { + result.s.high = 0; + result.s.low = input.s.high >> (b - bits_in_word); + } else /* 0 <= b < bits_in_word */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); + } + return result.all; +} + +#if defined(__ARM_EABI__) +COMPILER_RT_ALIAS(__lshrdi3, __aeabi_llsr) +#endif diff --git a/lib/compiler-rt/builtins/popcountdi2.c b/lib/compiler-rt/builtins/popcountdi2.c new file mode 100644 index 0000000..20dd0b0 --- /dev/null +++ b/lib/compiler-rt/builtins/popcountdi2.c @@ -0,0 +1,32 @@ +//===-- popcountdi2.c - Implement __popcountdi2 ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __popcountdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// + +#include "int_lib.h" + +// Returns: count of 1 bits + +COMPILER_RT_ABI int __popcountdi2(di_int a) { + du_int x2 = (du_int)a; + x2 = x2 - ((x2 >> 1) & 0x5555555555555555uLL); + // Every 2 bits holds the sum of every pair of bits (32) + x2 = ((x2 >> 2) & 0x3333333333333333uLL) + (x2 & 0x3333333333333333uLL); + // Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (16) + x2 = (x2 + (x2 >> 4)) & 0x0F0F0F0F0F0F0F0FuLL; + // Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (8) + su_int x = (su_int)(x2 + (x2 >> 32)); + // The lower 32 bits hold four 16 bit sums (5 significant bits). + // Upper 32 bits are garbage + x = x + (x >> 16); + // The lower 16 bits hold two 32 bit sums (6 significant bits). + // Upper 16 bits are garbage + return (x + (x >> 8)) & 0x0000007F; // (7 significant bits) +} diff --git a/lib/compiler-rt/builtins/popcountsi2.c b/lib/compiler-rt/builtins/popcountsi2.c new file mode 100644 index 0000000..4d346c4 --- /dev/null +++ b/lib/compiler-rt/builtins/popcountsi2.c @@ -0,0 +1,29 @@ +//===-- popcountsi2.c - Implement __popcountsi2 ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __popcountsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// + +#include "int_lib.h" + +// Returns: count of 1 bits + +COMPILER_RT_ABI int __popcountsi2(si_int a) { + su_int x = (su_int)a; + x = x - ((x >> 1) & 0x55555555); + // Every 2 bits holds the sum of every pair of bits + x = ((x >> 2) & 0x33333333) + (x & 0x33333333); + // Every 4 bits holds the sum of every 4-set of bits (3 significant bits) + x = (x + (x >> 4)) & 0x0F0F0F0F; + // Every 8 bits holds the sum of every 8-set of bits (4 significant bits) + x = (x + (x >> 16)); + // The lower 16 bits hold two 8 bit sums (5 significant bits). + // Upper 16 bits are garbage + return (x + (x >> 8)) & 0x0000003F; // (6 significant bits) +} diff --git a/lib/compiler-rt/builtins/udivmoddi4.c b/lib/compiler-rt/builtins/udivmoddi4.c new file mode 100644 index 0000000..123e5fb --- /dev/null +++ b/lib/compiler-rt/builtins/udivmoddi4.c @@ -0,0 +1,200 @@ +//===-- udivmoddi4.c - Implement __udivmoddi4 -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivmoddi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// + +#include "int_lib.h" + +// Effects: if rem != 0, *rem = a % b +// Returns: a / b + +// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide + +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC throws a warning about mod 0 here, disable it for builds that +// warn-as-error +#pragma warning(push) +#pragma warning(disable : 4723 4724) +#endif + +COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem) { + const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; + const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; + udwords n; + n.all = a; + udwords d; + d.all = b; + udwords q; + udwords r; + unsigned sr; + // special cases, X is unknown, K != 0 + if (n.s.high == 0) { + if (d.s.high == 0) { + // 0 X + // --- + // 0 X + if (rem) + *rem = n.s.low % d.s.low; + return n.s.low / d.s.low; + } + // 0 X + // --- + // K X + if (rem) + *rem = n.s.low; + return 0; + } + // n.s.high != 0 + if (d.s.low == 0) { + if (d.s.high == 0) { + // K X + // --- + // 0 0 + if (rem) + *rem = n.s.high % d.s.low; + return n.s.high / d.s.low; + } + // d.s.high != 0 + if (n.s.low == 0) { + // K 0 + // --- + // K 0 + if (rem) { + r.s.high = n.s.high % d.s.high; + r.s.low = 0; + *rem = r.all; + } + return n.s.high / d.s.high; + } + // K K + // --- + // K 0 + if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ { + if (rem) { + r.s.low = n.s.low; + r.s.high = n.s.high & (d.s.high - 1); + *rem = r.all; + } + return n.s.high >> ctzsi(d.s.high); + } + // K K + // --- + // K 0 + sr = clzsi(d.s.high) - clzsi(n.s.high); + // 0 <= sr <= n_uword_bits - 2 or sr large + if (sr > n_uword_bits - 2) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_uword_bits - 1 + // q.all = n.all << (n_udword_bits - sr); + q.s.low = 0; + q.s.high = n.s.low << (n_uword_bits - sr); + // r.all = n.all >> sr; + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } else /* d.s.low != 0 */ { + if (d.s.high == 0) { + // K X + // --- + // 0 K + if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ { + if (rem) + *rem = n.s.low & (d.s.low - 1); + if (d.s.low == 1) + return n.all; + sr = ctzsi(d.s.low); + q.s.high = n.s.high >> sr; + q.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + return q.all; + } + // K X + // --- + // 0 K + sr = 1 + n_uword_bits + clzsi(d.s.low) - clzsi(n.s.high); + // 2 <= sr <= n_udword_bits - 1 + // q.all = n.all << (n_udword_bits - sr); + // r.all = n.all >> sr; + if (sr == n_uword_bits) { + q.s.low = 0; + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else if (sr < n_uword_bits) /* 2 <= sr <= n_uword_bits - 1 */ { + q.s.low = 0; + q.s.high = n.s.low << (n_uword_bits - sr); + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } else /* n_uword_bits + 1 <= sr <= n_udword_bits - 1 */ { + q.s.low = n.s.low << (n_udword_bits - sr); + q.s.high = (n.s.high << (n_udword_bits - sr)) | + (n.s.low >> (sr - n_uword_bits)); + r.s.high = 0; + r.s.low = n.s.high >> (sr - n_uword_bits); + } + } else { + // K X + // --- + // K K + sr = clzsi(d.s.high) - clzsi(n.s.high); + // 0 <= sr <= n_uword_bits - 1 or sr large + if (sr > n_uword_bits - 1) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_uword_bits + // q.all = n.all << (n_udword_bits - sr); + q.s.low = 0; + if (sr == n_uword_bits) { + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else { + q.s.high = n.s.low << (n_uword_bits - sr); + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } + } + } + // Not a special case + // q and r are initialized with: + // q.all = n.all << (n_udword_bits - sr); + // r.all = n.all >> sr; + // 1 <= sr <= n_udword_bits - 1 + su_int carry = 0; + for (; sr > 0; --sr) { + // r:q = ((r:q) << 1) | carry + r.s.high = (r.s.high << 1) | (r.s.low >> (n_uword_bits - 1)); + r.s.low = (r.s.low << 1) | (q.s.high >> (n_uword_bits - 1)); + q.s.high = (q.s.high << 1) | (q.s.low >> (n_uword_bits - 1)); + q.s.low = (q.s.low << 1) | carry; + // carry = 0; + // if (r.all >= d.all) + // { + // r.all -= d.all; + // carry = 1; + // } + const di_int s = (di_int)(d.all - r.all - 1) >> (n_udword_bits - 1); + carry = s & 1; + r.all -= d.all & s; + } + q.all = (q.all << 1) | carry; + if (rem) + *rem = r.all; + return q.all; +} + +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(pop) +#endif diff --git a/lib/compiler-rt/compiler-rt.mk b/lib/compiler-rt/compiler-rt.mk new file mode 100644 index 0000000..2b16504 --- /dev/null +++ b/lib/compiler-rt/compiler-rt.mk @@ -0,0 +1,44 @@ +# +# Copyright (c) 2017-2019, Arm Limited and Contributors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# Neither the name of ARM nor the names of its contributors may be used +# to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +COMPILER_RT_SRCS := lib/compiler-rt/builtins/popcountdi2.c \ + lib/compiler-rt/builtins/popcountsi2.c + +ifeq (${ARCH},aarch32) +COMPILER_RT_SRCS += lib/compiler-rt/builtins/arm/aeabi_ldivmod.S \ + lib/compiler-rt/builtins/arm/aeabi_uldivmod.S \ + lib/compiler-rt/builtins/arm/aeabi_memcpy.S \ + lib/compiler-rt/builtins/arm/aeabi_memset.S \ + lib/compiler-rt/builtins/ctzdi2.c \ + lib/compiler-rt/builtins/divdi3.c \ + lib/compiler-rt/builtins/divmoddi4.c \ + lib/compiler-rt/builtins/lshrdi3.c \ + lib/compiler-rt/builtins/udivmoddi4.c +endif diff --git a/lib/coreboot/coreboot.mk b/lib/coreboot/coreboot.mk new file mode 100644 index 0000000..e07d28c --- /dev/null +++ b/lib/coreboot/coreboot.mk @@ -0,0 +1,22 @@ +# +# Copyright (c) 2017, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +COREBOOT := 0 +$(eval $(call assert_boolean,COREBOOT)) +$(eval $(call add_define,COREBOOT)) + +ifeq (${COREBOOT},1) + +ifneq (${ARCH},aarch64) +$(error "coreboot only supports Trusted Firmware on AArch64.") +endif + +BL31_SOURCES += $(addprefix lib/coreboot/, \ + coreboot_table.c) + +BL31_SOURCES += drivers/coreboot/cbmem_console/${ARCH}/cbmem_console.S + +endif # COREBOOT diff --git a/lib/coreboot/coreboot_table.c b/lib/coreboot/coreboot_table.c new file mode 100644 index 0000000..c954e97 --- /dev/null +++ b/lib/coreboot/coreboot_table.c @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2017-2020, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <string.h> + +#include <drivers/coreboot/cbmem_console.h> +#include <common/debug.h> +#include <lib/coreboot.h> +#include <lib/mmio.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +/* + * Structures describing coreboot's in-memory descriptor tables. See + * <coreboot>/src/commonlib/include/commonlib/coreboot_tables.h for + * canonical implementation. + */ + +typedef struct { + char signature[4]; + uint32_t header_bytes; + uint32_t header_checksum; + uint32_t table_bytes; + uint32_t table_checksum; + uint32_t table_entries; +} cb_header_t; + +typedef enum { + CB_TAG_MEMORY = 0x1, + CB_TAG_SERIAL = 0xf, + CB_TAG_CBMEM_CONSOLE = 0x17, +} cb_tag_t; + +typedef struct { + uint32_t tag; + uint32_t size; + union { + coreboot_memrange_t memranges[COREBOOT_MAX_MEMRANGES]; + coreboot_serial_t serial; + uint64_t uint64; + }; +} cb_entry_t; + +coreboot_memrange_t coreboot_memranges[COREBOOT_MAX_MEMRANGES]; +coreboot_serial_t coreboot_serial; +uint64_t coreboot_table_addr; +uint32_t coreboot_table_size; + +/* + * The coreboot table is parsed before the MMU is enabled (i.e. with strongly + * ordered memory), so we cannot make unaligned accesses. The table entries + * immediately follow one another without padding, so nothing after the header + * is guaranteed to be naturally aligned. Therefore, we need to define safety + * functions that can read unaligned integers. + */ +static uint32_t read_le32(uint32_t *p) +{ + uintptr_t addr = (uintptr_t)p; + return mmio_read_8(addr) | + mmio_read_8(addr + 1) << 8 | + mmio_read_8(addr + 2) << 16 | + mmio_read_8(addr + 3) << 24; +} +static uint64_t read_le64(uint64_t *p) +{ + return read_le32((void *)p) | (uint64_t)read_le32((void *)p + 4) << 32; +} + +static void expand_and_mmap(uintptr_t baseaddr, size_t size) +{ + uintptr_t pageaddr = round_down(baseaddr, PAGE_SIZE); + size_t expanded = round_up(baseaddr - pageaddr + size, PAGE_SIZE); + mmap_add_region(pageaddr, pageaddr, expanded, + MT_MEMORY | MT_RW | MT_NS | MT_EXECUTE_NEVER); +} + +static void setup_cbmem_console(uintptr_t baseaddr) +{ + static console_cbmc_t console; + assert(!console.console.base); /* should only have one CBMEM console */ + + /* CBMEM console structure stores its size in first header field. */ + uint32_t size = *(uint32_t *)baseaddr; + expand_and_mmap(baseaddr, size); + console_cbmc_register(baseaddr, &console); + console_set_scope(&console.console, CONSOLE_FLAG_BOOT | + CONSOLE_FLAG_RUNTIME | + CONSOLE_FLAG_CRASH); +} + +coreboot_memory_t coreboot_get_memory_type(uintptr_t start, size_t size) +{ + int i; + + for (i = 0; i < COREBOOT_MAX_MEMRANGES; i++) { + coreboot_memrange_t *range = &coreboot_memranges[i]; + + if (range->type == CB_MEM_NONE) + break; /* end of table reached */ + if ((start >= range->start) && + (start - range->start < range->size) && + (size <= range->size - (start - range->start))) { + return range->type; + } + } + + return CB_MEM_NONE; +} + +void coreboot_get_table_location(uint64_t *address, uint32_t *size) +{ + *address = coreboot_table_addr; + *size = coreboot_table_size; +} + +void coreboot_table_setup(void *base) +{ + cb_header_t *header = base; + void *ptr; + int i; + + if (strncmp(header->signature, "LBIO", 4)) { + ERROR("coreboot table signature corrupt!\n"); + return; + } + coreboot_table_addr = (uint64_t) base; + coreboot_table_size = header->header_bytes + header->table_bytes; + + ptr = base + header->header_bytes; + for (i = 0; i < header->table_entries; i++) { + size_t size; + cb_entry_t *entry = ptr; + + if (ptr - base >= header->header_bytes + header->table_bytes) { + ERROR("coreboot table exceeds its bounds!\n"); + break; + } + + switch (read_le32(&entry->tag)) { + case CB_TAG_MEMORY: + size = read_le32(&entry->size) - + offsetof(cb_entry_t, memranges); + if (size > sizeof(coreboot_memranges)) { + ERROR("Need to truncate coreboot memranges!\n"); + size = sizeof(coreboot_memranges); + } + memcpy(&coreboot_memranges, &entry->memranges, size); + break; + case CB_TAG_SERIAL: + memcpy(&coreboot_serial, &entry->serial, + sizeof(coreboot_serial)); + break; + case CB_TAG_CBMEM_CONSOLE: + setup_cbmem_console(read_le64(&entry->uint64)); + break; + default: + /* There are many tags TF doesn't need to care about. */ + break; + } + + ptr += read_le32(&entry->size); + } +} diff --git a/lib/cpus/aarch32/aem_generic.S b/lib/cpus/aarch32/aem_generic.S new file mode 100644 index 0000000..9f45e38 --- /dev/null +++ b/lib/cpus/aarch32/aem_generic.S @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016-2017, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <aem_generic.h> +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cpu_macros.S> + +func aem_generic_core_pwr_dwn + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + /* --------------------------------------------- + * Flush L1 cache to PoU. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + b dcsw_op_louis +endfunc aem_generic_core_pwr_dwn + + +func aem_generic_cluster_pwr_dwn + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + /* --------------------------------------------- + * Flush L1 and L2 caches to PoC. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + b dcsw_op_all +endfunc aem_generic_cluster_pwr_dwn + +#if REPORT_ERRATA +/* + * Errata printing function for AEM. Must follow AAPCS. + */ +func aem_generic_errata_report + bx lr +endfunc aem_generic_errata_report +#endif + +/* cpu_ops for Base AEM FVP */ +declare_cpu_ops aem_generic, BASE_AEM_MIDR, CPU_NO_RESET_FUNC, \ + aem_generic_core_pwr_dwn, \ + aem_generic_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a12.S b/lib/cpus/aarch32/cortex_a12.S new file mode 100644 index 0000000..8eec27c --- /dev/null +++ b/lib/cpus/aarch32/cortex_a12.S @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cortex_a12.h> +#include <cpu_macros.S> + + .macro assert_cache_enabled +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + .endm + +func cortex_a12_disable_smp + ldcopr r0, ACTLR + bic r0, #CORTEX_A12_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + dsb sy + bx lr +endfunc cortex_a12_disable_smp + +func cortex_a12_enable_smp + ldcopr r0, ACTLR + orr r0, #CORTEX_A12_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + bx lr +endfunc cortex_a12_enable_smp + +func cortex_a12_reset_func + b cortex_a12_enable_smp +endfunc cortex_a12_reset_func + +func cortex_a12_core_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 cache */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a12_disable_smp +endfunc cortex_a12_core_pwr_dwn + +func cortex_a12_cluster_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + bl plat_disable_acp + + /* Flush L2 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level2 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a12_disable_smp +endfunc cortex_a12_cluster_pwr_dwn + +errata_report_shim cortex_a12 + +declare_cpu_ops cortex_a12, CORTEX_A12_MIDR, \ + cortex_a12_reset_func, \ + cortex_a12_core_pwr_dwn, \ + cortex_a12_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a15.S b/lib/cpus/aarch32/cortex_a15.S new file mode 100644 index 0000000..b41676d --- /dev/null +++ b/lib/cpus/aarch32/cortex_a15.S @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cortex_a15.h> +#include <cpu_macros.S> + +/* + * Cortex-A15 support LPAE and Virtualization Extensions. + * Don't care if confiugration uses or not LPAE and VE. + * Therefore, where we don't check ARCH_IS_ARMV7_WITH_LPAE/VE + */ + + .macro assert_cache_enabled +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + .endm + +func cortex_a15_disable_smp + ldcopr r0, ACTLR + bic r0, #CORTEX_A15_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb +#if ERRATA_A15_816470 + /* + * Invalidate any TLB address + */ + mov r0, #0 + stcopr r0, TLBIMVA +#endif + dsb sy + bx lr +endfunc cortex_a15_disable_smp + +func cortex_a15_enable_smp + ldcopr r0, ACTLR + orr r0, #CORTEX_A15_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + bx lr +endfunc cortex_a15_enable_smp + + /* ---------------------------------------------------- + * Errata Workaround for Cortex A15 Errata #816470. + * This applies only to revision >= r3p0 of Cortex A15. + * ---------------------------------------------------- + */ +func check_errata_816470 + /* + * Even though this is only needed for revision >= r3p0, it is always + * applied because of the low cost of the workaround. + */ + mov r0, #ERRATA_APPLIES + bx lr +endfunc check_errata_816470 + +add_erratum_entry cortex_a15, ERRATUM(816470), ERRATA_A15_816470 + /* ---------------------------------------------------- + * Errata Workaround for Cortex A15 Errata #827671. + * This applies only to revision >= r3p0 of Cortex A15. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * ---------------------------------------------------- + */ +func errata_a15_827671_wa + /* + * Compare r0 against revision r3p0 + */ + mov r2, lr + bl check_errata_827671 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr r0, CORTEX_A15_ACTLR2 + orr r0, #CORTEX_A15_ACTLR2_INV_DCC_BIT + stcopr r0, CORTEX_A15_ACTLR2 + isb +1: + bx r2 +endfunc errata_a15_827671_wa + +func check_errata_827671 + mov r1, #0x30 + b cpu_rev_var_hs +endfunc check_errata_827671 + +add_erratum_entry cortex_a15, ERRATUM(827671), ERRATA_A15_827671 + +func check_errata_cve_2017_5715 +#if WORKAROUND_CVE_2017_5715 + mov r0, #ERRATA_APPLIES +#else + mov r0, #ERRATA_MISSING +#endif + bx lr +endfunc check_errata_cve_2017_5715 + +add_erratum_entry cortex_a15, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 + +func check_errata_cve_2022_23960 +#if WORKAROUND_CVE_2022_23960 + mov r0, #ERRATA_APPLIES +#else + mov r0, #ERRATA_MISSING +#endif + bx lr +endfunc check_errata_cve_2022_23960 + +add_erratum_entry cortex_a15, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +func cortex_a15_reset_func + mov r5, lr + bl cpu_get_rev_var + +#if ERRATA_A15_827671 + bl errata_a15_827671_wa +#endif + +#if IMAGE_BL32 && (WORKAROUND_CVE_2017_5715 || WORKAROUND_CVE_2022_23960) + ldcopr r0, ACTLR + orr r0, #CORTEX_A15_ACTLR_INV_BTB_BIT + stcopr r0, ACTLR + ldr r0, =wa_cve_2017_5715_icache_inv_vbar + stcopr r0, VBAR + stcopr r0, MVBAR + /* isb will be applied in the course of the reset func */ +#endif + + mov lr, r5 + b cortex_a15_enable_smp +endfunc cortex_a15_reset_func + +func cortex_a15_core_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 cache */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a15_disable_smp +endfunc cortex_a15_core_pwr_dwn + +func cortex_a15_cluster_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + bl plat_disable_acp + + /* Flush L2 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level2 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a15_disable_smp +endfunc cortex_a15_cluster_pwr_dwn + +errata_report_shim cortex_a15 + +declare_cpu_ops cortex_a15, CORTEX_A15_MIDR, \ + cortex_a15_reset_func, \ + cortex_a15_core_pwr_dwn, \ + cortex_a15_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a17.S b/lib/cpus/aarch32/cortex_a17.S new file mode 100644 index 0000000..1877570 --- /dev/null +++ b/lib/cpus/aarch32/cortex_a17.S @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cortex_a17.h> +#include <cpu_macros.S> + + .macro assert_cache_enabled +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + .endm + +func cortex_a17_disable_smp + ldcopr r0, ACTLR + bic r0, #CORTEX_A17_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + dsb sy + bx lr +endfunc cortex_a17_disable_smp + +func cortex_a17_enable_smp + ldcopr r0, ACTLR + orr r0, #CORTEX_A17_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + bx lr +endfunc cortex_a17_enable_smp + + /* ---------------------------------------------------- + * Errata Workaround for Cortex A17 Errata #852421. + * This applies only to revision <= r1p2 of Cortex A17. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * ---------------------------------------------------- + */ +func errata_a17_852421_wa + /* + * Compare r0 against revision r1p2 + */ + mov r2, lr + bl check_errata_852421 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr r0, CORTEX_A17_IMP_DEF_REG1 + orr r0, r0, #(1<<24) + stcopr r0, CORTEX_A17_IMP_DEF_REG1 +1: + bx r2 +endfunc errata_a17_852421_wa + +func check_errata_852421 + mov r1, #0x12 + b cpu_rev_var_ls +endfunc check_errata_852421 + +add_erratum_entry cortex_a17, ERRATUM(852421), ERRATA_A17_852421 + + /* ---------------------------------------------------- + * Errata Workaround for Cortex A17 Errata #852423. + * This applies only to revision <= r1p2 of Cortex A17. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * ---------------------------------------------------- + */ +func errata_a17_852423_wa + /* + * Compare r0 against revision r1p2 + */ + mov r2, lr + bl check_errata_852423 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr r0, CORTEX_A17_IMP_DEF_REG1 + orr r0, r0, #(1<<12) + stcopr r0, CORTEX_A17_IMP_DEF_REG1 +1: + bx r2 +endfunc errata_a17_852423_wa + +func check_errata_852423 + mov r1, #0x12 + b cpu_rev_var_ls +endfunc check_errata_852423 + +add_erratum_entry cortex_a17, ERRATUM(852423), ERRATA_A17_852423 + +func check_errata_cve_2017_5715 +#if WORKAROUND_CVE_2017_5715 + mov r0, #ERRATA_APPLIES +#else + mov r0, #ERRATA_MISSING +#endif + bx lr +endfunc check_errata_cve_2017_5715 + +add_erratum_entry cortex_a17, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 + +errata_report_shim cortex_a17 + +func cortex_a17_reset_func + mov r5, lr + bl cpu_get_rev_var + mov r4, r0 + +#if ERRATA_A17_852421 + mov r0, r4 + bl errata_a17_852421_wa +#endif + +#if ERRATA_A17_852423 + mov r0, r4 + bl errata_a17_852423_wa +#endif + +#if IMAGE_BL32 && WORKAROUND_CVE_2017_5715 + ldr r0, =wa_cve_2017_5715_bpiall_vbar + stcopr r0, VBAR + stcopr r0, MVBAR + /* isb will be applied in the course of the reset func */ +#endif + + mov lr, r5 + b cortex_a17_enable_smp +endfunc cortex_a17_reset_func + +func cortex_a17_core_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 cache */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a17_disable_smp +endfunc cortex_a17_core_pwr_dwn + +func cortex_a17_cluster_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + bl plat_disable_acp + + /* Flush L2 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level2 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a17_disable_smp +endfunc cortex_a17_cluster_pwr_dwn + +declare_cpu_ops cortex_a17, CORTEX_A17_MIDR, \ + cortex_a17_reset_func, \ + cortex_a17_core_pwr_dwn, \ + cortex_a17_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a32.S b/lib/cpus/aarch32/cortex_a32.S new file mode 100644 index 0000000..d08b4ff --- /dev/null +++ b/lib/cpus/aarch32/cortex_a32.S @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cortex_a32.h> +#include <cpu_macros.S> + + + /* --------------------------------------------- + * Disable intra-cluster coherency + * Clobbers: r0-r1 + * --------------------------------------------- + */ +func cortex_a32_disable_smp + ldcopr16 r0, r1, CORTEX_A32_CPUECTLR_EL1 + bic r0, r0, #CORTEX_A32_CPUECTLR_SMPEN_BIT + stcopr16 r0, r1, CORTEX_A32_CPUECTLR_EL1 + isb + dsb sy + bx lr +endfunc cortex_a32_disable_smp + + /* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A32. + * Clobbers: r0-r1 + * ------------------------------------------------- + */ +func cortex_a32_reset_func + /* --------------------------------------------- + * Enable the SMP bit. + * --------------------------------------------- + */ + ldcopr16 r0, r1, CORTEX_A32_CPUECTLR_EL1 + orr r0, r0, #CORTEX_A32_CPUECTLR_SMPEN_BIT + stcopr16 r0, r1, CORTEX_A32_CPUECTLR_EL1 + isb + bx lr +endfunc cortex_a32_reset_func + + /* ---------------------------------------------------- + * The CPU Ops core power down function for Cortex-A32. + * Clobbers: r0-r3 + * ---------------------------------------------------- + */ +func cortex_a32_core_pwr_dwn + /* r12 is pushed to meet the 8 byte stack alignment requirement */ + push {r12, lr} + + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + pop {r12, lr} + b cortex_a32_disable_smp +endfunc cortex_a32_core_pwr_dwn + + /* ------------------------------------------------------- + * The CPU Ops cluster power down function for Cortex-A32. + * Clobbers: r0-r3 + * ------------------------------------------------------- + */ +func cortex_a32_cluster_pwr_dwn + /* r12 is pushed to meet the 8 byte stack alignment requirement */ + push {r12, lr} + + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + + /* --------------------------------------------- + * Flush L1 cache. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* --------------------------------------------- + * Flush L2 cache. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + pop {r12, lr} + b cortex_a32_disable_smp +endfunc cortex_a32_cluster_pwr_dwn + +errata_report_shim cortex_a32 + +declare_cpu_ops cortex_a32, CORTEX_A32_MIDR, \ + cortex_a32_reset_func, \ + cortex_a32_core_pwr_dwn, \ + cortex_a32_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a5.S b/lib/cpus/aarch32/cortex_a5.S new file mode 100644 index 0000000..625ea7b --- /dev/null +++ b/lib/cpus/aarch32/cortex_a5.S @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cortex_a5.h> +#include <cpu_macros.S> + + .macro assert_cache_enabled +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + .endm + +func cortex_a5_disable_smp + ldcopr r0, ACTLR + bic r0, #CORTEX_A5_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + dsb sy + bx lr +endfunc cortex_a5_disable_smp + +func cortex_a5_enable_smp + ldcopr r0, ACTLR + orr r0, #CORTEX_A5_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + bx lr +endfunc cortex_a5_enable_smp + +func cortex_a5_reset_func + b cortex_a5_enable_smp +endfunc cortex_a5_reset_func + +func cortex_a5_core_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 cache */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a5_disable_smp +endfunc cortex_a5_core_pwr_dwn + +func cortex_a5_cluster_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + bl plat_disable_acp + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a5_disable_smp +endfunc cortex_a5_cluster_pwr_dwn + +errata_report_shim cortex_a5 + +declare_cpu_ops cortex_a5, CORTEX_A5_MIDR, \ + cortex_a5_reset_func, \ + cortex_a5_core_pwr_dwn, \ + cortex_a5_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a53.S b/lib/cpus/aarch32/cortex_a53.S new file mode 100644 index 0000000..89b238a --- /dev/null +++ b/lib/cpus/aarch32/cortex_a53.S @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <common/debug.h> +#include <cortex_a53.h> +#include <cpu_macros.S> + +#if A53_DISABLE_NON_TEMPORAL_HINT +#undef ERRATA_A53_836870 +#define ERRATA_A53_836870 1 +#endif + + /* --------------------------------------------- + * Disable intra-cluster coherency + * --------------------------------------------- + */ +func cortex_a53_disable_smp + ldcopr16 r0, r1, CORTEX_A53_ECTLR + bic64_imm r0, r1, CORTEX_A53_ECTLR_SMP_BIT + stcopr16 r0, r1, CORTEX_A53_ECTLR + isb + dsb sy + bx lr +endfunc cortex_a53_disable_smp + + /* --------------------------------------------------- + * Errata Workaround for Cortex A53 Errata #819472. + * This applies only to revision <= r0p1 of Cortex A53. + * --------------------------------------------------- + */ +func check_errata_819472 + /* + * Even though this is only needed for revision <= r0p1, it + * is always applied due to limitations of the current + * errata framework. + */ + mov r0, #ERRATA_APPLIES + bx lr +endfunc check_errata_819472 + +add_erratum_entry cortex_a53, ERRATUM(819472), ERRATA_A53_819472 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A53 Errata #824069. + * This applies only to revision <= r0p2 of Cortex A53. + * --------------------------------------------------- + */ +func check_errata_824069 + /* + * Even though this is only needed for revision <= r0p2, it + * is always applied due to limitations of the current + * errata framework. + */ + mov r0, #ERRATA_APPLIES + bx lr +endfunc check_errata_824069 + +add_erratum_entry cortex_a53, ERRATUM(824069), ERRATA_A53_824069 + + /* -------------------------------------------------- + * Errata Workaround for Cortex A53 Errata #826319. + * This applies only to revision <= r0p2 of Cortex A53. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * -------------------------------------------------- + */ +func errata_a53_826319_wa + /* + * Compare r0 against revision r0p2 + */ + mov r2, lr + bl check_errata_826319 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr r0, CORTEX_A53_L2ACTLR + bic r0, #CORTEX_A53_L2ACTLR_ENABLE_UNIQUECLEAN + orr r0, #CORTEX_A53_L2ACTLR_DISABLE_CLEAN_PUSH + stcopr r0, CORTEX_A53_L2ACTLR +1: + bx lr +endfunc errata_a53_826319_wa + +func check_errata_826319 + mov r1, #0x02 + b cpu_rev_var_ls +endfunc check_errata_826319 + +add_erratum_entry cortex_a53, ERRATUM(826319), ERRATA_A53_826319 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A53 Errata #827319. + * This applies only to revision <= r0p2 of Cortex A53. + * --------------------------------------------------- + */ +func check_errata_827319 + /* + * Even though this is only needed for revision <= r0p2, it + * is always applied due to limitations of the current + * errata framework. + */ + mov r0, #ERRATA_APPLIES + bx lr +endfunc check_errata_827319 + +add_erratum_entry cortex_a53, ERRATUM(827319), ERRATA_A53_827319 + + /* --------------------------------------------------------------------- + * Disable the cache non-temporal hint. + * + * This ignores the Transient allocation hint in the MAIR and treats + * allocations the same as non-transient allocation types. As a result, + * the LDNP and STNP instructions in AArch64 behave the same as the + * equivalent LDP and STP instructions. + * + * This is relevant only for revisions <= r0p3 of Cortex-A53. + * From r0p4 and onwards, the bit to disable the hint is enabled by + * default at reset. + * + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------------------------- + */ +func a53_disable_non_temporal_hint + /* + * Compare r0 against revision r0p3 + */ + mov r2, lr + bl check_errata_disable_non_temporal_hint + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A53_CPUACTLR + orr64_imm r0, r1, CORTEX_A53_CPUACTLR_DTAH + stcopr16 r0, r1, CORTEX_A53_CPUACTLR +1: + bx lr +endfunc a53_disable_non_temporal_hint + +func check_errata_disable_non_temporal_hint + mov r1, #0x03 + b cpu_rev_var_ls +endfunc check_errata_disable_non_temporal_hint + +add_erratum_entry cortex_a53, ERRATUM(836870), ERRATA_A53_836870 | A53_DISABLE_NON_TEMPORAL_HINT, \ + disable_non_temporal_hint + + /* -------------------------------------------------- + * Errata Workaround for Cortex A53 Errata #855873. + * + * This applies only to revisions >= r0p3 of Cortex A53. + * Earlier revisions of the core are affected as well, but don't + * have the chicken bit in the CPUACTLR register. It is expected that + * the rich OS takes care of that, especially as the workaround is + * shared with other erratas in those revisions of the CPU. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * -------------------------------------------------- + */ +func errata_a53_855873_wa + /* + * Compare r0 against revision r0p3 and higher + */ + mov r2, lr + bl check_errata_855873 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A53_CPUACTLR + orr64_imm r0, r1, CORTEX_A53_CPUACTLR_ENDCCASCI + stcopr16 r0, r1, CORTEX_A53_CPUACTLR +1: + bx lr +endfunc errata_a53_855873_wa + +func check_errata_855873 + mov r1, #0x03 + b cpu_rev_var_hs +endfunc check_errata_855873 + +add_erratum_entry cortex_a53, ERRATUM(855873), ERRATA_A53_855873 + + /* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A53. + * Shall clobber: r0-r6 + * ------------------------------------------------- + */ +func cortex_a53_reset_func + mov r5, lr + bl cpu_get_rev_var + mov r4, r0 + +#if ERRATA_A53_826319 + mov r0, r4 + bl errata_a53_826319_wa +#endif + +#if ERRATA_A53_836870 + mov r0, r4 + bl a53_disable_non_temporal_hint +#endif + +#if ERRATA_A53_855873 + mov r0, r4 + bl errata_a53_855873_wa +#endif + + /* --------------------------------------------- + * Enable the SMP bit. + * --------------------------------------------- + */ + ldcopr16 r0, r1, CORTEX_A53_ECTLR + orr64_imm r0, r1, CORTEX_A53_ECTLR_SMP_BIT + stcopr16 r0, r1, CORTEX_A53_ECTLR + isb + bx r5 +endfunc cortex_a53_reset_func + + /* ---------------------------------------------------- + * The CPU Ops core power down function for Cortex-A53. + * ---------------------------------------------------- + */ +func cortex_a53_core_pwr_dwn + push {r12, lr} + + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + pop {r12, lr} + b cortex_a53_disable_smp +endfunc cortex_a53_core_pwr_dwn + + /* ------------------------------------------------------- + * The CPU Ops cluster power down function for Cortex-A53. + * Clobbers: r0-r3 + * ------------------------------------------------------- + */ +func cortex_a53_cluster_pwr_dwn + push {r12, lr} + + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* --------------------------------------------- + * Flush L2 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + pop {r12, lr} + b cortex_a53_disable_smp +endfunc cortex_a53_cluster_pwr_dwn + +errata_report_shim cortex_a53 + +declare_cpu_ops cortex_a53, CORTEX_A53_MIDR, \ + cortex_a53_reset_func, \ + cortex_a53_core_pwr_dwn, \ + cortex_a53_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a57.S b/lib/cpus/aarch32/cortex_a57.S new file mode 100644 index 0000000..1e5377b --- /dev/null +++ b/lib/cpus/aarch32/cortex_a57.S @@ -0,0 +1,614 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <common/debug.h> +#include <cortex_a57.h> +#include <cpu_macros.S> + + /* --------------------------------------------- + * Disable intra-cluster coherency + * Clobbers: r0-r1 + * --------------------------------------------- + */ +func cortex_a57_disable_smp + ldcopr16 r0, r1, CORTEX_A57_ECTLR + bic64_imm r0, r1, CORTEX_A57_ECTLR_SMP_BIT + stcopr16 r0, r1, CORTEX_A57_ECTLR + bx lr +endfunc cortex_a57_disable_smp + + /* --------------------------------------------- + * Disable all types of L2 prefetches. + * Clobbers: r0-r2 + * --------------------------------------------- + */ +func cortex_a57_disable_l2_prefetch + ldcopr16 r0, r1, CORTEX_A57_ECTLR + orr64_imm r0, r1, CORTEX_A57_ECTLR_DIS_TWD_ACC_PFTCH_BIT + bic64_imm r0, r1, (CORTEX_A57_ECTLR_L2_IPFTCH_DIST_MASK | \ + CORTEX_A57_ECTLR_L2_DPFTCH_DIST_MASK) + stcopr16 r0, r1, CORTEX_A57_ECTLR + isb + dsb ish + bx lr +endfunc cortex_a57_disable_l2_prefetch + + /* --------------------------------------------- + * Disable debug interfaces + * --------------------------------------------- + */ +func cortex_a57_disable_ext_debug + mov r0, #1 + stcopr r0, DBGOSDLR + isb +#if ERRATA_A57_817169 + /* + * Invalidate any TLB address + */ + mov r0, #0 + stcopr r0, TLBIMVA +#endif + dsb sy + bx lr +endfunc cortex_a57_disable_ext_debug + + /* -------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #806969. + * This applies only to revision r0p0 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * -------------------------------------------------- + */ +func errata_a57_806969_wa + /* + * Compare r0 against revision r0p0 + */ + mov r2, lr + bl check_errata_806969 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_NO_ALLOC_WBWA + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc errata_a57_806969_wa + +func check_errata_806969 + mov r1, #0x00 + b cpu_rev_var_ls +endfunc check_errata_806969 + +add_erratum_entry cortex_a57, ERRATUM(806969), ERRATA_A57_806969 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #813419. + * This applies only to revision r0p0 of Cortex A57. + * --------------------------------------------------- + */ +func check_errata_813419 + /* + * Even though this is only needed for revision r0p0, it + * is always applied due to limitations of the current + * errata framework. + */ + mov r0, #ERRATA_APPLIES + bx lr +endfunc check_errata_813419 + +add_erratum_entry cortex_a57, ERRATUM(813419), ERRATA_A57_813419 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #813420. + * This applies only to revision r0p0 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a57_813420_wa + /* + * Compare r0 against revision r0p0 + */ + mov r2, lr + bl check_errata_813420 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_DCC_AS_DCCI + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc errata_a57_813420_wa + +func check_errata_813420 + mov r1, #0x00 + b cpu_rev_var_ls +endfunc check_errata_813420 + +add_erratum_entry cortex_a57, ERRATUM(813420), ERRATA_A57_813420 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #814670. + * This applies only to revision r0p0 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a57_814670_wa + /* + * Compare r0 against revision r0p0 + */ + mov r2, lr + bl check_errata_814670 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_DIS_DMB_NULLIFICATION + stcopr16 r0, r1, CORTEX_A57_CPUACTLR + isb +1: + bx r2 +endfunc errata_a57_814670_wa + +func check_errata_814670 + mov r1, #0x00 + b cpu_rev_var_ls +endfunc check_errata_814670 + +add_erratum_entry cortex_a57, ERRATUM(814670), ERRATA_A57_814670 + + /* ---------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #817169. + * This applies only to revision <= r0p1 of Cortex A57. + * ---------------------------------------------------- + */ +func check_errata_817169 + /* + * Even though this is only needed for revision <= r0p1, it + * is always applied because of the low cost of the workaround. + */ + mov r0, #ERRATA_APPLIES + bx lr +endfunc check_errata_817169 + +add_erratum_entry cortex_a57, ERRATUM(817169), ERRATA_A57_817169 + + /* -------------------------------------------------------------------- + * Disable the over-read from the LDNP instruction. + * + * This applies to all revisions <= r1p2. The performance degradation + * observed with LDNP/STNP has been fixed on r1p3 and onwards. + * + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------------------------- + */ +func a57_disable_ldnp_overread + /* + * Compare r0 against revision r1p2 + */ + mov r2, lr + bl check_errata_disable_ldnp_overread + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_DIS_OVERREAD + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc a57_disable_ldnp_overread + +func check_errata_disable_ldnp_overread + mov r1, #0x12 + b cpu_rev_var_ls +endfunc check_errata_disable_ldnp_overread + +add_erratum_entry cortex_a57, ERRATUM(1), A57_DISABLE_NON_TEMPORAL_HINT, disable_ldnp_overread + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #826974. + * This applies only to revision <= r1p1 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a57_826974_wa + /* + * Compare r0 against revision r1p1 + */ + mov r2, lr + bl check_errata_826974 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_DIS_LOAD_PASS_DMB + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc errata_a57_826974_wa + +func check_errata_826974 + mov r1, #0x11 + b cpu_rev_var_ls +endfunc check_errata_826974 + +add_erratum_entry cortex_a57, ERRATUM(826974), ERRATA_A57_826974 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #826977. + * This applies only to revision <= r1p1 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a57_826977_wa + /* + * Compare r0 against revision r1p1 + */ + mov r2, lr + bl check_errata_826977 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_GRE_NGRE_AS_NGNRE + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc errata_a57_826977_wa + +func check_errata_826977 + mov r1, #0x11 + b cpu_rev_var_ls +endfunc check_errata_826977 + +add_erratum_entry cortex_a57, ERRATUM(826977), ERRATA_A57_826977 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #828024. + * This applies only to revision <= r1p1 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a57_828024_wa + /* + * Compare r0 against revision r1p1 + */ + mov r2, lr + bl check_errata_828024 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + /* + * Setting the relevant bits in CORTEX_A57_CPUACTLR has to be done in 2 + * instructions here because the resulting bitmask doesn't fit in a + * 16-bit value so it cannot be encoded in a single instruction. + */ + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_NO_ALLOC_WBWA + orr64_imm r0, r1, (CORTEX_A57_CPUACTLR_DIS_L1_STREAMING | CORTEX_A57_CPUACTLR_DIS_STREAMING) + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc errata_a57_828024_wa + +func check_errata_828024 + mov r1, #0x11 + b cpu_rev_var_ls +endfunc check_errata_828024 + +add_erratum_entry cortex_a57, ERRATUM(828024), ERRATA_A57_828024 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #829520. + * This applies only to revision <= r1p2 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a57_829520_wa + /* + * Compare r0 against revision r1p2 + */ + mov r2, lr + bl check_errata_829520 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_DIS_INDIRECT_PREDICTOR + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc errata_a57_829520_wa + +func check_errata_829520 + mov r1, #0x12 + b cpu_rev_var_ls +endfunc check_errata_829520 + +add_erratum_entry cortex_a57, ERRATUM(829520), ERRATA_A57_829520 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #833471. + * This applies only to revision <= r1p2 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a57_833471_wa + /* + * Compare r0 against revision r1p2 + */ + mov r2, lr + bl check_errata_833471 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r1, r1, CORTEX_A57_CPUACTLR_FORCE_FPSCR_FLUSH + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc errata_a57_833471_wa + +func check_errata_833471 + mov r1, #0x12 + b cpu_rev_var_ls +endfunc check_errata_833471 + +add_erratum_entry cortex_a57, ERRATUM(833471), ERRATA_A57_833471 + + /* --------------------------------------------------- + * Errata Workaround for Cortex A57 Errata #859972. + * This applies only to revision <= r1p3 of Cortex A57. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a57_859972_wa + mov r2, lr + bl check_errata_859972 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r1, r1, CORTEX_A57_CPUACTLR_DIS_INSTR_PREFETCH + stcopr16 r0, r1, CORTEX_A57_CPUACTLR +1: + bx lr +endfunc errata_a57_859972_wa + +func check_errata_859972 + mov r1, #0x13 + b cpu_rev_var_ls +endfunc check_errata_859972 + +add_erratum_entry cortex_a57, ERRATUM(859972), ERRATA_A57_859972 + +func check_errata_cve_2017_5715 + mov r0, #ERRATA_MISSING + bx lr +endfunc check_errata_cve_2017_5715 + +add_erratum_entry cortex_a57, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 + +func check_errata_cve_2018_3639 +#if WORKAROUND_CVE_2018_3639 + mov r0, #ERRATA_APPLIES +#else + mov r0, #ERRATA_MISSING +#endif + bx lr +endfunc check_errata_cve_2018_3639 + +add_erratum_entry cortex_a57, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + +func check_errata_cve_2022_23960 + mov r0, #ERRATA_MISSING + bx lr +endfunc check_errata_cve_2022_23960 + +add_erratum_entry cortex_a57, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + + /* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A57. + * Shall clobber: r0-r6 + * ------------------------------------------------- + */ +func cortex_a57_reset_func + mov r5, lr + bl cpu_get_rev_var + mov r4, r0 + +#if ERRATA_A57_806969 + mov r0, r4 + bl errata_a57_806969_wa +#endif + +#if ERRATA_A57_813420 + mov r0, r4 + bl errata_a57_813420_wa +#endif + +#if ERRATA_A57_814670 + mov r0, r4 + bl errata_a57_814670_wa +#endif + +#if A57_DISABLE_NON_TEMPORAL_HINT + mov r0, r4 + bl a57_disable_ldnp_overread +#endif + +#if ERRATA_A57_826974 + mov r0, r4 + bl errata_a57_826974_wa +#endif + +#if ERRATA_A57_826977 + mov r0, r4 + bl errata_a57_826977_wa +#endif + +#if ERRATA_A57_828024 + mov r0, r4 + bl errata_a57_828024_wa +#endif + +#if ERRATA_A57_829520 + mov r0, r4 + bl errata_a57_829520_wa +#endif + +#if ERRATA_A57_833471 + mov r0, r4 + bl errata_a57_833471_wa +#endif + +#if ERRATA_A57_859972 + mov r0, r4 + bl errata_a57_859972_wa +#endif + +#if WORKAROUND_CVE_2018_3639 + ldcopr16 r0, r1, CORTEX_A57_CPUACTLR + orr64_imm r0, r1, CORTEX_A57_CPUACTLR_DIS_LOAD_PASS_STORE + stcopr16 r0, r1, CORTEX_A57_CPUACTLR + isb + dsb sy +#endif + + /* --------------------------------------------- + * Enable the SMP bit. + * --------------------------------------------- + */ + ldcopr16 r0, r1, CORTEX_A57_ECTLR + orr64_imm r0, r1, CORTEX_A57_ECTLR_SMP_BIT + stcopr16 r0, r1, CORTEX_A57_ECTLR + isb + bx r5 +endfunc cortex_a57_reset_func + + /* ---------------------------------------------------- + * The CPU Ops core power down function for Cortex-A57. + * ---------------------------------------------------- + */ +func cortex_a57_core_pwr_dwn + push {r12, lr} + + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + + /* --------------------------------------------- + * Disable the L2 prefetches. + * --------------------------------------------- + */ + bl cortex_a57_disable_l2_prefetch + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + bl cortex_a57_disable_smp + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + pop {r12, lr} + b cortex_a57_disable_ext_debug +endfunc cortex_a57_core_pwr_dwn + + /* ------------------------------------------------------- + * The CPU Ops cluster power down function for Cortex-A57. + * Clobbers: r0-r3 + * ------------------------------------------------------- + */ +func cortex_a57_cluster_pwr_dwn + push {r12, lr} + + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + + /* --------------------------------------------- + * Disable the L2 prefetches. + * --------------------------------------------- + */ + bl cortex_a57_disable_l2_prefetch + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* --------------------------------------------- + * Flush L2 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + bl cortex_a57_disable_smp + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + pop {r12, lr} + b cortex_a57_disable_ext_debug +endfunc cortex_a57_cluster_pwr_dwn + +errata_report_shim cortex_a57 + +declare_cpu_ops cortex_a57, CORTEX_A57_MIDR, \ + cortex_a57_reset_func, \ + cortex_a57_core_pwr_dwn, \ + cortex_a57_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a7.S b/lib/cpus/aarch32/cortex_a7.S new file mode 100644 index 0000000..4842ca6 --- /dev/null +++ b/lib/cpus/aarch32/cortex_a7.S @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cortex_a7.h> +#include <cpu_macros.S> + + .macro assert_cache_enabled +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + .endm + +func cortex_a7_disable_smp + ldcopr r0, ACTLR + bic r0, #CORTEX_A7_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + dsb sy + bx lr +endfunc cortex_a7_disable_smp + +func cortex_a7_enable_smp + ldcopr r0, ACTLR + orr r0, #CORTEX_A7_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + bx lr +endfunc cortex_a7_enable_smp + +func cortex_a7_reset_func + b cortex_a7_enable_smp +endfunc cortex_a7_reset_func + +func cortex_a7_core_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 cache */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a7_disable_smp +endfunc cortex_a7_core_pwr_dwn + +func cortex_a7_cluster_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + bl plat_disable_acp + + /* Flush L2 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level2 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a7_disable_smp +endfunc cortex_a7_cluster_pwr_dwn + +errata_report_shim cortex_a7 + +declare_cpu_ops cortex_a7, CORTEX_A7_MIDR, \ + cortex_a7_reset_func, \ + cortex_a7_core_pwr_dwn, \ + cortex_a7_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a72.S b/lib/cpus/aarch32/cortex_a72.S new file mode 100644 index 0000000..77cf84d --- /dev/null +++ b/lib/cpus/aarch32/cortex_a72.S @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <common/debug.h> +#include <cortex_a72.h> +#include <cpu_macros.S> + + /* --------------------------------------------- + * Disable all types of L2 prefetches. + * --------------------------------------------- + */ +func cortex_a72_disable_l2_prefetch + ldcopr16 r0, r1, CORTEX_A72_ECTLR + orr64_imm r0, r1, CORTEX_A72_ECTLR_DIS_TWD_ACC_PFTCH_BIT + bic64_imm r0, r1, (CORTEX_A72_ECTLR_L2_IPFTCH_DIST_MASK | \ + CORTEX_A72_ECTLR_L2_DPFTCH_DIST_MASK) + stcopr16 r0, r1, CORTEX_A72_ECTLR + isb + bx lr +endfunc cortex_a72_disable_l2_prefetch + + /* --------------------------------------------- + * Disable the load-store hardware prefetcher. + * --------------------------------------------- + */ +func cortex_a72_disable_hw_prefetcher + ldcopr16 r0, r1, CORTEX_A72_CPUACTLR + orr64_imm r0, r1, CORTEX_A72_CPUACTLR_DISABLE_L1_DCACHE_HW_PFTCH + stcopr16 r0, r1, CORTEX_A72_CPUACTLR + isb + dsb ish + bx lr +endfunc cortex_a72_disable_hw_prefetcher + + /* --------------------------------------------- + * Disable intra-cluster coherency + * Clobbers: r0-r1 + * --------------------------------------------- + */ +func cortex_a72_disable_smp + ldcopr16 r0, r1, CORTEX_A72_ECTLR + bic64_imm r0, r1, CORTEX_A72_ECTLR_SMP_BIT + stcopr16 r0, r1, CORTEX_A72_ECTLR + bx lr +endfunc cortex_a72_disable_smp + + /* --------------------------------------------- + * Disable debug interfaces + * --------------------------------------------- + */ +func cortex_a72_disable_ext_debug + mov r0, #1 + stcopr r0, DBGOSDLR + isb + dsb sy + bx lr +endfunc cortex_a72_disable_ext_debug + + /* --------------------------------------------------- + * Errata Workaround for Cortex A72 Errata #859971. + * This applies only to revision <= r0p3 of Cortex A72. + * Inputs: + * r0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: r0-r3 + * --------------------------------------------------- + */ +func errata_a72_859971_wa + mov r2,lr + bl check_errata_859971 + mov lr, r2 + cmp r0, #ERRATA_NOT_APPLIES + beq 1f + ldcopr16 r0, r1, CORTEX_A72_CPUACTLR + orr64_imm r1, r1, CORTEX_A72_CPUACTLR_DIS_INSTR_PREFETCH + stcopr16 r0, r1, CORTEX_A72_CPUACTLR +1: + bx lr +endfunc errata_a72_859971_wa + +func check_errata_859971 + mov r1, #0x03 + b cpu_rev_var_ls +endfunc check_errata_859971 + +add_erratum_entry cortex_a72, ERRATUM(859971), ERRATA_A72_859971 + +func check_errata_cve_2017_5715 + mov r0, #ERRATA_MISSING + bx lr +endfunc check_errata_cve_2017_5715 + +add_erratum_entry cortex_a72, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 + +func check_errata_cve_2018_3639 +#if WORKAROUND_CVE_2018_3639 + mov r0, #ERRATA_APPLIES +#else + mov r0, #ERRATA_MISSING +#endif + bx lr +endfunc check_errata_cve_2018_3639 + +add_erratum_entry cortex_a72, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + +func check_errata_cve_2022_23960 + mov r0, #ERRATA_MISSING + bx lr +endfunc check_errata_cve_2022_23960 + +add_erratum_entry cortex_a72, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + + /* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A72. + * ------------------------------------------------- + */ +func cortex_a72_reset_func + mov r5, lr + bl cpu_get_rev_var + mov r4, r0 + +#if ERRATA_A72_859971 + mov r0, r4 + bl errata_a72_859971_wa +#endif + +#if WORKAROUND_CVE_2018_3639 + ldcopr16 r0, r1, CORTEX_A72_CPUACTLR + orr64_imm r0, r1, CORTEX_A72_CPUACTLR_DIS_LOAD_PASS_STORE + stcopr16 r0, r1, CORTEX_A72_CPUACTLR + isb + dsb sy +#endif + + /* --------------------------------------------- + * Enable the SMP bit. + * --------------------------------------------- + */ + ldcopr16 r0, r1, CORTEX_A72_ECTLR + orr64_imm r0, r1, CORTEX_A72_ECTLR_SMP_BIT + stcopr16 r0, r1, CORTEX_A72_ECTLR + isb + bx r5 +endfunc cortex_a72_reset_func + + /* ---------------------------------------------------- + * The CPU Ops core power down function for Cortex-A72. + * ---------------------------------------------------- + */ +func cortex_a72_core_pwr_dwn + push {r12, lr} + + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + + /* --------------------------------------------- + * Disable the L2 prefetches. + * --------------------------------------------- + */ + bl cortex_a72_disable_l2_prefetch + + /* --------------------------------------------- + * Disable the load-store hardware prefetcher. + * --------------------------------------------- + */ + bl cortex_a72_disable_hw_prefetcher + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + bl cortex_a72_disable_smp + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + pop {r12, lr} + b cortex_a72_disable_ext_debug +endfunc cortex_a72_core_pwr_dwn + + /* ------------------------------------------------------- + * The CPU Ops cluster power down function for Cortex-A72. + * ------------------------------------------------------- + */ +func cortex_a72_cluster_pwr_dwn + push {r12, lr} + + /* Assert if cache is enabled */ +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + + /* --------------------------------------------- + * Disable the L2 prefetches. + * --------------------------------------------- + */ + bl cortex_a72_disable_l2_prefetch + + /* --------------------------------------------- + * Disable the load-store hardware prefetcher. + * --------------------------------------------- + */ + bl cortex_a72_disable_hw_prefetcher + +#if !SKIP_A72_L1_FLUSH_PWR_DWN + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 +#endif + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* ------------------------------------------------- + * Flush the L2 caches. + * ------------------------------------------------- + */ + mov r0, #DC_OP_CISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + bl cortex_a72_disable_smp + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + pop {r12, lr} + b cortex_a72_disable_ext_debug +endfunc cortex_a72_cluster_pwr_dwn + +errata_report_shim cortex_a72 + +declare_cpu_ops cortex_a72, CORTEX_A72_MIDR, \ + cortex_a72_reset_func, \ + cortex_a72_core_pwr_dwn, \ + cortex_a72_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cortex_a9.S b/lib/cpus/aarch32/cortex_a9.S new file mode 100644 index 0000000..1e9757a --- /dev/null +++ b/lib/cpus/aarch32/cortex_a9.S @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cortex_a9.h> +#include <cpu_macros.S> + + .macro assert_cache_enabled +#if ENABLE_ASSERTIONS + ldcopr r0, SCTLR + tst r0, #SCTLR_C_BIT + ASM_ASSERT(eq) +#endif + .endm + +func cortex_a9_disable_smp + ldcopr r0, ACTLR + bic r0, #CORTEX_A9_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + dsb sy + bx lr +endfunc cortex_a9_disable_smp + +func cortex_a9_enable_smp + ldcopr r0, ACTLR + orr r0, #CORTEX_A9_ACTLR_SMP_BIT + stcopr r0, ACTLR + isb + bx lr +endfunc cortex_a9_enable_smp + +func check_errata_794073 +#if ERRATA_A9_794073 + mov r0, #ERRATA_APPLIES +#else + mov r0, #ERRATA_MISSING +#endif + bx lr +endfunc check_errata_794073 + +add_erratum_entry cortex_a9, ERRATUM(794073), ERRATA_A9_794073 + +func check_errata_cve_2017_5715 +#if WORKAROUND_CVE_2017_5715 + mov r0, #ERRATA_APPLIES +#else + mov r0, #ERRATA_MISSING +#endif + bx lr +endfunc check_errata_cve_2017_5715 + +add_erratum_entry cortex_a9, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 + +errata_report_shim cortex_a9 + +func cortex_a9_reset_func +#if IMAGE_BL32 && WORKAROUND_CVE_2017_5715 + ldr r0, =wa_cve_2017_5715_bpiall_vbar + stcopr r0, VBAR + stcopr r0, MVBAR + /* isb will be applied in the course of the reset func */ +#endif + b cortex_a9_enable_smp +endfunc cortex_a9_reset_func + +func cortex_a9_core_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 cache */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a9_disable_smp +endfunc cortex_a9_core_pwr_dwn + +func cortex_a9_cluster_pwr_dwn + push {r12, lr} + + assert_cache_enabled + + /* Flush L1 caches */ + mov r0, #DC_OP_CISW + bl dcsw_op_level1 + + bl plat_disable_acp + + /* Exit cluster coherency */ + pop {r12, lr} + b cortex_a9_disable_smp +endfunc cortex_a9_cluster_pwr_dwn + +declare_cpu_ops cortex_a9, CORTEX_A9_MIDR, \ + cortex_a9_reset_func, \ + cortex_a9_core_pwr_dwn, \ + cortex_a9_cluster_pwr_dwn diff --git a/lib/cpus/aarch32/cpu_helpers.S b/lib/cpus/aarch32/cpu_helpers.S new file mode 100644 index 0000000..83e3e49 --- /dev/null +++ b/lib/cpus/aarch32/cpu_helpers.S @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cpu_macros.S> +#include <common/bl_common.h> +#include <lib/cpus/cpu_ops.h> +#include <lib/el3_runtime/cpu_data.h> + +#if defined(IMAGE_BL1) || defined(IMAGE_BL32) || \ + (defined(IMAGE_BL2) && RESET_TO_BL2) + /* + * The reset handler common to all platforms. After a matching + * cpu_ops structure entry is found, the correponding reset_handler + * in the cpu_ops is invoked. The reset handler is invoked very early + * in the boot sequence and it is assumed that we can clobber r0 - r10 + * without the need to follow AAPCS. + * Clobbers: r0 - r10 + */ + .globl reset_handler +func reset_handler + mov r8, lr + + /* The plat_reset_handler can clobber r0 - r7 */ + bl plat_reset_handler + + /* Get the matching cpu_ops pointer (clobbers: r0 - r5) */ + bl get_cpu_ops_ptr + +#if ENABLE_ASSERTIONS + cmp r0, #0 + ASM_ASSERT(ne) +#endif + + /* Get the cpu_ops reset handler */ + ldr r1, [r0, #CPU_RESET_FUNC] + cmp r1, #0 + mov lr, r8 + bxne r1 + bx lr +endfunc reset_handler + +#endif + +#ifdef IMAGE_BL32 /* The power down core and cluster is needed only in BL32 */ + /* + * void prepare_cpu_pwr_dwn(unsigned int power_level) + * + * Prepare CPU power down function for all platforms. The function takes + * a domain level to be powered down as its parameter. After the cpu_ops + * pointer is retrieved from cpu_data, the handler for requested power + * level is called. + */ + .globl prepare_cpu_pwr_dwn +func prepare_cpu_pwr_dwn + /* + * If the given power level exceeds CPU_MAX_PWR_DWN_OPS, we call the + * power down handler for the last power level + */ + mov r2, #(CPU_MAX_PWR_DWN_OPS - 1) + cmp r0, r2 + movhi r0, r2 + + push {r0, lr} + bl _cpu_data + pop {r2, lr} + + ldr r0, [r0, #CPU_DATA_CPU_OPS_PTR] +#if ENABLE_ASSERTIONS + cmp r0, #0 + ASM_ASSERT(ne) +#endif + + /* Get the appropriate power down handler */ + mov r1, #CPU_PWR_DWN_OPS + add r1, r1, r2, lsl #2 + ldr r1, [r0, r1] +#if ENABLE_ASSERTIONS + cmp r1, #0 + ASM_ASSERT(ne) +#endif + bx r1 +endfunc prepare_cpu_pwr_dwn + + /* + * Initializes the cpu_ops_ptr if not already initialized + * in cpu_data. This must only be called after the data cache + * is enabled. AAPCS is followed. + */ + .globl init_cpu_ops +func init_cpu_ops + push {r4 - r6, lr} + bl _cpu_data + mov r6, r0 + ldr r1, [r0, #CPU_DATA_CPU_OPS_PTR] + cmp r1, #0 + bne 1f + bl get_cpu_ops_ptr +#if ENABLE_ASSERTIONS + cmp r0, #0 + ASM_ASSERT(ne) +#endif + str r0, [r6, #CPU_DATA_CPU_OPS_PTR]! +1: + pop {r4 - r6, pc} +endfunc init_cpu_ops + +#endif /* IMAGE_BL32 */ + + /* + * The below function returns the cpu_ops structure matching the + * midr of the core. It reads the MIDR and finds the matching + * entry in cpu_ops entries. Only the implementation and part number + * are used to match the entries. + * Return : + * r0 - The matching cpu_ops pointer on Success + * r0 - 0 on failure. + * Clobbers: r0 - r5 + */ + .globl get_cpu_ops_ptr +func get_cpu_ops_ptr + /* Get the cpu_ops start and end locations */ + ldr r4, =(__CPU_OPS_START__ + CPU_MIDR) + ldr r5, =(__CPU_OPS_END__ + CPU_MIDR) + + /* Initialize the return parameter */ + mov r0, #0 + + /* Read the MIDR_EL1 */ + ldcopr r2, MIDR + ldr r3, =CPU_IMPL_PN_MASK + + /* Retain only the implementation and part number using mask */ + and r2, r2, r3 +1: + /* Check if we have reached end of list */ + cmp r4, r5 + bhs error_exit + + /* load the midr from the cpu_ops */ + ldr r1, [r4], #CPU_OPS_SIZE + and r1, r1, r3 + + /* Check if midr matches to midr of this core */ + cmp r1, r2 + bne 1b + + /* Subtract the increment and offset to get the cpu-ops pointer */ + sub r0, r4, #(CPU_OPS_SIZE + CPU_MIDR) +#if ENABLE_ASSERTIONS + cmp r0, #0 + ASM_ASSERT(ne) +#endif +error_exit: + bx lr +endfunc get_cpu_ops_ptr + +/* + * Extract CPU revision and variant, and combine them into a single numeric for + * easier comparison. + */ + .globl cpu_get_rev_var +func cpu_get_rev_var + ldcopr r1, MIDR + + /* + * Extract the variant[23:20] and revision[3:0] from r1 and pack it in + * r0[0:7] as variant[7:4] and revision[3:0]: + * + * First extract r1[23:16] to r0[7:0] and zero fill the rest. Then + * extract r1[3:0] into r0[3:0] retaining other bits. + */ + ubfx r0, r1, #(MIDR_VAR_SHIFT - MIDR_REV_BITS), #(MIDR_REV_BITS + MIDR_VAR_BITS) + bfi r0, r1, #MIDR_REV_SHIFT, #MIDR_REV_BITS + bx lr +endfunc cpu_get_rev_var + +/* + * Compare the CPU's revision-variant (r0) with a given value (r1), for errata + * application purposes. If the revision-variant is less than or same as a given + * value, indicates that errata applies; otherwise not. + */ + .globl cpu_rev_var_ls +func cpu_rev_var_ls + cmp r0, r1 + movls r0, #ERRATA_APPLIES + movhi r0, #ERRATA_NOT_APPLIES + bx lr +endfunc cpu_rev_var_ls + +/* + * Compare the CPU's revision-variant (r0) with a given value (r1), for errata + * application purposes. If the revision-variant is higher than or same as a + * given value, indicates that errata applies; otherwise not. + */ + .globl cpu_rev_var_hs +func cpu_rev_var_hs + cmp r0, r1 + movge r0, #ERRATA_APPLIES + movlt r0, #ERRATA_NOT_APPLIES + bx lr +endfunc cpu_rev_var_hs diff --git a/lib/cpus/aarch64/a64fx.S b/lib/cpus/aarch64/a64fx.S new file mode 100644 index 0000000..54c20c3 --- /dev/null +++ b/lib/cpus/aarch64/a64fx.S @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022, Fujitsu Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <a64fx.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +func a64fx_core_pwr_dwn +endfunc a64fx_core_pwr_dwn + +func a64fx_cluster_pwr_dwn +endfunc a64fx_cluster_pwr_dwn + +#if REPORT_ERRATA +/* + * Errata printing function for A64FX. Must follow AAPCS. + */ +func a64fx_errata_report + ret +endfunc a64fx_errata_report +#endif + + /* --------------------------------------------- + * This function provides cpu specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.a64fx_regs, "aS" +a64fx_regs: /* The ascii list of register names to be reported */ + .asciz "" + +func a64fx_cpu_reg_dump + adr x6, a64fx_regs + ret +endfunc a64fx_cpu_reg_dump + +declare_cpu_ops a64fx, A64FX_MIDR, CPU_NO_RESET_FUNC \ + a64fx_core_pwr_dwn, \ + a64fx_cluster_pwr_dwn + diff --git a/lib/cpus/aarch64/aem_generic.S b/lib/cpus/aarch64/aem_generic.S new file mode 100644 index 0000000..d47279a --- /dev/null +++ b/lib/cpus/aarch64/aem_generic.S @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2014-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <aem_generic.h> +#include <arch.h> +#include <asm_macros.S> +#include <cpu_macros.S> + +func aem_generic_core_pwr_dwn + /* --------------------------------------------- + * Disable the Data Cache. + * --------------------------------------------- + */ + mrs x1, sctlr_el3 + bic x1, x1, #SCTLR_C_BIT + msr sctlr_el3, x1 + isb + + /* --------------------------------------------- + * AEM model supports L3 caches in which case L2 + * will be private per core caches and flush + * from L1 to L2 is not sufficient. + * --------------------------------------------- + */ + mrs x1, clidr_el1 + + /* --------------------------------------------- + * Check if L3 cache is implemented. + * --------------------------------------------- + */ + tst x1, ((1 << CLIDR_FIELD_WIDTH) - 1) << CTYPE_SHIFT(3) + + /* --------------------------------------------- + * There is no L3 cache, flush L1 to L2 only. + * --------------------------------------------- + */ + mov x0, #DCCISW + b.eq dcsw_op_level1 + + mov x18, x30 + + /* --------------------------------------------- + * Flush L1 cache to L2. + * --------------------------------------------- + */ + bl dcsw_op_level1 + mov x30, x18 + + /* --------------------------------------------- + * Flush L2 cache to L3. + * --------------------------------------------- + */ + mov x0, #DCCISW + b dcsw_op_level2 +endfunc aem_generic_core_pwr_dwn + +func aem_generic_cluster_pwr_dwn + /* --------------------------------------------- + * Disable the Data Cache. + * --------------------------------------------- + */ + mrs x1, sctlr_el3 + bic x1, x1, #SCTLR_C_BIT + msr sctlr_el3, x1 + isb + + /* --------------------------------------------- + * Flush all caches to PoC. + * --------------------------------------------- + */ + mov x0, #DCCISW + b dcsw_op_all +endfunc aem_generic_cluster_pwr_dwn + +#if REPORT_ERRATA +/* + * Errata printing function for AEM. Must follow AAPCS. + */ +func aem_generic_errata_report + ret +endfunc aem_generic_errata_report +#endif + + /* --------------------------------------------- + * This function provides cpu specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.aem_generic_regs, "aS" +aem_generic_regs: /* The ascii list of register names to be reported */ + .asciz "" /* no registers to report */ + +func aem_generic_cpu_reg_dump + adr x6, aem_generic_regs + ret +endfunc aem_generic_cpu_reg_dump + + +/* cpu_ops for Base AEM FVP */ +declare_cpu_ops aem_generic, BASE_AEM_MIDR, CPU_NO_RESET_FUNC, \ + aem_generic_core_pwr_dwn, \ + aem_generic_cluster_pwr_dwn + +/* cpu_ops for Foundation FVP */ +declare_cpu_ops aem_generic, FOUNDATION_AEM_MIDR, CPU_NO_RESET_FUNC, \ + aem_generic_core_pwr_dwn, \ + aem_generic_cluster_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a35.S b/lib/cpus/aarch64/cortex_a35.S new file mode 100644 index 0000000..6ffb944 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a35.S @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a35.h> +#include <cpu_macros.S> +#include <plat_macros.S> + + /* --------------------------------------------- + * Disable L1 data cache and unified L2 cache + * --------------------------------------------- + */ +func cortex_a35_disable_dcache + sysreg_bit_clear sctlr_el3, SCTLR_C_BIT + isb + ret +endfunc cortex_a35_disable_dcache + + /* --------------------------------------------- + * Disable intra-cluster coherency + * --------------------------------------------- + */ +func cortex_a35_disable_smp + sysreg_bit_clear CORTEX_A35_CPUECTLR_EL1, CORTEX_A35_CPUECTLR_SMPEN_BIT + isb + dsb sy + ret +endfunc cortex_a35_disable_smp + +workaround_reset_start cortex_a35, ERRATUM(855472), ERRATA_A35_855472 + sysreg_bit_set CORTEX_A35_CPUACTLR_EL1, CORTEX_A35_CPUACTLR_EL1_ENDCCASCI +workaround_reset_end cortex_a35, ERRATUM(855472) + +check_erratum_ls cortex_a35, ERRATUM(855472), CPU_REV(0, 0) + + /* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A35. + * ------------------------------------------------- + */ +cpu_reset_func_start cortex_a35 + /* --------------------------------------------- + * Enable the SMP bit. + * --------------------------------------------- + */ + sysreg_bit_set CORTEX_A35_CPUECTLR_EL1, CORTEX_A35_CPUECTLR_SMPEN_BIT +cpu_reset_func_end cortex_a35 + +func cortex_a35_core_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a35_disable_dcache + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a35_disable_smp +endfunc cortex_a35_core_pwr_dwn + +func cortex_a35_cluster_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a35_disable_dcache + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* --------------------------------------------- + * Flush L2 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a35_disable_smp +endfunc cortex_a35_cluster_pwr_dwn + +errata_report_shim cortex_a35 + + /* --------------------------------------------- + * This function provides cortex_a35 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a35_regs, "aS" +cortex_a35_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a35_cpu_reg_dump + adr x6, cortex_a35_regs + mrs x8, CORTEX_A35_CPUECTLR_EL1 + ret +endfunc cortex_a35_cpu_reg_dump + +declare_cpu_ops cortex_a35, CORTEX_A35_MIDR, \ + cortex_a35_reset_func, \ + cortex_a35_core_pwr_dwn, \ + cortex_a35_cluster_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a510.S b/lib/cpus/aarch64/cortex_a510.S new file mode 100644 index 0000000..a59b92c --- /dev/null +++ b/lib/cpus/aarch64/cortex_a510.S @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a510.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-A510 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex-A510 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +workaround_reset_start cortex_a510, ERRATUM(1922240), ERRATA_A510_1922240 + /* Apply the workaround by setting IMP_CMPXACTLR_EL1[11:10] = 0b11. */ + sysreg_bitfield_insert CORTEX_A510_CMPXACTLR_EL1, CORTEX_A510_CMPXACTLR_EL1_SNPPREFERUNIQUE_DISABLE, \ + CORTEX_A510_CMPXACTLR_EL1_SNPPREFERUNIQUE_SHIFT, CORTEX_A510_CMPXACTLR_EL1_SNPPREFERUNIQUE_WIDTH +workaround_reset_end cortex_a510, ERRATUM(1922240) + +check_erratum_ls cortex_a510, ERRATUM(1922240), CPU_REV(0, 0) + +workaround_reset_start cortex_a510, ERRATUM(2041909), ERRATA_A510_2041909 + /* Apply workaround */ + mov x0, xzr + msr S3_6_C15_C4_0, x0 + isb + + mov x0, #0x8500000 + msr S3_6_C15_C4_2, x0 + + mov x0, #0x1F700000 + movk x0, #0x8, lsl #32 + msr S3_6_C15_C4_3, x0 + + mov x0, #0x3F1 + movk x0, #0x110, lsl #16 + msr S3_6_C15_C4_1, x0 +workaround_reset_end cortex_a510, ERRATUM(2041909) + +check_erratum_range cortex_a510, ERRATUM(2041909), CPU_REV(0, 2), CPU_REV(0, 2) + +workaround_reset_start cortex_a510, ERRATUM(2042739), ERRATA_A510_2042739 + /* Apply the workaround by disabling ReadPreferUnique. */ + sysreg_bitfield_insert CORTEX_A510_CPUECTLR_EL1, CORTEX_A510_CPUECTLR_EL1_READPREFERUNIQUE_DISABLE, \ + CORTEX_A510_CPUECTLR_EL1_READPREFERUNIQUE_SHIFT, CORTEX_A510_CPUECTLR_EL1_READPREFERUNIQUE_WIDTH +workaround_reset_end cortex_a510, ERRATUM(2042739) + +check_erratum_ls cortex_a510, ERRATUM(2042739), CPU_REV(0, 2) + +workaround_reset_start cortex_a510, ERRATUM(2080326), ERRATA_A510_2080326 + /* Apply workaround */ + mov x0, #1 + msr S3_6_C15_C4_0, x0 + isb + + mov x0, #0x0100 + movk x0, #0x0E08, lsl #16 + msr S3_6_C15_C4_2, x0 + + mov x0, #0x0300 + movk x0, #0x0F1F, lsl #16 + movk x0, #0x0008, lsl #32 + msr S3_6_C15_C4_3, x0 + + mov x0, #0x03F1 + movk x0, #0x00C0, lsl #16 + msr S3_6_C15_C4_1, x0 + + isb +workaround_reset_end cortex_a510, ERRATUM(2080326) + +check_erratum_range cortex_a510, ERRATUM(2080326), CPU_REV(0, 2), CPU_REV(0, 2) + +workaround_reset_start cortex_a510, ERRATUM(2172148), ERRATA_A510_2172148 + /* + * Force L2 allocation of transient lines by setting + * CPUECTLR_EL1.RSCTL=0b01 and CPUECTLR_EL1.NTCTL=0b01. + */ + mrs x0, CORTEX_A510_CPUECTLR_EL1 + mov x1, #1 + bfi x0, x1, #CORTEX_A510_CPUECTLR_EL1_RSCTL_SHIFT, #2 + bfi x0, x1, #CORTEX_A510_CPUECTLR_EL1_NTCTL_SHIFT, #2 + msr CORTEX_A510_CPUECTLR_EL1, x0 +workaround_reset_end cortex_a510, ERRATUM(2172148) + +check_erratum_ls cortex_a510, ERRATUM(2172148), CPU_REV(1, 0) + +workaround_reset_start cortex_a510, ERRATUM(2218950), ERRATA_A510_2218950 + /* Set bit 18 in CPUACTLR_EL1 */ + sysreg_bitfield_insert CORTEX_A510_CPUACTLR_EL1, CORTEX_A510_CPUACTLR_EL1_ALIAS_LOADSTORE_DISABLE, \ + CORTEX_A510_CPUACTLR_EL1_ALIAS_LOADSTORE_SHIFT, CORTEX_A510_CPUACTLR_EL1_ALIAS_LOADSTORE_WIDTH + + /* Set bit 25 in CMPXACTLR_EL1 */ + sysreg_bitfield_insert CORTEX_A510_CMPXACTLR_EL1, CORTEX_A510_CMPXACTLR_EL1_ALIAS_LOADSTORE_DISABLE, \ + CORTEX_A510_CMPXACTLR_EL1_ALIAS_LOADSTORE_SHIFT, CORTEX_A510_CMPXACTLR_EL1_ALIAS_LOADSTORE_WIDTH + +workaround_reset_end cortex_a510, ERRATUM(2218950) + +check_erratum_ls cortex_a510, ERRATUM(2218950), CPU_REV(1, 0) + + /* -------------------------------------------------- + * This workaround is not a typical errata fix. MPMM + * is disabled here, but this conflicts with the BL31 + * MPMM support. So in addition to simply disabling + * the feature, a flag is set in the MPMM library + * indicating that it should not be enabled even if + * ENABLE_MPMM=1. + * -------------------------------------------------- + */ +workaround_reset_start cortex_a510, ERRATUM(2250311), ERRATA_A510_2250311 + /* Disable MPMM */ + mrs x0, CPUMPMMCR_EL3 + bfm x0, xzr, #0, #0 /* bfc instruction does not work in GCC */ + msr CPUMPMMCR_EL3, x0 + +#if ENABLE_MPMM && IMAGE_BL31 + /* If ENABLE_MPMM is set, tell the runtime lib to skip enabling it. */ + bl mpmm_errata_disable +#endif +workaround_reset_end cortex_a510, ERRATUM(2250311) + +check_erratum_ls cortex_a510, ERRATUM(2250311), CPU_REV(1, 0) + +workaround_reset_start cortex_a510, ERRATUM(2288014), ERRATA_A510_2288014 + /* Apply the workaround by setting IMP_CPUACTLR_EL1[18] = 0b1. */ + sysreg_bitfield_insert CORTEX_A510_CPUACTLR_EL1, CORTEX_A510_CPUACTLR_EL1_DATA_CORRUPT_DISABLE, \ + CORTEX_A510_CPUACTLR_EL1_DATA_CORRUPT_SHIFT, CORTEX_A510_CPUACTLR_EL1_DATA_CORRUPT_WIDTH +workaround_reset_end cortex_a510, ERRATUM(2288014) + +check_erratum_ls cortex_a510, ERRATUM(2288014), CPU_REV(1, 0) + +workaround_reset_start cortex_a510, ERRATUM(2347730), ERRATA_A510_2347730 + /* + * Set CPUACTLR_EL1[17] to 1'b1, which disables + * specific microarchitectural clock gating + * behaviour. + */ + sysreg_bit_set CORTEX_A510_CPUACTLR_EL1, CORTEX_A510_CPUACTLR_EL1_BIT_17 +workaround_reset_end cortex_a510, ERRATUM(2347730) + +check_erratum_ls cortex_a510, ERRATUM(2347730), CPU_REV(1, 1) + +workaround_reset_start cortex_a510, ERRATUM(2371937), ERRATA_A510_2371937 + /* + * Cacheable atomic operations can be forced + * to be executed near by setting + * IMP_CPUECTLR_EL1.ATOM=0b010. ATOM is found + * in [40:38] of CPUECTLR_EL1. + */ + sysreg_bitfield_insert CORTEX_A510_CPUECTLR_EL1, CORTEX_A510_CPUECTLR_EL1_ATOM_EXECALLINSTRNEAR, \ + CORTEX_A510_CPUECTLR_EL1_ATOM_SHIFT, CORTEX_A510_CPUECTLR_EL1_ATOM_WIDTH +workaround_reset_end cortex_a510, ERRATUM(2371937) + +check_erratum_ls cortex_a510, ERRATUM(2371937), CPU_REV(1, 1) + +workaround_reset_start cortex_a510, ERRATUM(2666669), ERRATA_A510_2666669 + sysreg_bit_set CORTEX_A510_CPUACTLR_EL1, CORTEX_A510_CPUACTLR_EL1_BIT_38 +workaround_reset_end cortex_a510, ERRATUM(2666669) + +check_erratum_ls cortex_a510, ERRATUM(2666669), CPU_REV(1, 1) + +.global erratum_cortex_a510_2684597_wa +workaround_runtime_start cortex_a510, ERRATUM(2684597), ERRATA_A510_2684597, CORTEX_A510_MIDR + /* + * Many assemblers do not yet understand the "tsb csync" mnemonic, + * so use the equivalent hint instruction. + */ + hint #18 /* tsb csync */ +workaround_runtime_end cortex_a510, ERRATUM(2684597) + +check_erratum_ls cortex_a510, ERRATUM(2684597), CPU_REV(1, 2) + +/* + * ERRATA_DSU_2313941 : + * The errata is defined in dsu_helpers.S but applies to cortex_a510 + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_a510_2313941, check_errata_dsu_2313941 +.equ erratum_cortex_a510_2313941_wa, errata_dsu_2313941_wa +add_erratum_entry cortex_a510, ERRATUM(2313941), ERRATA_DSU_2313941, APPLY_AT_RESET + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_a510_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_A510_CPUPWRCTLR_EL1, CORTEX_A510_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc cortex_a510_core_pwr_dwn + +errata_report_shim cortex_a510 + +cpu_reset_func_start cortex_a510 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_a510 + + /* --------------------------------------------- + * This function provides Cortex-A510 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a510_regs, "aS" +cortex_a510_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a510_cpu_reg_dump + adr x6, cortex_a510_regs + mrs x8, CORTEX_A510_CPUECTLR_EL1 + ret +endfunc cortex_a510_cpu_reg_dump + +declare_cpu_ops cortex_a510, CORTEX_A510_MIDR, \ + cortex_a510_reset_func, \ + cortex_a510_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a520.S b/lib/cpus/aarch64/cortex_a520.S new file mode 100644 index 0000000..6c2f33e --- /dev/null +++ b/lib/cpus/aarch64/cortex_a520.S @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a520.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex A520 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex A520 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_a520_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_A520_CPUPWRCTLR_EL1, CORTEX_A520_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc cortex_a520_core_pwr_dwn + +errata_report_shim cortex_a520 + +cpu_reset_func_start cortex_a520 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_a520 + + /* --------------------------------------------- + * This function provides Cortex A520 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a520_regs, "aS" +cortex_a520_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a520_cpu_reg_dump + adr x6, cortex_a520_regs + mrs x8, CORTEX_A520_CPUECTLR_EL1 + ret +endfunc cortex_a520_cpu_reg_dump + +declare_cpu_ops cortex_a520, CORTEX_A520_MIDR, \ + cortex_a520_reset_func, \ + cortex_a520_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a53.S b/lib/cpus/aarch64/cortex_a53.S new file mode 100644 index 0000000..e6fb08a --- /dev/null +++ b/lib/cpus/aarch64/cortex_a53.S @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2014-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <common/debug.h> +#include <cortex_a53.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include <lib/cpus/errata.h> + + /* --------------------------------------------- + * Disable L1 data cache and unified L2 cache + * --------------------------------------------- + */ +func cortex_a53_disable_dcache + sysreg_bit_clear sctlr_el3, SCTLR_C_BIT + isb + ret +endfunc cortex_a53_disable_dcache + + /* --------------------------------------------- + * Disable intra-cluster coherency + * --------------------------------------------- + */ +func cortex_a53_disable_smp + sysreg_bit_clear CORTEX_A53_ECTLR_EL1, CORTEX_A53_ECTLR_SMP_BIT + isb + dsb sy + ret +endfunc cortex_a53_disable_smp + +/* Due to the nature of the errata it is applied unconditionally when chosen */ +check_erratum_ls cortex_a53, ERRATUM(819472), CPU_REV(0, 1) +/* erratum workaround is interleaved with generic code */ +add_erratum_entry cortex_a53, ERRATUM(819472), ERRATUM_ALWAYS_CHOSEN, NO_APPLY_AT_RESET + +/* Due to the nature of the errata it is applied unconditionally when chosen */ +check_erratum_ls cortex_a53, ERRATUM(824069), CPU_REV(0, 2) +/* erratum workaround is interleaved with generic code */ +add_erratum_entry cortex_a53, ERRATUM(824069), ERRATUM_ALWAYS_CHOSEN, NO_APPLY_AT_RESET + +workaround_reset_start cortex_a53, ERRATUM(826319), ERRATA_A53_826319 + mrs x1, CORTEX_A53_L2ACTLR_EL1 + bic x1, x1, #CORTEX_A53_L2ACTLR_ENABLE_UNIQUECLEAN + orr x1, x1, #CORTEX_A53_L2ACTLR_DISABLE_CLEAN_PUSH + msr CORTEX_A53_L2ACTLR_EL1, x1 +workaround_reset_end cortex_a53, ERRATUM(826319) + +check_erratum_ls cortex_a53, ERRATUM(826319), CPU_REV(0, 2) + +/* Due to the nature of the errata it is applied unconditionally when chosen */ +check_erratum_ls cortex_a53, ERRATUM(827319), CPU_REV(0, 2) +/* erratum workaround is interleaved with generic code */ +add_erratum_entry cortex_a53, ERRATUM(827319), ERRATUM_ALWAYS_CHOSEN, NO_APPLY_AT_RESET + +check_erratum_custom_start cortex_a53, ERRATUM(835769) + cmp x0, CPU_REV(0, 4) + b.hi errata_not_applies + /* + * Fix potentially available for revisions r0p2, r0p3 and r0p4. + * If r0p2, r0p3 or r0p4; check for fix in REVIDR, else exit. + */ + cmp x0, #0x01 + mov x0, #ERRATA_APPLIES + b.ls exit_check_errata_835769 + /* Load REVIDR. */ + mrs x1, revidr_el1 + /* If REVIDR[7] is set (fix exists) set ERRATA_NOT_APPLIES, else exit. */ + tbz x1, #7, exit_check_errata_835769 +errata_not_applies: + mov x0, #ERRATA_NOT_APPLIES +exit_check_errata_835769: + ret +check_erratum_custom_end cortex_a53, ERRATUM(835769) + +/* workaround at build time */ +add_erratum_entry cortex_a53, ERRATUM(835769), ERRATA_A53_835769, NO_APPLY_AT_RESET + + /* + * Disable the cache non-temporal hint. + * + * This ignores the Transient allocation hint in the MAIR and treats + * allocations the same as non-transient allocation types. As a result, + * the LDNP and STNP instructions in AArch64 behave the same as the + * equivalent LDP and STP instructions. + */ +workaround_reset_start cortex_a53, ERRATUM(836870), ERRATA_A53_836870 | A53_DISABLE_NON_TEMPORAL_HINT + sysreg_bit_set CORTEX_A53_CPUACTLR_EL1, CORTEX_A53_CPUACTLR_EL1_DTAH +workaround_reset_end cortex_a53, ERRATUM(836870) + +check_erratum_ls cortex_a53, ERRATUM(836870), CPU_REV(0, 3) + +check_erratum_custom_start cortex_a53, ERRATUM(843419) + mov x1, #ERRATA_APPLIES + mov x2, #ERRATA_NOT_APPLIES + cmp x0, CPU_REV(0, 4) + csel x0, x1, x2, ls + /* + * Fix potentially available for revision r0p4. + * If r0p4 check for fix in REVIDR, else exit. + */ + b.ne exit_check_errata_843419 + /* Load REVIDR. */ + mrs x3, revidr_el1 + /* If REVIDR[8] is set (fix exists) set ERRATA_NOT_APPLIES, else exit. */ + tbz x3, #8, exit_check_errata_843419 + mov x0, x2 +exit_check_errata_843419: + ret +check_erratum_custom_end cortex_a53, ERRATUM(843419) + +/* workaround at build time */ +add_erratum_entry cortex_a53, ERRATUM(843419), ERRATA_A53_843419, NO_APPLY_AT_RESET + + /* + * Earlier revisions of the core are affected as well, but don't + * have the chicken bit in the CPUACTLR register. It is expected that + * the rich OS takes care of that, especially as the workaround is + * shared with other erratas in those revisions of the CPU. + */ +workaround_reset_start cortex_a53, ERRATUM(855873), ERRATA_A53_855873 + sysreg_bit_set CORTEX_A53_CPUACTLR_EL1, CORTEX_A53_CPUACTLR_EL1_ENDCCASCI +workaround_reset_end cortex_a53, ERRATUM(855873) + +check_erratum_hs cortex_a53, ERRATUM(855873), CPU_REV(0, 3) + +check_erratum_chosen cortex_a53, ERRATUM(1530924), ERRATA_A53_1530924 + +/* erratum has no workaround in the cpu. Generic code must take care */ +add_erratum_entry cortex_a53, ERRATUM(1530924), ERRATA_A53_1530924, NO_APPLY_AT_RESET + +cpu_reset_func_start cortex_a53 + /* Enable the SMP bit. */ + sysreg_bit_set CORTEX_A53_ECTLR_EL1, CORTEX_A53_ECTLR_SMP_BIT +cpu_reset_func_end cortex_a53 + +func cortex_a53_core_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a53_disable_dcache + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a53_disable_smp +endfunc cortex_a53_core_pwr_dwn + +func cortex_a53_cluster_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a53_disable_dcache + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* --------------------------------------------- + * Flush L2 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a53_disable_smp +endfunc cortex_a53_cluster_pwr_dwn + +errata_report_shim cortex_a53 + + /* --------------------------------------------- + * This function provides cortex_a53 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a53_regs, "aS" +cortex_a53_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "cpumerrsr_el1", "l2merrsr_el1", \ + "cpuactlr_el1", "" + +func cortex_a53_cpu_reg_dump + adr x6, cortex_a53_regs + mrs x8, CORTEX_A53_ECTLR_EL1 + mrs x9, CORTEX_A53_MERRSR_EL1 + mrs x10, CORTEX_A53_L2MERRSR_EL1 + mrs x11, CORTEX_A53_CPUACTLR_EL1 + ret +endfunc cortex_a53_cpu_reg_dump + +declare_cpu_ops cortex_a53, CORTEX_A53_MIDR, \ + cortex_a53_reset_func, \ + cortex_a53_core_pwr_dwn, \ + cortex_a53_cluster_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a55.S b/lib/cpus/aarch64/cortex_a55.S new file mode 100644 index 0000000..712b6e0 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a55.S @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a55.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-A55 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + + .globl cortex_a55_reset_func + .globl cortex_a55_core_pwr_dwn + +/* ERRATA_DSU_798953: + * The errata is defined in dsu_helpers.S but applies to cortex_a55 + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_a55_798953, check_errata_dsu_798953 +.equ erratum_cortex_a55_798953_wa, errata_dsu_798953_wa +add_erratum_entry cortex_a55, ERRATUM(798953), ERRATA_DSU_798953, APPLY_AT_RESET + +/* ERRATA_DSU_936184: + * The errata is defined in dsu_helpers.S but applies to cortex_a55 + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_a55_936184, check_errata_dsu_936184 +.equ erratum_cortex_a55_936184_wa, errata_dsu_936184_wa +add_erratum_entry cortex_a55, ERRATUM(936184), ERRATA_DSU_936184, APPLY_AT_RESET + +workaround_reset_start cortex_a55, ERRATUM(768277), ERRATA_A55_768277 + sysreg_bit_set CORTEX_A55_CPUACTLR_EL1, CORTEX_A55_CPUACTLR_EL1_DISABLE_DUAL_ISSUE +workaround_reset_end cortex_a55, ERRATUM(768277) + +check_erratum_ls cortex_a55, ERRATUM(768277), CPU_REV(0, 0) + +workaround_reset_start cortex_a55, ERRATUM(778703), ERRATA_A55_778703 + sysreg_bit_set CORTEX_A55_CPUECTLR_EL1, CORTEX_A55_CPUECTLR_EL1_L1WSCTL + sysreg_bit_set CORTEX_A55_CPUACTLR_EL1, CORTEX_A55_CPUACTLR_EL1_DISABLE_WRITE_STREAMING +workaround_reset_end cortex_a55, ERRATUM(778703) + +check_erratum_custom_start cortex_a55, ERRATUM(778703) + mov x16, x30 + mov x1, #0x00 + bl cpu_rev_var_ls + /* + * Check that no private L2 cache is configured + */ + mrs x1, CORTEX_A55_CLIDR_EL1 + and x1, x1, CORTEX_A55_CLIDR_EL1_CTYPE3 + cmp x1, #0 + mov x2, #ERRATA_NOT_APPLIES + csel x0, x0, x2, eq + ret x16 +check_erratum_custom_end cortex_a55, ERRATUM(778703) + +workaround_reset_start cortex_a55, ERRATUM(798797), ERRATA_A55_798797 + sysreg_bit_set CORTEX_A55_CPUACTLR_EL1, CORTEX_A55_CPUACTLR_EL1_DISABLE_L1_PAGEWALKS +workaround_reset_end cortex_a55, ERRATUM(798797) + +check_erratum_ls cortex_a55, ERRATUM(798797), CPU_REV(0, 0) + +workaround_reset_start cortex_a55, ERRATUM(846532), ERRATA_A55_846532 + sysreg_bit_set CORTEX_A55_CPUACTLR_EL1, CORTEX_A55_CPUACTLR_EL1_DISABLE_DUAL_ISSUE +workaround_reset_end cortex_a55, ERRATUM(846532) + +check_erratum_ls cortex_a55, ERRATUM(846532), CPU_REV(0, 1) + +workaround_reset_start cortex_a55, ERRATUM(903758), ERRATA_A55_903758 + sysreg_bit_set CORTEX_A55_CPUACTLR_EL1, CORTEX_A55_CPUACTLR_EL1_DISABLE_L1_PAGEWALKS +workaround_reset_end cortex_a55, ERRATUM(903758) + +check_erratum_ls cortex_a55, ERRATUM(903758), CPU_REV(0, 1) + +workaround_reset_start cortex_a55, ERRATUM(1221012), ERRATA_A55_1221012 + mov x0, #0x0020 + movk x0, #0x0850, lsl #16 + msr CPUPOR_EL3, x0 + mov x0, #0x0000 + movk x0, #0x1FF0, lsl #16 + movk x0, #0x2, lsl #32 + msr CPUPMR_EL3, x0 + mov x0, #0x03fd + movk x0, #0x0110, lsl #16 + msr CPUPCR_EL3, x0 + mov x0, #0x1 + msr CPUPSELR_EL3, x0 + mov x0, #0x0040 + movk x0, #0x08D0, lsl #16 + msr CPUPOR_EL3, x0 + mov x0, #0x0040 + movk x0, #0x1FF0, lsl #16 + movk x0, #0x2, lsl #32 + msr CPUPMR_EL3, x0 + mov x0, #0x03fd + movk x0, #0x0110, lsl #16 + msr CPUPCR_EL3, x0 +workaround_reset_end cortex_a55, ERRATUM(1221012) + +check_erratum_ls cortex_a55, ERRATUM(1221012), CPU_REV(1, 0) + +check_erratum_chosen cortex_a55, ERRATUM(1530923), ERRATA_A55_1530923 + +/* erratum has no workaround in the cpu. Generic code must take care */ +add_erratum_entry cortex_a55, ERRATUM(1530923), ERRATA_A55_1530923, NO_APPLY_AT_RESET + +cpu_reset_func_start cortex_a55 +cpu_reset_func_end cortex_a55 + +errata_report_shim cortex_a55 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func cortex_a55_core_pwr_dwn + sysreg_bit_set CORTEX_A55_CPUPWRCTLR_EL1, CORTEX_A55_CORE_PWRDN_EN_MASK + isb + ret +endfunc cortex_a55_core_pwr_dwn + + /* --------------------------------------------- + * This function provides cortex_a55 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a55_regs, "aS" +cortex_a55_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a55_cpu_reg_dump + adr x6, cortex_a55_regs + mrs x8, CORTEX_A55_CPUECTLR_EL1 + ret +endfunc cortex_a55_cpu_reg_dump + +declare_cpu_ops cortex_a55, CORTEX_A55_MIDR, \ + cortex_a55_reset_func, \ + cortex_a55_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a57.S b/lib/cpus/aarch64/cortex_a57.S new file mode 100644 index 0000000..8fafaca --- /dev/null +++ b/lib/cpus/aarch64/cortex_a57.S @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2014-2023, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <common/bl_common.h> +#include <common/debug.h> +#include <cortex_a57.h> +#include <cpu_macros.S> +#include <plat_macros.S> + + /* --------------------------------------------- + * Disable L1 data cache and unified L2 cache + * --------------------------------------------- + */ +func cortex_a57_disable_dcache + sysreg_bit_clear sctlr_el3, SCTLR_C_BIT + isb + ret +endfunc cortex_a57_disable_dcache + + /* --------------------------------------------- + * Disable all types of L2 prefetches. + * --------------------------------------------- + */ +func cortex_a57_disable_l2_prefetch + mrs x0, CORTEX_A57_ECTLR_EL1 + orr x0, x0, #CORTEX_A57_ECTLR_DIS_TWD_ACC_PFTCH_BIT + mov x1, #CORTEX_A57_ECTLR_L2_IPFTCH_DIST_MASK + orr x1, x1, #CORTEX_A57_ECTLR_L2_DPFTCH_DIST_MASK + bic x0, x0, x1 + msr CORTEX_A57_ECTLR_EL1, x0 + isb + dsb ish + ret +endfunc cortex_a57_disable_l2_prefetch + + /* --------------------------------------------- + * Disable intra-cluster coherency + * --------------------------------------------- + */ +func cortex_a57_disable_smp + sysreg_bit_clear CORTEX_A57_ECTLR_EL1, CORTEX_A57_ECTLR_SMP_BIT + ret +endfunc cortex_a57_disable_smp + + /* --------------------------------------------- + * Disable debug interfaces + * --------------------------------------------- + */ +func cortex_a57_disable_ext_debug + mov x0, #1 + msr osdlr_el1, x0 + isb + + apply_erratum cortex_a57, ERRATUM(817169), ERRATA_A57_817169 + + dsb sy + ret +endfunc cortex_a57_disable_ext_debug + +/* + * Disable the over-read from the LDNP/STNP instruction. The SDEN doesn't + * provide and erratum number, so assign it an obvious 1 + */ +workaround_reset_start cortex_a57, ERRATUM(1), A57_DISABLE_NON_TEMPORAL_HINT + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_DIS_OVERREAD +workaround_reset_end cortex_a57, ERRATUM(1) + +check_erratum_ls cortex_a57, ERRATUM(1), CPU_REV(1, 2) + +workaround_reset_start cortex_a57, ERRATUM(806969), ERRATA_A57_806969 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_NO_ALLOC_WBWA +workaround_reset_end cortex_a57, ERRATUM(806969) + +check_erratum_ls cortex_a57, ERRATUM(806969), CPU_REV(0, 0) + +/* erratum always worked around, but report it correctly */ +check_erratum_ls cortex_a57, ERRATUM(813419), CPU_REV(0, 0) +add_erratum_entry cortex_a57, ERRATUM(813419), ERRATUM_ALWAYS_CHOSEN, NO_APPLY_AT_RESET + +workaround_reset_start cortex_a57, ERRATUM(813420), ERRATA_A57_813420 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_DCC_AS_DCCI +workaround_reset_end cortex_a57, ERRATUM(813420) + +check_erratum_ls cortex_a57, ERRATUM(813420), CPU_REV(0, 0) + +workaround_reset_start cortex_a57, ERRATUM(814670), ERRATA_A57_814670 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_DIS_DMB_NULLIFICATION +workaround_reset_end cortex_a57, ERRATUM(814670) + +check_erratum_ls cortex_a57, ERRATUM(814670), CPU_REV(0, 0) + +workaround_runtime_start cortex_a57, ERRATUM(817169), ERRATA_A57_817169, CORTEX_A57_MIDR + /* Invalidate any TLB address */ + mov x0, #0 + tlbi vae3, x0 +workaround_runtime_end cortex_a57, ERRATUM(817169), NO_ISB + +check_erratum_ls cortex_a57, ERRATUM(817169), CPU_REV(0, 1) + +workaround_reset_start cortex_a57, ERRATUM(826974), ERRATA_A57_826974 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_DIS_LOAD_PASS_DMB +workaround_reset_end cortex_a57, ERRATUM(826974) + +check_erratum_ls cortex_a57, ERRATUM(826974), CPU_REV(1, 1) + +workaround_reset_start cortex_a57, ERRATUM(826977), ERRATA_A57_826977 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_GRE_NGRE_AS_NGNRE +workaround_reset_end cortex_a57, ERRATUM(826977) + +check_erratum_ls cortex_a57, ERRATUM(826977), CPU_REV(1, 1) + +workaround_reset_start cortex_a57, ERRATUM(828024), ERRATA_A57_828024 + mrs x1, CORTEX_A57_CPUACTLR_EL1 + /* + * Setting the relevant bits in CPUACTLR_EL1 has to be done in 2 + * instructions here because the resulting bitmask doesn't fit in a + * 16-bit value so it cannot be encoded in a single instruction. + */ + orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_NO_ALLOC_WBWA + orr x1, x1, #(CORTEX_A57_CPUACTLR_EL1_DIS_L1_STREAMING | \ + CORTEX_A57_CPUACTLR_EL1_DIS_STREAMING) + msr CORTEX_A57_CPUACTLR_EL1, x1 +workaround_reset_end cortex_a57, ERRATUM(828024) + +check_erratum_ls cortex_a57, ERRATUM(828024), CPU_REV(1, 1) + +workaround_reset_start cortex_a57, ERRATUM(829520), ERRATA_A57_829520 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_DIS_INDIRECT_PREDICTOR +workaround_reset_end cortex_a57, ERRATUM(829520) + +check_erratum_ls cortex_a57, ERRATUM(829520), CPU_REV(1, 2) + +workaround_reset_start cortex_a57, ERRATUM(833471), ERRATA_A57_833471 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_FORCE_FPSCR_FLUSH +workaround_reset_end cortex_a57, ERRATUM(833471) + +check_erratum_ls cortex_a57, ERRATUM(833471), CPU_REV(1, 2) + +workaround_reset_start cortex_a57, ERRATUM(859972), ERRATA_A57_859972 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_DIS_INSTR_PREFETCH +workaround_reset_end cortex_a57, ERRATUM(859972) + +check_erratum_ls cortex_a57, ERRATUM(859972), CPU_REV(1, 3) + +check_erratum_chosen cortex_a57, ERRATUM(1319537), ERRATA_A57_1319537 +/* erratum has no workaround in the cpu. Generic code must take care */ +add_erratum_entry cortex_a57, ERRATUM(1319537), ERRATA_A57_1319537, NO_APPLY_AT_RESET + +workaround_reset_start cortex_a57, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 +#if IMAGE_BL31 + override_vector_table wa_cve_2017_5715_mmu_vbar +#endif +workaround_reset_end cortex_a57, CVE(2017, 5715) + +check_erratum_chosen cortex_a57, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 + +workaround_reset_start cortex_a57, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_DIS_LOAD_PASS_STORE + isb + dsb sy +workaround_reset_end cortex_a57, CVE(2018, 3639) + +check_erratum_chosen cortex_a57, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + +workaround_reset_start cortex_a57, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + override_vector_table wa_cve_2017_5715_mmu_vbar +#endif +workaround_reset_end cortex_a57, CVE(2022, 23960) + +check_erratum_chosen cortex_a57, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +cpu_reset_func_start cortex_a57 +#if A57_ENABLE_NONCACHEABLE_LOAD_FWD + /* Enable higher performance non-cacheable load forwarding */ + sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_EN_NC_LOAD_FWD +#endif + /* Enable the SMP bit. */ + sysreg_bit_set CORTEX_A57_ECTLR_EL1, CORTEX_A57_ECTLR_SMP_BIT +cpu_reset_func_end cortex_a57 + +func check_smccc_arch_workaround_3 + mov x0, #ERRATA_APPLIES + ret +endfunc check_smccc_arch_workaround_3 + + /* ---------------------------------------------------- + * The CPU Ops core power down function for Cortex-A57. + * ---------------------------------------------------- + */ +func cortex_a57_core_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a57_disable_dcache + + /* --------------------------------------------- + * Disable the L2 prefetches. + * --------------------------------------------- + */ + bl cortex_a57_disable_l2_prefetch + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + bl cortex_a57_disable_smp + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a57_disable_ext_debug +endfunc cortex_a57_core_pwr_dwn + + /* ------------------------------------------------------- + * The CPU Ops cluster power down function for Cortex-A57. + * ------------------------------------------------------- + */ +func cortex_a57_cluster_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a57_disable_dcache + + /* --------------------------------------------- + * Disable the L2 prefetches. + * --------------------------------------------- + */ + bl cortex_a57_disable_l2_prefetch + +#if !SKIP_A57_L1_FLUSH_PWR_DWN + /* ------------------------------------------------- + * Flush the L1 caches. + * ------------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 +#endif + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* ------------------------------------------------- + * Flush the L2 caches. + * ------------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + bl cortex_a57_disable_smp + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a57_disable_ext_debug +endfunc cortex_a57_cluster_pwr_dwn + +errata_report_shim cortex_a57 + + /* --------------------------------------------- + * This function provides cortex_a57 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a57_regs, "aS" +cortex_a57_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "cpumerrsr_el1", "l2merrsr_el1", "" + +func cortex_a57_cpu_reg_dump + adr x6, cortex_a57_regs + mrs x8, CORTEX_A57_ECTLR_EL1 + mrs x9, CORTEX_A57_MERRSR_EL1 + mrs x10, CORTEX_A57_L2MERRSR_EL1 + ret +endfunc cortex_a57_cpu_reg_dump + +declare_cpu_ops_wa cortex_a57, CORTEX_A57_MIDR, \ + cortex_a57_reset_func, \ + check_erratum_cortex_a57_5715, \ + CPU_NO_EXTRA2_FUNC, \ + check_smccc_arch_workaround_3, \ + cortex_a57_core_pwr_dwn, \ + cortex_a57_cluster_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a65.S b/lib/cpus/aarch64/cortex_a65.S new file mode 100644 index 0000000..666324c --- /dev/null +++ b/lib/cpus/aarch64/cortex_a65.S @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> + +#include <asm_macros.S> +#include <common/bl_common.h> +#include <common/debug.h> +#include <cortex_a65.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if !HW_ASSISTED_COHERENCY +#error "Cortex-A65 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS +#error "Cortex-A65 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +/* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A65. + * Shall clobber: x0-x19 + * ------------------------------------------------- + */ +func cortex_a65_reset_func + mov x19, x30 + +#if ERRATA_DSU_936184 + bl errata_dsu_936184_wa +#endif + + ret x19 +endfunc cortex_a65_reset_func + +func cortex_a65_cpu_pwr_dwn + mrs x0, CORTEX_A65_CPUPWRCTLR_EL1 + orr x0, x0, #CORTEX_A65_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + msr CORTEX_A65_CPUPWRCTLR_EL1, x0 + isb + ret +endfunc cortex_a65_cpu_pwr_dwn + +#if REPORT_ERRATA +/* + * Errata printing function for Cortex-A65. Must follow AAPCS. + */ +func cortex_a65_errata_report + stp x8, x30, [sp, #-16]! + + bl cpu_get_rev_var + mov x8, x0 + + /* + * Report all errata. The revision-variant information is passed to + * checking functions of each errata. + */ + report_errata ERRATA_DSU_936184, cortex_a65, dsu_936184 + + ldp x8, x30, [sp], #16 + ret +endfunc cortex_a65_errata_report +#endif + +.section .rodata.cortex_a65_regs, "aS" +cortex_a65_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a65_cpu_reg_dump + adr x6, cortex_a65_regs + mrs x8, CORTEX_A65_ECTLR_EL1 + ret +endfunc cortex_a65_cpu_reg_dump + +declare_cpu_ops cortex_a65, CORTEX_A65_MIDR, \ + cortex_a65_reset_func, \ + cortex_a65_cpu_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a65ae.S b/lib/cpus/aarch64/cortex_a65ae.S new file mode 100644 index 0000000..85d1894 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a65ae.S @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> + +#include <asm_macros.S> +#include <common/bl_common.h> +#include <common/debug.h> +#include <cortex_a65ae.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if !HW_ASSISTED_COHERENCY +#error "Cortex-A65AE must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS +#error "Cortex-A65AE supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + + /* + * ERRATA_DSU_936184 : + * The errata is defined in dsu_helpers.S but applies to cortex_a65ae + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_a65ae_936184, check_errata_dsu_936184 +.equ erratum_cortex_a65ae_936184_wa, errata_dsu_936184_wa +add_erratum_entry cortex_a65ae, ERRATUM(936184), ERRATA_DSU_936184, APPLY_AT_RESET + +cpu_reset_func_start cortex_a65ae +cpu_reset_func_end cortex_a65ae + +func cortex_a65ae_cpu_pwr_dwn + sysreg_bit_set CORTEX_A65AE_CPUPWRCTLR_EL1, CORTEX_A65AE_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc cortex_a65ae_cpu_pwr_dwn + +errata_report_shim cortex_a65ae + +.section .rodata.cortex_a65ae_regs, "aS" +cortex_a65ae_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a65ae_cpu_reg_dump + adr x6, cortex_a65ae_regs + mrs x8, CORTEX_A65AE_ECTLR_EL1 + ret +endfunc cortex_a65ae_cpu_reg_dump + +declare_cpu_ops cortex_a65ae, CORTEX_A65AE_MIDR, \ + cortex_a65ae_reset_func, \ + cortex_a65ae_cpu_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a710.S b/lib/cpus/aarch64/cortex_a710.S new file mode 100644 index 0000000..f3931d7 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a710.S @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a710.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex A710 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex A710 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A710_BHB_LOOP_COUNT, cortex_a710 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_a710, ERRATUM(1987031), ERRATA_A710_1987031 + ldr x0,=0x6 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xF3A08002 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFF0F7FE + msr S3_6_c15_c8_3,x0 + ldr x0,=0x40000001003ff + msr S3_6_c15_c8_1,x0 + ldr x0,=0x7 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xBF200000 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFEF0000 + msr S3_6_c15_c8_3,x0 + ldr x0,=0x40000001003f3 + msr S3_6_c15_c8_1,x0 +workaround_reset_end cortex_a710, ERRATUM(1987031) + +check_erratum_ls cortex_a710, ERRATUM(1987031), CPU_REV(2, 0) + +workaround_runtime_start cortex_a710, ERRATUM(2008768), ERRATA_A710_2008768 + /* Stash ERRSELR_EL1 in x2 */ + mrs x2, ERRSELR_EL1 + + /* Select error record 0 and clear ED bit */ + msr ERRSELR_EL1, xzr + mrs x1, ERXCTLR_EL1 + bfi x1, xzr, #ERXCTLR_ED_SHIFT, #1 + msr ERXCTLR_EL1, x1 + + /* Select error record 1 and clear ED bit */ + mov x0, #1 + msr ERRSELR_EL1, x0 + mrs x1, ERXCTLR_EL1 + bfi x1, xzr, #ERXCTLR_ED_SHIFT, #1 + msr ERXCTLR_EL1, x1 + + /* Restore ERRSELR_EL1 from x2 */ + msr ERRSELR_EL1, x2 +workaround_runtime_end cortex_a710, ERRATUM(2008768), NO_ISB + +check_erratum_ls cortex_a710, ERRATUM(2008768), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2017096), ERRATA_A710_2017096 + sysreg_bit_set CORTEX_A710_CPUECTLR_EL1, CORTEX_A710_CPUECTLR_EL1_PFSTIDIS_BIT +workaround_reset_end cortex_a710, ERRATUM(2017096) + +check_erratum_ls cortex_a710, ERRATUM(2017096), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2055002), ERRATA_A710_2055002 + sysreg_bit_set CORTEX_A710_CPUACTLR_EL1, CORTEX_A710_CPUACTLR_EL1_BIT_46 +workaround_reset_end cortex_a710, ERRATUM(2055002) + +check_erratum_range cortex_a710, ERRATUM(2055002), CPU_REV(1, 0), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2058056), ERRATA_A710_2058056 + sysreg_bitfield_insert CORTEX_A710_CPUECTLR2_EL1, CORTEX_A710_CPUECTLR2_EL1_PF_MODE_CNSRV, \ + CPUECTLR2_EL1_PF_MODE_LSB, CPUECTLR2_EL1_PF_MODE_WIDTH +workaround_reset_end cortex_a710, ERRATUM(2058056) + +check_erratum_ls cortex_a710, ERRATUM(2058056), CPU_REV(2, 1) + +workaround_reset_start cortex_a710, ERRATUM(2081180), ERRATA_A710_2081180 + ldr x0,=0x3 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xF3A08002 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFF0F7FE + msr S3_6_c15_c8_3,x0 + ldr x0,=0x10002001003FF + msr S3_6_c15_c8_1,x0 + ldr x0,=0x4 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xBF200000 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFEF0000 + msr S3_6_c15_c8_3,x0 + ldr x0,=0x10002001003F3 + msr S3_6_c15_c8_1,x0 +workaround_reset_end cortex_a710, ERRATUM(2081180) + +check_erratum_ls cortex_a710, ERRATUM(2081180), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2083908), ERRATA_A710_2083908 + sysreg_bit_set CORTEX_A710_CPUACTLR5_EL1, CORTEX_A710_CPUACTLR5_EL1_BIT_13 +workaround_reset_end cortex_a710, ERRATUM(2083908) + +check_erratum_range cortex_a710, ERRATUM(2083908), CPU_REV(2, 0), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2136059), ERRATA_A710_2136059 + sysreg_bit_set CORTEX_A710_CPUACTLR5_EL1, CORTEX_A710_CPUACTLR5_EL1_BIT_44 +workaround_reset_end cortex_a710, ERRATUM(2136059) + +check_erratum_ls cortex_a710, ERRATUM(2136059), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2147715), ERRATA_A710_2147715 + sysreg_bit_set CORTEX_A710_CPUACTLR_EL1, CORTEX_A710_CPUACTLR_EL1_BIT_22 +workaround_reset_end cortex_a710, ERRATUM(2147715) + +check_erratum_range cortex_a710, ERRATUM(2147715), CPU_REV(2, 0), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2216384), ERRATA_A710_2216384 + sysreg_bit_set CORTEX_A710_CPUACTLR5_EL1, CORTEX_A710_CPUACTLR5_EL1_BIT_17 + + ldr x0,=0x5 + msr CORTEX_A710_CPUPSELR_EL3, x0 + ldr x0,=0x10F600E000 + msr CORTEX_A710_CPUPOR_EL3, x0 + ldr x0,=0x10FF80E000 + msr CORTEX_A710_CPUPMR_EL3, x0 + ldr x0,=0x80000000003FF + msr CORTEX_A710_CPUPCR_EL3, x0 +workaround_reset_end cortex_a710, ERRATUM(2216384) + +check_erratum_ls cortex_a710, ERRATUM(2216384), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2267065), ERRATA_A710_2267065 + sysreg_bit_set CORTEX_A710_CPUACTLR_EL1, CORTEX_A710_CPUACTLR_EL1_BIT_22 +workaround_reset_end cortex_a710, ERRATUM(2267065) + +check_erratum_ls cortex_a710, ERRATUM(2267065), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2282622), ERRATA_A710_2282622 + sysreg_bit_set CORTEX_A710_CPUACTLR2_EL1, BIT(0) +workaround_reset_end cortex_a710, ERRATUM(2282622) + +check_erratum_ls cortex_a710, ERRATUM(2282622), CPU_REV(2, 1) + +workaround_runtime_start cortex_a710, ERRATUM(2291219), ERRATA_A710_2291219 + /* Set bit 36 in ACTLR2_EL1 */ + sysreg_bit_set CORTEX_A710_CPUACTLR2_EL1, CORTEX_A710_CPUACTLR2_EL1_BIT_36 +workaround_runtime_end cortex_a710, ERRATUM(2291219), NO_ISB + +check_erratum_ls cortex_a710, ERRATUM(2291219), CPU_REV(2, 0) + +/* + * ERRATA_DSU_2313941 is defined in dsu_helpers.S but applies to Cortex-A710 as + * well. Create a symbollic link to existing errata workaround to get them + * registered under the Errata Framework. + */ +.equ check_erratum_cortex_a710_2313941, check_errata_dsu_2313941 +.equ erratum_cortex_a710_2313941_wa, errata_dsu_2313941_wa +add_erratum_entry cortex_a710, ERRATUM(2313941), ERRATA_DSU_2313941, APPLY_AT_RESET + +workaround_reset_start cortex_a710, ERRATUM(2371105), ERRATA_A710_2371105 + /* Set bit 40 in CPUACTLR2_EL1 */ + sysreg_bit_set CORTEX_A710_CPUACTLR2_EL1, CORTEX_A710_CPUACTLR2_EL1_BIT_40 +workaround_reset_end cortex_a710, ERRATUM(2371105) + +check_erratum_ls cortex_a710, ERRATUM(2371105), CPU_REV(2, 0) + +workaround_reset_start cortex_a710, ERRATUM(2742423), ERRATA_A710_2742423 + /* Set CPUACTLR5_EL1[56:55] to 2'b01 */ + sysreg_bit_set CORTEX_A710_CPUACTLR5_EL1, BIT(55) + sysreg_bit_clear CORTEX_A710_CPUACTLR5_EL1, BIT(56) +workaround_reset_end cortex_a710, ERRATUM(2742423) + +check_erratum_ls cortex_a710, ERRATUM(2742423), CPU_REV(2, 1) + +workaround_runtime_start cortex_a710, ERRATUM(2768515), ERRATA_A710_2768515 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end cortex_a710, ERRATUM(2768515), NO_ISB + +check_erratum_ls cortex_a710, ERRATUM(2768515), CPU_REV(2, 1) + +workaround_reset_start cortex_a710, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-A710 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_a710 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a710, CVE(2022, 23960) + +check_erratum_chosen cortex_a710, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_a710_core_pwr_dwn + apply_erratum cortex_a710, ERRATUM(2008768), ERRATA_A710_2008768 + apply_erratum cortex_a710, ERRATUM(2291219), ERRATA_A710_2291219, NO_GET_CPU_REV + + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_A710_CPUPWRCTLR_EL1, CORTEX_A710_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + apply_erratum cortex_a710, ERRATUM(2768515), ERRATA_A710_2768515, NO_GET_CPU_REV + isb + ret +endfunc cortex_a710_core_pwr_dwn + +errata_report_shim cortex_a710 + +cpu_reset_func_start cortex_a710 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_a710 + + /* --------------------------------------------- + * This function provides Cortex-A710 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a710_regs, "aS" +cortex_a710_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a710_cpu_reg_dump + adr x6, cortex_a710_regs + mrs x8, CORTEX_A710_CPUECTLR_EL1 + ret +endfunc cortex_a710_cpu_reg_dump + +declare_cpu_ops cortex_a710, CORTEX_A710_MIDR, \ + cortex_a710_reset_func, \ + cortex_a710_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a715.S b/lib/cpus/aarch64/cortex_a715.S new file mode 100644 index 0000000..dd4c307 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a715.S @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a715.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-A715 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex-A715 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A715_BHB_LOOP_COUNT, cortex_a715 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_a715, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-A715 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_a715 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a715, CVE(2022, 23960) + +check_erratum_chosen cortex_a715, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +cpu_reset_func_start cortex_a715 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_a715 + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_a715_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + mrs x0, CORTEX_A715_CPUPWRCTLR_EL1 + orr x0, x0, #CORTEX_A715_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + msr CORTEX_A715_CPUPWRCTLR_EL1, x0 + isb + ret +endfunc cortex_a715_core_pwr_dwn + +errata_report_shim cortex_a715 + + /* --------------------------------------------- + * This function provides Cortex-A715 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a715_regs, "aS" +cortex_a715_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a715_cpu_reg_dump + adr x6, cortex_a715_regs + mrs x8, CORTEX_A715_CPUECTLR_EL1 + ret +endfunc cortex_a715_cpu_reg_dump + +declare_cpu_ops cortex_a715, CORTEX_A715_MIDR, \ + cortex_a715_reset_func, \ + cortex_a715_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a72.S b/lib/cpus/aarch64/cortex_a72.S new file mode 100644 index 0000000..997f261 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a72.S @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2015-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <cortex_a72.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A72_BHB_LOOP_COUNT, cortex_a72 +#endif /* WORKAROUND_CVE_2022_23960 */ + + /* --------------------------------------------- + * Disable L1 data cache and unified L2 cache + * --------------------------------------------- + */ +func cortex_a72_disable_dcache + mrs x1, sctlr_el3 + bic x1, x1, #SCTLR_C_BIT + msr sctlr_el3, x1 + isb + ret +endfunc cortex_a72_disable_dcache + + /* --------------------------------------------- + * Disable all types of L2 prefetches. + * --------------------------------------------- + */ +func cortex_a72_disable_l2_prefetch + mrs x0, CORTEX_A72_ECTLR_EL1 + orr x0, x0, #CORTEX_A72_ECTLR_DIS_TWD_ACC_PFTCH_BIT + mov x1, #CORTEX_A72_ECTLR_L2_IPFTCH_DIST_MASK + orr x1, x1, #CORTEX_A72_ECTLR_L2_DPFTCH_DIST_MASK + bic x0, x0, x1 + msr CORTEX_A72_ECTLR_EL1, x0 + isb + ret +endfunc cortex_a72_disable_l2_prefetch + + /* --------------------------------------------- + * Disable the load-store hardware prefetcher. + * --------------------------------------------- + */ +func cortex_a72_disable_hw_prefetcher + sysreg_bit_set CORTEX_A72_CPUACTLR_EL1, CORTEX_A72_CPUACTLR_EL1_DISABLE_L1_DCACHE_HW_PFTCH + isb + dsb ish + ret +endfunc cortex_a72_disable_hw_prefetcher + + /* --------------------------------------------- + * Disable intra-cluster coherency + * --------------------------------------------- + */ +func cortex_a72_disable_smp + sysreg_bit_clear CORTEX_A72_ECTLR_EL1, CORTEX_A72_ECTLR_SMP_BIT + ret +endfunc cortex_a72_disable_smp + + /* --------------------------------------------- + * Disable debug interfaces + * --------------------------------------------- + */ +func cortex_a72_disable_ext_debug + mov x0, #1 + msr osdlr_el1, x0 + isb + dsb sy + ret +endfunc cortex_a72_disable_ext_debug + +func check_smccc_arch_workaround_3 + cpu_check_csv2 x0, 1f + mov x0, #ERRATA_APPLIES + ret +1: + mov x0, #ERRATA_NOT_APPLIES + ret +endfunc check_smccc_arch_workaround_3 + +workaround_reset_start cortex_a72, ERRATUM(859971), ERRATA_A72_859971 + sysreg_bit_set CORTEX_A72_CPUACTLR_EL1, CORTEX_A72_CPUACTLR_EL1_DIS_INSTR_PREFETCH +workaround_reset_end cortex_a72, ERRATUM(859971) + +check_erratum_ls cortex_a72, ERRATUM(859971), CPU_REV(0, 3) + +/* Due to the nature of the errata it is applied unconditionally when chosen */ +check_erratum_chosen cortex_a72, ERRATUM(1319367), ERRATA_A72_1319367 +/* erratum workaround is interleaved with generic code */ +add_erratum_entry cortex_a72, ERRATUM(1319367), ERRATA_A72_1319367, NO_APPLY_AT_RESET + +workaround_reset_start cortex_a72, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 +#if IMAGE_BL31 + override_vector_table wa_cve_2017_5715_mmu_vbar +#endif +workaround_reset_end cortex_a72, CVE(2017, 5715) + +check_erratum_custom_start cortex_a72, CVE(2017, 5715) + cpu_check_csv2 x0, 1f +#if WORKAROUND_CVE_2017_5715 + mov x0, #ERRATA_APPLIES +#else + mov x0, #ERRATA_MISSING +#endif + ret +1: + mov x0, #ERRATA_NOT_APPLIES + ret +check_erratum_custom_end cortex_a72, CVE(2017, 5715) + +workaround_reset_start cortex_a72, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + sysreg_bit_set CORTEX_A72_CPUACTLR_EL1, CORTEX_A72_CPUACTLR_EL1_DIS_LOAD_PASS_STORE + isb + dsb sy +workaround_reset_end cortex_a72, CVE(2018, 3639) +check_erratum_chosen cortex_a72, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + +workaround_reset_start cortex_a72, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* Skip installing vector table again if already done for CVE(2017, 5715) */ + /* + * The Cortex-A72 generic vectors are overridden to apply the + * mitigation on exception entry from lower ELs for revisions >= r1p0 + * which has CSV2 implemented. + */ + adr x0, wa_cve_vbar_cortex_a72 + mrs x1, vbar_el3 + cmp x0, x1 + b.eq 1f + msr vbar_el3, x0 +1: +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a72, CVE(2022, 23960) + +check_erratum_custom_start cortex_a72, CVE(2022, 23960) +#if WORKAROUND_CVE_2017_5715 || WORKAROUND_CVE_2022_23960 + cpu_check_csv2 x0, 1f + mov x0, #ERRATA_APPLIES + ret +1: +#if WORKAROUND_CVE_2022_23960 + mov x0, #ERRATA_APPLIES +#else + mov x0, #ERRATA_MISSING +#endif /* WORKAROUND_CVE_2022_23960 */ + ret +#endif /* WORKAROUND_CVE_2017_5715 || WORKAROUND_CVE_2022_23960 */ + mov x0, #ERRATA_MISSING + ret +check_erratum_custom_end cortex_a72, CVE(2022, 23960) + +cpu_reset_func_start cortex_a72 + + /* --------------------------------------------- + * Enable the SMP bit. + * --------------------------------------------- + */ + sysreg_bit_set CORTEX_A72_ECTLR_EL1, CORTEX_A72_ECTLR_SMP_BIT + +cpu_reset_func_end cortex_a72 + + /* ---------------------------------------------------- + * The CPU Ops core power down function for Cortex-A72. + * ---------------------------------------------------- + */ +func cortex_a72_core_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a72_disable_dcache + + /* --------------------------------------------- + * Disable the L2 prefetches. + * --------------------------------------------- + */ + bl cortex_a72_disable_l2_prefetch + + /* --------------------------------------------- + * Disable the load-store hardware prefetcher. + * --------------------------------------------- + */ + bl cortex_a72_disable_hw_prefetcher + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + bl cortex_a72_disable_smp + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a72_disable_ext_debug +endfunc cortex_a72_core_pwr_dwn + + /* ------------------------------------------------------- + * The CPU Ops cluster power down function for Cortex-A72. + * ------------------------------------------------------- + */ +func cortex_a72_cluster_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a72_disable_dcache + + /* --------------------------------------------- + * Disable the L2 prefetches. + * --------------------------------------------- + */ + bl cortex_a72_disable_l2_prefetch + + /* --------------------------------------------- + * Disable the load-store hardware prefetcher. + * --------------------------------------------- + */ + bl cortex_a72_disable_hw_prefetcher + +#if !SKIP_A72_L1_FLUSH_PWR_DWN + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 +#endif + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* ------------------------------------------------- + * Flush the L2 caches. + * ------------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + bl cortex_a72_disable_smp + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a72_disable_ext_debug +endfunc cortex_a72_cluster_pwr_dwn + +errata_report_shim cortex_a72 + + /* --------------------------------------------- + * This function provides cortex_a72 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a72_regs, "aS" +cortex_a72_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "cpumerrsr_el1", "l2merrsr_el1", "" + +func cortex_a72_cpu_reg_dump + adr x6, cortex_a72_regs + mrs x8, CORTEX_A72_ECTLR_EL1 + mrs x9, CORTEX_A72_MERRSR_EL1 + mrs x10, CORTEX_A72_L2MERRSR_EL1 + ret +endfunc cortex_a72_cpu_reg_dump + +declare_cpu_ops_wa cortex_a72, CORTEX_A72_MIDR, \ + cortex_a72_reset_func, \ + check_erratum_cortex_a72_5715, \ + CPU_NO_EXTRA2_FUNC, \ + check_smccc_arch_workaround_3, \ + cortex_a72_core_pwr_dwn, \ + cortex_a72_cluster_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a720.S b/lib/cpus/aarch64/cortex_a720.S new file mode 100644 index 0000000..4b28fdb --- /dev/null +++ b/lib/cpus/aarch64/cortex_a720.S @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a720.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex A720 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex A720 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A720_BHB_LOOP_COUNT, cortex_a720 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_a720, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex A720 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_a720 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a720, CVE(2022, 23960) + +check_erratum_chosen cortex_a720, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +cpu_reset_func_start cortex_a720 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_a720 + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_a720_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_A720_CPUPWRCTLR_EL1, CORTEX_A720_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + + isb + ret +endfunc cortex_a720_core_pwr_dwn + +errata_report_shim cortex_a720 + + /* --------------------------------------------- + * This function provides Cortex A720-specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a720_regs, "aS" +cortex_a720_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a720_cpu_reg_dump + adr x6, cortex_a720_regs + mrs x8, CORTEX_A720_CPUECTLR_EL1 + ret +endfunc cortex_a720_cpu_reg_dump + +declare_cpu_ops cortex_a720, CORTEX_A720_MIDR, \ + cortex_a720_reset_func, \ + cortex_a720_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a73.S b/lib/cpus/aarch64/cortex_a73.S new file mode 100644 index 0000000..3a6b922 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a73.S @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a73.h> +#include <cpu_macros.S> +#include <plat_macros.S> + + /* --------------------------------------------- + * Disable L1 data cache + * --------------------------------------------- + */ +func cortex_a73_disable_dcache + sysreg_bit_clear sctlr_el3, SCTLR_C_BIT + isb + ret +endfunc cortex_a73_disable_dcache + + /* --------------------------------------------- + * Disable intra-cluster coherency + * --------------------------------------------- + */ +func cortex_a73_disable_smp + sysreg_bit_clear CORTEX_A73_CPUECTLR_EL1, CORTEX_A73_CPUECTLR_SMP_BIT + isb + dsb sy + ret +endfunc cortex_a73_disable_smp + +func check_smccc_arch_workaround_3 + mov x0, #ERRATA_APPLIES + ret +endfunc check_smccc_arch_workaround_3 + +workaround_reset_start cortex_a73, ERRATUM(852427), ERRATA_A73_852427 + sysreg_bit_set CORTEX_A73_DIAGNOSTIC_REGISTER, BIT(12) +workaround_reset_end cortex_a73, ERRATUM(852427) + +check_erratum_ls cortex_a73, ERRATUM(852427), CPU_REV(0, 0) + +workaround_reset_start cortex_a73, ERRATUM(855423), ERRATA_A73_855423 + sysreg_bit_set CORTEX_A73_IMP_DEF_REG2, BIT(7) +workaround_reset_end cortex_a73, ERRATUM(855423) + +check_erratum_ls cortex_a73, ERRATUM(855423), CPU_REV(0, 1) + +workaround_reset_start cortex_a73, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 +#if IMAGE_BL31 + override_vector_table wa_cve_2017_5715_bpiall_vbar +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a73, CVE(2017, 5715) + +check_erratum_custom_start cortex_a73, CVE(2017, 5715) + cpu_check_csv2 x0, 1f +#if WORKAROUND_CVE_2017_5715 + mov x0, #ERRATA_APPLIES +#else + mov x0, #ERRATA_MISSING +#endif + ret +1: + mov x0, #ERRATA_NOT_APPLIES + ret +check_erratum_custom_end cortex_a73, CVE(2017, 5715) + +workaround_reset_start cortex_a73, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + sysreg_bit_set CORTEX_A73_IMP_DEF_REG1, CORTEX_A73_IMP_DEF_REG1_DISABLE_LOAD_PASS_STORE +workaround_reset_end cortex_a73, CVE(2018, 3639) + +check_erratum_chosen cortex_a73, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + +workaround_reset_start cortex_a73, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* Skip installing vector table again for CVE_2022_23960 */ + adr x0, wa_cve_2017_5715_bpiall_vbar + mrs x1, vbar_el3 + + cmp x0, x1 + b.eq 1f + msr vbar_el3, x0 +1: +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a73, CVE(2022, 23960) + +check_erratum_custom_start cortex_a73, CVE(2022, 23960) +#if WORKAROUND_CVE_2017_5715 || WORKAROUND_CVE_2022_23960 + cpu_check_csv2 x0, 1f + mov x0, #ERRATA_APPLIES + ret + 1: +#if WORKAROUND_CVE_2022_23960 + mov x0, #ERRATA_APPLIES +#else + mov x0, #ERRATA_MISSING +#endif /* WORKAROUND_CVE_2022_23960 */ + ret +#endif /* WORKAROUND_CVE_2017_5715 || WORKAROUND_CVE_2022_23960 */ + mov x0, #ERRATA_MISSING + ret +check_erratum_custom_end cortex_a73, CVE(2022, 23960) + + /* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A73. + * ------------------------------------------------- + */ + +cpu_reset_func_start cortex_a73 + /* --------------------------------------------- + * Enable the SMP bit. + * Clobbers : x0 + * --------------------------------------------- + */ + sysreg_bit_set CORTEX_A73_CPUECTLR_EL1, CORTEX_A73_CPUECTLR_SMP_BIT +cpu_reset_func_end cortex_a73 + +func cortex_a73_core_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a73_disable_dcache + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a73_disable_smp +endfunc cortex_a73_core_pwr_dwn + +func cortex_a73_cluster_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl cortex_a73_disable_dcache + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* --------------------------------------------- + * Flush L2 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level2 + + /* --------------------------------------------- + * Come out of intra cluster coherency + * --------------------------------------------- + */ + mov x30, x18 + b cortex_a73_disable_smp +endfunc cortex_a73_cluster_pwr_dwn + + +errata_report_shim cortex_a73 + + /* --------------------------------------------- + * This function provides cortex_a73 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a73_regs, "aS" +cortex_a73_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "l2merrsr_el1", "" + +func cortex_a73_cpu_reg_dump + adr x6, cortex_a73_regs + mrs x8, CORTEX_A73_CPUECTLR_EL1 + mrs x9, CORTEX_A73_L2MERRSR_EL1 + ret +endfunc cortex_a73_cpu_reg_dump + +declare_cpu_ops_wa cortex_a73, CORTEX_A73_MIDR, \ + cortex_a73_reset_func, \ + check_erratum_cortex_a73_5715, \ + CPU_NO_EXTRA2_FUNC, \ + check_smccc_arch_workaround_3, \ + cortex_a73_core_pwr_dwn, \ + cortex_a73_cluster_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a75.S b/lib/cpus/aarch64/cortex_a75.S new file mode 100644 index 0000000..c90be67 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a75.S @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <cortex_a75.h> +#include <cpuamu.h> +#include <cpu_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-A75 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +workaround_reset_start cortex_a75, ERRATUM(764081), ERRATA_A75_764081 + sysreg_bit_set sctlr_el3, SCTLR_IESB_BIT +workaround_reset_end cortex_a75, ERRATUM(764081) + +check_erratum_ls cortex_a75, ERRATUM(764081), CPU_REV(0, 0) + +workaround_reset_start cortex_a75, ERRATUM(790748), ERRATA_A75_790748 + sysreg_bit_set CORTEX_A75_CPUACTLR_EL1, (1 << 13) +workaround_reset_end cortex_a75, ERRATUM(790748) + +check_erratum_ls cortex_a75, ERRATUM(790748), CPU_REV(0, 0) + +/* ERRATA_DSU_798953 : + * The errata is defined in dsu_helpers.S but applies to cortex_a75 + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_a75_798953, check_errata_dsu_798953 +.equ erratum_cortex_a75_798953_wa, errata_dsu_798953_wa +add_erratum_entry cortex_a75, ERRATUM(798953), ERRATA_DSU_798953, APPLY_AT_RESET + +/* ERRATA_DSU_936184 : + * The errata is defined in dsu_helpers.S but applies to cortex_a75 + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_a75_936184, check_errata_dsu_936184 +.equ erratum_cortex_a75_936184_wa, errata_dsu_936184_wa +add_erratum_entry cortex_a75, ERRATUM(936184), ERRATA_DSU_936184, APPLY_AT_RESET + +workaround_reset_start cortex_a75, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 +#if IMAGE_BL31 + override_vector_table wa_cve_2017_5715_bpiall_vbar +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a75, CVE(2017, 5715) + +check_erratum_custom_start cortex_a75, CVE(2017, 5715) + cpu_check_csv2 x0, 1f +#if WORKAROUND_CVE_2017_5715 + mov x0, #ERRATA_APPLIES +#else + mov x0, #ERRATA_MISSING +#endif + ret +1: + mov x0, #ERRATA_NOT_APPLIES + ret +check_erratum_custom_end cortex_a75, CVE(2017, 5715) + +workaround_reset_start cortex_a75, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + sysreg_bit_set CORTEX_A75_CPUACTLR_EL1, CORTEX_A75_CPUACTLR_EL1_DISABLE_LOAD_PASS_STORE +workaround_reset_end cortex_a75, CVE(2018, 3639) + +check_erratum_chosen cortex_a75, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + +workaround_reset_start cortex_a75, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* Skip installing vector table again if already done for CVE(2017, 5715) */ + adr x0, wa_cve_2017_5715_bpiall_vbar + mrs x1, vbar_el3 + cmp x0, x1 + b.eq 1f + msr vbar_el3, x0 +1: +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a75, CVE(2022, 23960) + +check_erratum_custom_start cortex_a75, CVE(2022, 23960) +#if WORKAROUND_CVE_2017_5715 || WORKAROUND_CVE_2022_23960 + cpu_check_csv2 x0, 1f + mov x0, #ERRATA_APPLIES + ret +1: +# if WORKAROUND_CVE_2022_23960 + mov x0, #ERRATA_APPLIES +# else + mov x0, #ERRATA_MISSING +# endif /* WORKAROUND_CVE_2022_23960 */ + ret +#endif /* WORKAROUND_CVE_2017_5715 || WORKAROUND_CVE_2022_23960 */ + mov x0, #ERRATA_MISSING + ret +check_erratum_custom_end cortex_a75, CVE(2022, 23960) + + /* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A75. + * ------------------------------------------------- + */ + +cpu_reset_func_start cortex_a75 +#if ENABLE_FEAT_AMU + /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ + sysreg_bit_set actlr_el3, CORTEX_A75_ACTLR_AMEN_BIT + isb + + /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ + sysreg_bit_set actlr_el2, CORTEX_A75_ACTLR_AMEN_BIT + isb + + /* Enable group0 counters */ + mov x0, #CORTEX_A75_AMU_GROUP0_MASK + msr CPUAMCNTENSET_EL0, x0 + isb + + /* Enable group1 counters */ + mov x0, #CORTEX_A75_AMU_GROUP1_MASK + msr CPUAMCNTENSET_EL0, x0 + /* isb included in cpu_reset_func_end macro */ +#endif +cpu_reset_func_end cortex_a75 + +func check_smccc_arch_workaround_3 + mov x0, #ERRATA_APPLIES + ret +endfunc check_smccc_arch_workaround_3 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func cortex_a75_core_pwr_dwn + /* --------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------- + */ + sysreg_bit_set CORTEX_A75_CPUPWRCTLR_EL1, \ + CORTEX_A75_CORE_PWRDN_EN_MASK + isb + ret +endfunc cortex_a75_core_pwr_dwn + +errata_report_shim cortex_a75 + + /* --------------------------------------------- + * This function provides cortex_a75 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a75_regs, "aS" +cortex_a75_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a75_cpu_reg_dump + adr x6, cortex_a75_regs + mrs x8, CORTEX_A75_CPUECTLR_EL1 + ret +endfunc cortex_a75_cpu_reg_dump + +declare_cpu_ops_wa cortex_a75, CORTEX_A75_MIDR, \ + cortex_a75_reset_func, \ + check_erratum_cortex_a75_5715, \ + CPU_NO_EXTRA2_FUNC, \ + check_smccc_arch_workaround_3, \ + cortex_a75_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a75_pubsub.c b/lib/cpus/aarch64/cortex_a75_pubsub.c new file mode 100644 index 0000000..11190c8 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a75_pubsub.c @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2017-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <cortex_a75.h> +#include <cpuamu.h> +#include <lib/el3_runtime/pubsub_events.h> + +static void *cortex_a75_context_save(const void *arg) +{ + if (midr_match(CORTEX_A75_MIDR) != 0) + cpuamu_context_save(CORTEX_A75_AMU_NR_COUNTERS); + + return (void *)0; +} + +static void *cortex_a75_context_restore(const void *arg) +{ + if (midr_match(CORTEX_A75_MIDR) != 0) + cpuamu_context_restore(CORTEX_A75_AMU_NR_COUNTERS); + + return (void *)0; +} + +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, cortex_a75_context_save); +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, cortex_a75_context_restore); diff --git a/lib/cpus/aarch64/cortex_a76.S b/lib/cpus/aarch64/cortex_a76.S new file mode 100644 index 0000000..8b3d730 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a76.S @@ -0,0 +1,540 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a76.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include <services/arm_arch_svc.h> +#include "wa_cve_2022_23960_bhb.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-A76 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + .globl cortex_a76_reset_func + .globl cortex_a76_core_pwr_dwn + .globl cortex_a76_disable_wa_cve_2018_3639 + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex-A76 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#define ESR_EL3_A64_SMC0 0x5e000000 +#define ESR_EL3_A32_SMC0 0x4e000000 + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + /* + * This macro applies the mitigation for CVE-2018-3639. + * It implements a fast path where `SMCCC_ARCH_WORKAROUND_2` + * SMC calls from a lower EL running in AArch32 or AArch64 + * will go through the fast and return early. + * + * The macro saves x2-x3 to the context. In the fast path + * x0-x3 registers do not need to be restored as the calling + * context will have saved them. The macro also saves + * x29-x30 to the context in the sync_exception path. + */ + .macro apply_cve_2018_3639_wa _is_sync_exception _esr_el3_val + stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + .if \_is_sync_exception + stp x29, x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X29] + mov_imm w2, \_esr_el3_val + bl apply_cve_2018_3639_sync_wa + ldp x29, x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X29] + .endif + /* + * Always enable v4 mitigation during EL3 execution. This is not + * required for the fast path above because it does not perform any + * memory loads. + */ + mrs x2, CORTEX_A76_CPUACTLR2_EL1 + orr x2, x2, #CORTEX_A76_CPUACTLR2_EL1_DISABLE_LOAD_PASS_STORE + msr CORTEX_A76_CPUACTLR2_EL1, x2 + isb + + /* + * The caller may have passed arguments to EL3 via x2-x3. + * Restore these registers from the context before jumping to the + * main runtime vector table entry. + */ + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + .endm +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 || WORKAROUND_CVE_2022_23960 +vector_base cortex_a76_wa_cve_vbar + + /* --------------------------------------------------------------------- + * Current EL with SP_EL0 : 0x0 - 0x200 + * --------------------------------------------------------------------- + */ +vector_entry cortex_a76_sync_exception_sp_el0 + b sync_exception_sp_el0 +end_vector_entry cortex_a76_sync_exception_sp_el0 + +vector_entry cortex_a76_irq_sp_el0 + b irq_sp_el0 +end_vector_entry cortex_a76_irq_sp_el0 + +vector_entry cortex_a76_fiq_sp_el0 + b fiq_sp_el0 +end_vector_entry cortex_a76_fiq_sp_el0 + +vector_entry cortex_a76_serror_sp_el0 + b serror_sp_el0 +end_vector_entry cortex_a76_serror_sp_el0 + + /* --------------------------------------------------------------------- + * Current EL with SP_ELx: 0x200 - 0x400 + * --------------------------------------------------------------------- + */ +vector_entry cortex_a76_sync_exception_sp_elx + b sync_exception_sp_elx +end_vector_entry cortex_a76_sync_exception_sp_elx + +vector_entry cortex_a76_irq_sp_elx + b irq_sp_elx +end_vector_entry cortex_a76_irq_sp_elx + +vector_entry cortex_a76_fiq_sp_elx + b fiq_sp_elx +end_vector_entry cortex_a76_fiq_sp_elx + +vector_entry cortex_a76_serror_sp_elx + b serror_sp_elx +end_vector_entry cortex_a76_serror_sp_elx + + /* --------------------------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x600 + * --------------------------------------------------------------------- + */ +vector_entry cortex_a76_sync_exception_aarch64 + +#if WORKAROUND_CVE_2022_23960 + apply_cve_2022_23960_bhb_wa CORTEX_A76_BHB_LOOP_COUNT +#endif /* WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + apply_cve_2018_3639_wa _is_sync_exception=1 _esr_el3_val=ESR_EL3_A64_SMC0 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639*/ + + b sync_exception_aarch64 +end_vector_entry cortex_a76_sync_exception_aarch64 + +vector_entry cortex_a76_irq_aarch64 + +#if WORKAROUND_CVE_2022_23960 + apply_cve_2022_23960_bhb_wa CORTEX_A76_BHB_LOOP_COUNT +#endif /* WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + apply_cve_2018_3639_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A64_SMC0 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639*/ + + b irq_aarch64 +end_vector_entry cortex_a76_irq_aarch64 + +vector_entry cortex_a76_fiq_aarch64 + +#if WORKAROUND_CVE_2022_23960 + apply_cve_2022_23960_bhb_wa CORTEX_A76_BHB_LOOP_COUNT +#endif /* WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + apply_cve_2018_3639_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A64_SMC0 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639*/ + + b fiq_aarch64 +end_vector_entry cortex_a76_fiq_aarch64 + +vector_entry cortex_a76_serror_aarch64 + +#if WORKAROUND_CVE_2022_23960 + apply_cve_2022_23960_bhb_wa CORTEX_A76_BHB_LOOP_COUNT +#endif /* WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + apply_cve_2018_3639_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A64_SMC0 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639*/ + + b serror_aarch64 +end_vector_entry cortex_a76_serror_aarch64 + + /* --------------------------------------------------------------------- + * Lower EL using AArch32 : 0x600 - 0x800 + * --------------------------------------------------------------------- + */ +vector_entry cortex_a76_sync_exception_aarch32 + +#if WORKAROUND_CVE_2022_23960 + apply_cve_2022_23960_bhb_wa CORTEX_A76_BHB_LOOP_COUNT +#endif /* WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + apply_cve_2018_3639_wa _is_sync_exception=1 _esr_el3_val=ESR_EL3_A32_SMC0 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639*/ + + b sync_exception_aarch32 +end_vector_entry cortex_a76_sync_exception_aarch32 + +vector_entry cortex_a76_irq_aarch32 + +#if WORKAROUND_CVE_2022_23960 + apply_cve_2022_23960_bhb_wa CORTEX_A76_BHB_LOOP_COUNT +#endif /* WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + apply_cve_2018_3639_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A32_SMC0 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639*/ + + b irq_aarch32 +end_vector_entry cortex_a76_irq_aarch32 + +vector_entry cortex_a76_fiq_aarch32 + +#if WORKAROUND_CVE_2022_23960 + apply_cve_2022_23960_bhb_wa CORTEX_A76_BHB_LOOP_COUNT +#endif /* WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + apply_cve_2018_3639_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A32_SMC0 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639*/ + + b fiq_aarch32 +end_vector_entry cortex_a76_fiq_aarch32 + +vector_entry cortex_a76_serror_aarch32 + +#if WORKAROUND_CVE_2022_23960 + apply_cve_2022_23960_bhb_wa CORTEX_A76_BHB_LOOP_COUNT +#endif /* WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + apply_cve_2018_3639_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A32_SMC0 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639*/ + + b serror_aarch32 +end_vector_entry cortex_a76_serror_aarch32 +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639 || WORKAROUND_CVE_2022_23960 */ + +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + /* + * ----------------------------------------------------------------- + * This function applies the mitigation for CVE-2018-3639 + * specifically for sync exceptions. It implements a fast path + * where `SMCCC_ARCH_WORKAROUND_2` SMC calls from a lower EL + * running in AArch64 will go through the fast and return early. + * + * In the fast path x0-x3 registers do not need to be restored as the + * calling context will have saved them. + * + * Caller must pass value of esr_el3 to compare via x2. + * Save and restore these registers outside of this function from the + * context before jumping to the main runtime vector table entry. + * + * Shall clobber: x0-x3, x30 + * ----------------------------------------------------------------- + */ +func apply_cve_2018_3639_sync_wa + /* + * Ensure SMC is coming from A64/A32 state on #0 + * with W0 = SMCCC_ARCH_WORKAROUND_2 + * + * This sequence evaluates as: + * (W0==SMCCC_ARCH_WORKAROUND_2) ? (ESR_EL3==SMC#0) : (NE) + * allowing use of a single branch operation + * X2 populated outside this function with the SMC FID. + */ + orr w3, wzr, #SMCCC_ARCH_WORKAROUND_2 + cmp x0, x3 + mrs x3, esr_el3 + + ccmp w2, w3, #0, eq + /* + * Static predictor will predict a fall-through, optimizing + * the `SMCCC_ARCH_WORKAROUND_2` fast path. + */ + bne 1f + + /* + * The sequence below implements the `SMCCC_ARCH_WORKAROUND_2` + * fast path. + */ + cmp x1, xzr /* enable/disable check */ + + /* + * When the calling context wants mitigation disabled, + * we program the mitigation disable function in the + * CPU context, which gets invoked on subsequent exits from + * EL3 via the `el3_exit` function. Otherwise NULL is + * programmed in the CPU context, which results in caller's + * inheriting the EL3 mitigation state (enabled) on subsequent + * `el3_exit`. + */ + mov x0, xzr + adr x1, cortex_a76_disable_wa_cve_2018_3639 + csel x1, x1, x0, eq + str x1, [sp, #CTX_CVE_2018_3639_OFFSET + CTX_CVE_2018_3639_DISABLE] + + mrs x2, CORTEX_A76_CPUACTLR2_EL1 + orr x1, x2, #CORTEX_A76_CPUACTLR2_EL1_DISABLE_LOAD_PASS_STORE + bic x3, x2, #CORTEX_A76_CPUACTLR2_EL1_DISABLE_LOAD_PASS_STORE + csel x3, x3, x1, eq + msr CORTEX_A76_CPUACTLR2_EL1, x3 + ldp x29, x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X29] + /* + * `SMCCC_ARCH_WORKAROUND_2`fast path return to lower EL. + */ + exception_return /* exception_return contains ISB */ +1: + ret +endfunc apply_cve_2018_3639_sync_wa +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639 */ + +workaround_reset_start cortex_a76, ERRATUM(1073348), ERRATA_A76_1073348 + sysreg_bit_set CORTEX_A76_CPUACTLR_EL1 ,CORTEX_A76_CPUACTLR_EL1_DISABLE_STATIC_PREDICTION +workaround_reset_end cortex_a76, ERRATUM(1073348) + +check_erratum_ls cortex_a76, ERRATUM(1073348), CPU_REV(1, 0) + +workaround_reset_start cortex_a76, ERRATUM(1130799), ERRATA_A76_1130799 + sysreg_bit_set CORTEX_A76_CPUACTLR2_EL1, CORTEX_A76_CPUACTLR2_EL1_BIT_59 + msr CORTEX_A76_CPUACTLR2_EL1, x1 +workaround_reset_end cortex_a76, ERRATUM(1130799) + +check_erratum_ls cortex_a76, ERRATUM(1130799), CPU_REV(2, 0) + +workaround_reset_start cortex_a76, ERRATUM(1220197), ERRATA_A76_1220197 + sysreg_bit_set CORTEX_A76_CPUECTLR_EL1, CORTEX_A76_CPUECTLR_EL1_WS_THR_L2 +workaround_reset_end cortex_a76, ERRATUM(1220197) + +check_erratum_ls cortex_a76, ERRATUM(1220197), CPU_REV(2, 0) + +workaround_reset_start cortex_a76, ERRATUM(1257314), ERRATA_A76_1257314 + sysreg_bit_set CORTEX_A76_CPUACTLR3_EL1, CORTEX_A76_CPUACTLR3_EL1_BIT_10 +workaround_reset_end cortex_a76, ERRATUM(1257314) + +check_erratum_ls cortex_a76, ERRATUM(1257314), CPU_REV(3, 0) + +workaround_reset_start cortex_a76, ERRATUM(1262606), ERRATA_A76_1262606 + sysreg_bit_set CORTEX_A76_CPUACTLR_EL1, CORTEX_A76_CPUACTLR_EL1_BIT_13 +workaround_reset_end cortex_a76, ERRATUM(1262606) + +check_erratum_ls cortex_a76, ERRATUM(1262606), CPU_REV(3, 0) + +workaround_reset_start cortex_a76, ERRATUM(1262888), ERRATA_A76_1262888 + sysreg_bit_set CORTEX_A76_CPUECTLR_EL1, CORTEX_A76_CPUECTLR_EL1_BIT_51 +workaround_reset_end cortex_a76, ERRATUM(1262888) + +check_erratum_ls cortex_a76, ERRATUM(1262888), CPU_REV(3, 0) + +workaround_reset_start cortex_a76, ERRATUM(1275112), ERRATA_A76_1275112 + sysreg_bit_set CORTEX_A76_CPUACTLR_EL1, CORTEX_A76_CPUACTLR_EL1_BIT_13 +workaround_reset_end cortex_a76, ERRATUM(1275112) + +check_erratum_ls cortex_a76, ERRATUM(1275112), CPU_REV(3, 0) + +check_erratum_custom_start cortex_a76, ERRATUM(1286807) +#if ERRATA_A76_1286807 + mov x0, #ERRATA_APPLIES + ret +#else + mov x1, #0x30 + b cpu_rev_var_ls +#endif +check_erratum_custom_end cortex_a76, ERRATUM(1286807) + +workaround_reset_start cortex_a76, ERRATUM(1791580), ERRATA_A76_1791580 + sysreg_bit_set CORTEX_A76_CPUACTLR2_EL1, CORTEX_A76_CPUACTLR2_EL1_BIT_2 +workaround_reset_end cortex_a76, ERRATUM(1791580) + +check_erratum_ls cortex_a76, ERRATUM(1791580), CPU_REV(4, 0) + +workaround_reset_start cortex_a76, ERRATUM(1868343), ERRATA_A76_1868343 + sysreg_bit_set CORTEX_A76_CPUACTLR_EL1, CORTEX_A76_CPUACTLR_EL1_BIT_13 +workaround_reset_end cortex_a76, ERRATUM(1868343) + +check_erratum_ls cortex_a76, ERRATUM(1868343), CPU_REV(4, 0) + +workaround_reset_start cortex_a76, ERRATUM(1946160), ERRATA_A76_1946160 + mov x0, #3 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3900002 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 + + mov x0, #4 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3800082 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 + + mov x0, #5 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3800200 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF003E0 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 +workaround_reset_end cortex_a76, ERRATUM(1946160) + +check_erratum_range cortex_a76, ERRATUM(1946160), CPU_REV(3, 0), CPU_REV(4, 1) + +workaround_runtime_start cortex_a76, ERRATUM(2743102), ERRATA_A76_2743102 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end cortex_a76, ERRATUM(2743102) + +check_erratum_ls cortex_a76, ERRATUM(2743102), CPU_REV(4, 1) + +check_erratum_chosen cortex_a76, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + +func cortex_a76_disable_wa_cve_2018_3639 + sysreg_bit_clear CORTEX_A76_CPUACTLR2_EL1, CORTEX_A76_CPUACTLR2_EL1_DISABLE_LOAD_PASS_STORE + isb + ret +endfunc cortex_a76_disable_wa_cve_2018_3639 + +/* -------------------------------------------------------------- + * Errata Workaround for Cortex A76 Errata #1165522. + * This applies only to revisions <= r3p0 of Cortex A76. + * Due to the nature of the errata it is applied unconditionally + * when built in, report it as applicable in this case + * -------------------------------------------------------------- + */ +check_erratum_custom_start cortex_a76, ERRATUM(1165522) +#if ERRATA_A76_1165522 + mov x0, #ERRATA_APPLIES + ret +#else + mov x1, #0x30 + b cpu_rev_var_ls +#endif +check_erratum_custom_end cortex_a76, ERRATUM(1165522) + +check_erratum_chosen cortex_a76, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +/* erratum has no workaround in the cpu. Generic code must take care */ +add_erratum_entry cortex_a76, CVE(2022, 23960), WORKAROUND_CVE_2022_23960, NO_APPLY_AT_RESET + +/* ERRATA_DSU_798953 : + * The errata is defined in dsu_helpers.S but applies to cortex_a76 + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_a76_798953, check_errata_dsu_798953 +.equ erratum_cortex_a76_798953_wa, errata_dsu_798953_wa +add_erratum_entry cortex_a76, ERRATUM(798953), ERRATA_DSU_798953, APPLY_AT_RESET + +/* ERRATA_DSU_936184 : + * The errata is defined in dsu_helpers.S but applies to cortex_a76 + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_a76_936184, check_errata_dsu_936184 +.equ erratum_cortex_a76_936184_wa, errata_dsu_936184_wa +add_erratum_entry cortex_a76, ERRATUM(936184), ERRATA_DSU_936184, APPLY_AT_RESET + +cpu_reset_func_start cortex_a76 + +#if WORKAROUND_CVE_2018_3639 + /* If the PE implements SSBS, we don't need the dynamic workaround */ + mrs x0, id_aa64pfr1_el1 + lsr x0, x0, #ID_AA64PFR1_EL1_SSBS_SHIFT + and x0, x0, #ID_AA64PFR1_EL1_SSBS_MASK +#if !DYNAMIC_WORKAROUND_CVE_2018_3639 && ENABLE_ASSERTIONS + cmp x0, 0 + ASM_ASSERT(ne) +#endif +#if DYNAMIC_WORKAROUND_CVE_2018_3639 + cbnz x0, 1f + sysreg_bit_set CORTEX_A76_CPUACTLR2_EL1, CORTEX_A76_CPUACTLR2_EL1_DISABLE_LOAD_PASS_STORE + isb + +#ifdef IMAGE_BL31 + /* + * The Cortex-A76 generic vectors are overwritten to use the vectors + * defined above. This is required in order to apply mitigation + * against CVE-2018-3639 on exception entry from lower ELs. + * If the below vector table is used, skip overriding it again for + * CVE_2022_23960 as both use the same vbar. + */ + override_vector_table cortex_a76_wa_cve_vbar + isb + b 2f +#endif /* IMAGE_BL31 */ + +1: +#endif /* DYNAMIC_WORKAROUND_CVE_2018_3639 */ +#endif /* WORKAROUND_CVE_2018_3639 */ + +#if IMAGE_BL31 && WORKAROUND_CVE_2022_23960 + /* + * The Cortex-A76 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. This will be bypassed + * if DYNAMIC_WORKAROUND_CVE_2018_3639 has overridden the vectors. + */ + override_vector_table cortex_a76_wa_cve_vbar + isb +#endif /* IMAGE_BL31 && WORKAROUND_CVE_2022_23960 */ +2: +cpu_reset_func_end cortex_a76 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func cortex_a76_core_pwr_dwn + /* --------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------- + */ + sysreg_bit_set CORTEX_A76_CPUPWRCTLR_EL1, CORTEX_A76_CORE_PWRDN_EN_MASK + + apply_erratum cortex_a76, ERRATUM(2743102), ERRATA_A76_2743102 + + isb + ret +endfunc cortex_a76_core_pwr_dwn + +errata_report_shim cortex_a76 + + /* --------------------------------------------- + * This function provides cortex_a76 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a76_regs, "aS" +cortex_a76_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a76_cpu_reg_dump + adr x6, cortex_a76_regs + mrs x8, CORTEX_A76_CPUECTLR_EL1 + ret +endfunc cortex_a76_cpu_reg_dump + +declare_cpu_ops_wa cortex_a76, CORTEX_A76_MIDR, \ + cortex_a76_reset_func, \ + CPU_NO_EXTRA1_FUNC, \ + cortex_a76_disable_wa_cve_2018_3639, \ + CPU_NO_EXTRA3_FUNC, \ + cortex_a76_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a76ae.S b/lib/cpus/aarch64/cortex_a76ae.S new file mode 100644 index 0000000..08a6ef9 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a76ae.S @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a76ae.h> +#include <cpu_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-A76AE must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex-A76AE supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A76AE_BHB_LOOP_COUNT, cortex_a76ae +#endif /* WORKAROUND_CVE_2022_23960 */ + +check_erratum_chosen cortex_a76ae, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +workaround_reset_start cortex_a76ae, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-A76ae generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_a76ae + isb +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a76ae, CVE(2022, 23960) + +cpu_reset_func_start cortex_a76ae +cpu_reset_func_end cortex_a76ae + +errata_report_shim cortex_a76ae + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_a76ae_core_pwr_dwn + sysreg_bit_set CORTEX_A76AE_CPUPWRCTLR_EL1, CORTEX_A76AE_CORE_PWRDN_EN_MASK + isb + ret +endfunc cortex_a76ae_core_pwr_dwn + + /* --------------------------------------------- + * This function provides cortex_a76ae specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a76ae_regs, "aS" +cortex_a76ae_regs: /* The ASCII list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a76ae_cpu_reg_dump + adr x6, cortex_a76ae_regs + mrs x8, CORTEX_A76AE_CPUECTLR_EL1 + ret +endfunc cortex_a76ae_cpu_reg_dump + +declare_cpu_ops cortex_a76ae, CORTEX_A76AE_MIDR, cortex_a76ae_reset_func, \ + cortex_a76ae_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a77.S b/lib/cpus/aarch64/cortex_a77.S new file mode 100644 index 0000000..86c2561 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a77.S @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2018-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a77.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-A77 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex-A77 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A77_BHB_LOOP_COUNT, cortex_a77 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_a77, ERRATUM(1508412), ERRATA_A77_1508412 + /* move cpu revision in again and compare against r0p0 */ + mov x0, x7 + mov x1, #CPU_REV(0, 0) + bl cpu_rev_var_ls + cbz x0, 1f + + ldr x0, =0x0 + msr CORTEX_A77_CPUPSELR_EL3, x0 + ldr x0, =0x00E8400000 + msr CORTEX_A77_CPUPOR_EL3, x0 + ldr x0, =0x00FFE00000 + msr CORTEX_A77_CPUPMR_EL3, x0 + ldr x0, =0x4004003FF + msr CORTEX_A77_CPUPCR_EL3, x0 + ldr x0, =0x1 + msr CORTEX_A77_CPUPSELR_EL3, x0 + ldr x0, =0x00E8C00040 + msr CORTEX_A77_CPUPOR_EL3, x0 + ldr x0, =0x00FFE00040 + msr CORTEX_A77_CPUPMR_EL3, x0 + b 2f +1: + ldr x0, =0x0 + msr CORTEX_A77_CPUPSELR_EL3, x0 + ldr x0, =0x00E8400000 + msr CORTEX_A77_CPUPOR_EL3, x0 + ldr x0, =0x00FF600000 + msr CORTEX_A77_CPUPMR_EL3, x0 + ldr x0, =0x00E8E00080 + msr CORTEX_A77_CPUPOR2_EL3, x0 + ldr x0, =0x00FFE000C0 + msr CORTEX_A77_CPUPMR2_EL3, x0 +2: + ldr x0, =0x04004003FF + msr CORTEX_A77_CPUPCR_EL3, x0 +workaround_reset_end cortex_a77, ERRATUM(1508412) + +check_erratum_ls cortex_a77, ERRATUM(1508412), CPU_REV(1, 0) + +workaround_reset_start cortex_a77, ERRATUM(1791578), ERRATA_A77_1791578 + sysreg_bit_set CORTEX_A77_ACTLR2_EL1, CORTEX_A77_ACTLR2_EL1_BIT_2 +workaround_reset_end cortex_a77, ERRATUM(1791578) + +check_erratum_ls cortex_a77, ERRATUM(1791578), CPU_REV(1, 1) + +workaround_reset_start cortex_a77, ERRATUM(1800714), ERRATA_A77_1800714 + /* Disable allocation of splintered pages in the L2 TLB */ + sysreg_bit_set CORTEX_A77_CPUECTLR_EL1, CORTEX_A77_CPUECTLR_EL1_BIT_53 +workaround_reset_end cortex_a77, ERRATUM(1800714) + +check_erratum_ls cortex_a77, ERRATUM(1800714), CPU_REV(1, 1) + +workaround_reset_start cortex_a77, ERRATUM(1925769), ERRATA_A77_1925769 + sysreg_bit_set CORTEX_A77_CPUECTLR_EL1, CORTEX_A77_CPUECTLR_EL1_BIT_8 +workaround_reset_end cortex_a77, ERRATUM(1925769) + +check_erratum_ls cortex_a77, ERRATUM(1925769), CPU_REV(1, 1) + +workaround_reset_start cortex_a77, ERRATUM(1946167), ERRATA_A77_1946167 + ldr x0,=0x4 + msr CORTEX_A77_CPUPSELR_EL3,x0 + ldr x0,=0x10E3900002 + msr CORTEX_A77_CPUPOR_EL3,x0 + ldr x0,=0x10FFF00083 + msr CORTEX_A77_CPUPMR_EL3,x0 + ldr x0,=0x2001003FF + msr CORTEX_A77_CPUPCR_EL3,x0 + + ldr x0,=0x5 + msr CORTEX_A77_CPUPSELR_EL3,x0 + ldr x0,=0x10E3800082 + msr CORTEX_A77_CPUPOR_EL3,x0 + ldr x0,=0x10FFF00083 + msr CORTEX_A77_CPUPMR_EL3,x0 + ldr x0,=0x2001003FF + msr CORTEX_A77_CPUPCR_EL3,x0 + + ldr x0,=0x6 + msr CORTEX_A77_CPUPSELR_EL3,x0 + ldr x0,=0x10E3800200 + msr CORTEX_A77_CPUPOR_EL3,x0 + ldr x0,=0x10FFF003E0 + msr CORTEX_A77_CPUPMR_EL3,x0 + ldr x0,=0x2001003FF + msr CORTEX_A77_CPUPCR_EL3,x0 +workaround_reset_end cortex_a77, ERRATUM(1946167) + +check_erratum_ls cortex_a77, ERRATUM(1946167), CPU_REV(1, 1) + +workaround_reset_start cortex_a77, ERRATUM(2356587), ERRATA_A77_2356587 + sysreg_bit_set CORTEX_A77_ACTLR2_EL1, CORTEX_A77_ACTLR2_EL1_BIT_0 +workaround_reset_end cortex_a77, ERRATUM(2356587) + +check_erratum_ls cortex_a77, ERRATUM(2356587), CPU_REV(1, 1) + +workaround_runtime_start cortex_a77, ERRATUM(2743100), ERRATA_A77_2743100 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end cortex_a77, ERRATUM(2743100), NO_ISB + +check_erratum_ls cortex_a77, ERRATUM(2743100), CPU_REV(1, 1) + +workaround_reset_start cortex_a77, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-A77 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + adr x0, wa_cve_vbar_cortex_a77 + msr vbar_el3, x0 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a77, CVE(2022, 23960) + +check_erratum_chosen cortex_a77, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + + /* ------------------------------------------------- + * The CPU Ops reset function for Cortex-A77. Must follow AAPCS. + * ------------------------------------------------- + */ +cpu_reset_func_start cortex_a77 +cpu_reset_func_end cortex_a77 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func cortex_a77_core_pwr_dwn + /* --------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------- + */ + sysreg_bit_set CORTEX_A77_CPUPWRCTLR_EL1, \ + CORTEX_A77_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + + apply_erratum cortex_a77, ERRATUM(2743100), ERRATA_A77_2743100 + + isb + ret +endfunc cortex_a77_core_pwr_dwn + +errata_report_shim cortex_a77 + /* --------------------------------------------- + * This function provides Cortex-A77 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a77_regs, "aS" +cortex_a77_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a77_cpu_reg_dump + adr x6, cortex_a77_regs + mrs x8, CORTEX_A77_CPUECTLR_EL1 + ret +endfunc cortex_a77_cpu_reg_dump + +declare_cpu_ops cortex_a77, CORTEX_A77_MIDR, \ + cortex_a77_reset_func, \ + cortex_a77_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a78.S b/lib/cpus/aarch64/cortex_a78.S new file mode 100644 index 0000000..b5c24e1 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a78.S @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2019-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a78.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "cortex_a78 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +.globl cortex_a78_reset_func +.globl cortex_a78_core_pwr_dwn + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A78_BHB_LOOP_COUNT, cortex_a78 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_a78, ERRATUM(1688305), ERRATA_A78_1688305 + sysreg_bit_set CORTEX_A78_ACTLR2_EL1, CORTEX_A78_ACTLR2_EL1_BIT_1 +workaround_reset_end cortex_a78, ERRATUM(1688305) + +check_erratum_ls cortex_a78, ERRATUM(1688305), CPU_REV(1, 0) + +workaround_reset_start cortex_a78, ERRATUM(1821534), ERRATA_A78_1821534 + sysreg_bit_set CORTEX_A78_ACTLR2_EL1, CORTEX_A78_ACTLR2_EL1_BIT_2 +workaround_reset_end cortex_a78, ERRATUM(1821534) + +check_erratum_ls cortex_a78, ERRATUM(1821534), CPU_REV(1, 0) + +workaround_reset_start cortex_a78, ERRATUM(1941498), ERRATA_A78_1941498 + sysreg_bit_set CORTEX_A78_CPUECTLR_EL1, CORTEX_A78_CPUECTLR_EL1_BIT_8 +workaround_reset_end cortex_a78, ERRATUM(1941498) + +check_erratum_ls cortex_a78, ERRATUM(1941498), CPU_REV(1, 1) + +workaround_reset_start cortex_a78, ERRATUM(1951500), ERRATA_A78_1951500 + msr S3_6_c15_c8_0, xzr + ldr x0, =0x10E3900002 + msr S3_6_c15_c8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_c15_c8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_c15_c8_1, x0 + + mov x0, #1 + msr S3_6_c15_c8_0, x0 + ldr x0, =0x10E3800082 + msr S3_6_c15_c8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_c15_c8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_c15_c8_1, x0 + + mov x0, #2 + msr S3_6_c15_c8_0, x0 + ldr x0, =0x10E3800200 + msr S3_6_c15_c8_2, x0 + ldr x0, =0x10FFF003E0 + msr S3_6_c15_c8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_c15_c8_1, x0 +workaround_reset_end cortex_a78, ERRATUM(1951500) + +check_erratum_range cortex_a78, ERRATUM(1951500), CPU_REV(1, 0), CPU_REV(1, 1) + +workaround_reset_start cortex_a78, ERRATUM(1952683), ERRATA_A78_1952683 + ldr x0,=0x5 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xEEE10A10 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFEF0FFF + msr S3_6_c15_c8_3,x0 + ldr x0,=0x0010F000 + msr S3_6_c15_c8_4,x0 + ldr x0,=0x0010F000 + msr S3_6_c15_c8_5,x0 + ldr x0,=0x40000080023ff + msr S3_6_c15_c8_1,x0 + ldr x0,=0x6 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xEE640F34 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFEF0FFF + msr S3_6_c15_c8_3,x0 + ldr x0,=0x40000080023ff + msr S3_6_c15_c8_1,x0 +workaround_reset_end cortex_a78, ERRATUM(1952683) + +check_erratum_ls cortex_a78, ERRATUM(1952683), CPU_REV(0, 0) + +workaround_reset_start cortex_a78, ERRATUM(2132060), ERRATA_A78_2132060 + /* Apply the workaround. */ + mrs x1, CORTEX_A78_CPUECTLR_EL1 + mov x0, #CORTEX_A78_CPUECTLR_EL1_PF_MODE_CNSRV + bfi x1, x0, #CPUECTLR_EL1_PF_MODE_LSB, #CPUECTLR_EL1_PF_MODE_WIDTH + msr CORTEX_A78_CPUECTLR_EL1, x1 +workaround_reset_end cortex_a78, ERRATUM(2132060) + +check_erratum_ls cortex_a78, ERRATUM(2132060), CPU_REV(1, 2) + +workaround_reset_start cortex_a78, ERRATUM(2242635), ERRATA_A78_2242635 + ldr x0, =0x5 + msr S3_6_c15_c8_0, x0 /* CPUPSELR_EL3 */ + ldr x0, =0x10F600E000 + msr S3_6_c15_c8_2, x0 /* CPUPOR_EL3 */ + ldr x0, =0x10FF80E000 + msr S3_6_c15_c8_3, x0 /* CPUPMR_EL3 */ + ldr x0, =0x80000000003FF + msr S3_6_c15_c8_1, x0 /* CPUPCR_EL3 */ +workaround_reset_end cortex_a78, ERRATUM(2242635) + +check_erratum_range cortex_a78, ERRATUM(2242635), CPU_REV(1, 0), CPU_REV(1, 2) + +workaround_reset_start cortex_a78, ERRATUM(2376745), ERRATA_A78_2376745 + sysreg_bit_set CORTEX_A78_ACTLR2_EL1, BIT(0) +workaround_reset_end cortex_a78, ERRATUM(2376745) + +check_erratum_ls cortex_a78, ERRATUM(2376745), CPU_REV(1, 2) + +workaround_reset_start cortex_a78, ERRATUM(2395406), ERRATA_A78_2395406 + sysreg_bit_set CORTEX_A78_ACTLR2_EL1, BIT(40) +workaround_reset_end cortex_a78, ERRATUM(2395406) + +check_erratum_ls cortex_a78, ERRATUM(2395406), CPU_REV(1, 2) + +workaround_reset_start cortex_a78, ERRATUM(2742426), ERRATA_A78_2742426 + /* Apply the workaround */ + mrs x1, CORTEX_A78_ACTLR5_EL1 + bic x1, x1, #BIT(56) + orr x1, x1, #BIT(55) + msr CORTEX_A78_ACTLR5_EL1, x1 +workaround_reset_end cortex_a78, ERRATUM(2742426) + +check_erratum_ls cortex_a78, ERRATUM(2742426), CPU_REV(1, 2) + +workaround_runtime_start cortex_a78, ERRATUM(2772019), ERRATA_A78_2772019 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end cortex_a78, ERRATUM(2772019) + +check_erratum_ls cortex_a78, ERRATUM(2772019), CPU_REV(1, 2) + +workaround_reset_start cortex_a78, ERRATUM(2779479), ERRATA_A78_2779479 + sysreg_bit_set CORTEX_A78_ACTLR3_EL1, BIT(47) +workaround_reset_end cortex_a78, ERRATUM(2779479) + +check_erratum_ls cortex_a78, ERRATUM(2779479), CPU_REV(1, 2) + +workaround_reset_start cortex_a78, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-X1 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_a78 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a78, CVE(2022, 23960) + +check_erratum_chosen cortex_a78, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +cpu_reset_func_start cortex_a78 +#if ENABLE_FEAT_AMU + /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ + sysreg_bit_clear actlr_el3, CORTEX_A78_ACTLR_TAM_BIT + + /* Make sure accesses from non-secure EL0/EL1 are not trapped to EL2 */ + sysreg_bit_clear actlr_el2, CORTEX_A78_ACTLR_TAM_BIT + + /* Enable group0 counters */ + mov x0, #CORTEX_A78_AMU_GROUP0_MASK + msr CPUAMCNTENSET0_EL0, x0 + + /* Enable group1 counters */ + mov x0, #CORTEX_A78_AMU_GROUP1_MASK + msr CPUAMCNTENSET1_EL0, x0 +#endif +cpu_reset_func_end cortex_a78 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func cortex_a78_core_pwr_dwn + sysreg_bit_set CORTEX_A78_CPUPWRCTLR_EL1, CORTEX_A78_CPUPWRCTLR_EL1_CORE_PWRDN_EN_BIT + + apply_erratum cortex_a78, ERRATUM(2772019), ERRATA_A78_2772019 + + isb + ret +endfunc cortex_a78_core_pwr_dwn + +errata_report_shim cortex_a78 + + /* --------------------------------------------- + * This function provides cortex_a78 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a78_regs, "aS" +cortex_a78_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a78_cpu_reg_dump + adr x6, cortex_a78_regs + mrs x8, CORTEX_A78_CPUECTLR_EL1 + ret +endfunc cortex_a78_cpu_reg_dump + +declare_cpu_ops cortex_a78, CORTEX_A78_MIDR, \ + cortex_a78_reset_func, \ + cortex_a78_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a78_ae.S b/lib/cpus/aarch64/cortex_a78_ae.S new file mode 100644 index 0000000..d3a3e5d --- /dev/null +++ b/lib/cpus/aarch64/cortex_a78_ae.S @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2019-2023, Arm Limited. All rights reserved. + * Copyright (c) 2021-2023, NVIDIA Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a78_ae.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "cortex_a78_ae must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A78_AE_BHB_LOOP_COUNT, cortex_a78_ae +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_a78_ae, ERRATUM(1941500), ERRATA_A78_AE_1941500 + sysreg_bit_set CORTEX_A78_AE_CPUECTLR_EL1, CORTEX_A78_AE_CPUECTLR_EL1_BIT_8 +workaround_reset_end cortex_a78_ae, ERRATUM(1941500) + +check_erratum_ls cortex_a78_ae, ERRATUM(1941500), CPU_REV(0, 1) + +workaround_reset_start cortex_a78_ae, ERRATUM(1951502), ERRATA_A78_AE_1951502 + msr S3_6_c15_c8_0, xzr + ldr x0, =0x10E3900002 + msr S3_6_c15_c8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_c15_c8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_c15_c8_1, x0 + + mov x0, #1 + msr S3_6_c15_c8_0, x0 + ldr x0, =0x10E3800082 + msr S3_6_c15_c8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_c15_c8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_c15_c8_1, x0 + + mov x0, #2 + msr S3_6_c15_c8_0, x0 + ldr x0, =0x10E3800200 + msr S3_6_c15_c8_2, x0 + ldr x0, =0x10FFF003E0 + msr S3_6_c15_c8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_c15_c8_1, x0 +workaround_reset_end cortex_a78_ae, ERRATUM(1951502) + +check_erratum_ls cortex_a78_ae, ERRATUM(1951502), CPU_REV(0, 1) + +workaround_reset_start cortex_a78_ae, ERRATUM(2376748), ERRATA_A78_AE_2376748 + /* ------------------------------------------------------- + * Set CPUACTLR2_EL1[0] to 1 to force PLDW/PFRM ST to + * behave like PLD/PRFM LD and not cause invalidations to + * other PE caches. There might be a small performance + * degradation to this workaround for certain workloads + * that share data. + * ------------------------------------------------------- + */ + sysreg_bit_set CORTEX_A78_AE_ACTLR2_EL1, CORTEX_A78_AE_ACTLR2_EL1_BIT_0 +workaround_reset_end cortex_a78_ae, ERRATUM(2376748) + +check_erratum_ls cortex_a78_ae, ERRATUM(2376748), CPU_REV(0, 2) + +workaround_reset_start cortex_a78_ae, ERRATUM(2395408), ERRATA_A78_AE_2395408 + /* -------------------------------------------------------- + * Disable folding of demand requests into older prefetches + * with L2 miss requests outstanding by setting the + * CPUACTLR2_EL1[40] to 1. + * -------------------------------------------------------- + */ + sysreg_bit_set CORTEX_A78_AE_ACTLR2_EL1, CORTEX_A78_AE_ACTLR2_EL1_BIT_40 +workaround_reset_end cortex_a78_ae, ERRATUM(2395408) + +check_erratum_ls cortex_a78_ae, ERRATUM(2395408), CPU_REV(0, 1) + +workaround_reset_start cortex_a78_ae, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-A78AE generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_a78_ae +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a78_ae, CVE(2022, 23960) + +check_erratum_chosen cortex_a78_ae, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +cpu_reset_func_start cortex_a78_ae +#if ENABLE_FEAT_AMU + /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ + sysreg_bit_clear actlr_el3, CORTEX_A78_ACTLR_TAM_BIT + + /* Make sure accesses from non-secure EL0/EL1 are not trapped to EL2 */ + sysreg_bit_clear actlr_el2, CORTEX_A78_ACTLR_TAM_BIT + + /* Enable group0 counters */ + mov x0, #CORTEX_A78_AMU_GROUP0_MASK + msr CPUAMCNTENSET0_EL0, x0 + + /* Enable group1 counters */ + mov x0, #CORTEX_A78_AMU_GROUP1_MASK + msr CPUAMCNTENSET1_EL0, x0 +#endif +cpu_reset_func_end cortex_a78_ae + + /* ------------------------------------------------------- + * HW will do the cache maintenance while powering down + * ------------------------------------------------------- + */ +func cortex_a78_ae_core_pwr_dwn + /* ------------------------------------------------------- + * Enable CPU power down bit in power control register + * ------------------------------------------------------- + */ + sysreg_bit_set CORTEX_A78_CPUPWRCTLR_EL1, CORTEX_A78_CPUPWRCTLR_EL1_CORE_PWRDN_EN_BIT + isb + ret +endfunc cortex_a78_ae_core_pwr_dwn + +errata_report_shim cortex_a78_ae + + /* ------------------------------------------------------- + * This function provides cortex_a78_ae specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * ------------------------------------------------------- + */ +.section .rodata.cortex_a78_ae_regs, "aS" +cortex_a78_ae_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a78_ae_cpu_reg_dump + adr x6, cortex_a78_ae_regs + mrs x8, CORTEX_A78_CPUECTLR_EL1 + ret +endfunc cortex_a78_ae_cpu_reg_dump + +declare_cpu_ops cortex_a78_ae, CORTEX_A78_AE_MIDR, \ + cortex_a78_ae_reset_func, \ + cortex_a78_ae_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_a78c.S b/lib/cpus/aarch64/cortex_a78c.S new file mode 100644 index 0000000..d19c693 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a78c.S @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_a78c.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "cortex_a78c must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_A78C_BHB_LOOP_COUNT, cortex_a78c +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_a78c, ERRATUM(1827430), ERRATA_A78C_1827430 + /* Disable allocation of splintered pages in the L2 TLB */ + sysreg_bit_set CORTEX_A78C_CPUECTLR_EL1, CORTEX_A78C_CPUECTLR_EL1_MM_ASP_EN +workaround_reset_end cortex_a78c, ERRATUM(1827430) + +check_erratum_ls cortex_a78c, ERRATUM(1827430), CPU_REV(0, 0) + +workaround_reset_start cortex_a78c, ERRATUM(1827440), ERRATA_A78C_1827440 + /* Force Atomic Store to WB memory be done in L1 data cache */ + sysreg_bit_set CORTEX_A78C_CPUACTLR2_EL1, BIT(2) +workaround_reset_end cortex_a78c, ERRATUM(1827440) + +check_erratum_ls cortex_a78c, ERRATUM(1827440), CPU_REV(0, 0) + +workaround_reset_start cortex_a78c, ERRATUM(2132064), ERRATA_A78C_2132064 + /* -------------------------------------------------------- + * Place the data prefetcher in the most conservative mode + * to reduce prefetches by writing the following bits to + * the value indicated: ecltr[7:6], PF_MODE = 2'b11 + * -------------------------------------------------------- + */ + sysreg_bit_set CORTEX_A78C_CPUECTLR_EL1, (CORTEX_A78C_CPUECTLR_EL1_BIT_6 | CORTEX_A78C_CPUECTLR_EL1_BIT_7) +workaround_reset_end cortex_a78c, ERRATUM(2132064) + +check_erratum_range cortex_a78c, ERRATUM(2132064), CPU_REV(0, 1), CPU_REV(0, 2) + +workaround_reset_start cortex_a78c, ERRATUM(2242638), ERRATA_A78C_2242638 + ldr x0, =0x5 + msr CORTEX_A78C_IMP_CPUPSELR_EL3, x0 + ldr x0, =0x10F600E000 + msr CORTEX_A78C_IMP_CPUPOR_EL3, x0 + ldr x0, =0x10FF80E000 + msr CORTEX_A78C_IMP_CPUPMR_EL3, x0 + ldr x0, =0x80000000003FF + msr CORTEX_A78C_IMP_CPUPCR_EL3, x0 +workaround_reset_end cortex_a78c, ERRATUM(2242638) + +check_erratum_range cortex_a78c, ERRATUM(2242638), CPU_REV(0, 1), CPU_REV(0, 2) + +workaround_reset_start cortex_a78c, ERRATUM(2376749), ERRATA_A78C_2376749 + sysreg_bit_set CORTEX_A78C_CPUACTLR2_EL1, CORTEX_A78C_CPUACTLR2_EL1_BIT_0 +workaround_reset_end cortex_a78c, ERRATUM(2376749) + +check_erratum_range cortex_a78c, ERRATUM(2376749), CPU_REV(0, 1), CPU_REV(0, 2) + +workaround_reset_start cortex_a78c, ERRATUM(2395411), ERRATA_A78C_2395411 + sysreg_bit_set CORTEX_A78C_CPUACTLR2_EL1, CORTEX_A78C_CPUACTLR2_EL1_BIT_40 +workaround_reset_end cortex_a78c, ERRATUM(2395411) + +check_erratum_range cortex_a78c, ERRATUM(2395411), CPU_REV(0, 1), CPU_REV(0, 2) + +workaround_runtime_start cortex_a78c, ERRATUM(2772121), ERRATA_A78C_2772121 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end cortex_a78c, ERRATUM(2772121) + +check_erratum_ls cortex_a78c, ERRATUM(2772121), CPU_REV(0, 2) + +workaround_reset_start cortex_a78c, ERRATUM(2779484), ERRATA_A78C_2779484 + sysreg_bit_set CORTEX_A78C_ACTLR3_EL1, BIT(47) +workaround_reset_end cortex_a78c, ERRATUM(2779484) + +check_erratum_range cortex_a78c, ERRATUM(2779484), CPU_REV(0, 1), CPU_REV(0, 2) + +check_erratum_chosen cortex_a78c, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +workaround_reset_start cortex_a78c, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-A78c generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_a78c +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_a78c, CVE(2022, 23960) + +cpu_reset_func_start cortex_a78c +cpu_reset_func_end cortex_a78c + +errata_report_shim cortex_a78c + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_a78c_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_A78C_CPUPWRCTLR_EL1, CORTEX_A78C_CPUPWRCTLR_EL1_CORE_PWRDN_EN_BIT + + apply_erratum cortex_a78c, ERRATUM(2772121), ERRATA_A78C_2772121 + + isb + ret +endfunc cortex_a78c_core_pwr_dwn + + /* --------------------------------------------- + * This function provides cortex_a78c specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_a78c_regs, "aS" +cortex_a78c_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_a78c_cpu_reg_dump + adr x6, cortex_a78c_regs + mrs x8, CORTEX_A78C_CPUECTLR_EL1 + ret +endfunc cortex_a78c_cpu_reg_dump + +declare_cpu_ops cortex_a78c, CORTEX_A78C_MIDR, \ + cortex_a78c_reset_func, \ + cortex_a78c_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_blackhawk.S b/lib/cpus/aarch64/cortex_blackhawk.S new file mode 100644 index 0000000..b7b7a2d --- /dev/null +++ b/lib/cpus/aarch64/cortex_blackhawk.S @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_blackhawk.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex blackhawk must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex blackhawk supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +cpu_reset_func_start cortex_blackhawk + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_blackhawk + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_blackhawk_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_BLACKHAWK_CPUPWRCTLR_EL1, CORTEX_BLACKHAWK_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc cortex_blackhawk_core_pwr_dwn + +errata_report_shim cortex_blackhawk + + /* --------------------------------------------- + * This function provides Cortex Blackhawk specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_blackhawk_regs, "aS" +cortex_blackhawk_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_blackhawk_cpu_reg_dump + adr x6, cortex_blackhawk_regs + mrs x8, CORTEX_BLACKHAWK_CPUECTLR_EL1 + ret +endfunc cortex_blackhawk_cpu_reg_dump + +declare_cpu_ops cortex_blackhawk, CORTEX_BLACKHAWK_MIDR, \ + cortex_blackhawk_reset_func, \ + cortex_blackhawk_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_chaberton.S b/lib/cpus/aarch64/cortex_chaberton.S new file mode 100644 index 0000000..596fe4a --- /dev/null +++ b/lib/cpus/aarch64/cortex_chaberton.S @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_chaberton.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex Chaberton must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex Chaberton supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +cpu_reset_func_start cortex_chaberton + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_chaberton + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_chaberton_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_CHABERTON_CPUPWRCTLR_EL1, CORTEX_CHABERTON_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc cortex_chaberton_core_pwr_dwn + +errata_report_shim cortex_chaberton + + /* --------------------------------------------- + * This function provides Cortex Chaberton specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_chaberton_regs, "aS" +cortex_chaberton_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_chaberton_cpu_reg_dump + adr x6, cortex_chaberton_regs + mrs x8, CORTEX_CHABERTON_CPUECTLR_EL1 + ret +endfunc cortex_chaberton_cpu_reg_dump + +declare_cpu_ops cortex_chaberton, CORTEX_CHABERTON_MIDR, \ + cortex_chaberton_reset_func, \ + cortex_chaberton_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_gelas.S b/lib/cpus/aarch64/cortex_gelas.S new file mode 100644 index 0000000..dc704f2 --- /dev/null +++ b/lib/cpus/aarch64/cortex_gelas.S @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_gelas.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Gelas must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Gelas supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +cpu_reset_func_start cortex_gelas + /* ---------------------------------------------------- + * Disable speculative loads + * ---------------------------------------------------- + */ + msr SSBS, xzr +cpu_reset_func_end cortex_gelas + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_gelas_core_pwr_dwn +#if ENABLE_SME_FOR_NS + /* --------------------------------------------------- + * Disable SME if enabled and supported + * --------------------------------------------------- + */ + mrs x0, ID_AA64PFR1_EL1 + ubfx x0, x0, #ID_AA64PFR1_EL1_SME_SHIFT, \ + #ID_AA64PFR1_EL1_SME_WIDTH + cmp x0, #ID_AA64PFR1_EL1_SME_NOT_SUPPORTED + b.eq 1f + msr CORTEX_GELAS_SVCRSM, xzr + msr CORTEX_GELAS_SVCRZA, xzr +1: +#endif + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_GELAS_CPUPWRCTLR_EL1, \ + CORTEX_GELAS_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc cortex_gelas_core_pwr_dwn + +errata_report_shim cortex_gelas + + /* --------------------------------------------- + * This function provides Gelas specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_gelas_regs, "aS" +cortex_gelas_regs: /* The ASCII list of register names to be reported */ + .asciz "imp_cpuectlr_el1", "" + +func cortex_gelas_cpu_reg_dump + adr x6, cortex_gelas_regs + mrs x8, CORTEX_GELAS_IMP_CPUECTLR_EL1 + ret +endfunc cortex_gelas_cpu_reg_dump + +declare_cpu_ops cortex_gelas, CORTEX_GELAS_MIDR, \ + cortex_gelas_reset_func, \ + cortex_gelas_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_x1.S b/lib/cpus/aarch64/cortex_x1.S new file mode 100644 index 0000000..42634f1 --- /dev/null +++ b/lib/cpus/aarch64/cortex_x1.S @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2022-2023, Google LLC. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <cortex_x1.h> +#include <cpu_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-X1 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex-X1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_X1_BHB_LOOP_COUNT, cortex_x1 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_x1, ERRATUM(1688305), ERRATA_X1_1688305 + sysreg_bit_set CORTEX_X1_ACTLR2_EL1, BIT(1) +workaround_reset_end cortex_x1, ERRATUM(1688305) + +check_erratum_ls cortex_x1, ERRATUM(1688305), CPU_REV(1, 0) + +workaround_reset_start cortex_x1, ERRATUM(1821534), ERRATA_X1_1821534 + sysreg_bit_set CORTEX_X1_ACTLR2_EL1, BIT(2) +workaround_reset_end cortex_x1, ERRATUM(1821534) + +check_erratum_ls cortex_x1, ERRATUM(1821534), CPU_REV(1, 0) + +workaround_reset_start cortex_x1, ERRATUM(1827429), ERRATA_X1_1827429 + sysreg_bit_set CORTEX_X1_CPUECTLR_EL1, BIT(53) +workaround_reset_end cortex_x1, ERRATUM(1827429) + +check_erratum_ls cortex_x1, ERRATUM(1827429), CPU_REV(1, 0) + +check_erratum_chosen cortex_x1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +workaround_reset_start cortex_x1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-X1 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_x1 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_x1, CVE(2022, 23960) + +cpu_reset_func_start cortex_x1 +cpu_reset_func_end cortex_x1 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func cortex_x1_core_pwr_dwn + sysreg_bit_set CORTEX_X1_CPUPWRCTLR_EL1, CORTEX_X1_CORE_PWRDN_EN_MASK + isb + ret +endfunc cortex_x1_core_pwr_dwn + +errata_report_shim cortex_x1 + + /* --------------------------------------------- + * This function provides Cortex X1 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_x1_regs, "aS" +cortex_x1_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_x1_cpu_reg_dump + adr x6, cortex_x1_regs + mrs x8, CORTEX_X1_CPUECTLR_EL1 + ret +endfunc cortex_x1_cpu_reg_dump + +declare_cpu_ops cortex_x1, CORTEX_X1_MIDR, \ + cortex_x1_reset_func, \ + cortex_x1_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_x2.S b/lib/cpus/aarch64/cortex_x2.S new file mode 100644 index 0000000..258288c --- /dev/null +++ b/lib/cpus/aarch64/cortex_x2.S @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_x2.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex X2 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex X2 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_X2_BHB_LOOP_COUNT, cortex_x2 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_x2, ERRATUM(2002765), ERRATA_X2_2002765 + ldr x0, =0x6 + msr S3_6_C15_C8_0, x0 /* CPUPSELR_EL3 */ + ldr x0, =0xF3A08002 + msr S3_6_C15_C8_2, x0 /* CPUPOR_EL3 */ + ldr x0, =0xFFF0F7FE + msr S3_6_C15_C8_3, x0 /* CPUPMR_EL3 */ + ldr x0, =0x40000001003ff + msr S3_6_C15_C8_1, x0 /* CPUPCR_EL3 */ +workaround_reset_end cortex_x2, ERRATUM(2002765) + +check_erratum_ls cortex_x2, ERRATUM(2002765), CPU_REV(2, 0) + +workaround_reset_start cortex_x2, ERRATUM(2017096), ERRATA_X2_2017096 + sysreg_bit_set CORTEX_X2_CPUECTLR_EL1, CORTEX_X2_CPUECTLR_EL1_PFSTIDIS_BIT +workaround_reset_end cortex_x2, ERRATUM(2017096) + +check_erratum_ls cortex_x2, ERRATUM(2017096), CPU_REV(2, 0) + +workaround_reset_start cortex_x2, ERRATUM(2058056), ERRATA_X2_2058056 + sysreg_bitfield_insert CORTEX_X2_CPUECTLR2_EL1, CORTEX_X2_CPUECTLR2_EL1_PF_MODE_CNSRV, \ + CORTEX_X2_CPUECTLR2_EL1_PF_MODE_SHIFT, CORTEX_X2_CPUECTLR2_EL1_PF_MODE_WIDTH +workaround_reset_end cortex_x2, ERRATUM(2058056) + +check_erratum_ls cortex_x2, ERRATUM(2058056), CPU_REV(2, 1) + +workaround_reset_start cortex_x2, ERRATUM(2081180), ERRATA_X2_2081180 + /* Apply instruction patching sequence */ + ldr x0, =0x3 + msr CORTEX_X2_IMP_CPUPSELR_EL3, x0 + ldr x0, =0xF3A08002 + msr CORTEX_X2_IMP_CPUPOR_EL3, x0 + ldr x0, =0xFFF0F7FE + msr CORTEX_X2_IMP_CPUPMR_EL3, x0 + ldr x0, =0x10002001003FF + msr CORTEX_X2_IMP_CPUPCR_EL3, x0 + ldr x0, =0x4 + msr CORTEX_X2_IMP_CPUPSELR_EL3, x0 + ldr x0, =0xBF200000 + msr CORTEX_X2_IMP_CPUPOR_EL3, x0 + ldr x0, =0xFFEF0000 + msr CORTEX_X2_IMP_CPUPMR_EL3, x0 + ldr x0, =0x10002001003F3 + msr CORTEX_X2_IMP_CPUPCR_EL3, x0 +workaround_reset_end cortex_x2, ERRATUM(2081180) + +check_erratum_ls cortex_x2, ERRATUM(2081180), CPU_REV(2, 0) + +workaround_reset_start cortex_x2, ERRATUM(2083908), ERRATA_X2_2083908 + /* Apply the workaround by setting bit 13 in CPUACTLR5_EL1. */ + sysreg_bit_set CORTEX_X2_CPUACTLR5_EL1, BIT(13) +workaround_reset_end cortex_x2, ERRATUM(2083908) + +check_erratum_range cortex_x2, ERRATUM(2083908), CPU_REV(2, 0), CPU_REV(2, 0) + +workaround_reset_start cortex_x2, ERRATUM(2147715), ERRATA_X2_2147715 + /* Apply the workaround by setting bit 22 in CPUACTLR_EL1. */ + sysreg_bit_set CORTEX_X2_CPUACTLR_EL1, CORTEX_X2_CPUACTLR_EL1_BIT_22 +workaround_reset_end cortex_x2, ERRATUM(2147715) + +check_erratum_range cortex_x2, ERRATUM(2147715), CPU_REV(2, 0), CPU_REV(2, 0) + +workaround_reset_start cortex_x2, ERRATUM(2216384), ERRATA_X2_2216384 + sysreg_bit_set CORTEX_X2_CPUACTLR5_EL1, CORTEX_X2_CPUACTLR5_EL1_BIT_17 + + /* Apply instruction patching sequence */ + ldr x0, =0x5 + msr CORTEX_X2_IMP_CPUPSELR_EL3, x0 + ldr x0, =0x10F600E000 + msr CORTEX_X2_IMP_CPUPOR_EL3, x0 + ldr x0, =0x10FF80E000 + msr CORTEX_X2_IMP_CPUPMR_EL3, x0 + ldr x0, =0x80000000003FF + msr CORTEX_X2_IMP_CPUPCR_EL3, x0 +workaround_reset_end cortex_x2, ERRATUM(2216384) + +check_erratum_ls cortex_x2, ERRATUM(2216384), CPU_REV(2, 0) + +workaround_reset_start cortex_x2, ERRATUM(2282622), ERRATA_X2_2282622 + /* Apply the workaround */ + sysreg_bit_set CORTEX_X2_CPUACTLR2_EL1, BIT(0) +workaround_reset_end cortex_x2, ERRATUM(2282622) + +check_erratum_ls cortex_x2, ERRATUM(2282622), CPU_REV(2, 1) + +workaround_reset_start cortex_x2, ERRATUM(2371105), ERRATA_X2_2371105 + /* Set bit 40 in CPUACTLR2_EL1 */ + sysreg_bit_set CORTEX_X2_CPUACTLR2_EL1, CORTEX_X2_CPUACTLR2_EL1_BIT_40 +workaround_reset_end cortex_x2, ERRATUM(2371105) + +check_erratum_ls cortex_x2, ERRATUM(2371105), CPU_REV(2, 0) + +workaround_reset_start cortex_x2, ERRATUM(2742423), ERRATA_X2_2742423 + /* Set CPUACTLR5_EL1[56:55] to 2'b01 */ + sysreg_bit_set CORTEX_X2_CPUACTLR5_EL1, BIT(55) + sysreg_bit_clear CORTEX_X2_CPUACTLR5_EL1, BIT(56) +workaround_reset_end cortex_x2, ERRATUM(2742423) + +check_erratum_ls cortex_x2, ERRATUM(2742423), CPU_REV(2, 1) + +workaround_reset_start cortex_x2, ERRATUM(2768515), ERRATA_X2_2768515 + /* dsb before isb of power down sequence */ + dsb sy +workaround_reset_end cortex_x2, ERRATUM(2768515) + +check_erratum_ls cortex_x2, ERRATUM(2768515), CPU_REV(2, 1) + +workaround_reset_start cortex_x2, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex-X2 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_x2 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_x2, CVE(2022, 23960) + +check_erratum_chosen cortex_x2, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +/* + * ERRATA_DSU_2313941 : + * The errata is defined in dsu_helpers.S but applies to cortex_x2 + * as well. Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_cortex_x2_2313941, check_errata_dsu_2313941 +.equ erratum_cortex_x2_2313941_wa, errata_dsu_2313941_wa +add_erratum_entry cortex_x2, ERRATUM(2313941), ERRATA_DSU_2313941, APPLY_AT_RESET + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_x2_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_X2_CPUPWRCTLR_EL1, CORTEX_X2_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + +#if ERRATA_X2_2768515 + mov x15, x30 + bl cpu_get_rev_var + bl erratum_cortex_x2_2768515_wa + mov x30, x15 +#endif /* ERRATA_X2_2768515 */ + isb + ret +endfunc cortex_x2_core_pwr_dwn + +errata_report_shim cortex_x2 + +cpu_reset_func_start cortex_x2 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_x2 + + /* --------------------------------------------- + * This function provides Cortex X2 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_x2_regs, "aS" +cortex_x2_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_x2_cpu_reg_dump + adr x6, cortex_x2_regs + mrs x8, CORTEX_X2_CPUECTLR_EL1 + ret +endfunc cortex_x2_cpu_reg_dump + +declare_cpu_ops cortex_x2, CORTEX_X2_MIDR, \ + cortex_x2_reset_func, \ + cortex_x2_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_x3.S b/lib/cpus/aarch64/cortex_x3.S new file mode 100644 index 0000000..0cb3b97 --- /dev/null +++ b/lib/cpus/aarch64/cortex_x3.S @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_x3.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex-X3 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex-X3 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_X3_BHB_LOOP_COUNT, cortex_x3 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_x3, ERRATUM(2070301), ERRATA_X3_2070301 + sysreg_bitfield_insert CORTEX_X3_CPUECTLR2_EL1, CORTEX_X3_CPUECTLR2_EL1_PF_MODE_CNSRV, \ + CORTEX_X3_CPUECTLR2_EL1_PF_MODE_LSB, CORTEX_X3_CPUECTLR2_EL1_PF_MODE_WIDTH +workaround_reset_end cortex_x3, ERRATUM(2070301) + +check_erratum_ls cortex_x3, ERRATUM(2070301), CPU_REV(1, 2) + +workaround_runtime_start cortex_x3, ERRATUM(2313909), ERRATA_X3_2313909 + sysreg_bit_set CORTEX_X3_CPUACTLR2_EL1, CORTEX_X3_CPUACTLR2_EL1_BIT_36 +workaround_runtime_end cortex_x3, ERRATUM(2313909), NO_ISB + +check_erratum_ls cortex_x3, ERRATUM(2313909), CPU_REV(1, 0) + +workaround_reset_start cortex_x3, ERRATUM(2615812), ERRATA_X3_2615812 + /* Disable retention control for WFI and WFE. */ + mrs x0, CORTEX_X3_CPUPWRCTLR_EL1 + bfi x0, xzr, #CORTEX_X3_CPUPWRCTLR_EL1_WFI_RET_CTRL_BITS_SHIFT, #3 + bfi x0, xzr, #CORTEX_X3_CPUPWRCTLR_EL1_WFE_RET_CTRL_BITS_SHIFT, #3 + msr CORTEX_X3_CPUPWRCTLR_EL1, x0 +workaround_reset_end cortex_x3, ERRATUM(2615812) + +check_erratum_ls cortex_x3, ERRATUM(2615812), CPU_REV(1, 1) + +workaround_reset_start cortex_x3, ERRATUM(2742421), ERRATA_X3_2742421 + /* Set CPUACTLR5_EL1[56:55] to 2'b01 */ + sysreg_bit_set CORTEX_X3_CPUACTLR5_EL1, CORTEX_X3_CPUACTLR5_EL1_BIT_55 + sysreg_bit_clear CORTEX_X3_CPUACTLR5_EL1, CORTEX_X3_CPUACTLR5_EL1_BIT_56 +workaround_reset_end cortex_x3, ERRATUM(2742421) + +check_erratum_ls cortex_x3, ERRATUM(2742421), CPU_REV(1, 1) + +workaround_reset_start cortex_x3, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + override_vector_table wa_cve_vbar_cortex_x3 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_x3, CVE(2022, 23960) + +check_erratum_chosen cortex_x3, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +cpu_reset_func_start cortex_x3 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_x3 + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_x3_core_pwr_dwn +apply_erratum cortex_x3, ERRATUM(2313909), ERRATA_X3_2313909 + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_X3_CPUPWRCTLR_EL1, CORTEX_X3_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc cortex_x3_core_pwr_dwn + +errata_report_shim cortex_x3 + + /* --------------------------------------------- + * This function provides Cortex-X3- + * specific register information for crash + * reporting. It needs to return with x6 + * pointing to a list of register names in ascii + * and x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_x3_regs, "aS" +cortex_x3_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_x3_cpu_reg_dump + adr x6, cortex_x3_regs + mrs x8, CORTEX_X3_CPUECTLR_EL1 + ret +endfunc cortex_x3_cpu_reg_dump + +declare_cpu_ops cortex_x3, CORTEX_X3_MIDR, \ + cortex_x3_reset_func, \ + cortex_x3_core_pwr_dwn diff --git a/lib/cpus/aarch64/cortex_x4.S b/lib/cpus/aarch64/cortex_x4.S new file mode 100644 index 0000000..7619f9c --- /dev/null +++ b/lib/cpus/aarch64/cortex_x4.S @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <cortex_x4.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Cortex X4 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Cortex X4 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table CORTEX_X4_BHB_LOOP_COUNT, cortex_x4 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start cortex_x4, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Cortex X4 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_cortex_x4 +#endif /* IMAGE_BL31 */ +workaround_reset_end cortex_x4, CVE(2022, 23960) + +check_erratum_chosen cortex_x4, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +cpu_reset_func_start cortex_x4 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end cortex_x4 + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func cortex_x4_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set CORTEX_X4_CPUPWRCTLR_EL1, CORTEX_X4_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc cortex_x4_core_pwr_dwn + +errata_report_shim cortex_x4 + + /* --------------------------------------------- + * This function provides Cortex X4-specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.cortex_x4_regs, "aS" +cortex_x4_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func cortex_x4_cpu_reg_dump + adr x6, cortex_x4_regs + mrs x8, CORTEX_X4_CPUECTLR_EL1 + ret +endfunc cortex_x4_cpu_reg_dump + +declare_cpu_ops cortex_x4, CORTEX_X4_MIDR, \ + cortex_x4_reset_func, \ + cortex_x4_core_pwr_dwn diff --git a/lib/cpus/aarch64/cpu_helpers.S b/lib/cpus/aarch64/cpu_helpers.S new file mode 100644 index 0000000..1ae3180 --- /dev/null +++ b/lib/cpus/aarch64/cpu_helpers.S @@ -0,0 +1,393 @@ +/* + * Copyright (c) 2014-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <common/bl_common.h> +#include <common/debug.h> +#include <cpu_macros.S> +#include <lib/cpus/cpu_ops.h> +#include <lib/cpus/errata.h> +#include <lib/el3_runtime/cpu_data.h> + + /* Reset fn is needed in BL at reset vector */ +#if defined(IMAGE_BL1) || defined(IMAGE_BL31) || \ + (defined(IMAGE_BL2) && RESET_TO_BL2) + /* + * The reset handler common to all platforms. After a matching + * cpu_ops structure entry is found, the correponding reset_handler + * in the cpu_ops is invoked. + * Clobbers: x0 - x19, x30 + */ + .globl reset_handler +func reset_handler + mov x19, x30 + + /* The plat_reset_handler can clobber x0 - x18, x30 */ + bl plat_reset_handler + + /* Get the matching cpu_ops pointer */ + bl get_cpu_ops_ptr + +#if ENABLE_ASSERTIONS + /* + * Assert if invalid cpu_ops obtained. If this is not valid, it may + * suggest that the proper CPU file hasn't been included. + */ + cmp x0, #0 + ASM_ASSERT(ne) +#endif + + /* Get the cpu_ops reset handler */ + ldr x2, [x0, #CPU_RESET_FUNC] + mov x30, x19 + cbz x2, 1f + + /* The cpu_ops reset handler can clobber x0 - x19, x30 */ + br x2 +1: + ret +endfunc reset_handler + +#endif + +#ifdef IMAGE_BL31 /* The power down core and cluster is needed only in BL31 */ + /* + * void prepare_cpu_pwr_dwn(unsigned int power_level) + * + * Prepare CPU power down function for all platforms. The function takes + * a domain level to be powered down as its parameter. After the cpu_ops + * pointer is retrieved from cpu_data, the handler for requested power + * level is called. + */ + .globl prepare_cpu_pwr_dwn +func prepare_cpu_pwr_dwn + /* + * If the given power level exceeds CPU_MAX_PWR_DWN_OPS, we call the + * power down handler for the last power level + */ + mov_imm x2, (CPU_MAX_PWR_DWN_OPS - 1) + cmp x0, x2 + csel x2, x2, x0, hi + + mrs x1, tpidr_el3 + ldr x0, [x1, #CPU_DATA_CPU_OPS_PTR] +#if ENABLE_ASSERTIONS + cmp x0, #0 + ASM_ASSERT(ne) +#endif + + /* Get the appropriate power down handler */ + mov x1, #CPU_PWR_DWN_OPS + add x1, x1, x2, lsl #3 + ldr x1, [x0, x1] +#if ENABLE_ASSERTIONS + cmp x1, #0 + ASM_ASSERT(ne) +#endif + br x1 +endfunc prepare_cpu_pwr_dwn + + + /* + * Initializes the cpu_ops_ptr if not already initialized + * in cpu_data. This can be called without a runtime stack, but may + * only be called after the MMU is enabled. + * clobbers: x0 - x6, x10 + */ + .globl init_cpu_ops +func init_cpu_ops + mrs x6, tpidr_el3 + ldr x0, [x6, #CPU_DATA_CPU_OPS_PTR] + cbnz x0, 1f + mov x10, x30 + bl get_cpu_ops_ptr + str x0, [x6, #CPU_DATA_CPU_OPS_PTR]! + mov x30, x10 +1: + ret +endfunc init_cpu_ops +#endif /* IMAGE_BL31 */ + +#if defined(IMAGE_BL31) && CRASH_REPORTING + /* + * The cpu specific registers which need to be reported in a crash + * are reported via cpu_ops cpu_reg_dump function. After a matching + * cpu_ops structure entry is found, the correponding cpu_reg_dump + * in the cpu_ops is invoked. + */ + .globl do_cpu_reg_dump +func do_cpu_reg_dump + mov x16, x30 + + /* Get the matching cpu_ops pointer */ + bl get_cpu_ops_ptr + cbz x0, 1f + + /* Get the cpu_ops cpu_reg_dump */ + ldr x2, [x0, #CPU_REG_DUMP] + cbz x2, 1f + blr x2 +1: + mov x30, x16 + ret +endfunc do_cpu_reg_dump +#endif + + /* + * The below function returns the cpu_ops structure matching the + * midr of the core. It reads the MIDR_EL1 and finds the matching + * entry in cpu_ops entries. Only the implementation and part number + * are used to match the entries. + * + * If cpu_ops for the MIDR_EL1 cannot be found and + * SUPPORT_UNKNOWN_MPID is enabled, it will try to look for a + * default cpu_ops with an MIDR value of 0. + * (Implementation number 0x0 should be reserved for software use + * and therefore no clashes should happen with that default value). + * + * Return : + * x0 - The matching cpu_ops pointer on Success + * x0 - 0 on failure. + * Clobbers : x0 - x5 + */ + .globl get_cpu_ops_ptr +func get_cpu_ops_ptr + /* Read the MIDR_EL1 */ + mrs x2, midr_el1 + mov_imm x3, CPU_IMPL_PN_MASK + + /* Retain only the implementation and part number using mask */ + and w2, w2, w3 + + /* Get the cpu_ops end location */ + adr x5, (__CPU_OPS_END__ + CPU_MIDR) + + /* Initialize the return parameter */ + mov x0, #0 +1: + /* Get the cpu_ops start location */ + adr x4, (__CPU_OPS_START__ + CPU_MIDR) + +2: + /* Check if we have reached end of list */ + cmp x4, x5 + b.eq search_def_ptr + + /* load the midr from the cpu_ops */ + ldr x1, [x4], #CPU_OPS_SIZE + and w1, w1, w3 + + /* Check if midr matches to midr of this core */ + cmp w1, w2 + b.ne 2b + + /* Subtract the increment and offset to get the cpu-ops pointer */ + sub x0, x4, #(CPU_OPS_SIZE + CPU_MIDR) +#if ENABLE_ASSERTIONS + cmp x0, #0 + ASM_ASSERT(ne) +#endif +#ifdef SUPPORT_UNKNOWN_MPID + cbnz x2, exit_mpid_found + /* Mark the unsupported MPID flag */ + adrp x1, unsupported_mpid_flag + add x1, x1, :lo12:unsupported_mpid_flag + str w2, [x1] +exit_mpid_found: +#endif + ret + + /* + * Search again for a default pointer (MIDR = 0x0) + * or return error if already searched. + */ +search_def_ptr: +#ifdef SUPPORT_UNKNOWN_MPID + cbz x2, error_exit + mov x2, #0 + b 1b +error_exit: +#endif + ret +endfunc get_cpu_ops_ptr + +/* + * Extract CPU revision and variant, and combine them into a single numeric for + * easier comparison. + */ + .globl cpu_get_rev_var +func cpu_get_rev_var + mrs x1, midr_el1 + + /* + * Extract the variant[23:20] and revision[3:0] from MIDR, and pack them + * as variant[7:4] and revision[3:0] of x0. + * + * First extract x1[23:16] to x0[7:0] and zero fill the rest. Then + * extract x1[3:0] into x0[3:0] retaining other bits. + */ + ubfx x0, x1, #(MIDR_VAR_SHIFT - MIDR_REV_BITS), #(MIDR_REV_BITS + MIDR_VAR_BITS) + bfxil x0, x1, #MIDR_REV_SHIFT, #MIDR_REV_BITS + ret +endfunc cpu_get_rev_var + +/* + * Compare the CPU's revision-variant (x0) with a given value (x1), for errata + * application purposes. If the revision-variant is less than or same as a given + * value, indicates that errata applies; otherwise not. + * + * Shall clobber: x0-x3 + */ + .globl cpu_rev_var_ls +func cpu_rev_var_ls + mov x2, #ERRATA_APPLIES + mov x3, #ERRATA_NOT_APPLIES + cmp x0, x1 + csel x0, x2, x3, ls + ret +endfunc cpu_rev_var_ls + +/* + * Compare the CPU's revision-variant (x0) with a given value (x1), for errata + * application purposes. If the revision-variant is higher than or same as a + * given value, indicates that errata applies; otherwise not. + * + * Shall clobber: x0-x3 + */ + .globl cpu_rev_var_hs +func cpu_rev_var_hs + mov x2, #ERRATA_APPLIES + mov x3, #ERRATA_NOT_APPLIES + cmp x0, x1 + csel x0, x2, x3, hs + ret +endfunc cpu_rev_var_hs + +/* + * Compare the CPU's revision-variant (x0) with a given range (x1 - x2), for errata + * application purposes. If the revision-variant is between or includes the given + * values, this indicates that errata applies; otherwise not. + * + * Shall clobber: x0-x4 + */ + .globl cpu_rev_var_range +func cpu_rev_var_range + mov x3, #ERRATA_APPLIES + mov x4, #ERRATA_NOT_APPLIES + cmp x0, x1 + csel x1, x3, x4, hs + cbz x1, 1f + cmp x0, x2 + csel x1, x3, x4, ls +1: + mov x0, x1 + ret +endfunc cpu_rev_var_range + +/* + * int check_wa_cve_2017_5715(void); + * + * This function returns: + * - ERRATA_APPLIES when firmware mitigation is required. + * - ERRATA_NOT_APPLIES when firmware mitigation is _not_ required. + * - ERRATA_MISSING when firmware mitigation would be required but + * is not compiled in. + * + * NOTE: Must be called only after cpu_ops have been initialized + * in per-CPU data. + */ + .globl check_wa_cve_2017_5715 +func check_wa_cve_2017_5715 + mrs x0, tpidr_el3 +#if ENABLE_ASSERTIONS + cmp x0, #0 + ASM_ASSERT(ne) +#endif + ldr x0, [x0, #CPU_DATA_CPU_OPS_PTR] +#if ENABLE_ASSERTIONS + cmp x0, #0 + ASM_ASSERT(ne) +#endif + ldr x0, [x0, #CPU_EXTRA1_FUNC] + /* + * If the reserved function pointer is NULL, this CPU + * is unaffected by CVE-2017-5715 so bail out. + */ + cmp x0, #CPU_NO_EXTRA1_FUNC + beq 1f + br x0 +1: + mov x0, #ERRATA_NOT_APPLIES + ret +endfunc check_wa_cve_2017_5715 + +/* + * void *wa_cve_2018_3639_get_disable_ptr(void); + * + * Returns a function pointer which is used to disable mitigation + * for CVE-2018-3639. + * The function pointer is only returned on cores that employ + * dynamic mitigation. If the core uses static mitigation or is + * unaffected by CVE-2018-3639 this function returns NULL. + * + * NOTE: Must be called only after cpu_ops have been initialized + * in per-CPU data. + */ + .globl wa_cve_2018_3639_get_disable_ptr +func wa_cve_2018_3639_get_disable_ptr + mrs x0, tpidr_el3 +#if ENABLE_ASSERTIONS + cmp x0, #0 + ASM_ASSERT(ne) +#endif + ldr x0, [x0, #CPU_DATA_CPU_OPS_PTR] +#if ENABLE_ASSERTIONS + cmp x0, #0 + ASM_ASSERT(ne) +#endif + ldr x0, [x0, #CPU_EXTRA2_FUNC] + ret +endfunc wa_cve_2018_3639_get_disable_ptr + +/* + * int check_smccc_arch_wa3_applies(void); + * + * This function checks whether SMCCC_ARCH_WORKAROUND_3 is enabled to mitigate + * CVE-2022-23960 for this CPU. It returns: + * - ERRATA_APPLIES when SMCCC_ARCH_WORKAROUND_3 can be invoked to mitigate + * the CVE. + * - ERRATA_NOT_APPLIES when SMCCC_ARCH_WORKAROUND_3 should not be invoked to + * mitigate the CVE. + * + * NOTE: Must be called only after cpu_ops have been initialized + * in per-CPU data. + */ + .globl check_smccc_arch_wa3_applies +func check_smccc_arch_wa3_applies + mrs x0, tpidr_el3 +#if ENABLE_ASSERTIONS + cmp x0, #0 + ASM_ASSERT(ne) +#endif + ldr x0, [x0, #CPU_DATA_CPU_OPS_PTR] +#if ENABLE_ASSERTIONS + cmp x0, #0 + ASM_ASSERT(ne) +#endif + ldr x0, [x0, #CPU_EXTRA3_FUNC] + /* + * If the reserved function pointer is NULL, this CPU + * is unaffected by CVE-2022-23960 so bail out. + */ + cmp x0, #CPU_NO_EXTRA3_FUNC + beq 1f + br x0 +1: + mov x0, #ERRATA_NOT_APPLIES + ret +endfunc check_smccc_arch_wa3_applies diff --git a/lib/cpus/aarch64/cpuamu.c b/lib/cpus/aarch64/cpuamu.c new file mode 100644 index 0000000..3a2fa81 --- /dev/null +++ b/lib/cpus/aarch64/cpuamu.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <cpuamu.h> +#include <lib/el3_runtime/pubsub_events.h> +#include <plat/common/platform.h> + +#define CPUAMU_NR_COUNTERS 5U + +struct cpuamu_ctx { + uint64_t cnts[CPUAMU_NR_COUNTERS]; + unsigned int mask; +}; + +static struct cpuamu_ctx cpuamu_ctxs[PLATFORM_CORE_COUNT]; + +int midr_match(unsigned int cpu_midr) +{ + unsigned int midr, midr_mask; + + midr = (unsigned int)read_midr(); + midr_mask = (MIDR_IMPL_MASK << MIDR_IMPL_SHIFT) | + (MIDR_PN_MASK << MIDR_PN_SHIFT); + return ((midr & midr_mask) == (cpu_midr & midr_mask)); +} + +void cpuamu_context_save(unsigned int nr_counters) +{ + struct cpuamu_ctx *ctx = &cpuamu_ctxs[plat_my_core_pos()]; + unsigned int i; + + assert(nr_counters <= CPUAMU_NR_COUNTERS); + + /* Save counter configuration */ + ctx->mask = cpuamu_read_cpuamcntenset_el0(); + + /* Disable counters */ + cpuamu_write_cpuamcntenclr_el0(ctx->mask); + isb(); + + /* Save counters */ + for (i = 0; i < nr_counters; i++) + ctx->cnts[i] = cpuamu_cnt_read(i); +} + +void cpuamu_context_restore(unsigned int nr_counters) +{ + struct cpuamu_ctx *ctx = &cpuamu_ctxs[plat_my_core_pos()]; + unsigned int i; + + assert(nr_counters <= CPUAMU_NR_COUNTERS); + + /* + * Disable counters. They were enabled early in the + * CPU reset function. + */ + cpuamu_write_cpuamcntenclr_el0(ctx->mask); + isb(); + + /* Restore counters */ + for (i = 0; i < nr_counters; i++) + cpuamu_cnt_write(i, ctx->cnts[i]); + isb(); + + /* Restore counter configuration */ + cpuamu_write_cpuamcntenset_el0(ctx->mask); +} diff --git a/lib/cpus/aarch64/cpuamu_helpers.S b/lib/cpus/aarch64/cpuamu_helpers.S new file mode 100644 index 0000000..e71e345 --- /dev/null +++ b/lib/cpus/aarch64/cpuamu_helpers.S @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2018-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <cpuamu.h> + + .globl cpuamu_cnt_read + .globl cpuamu_cnt_write + .globl cpuamu_read_cpuamcntenset_el0 + .globl cpuamu_read_cpuamcntenclr_el0 + .globl cpuamu_write_cpuamcntenset_el0 + .globl cpuamu_write_cpuamcntenclr_el0 + +/* + * uint64_t cpuamu_cnt_read(unsigned int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `x0`. + */ +func cpuamu_cnt_read + adr x1, 1f + add x1, x1, x0, lsl #3 /* each mrs/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x1, x1, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x1 + +1: read CPUAMEVCNTR0_EL0 + read CPUAMEVCNTR1_EL0 + read CPUAMEVCNTR2_EL0 + read CPUAMEVCNTR3_EL0 + read CPUAMEVCNTR4_EL0 +endfunc cpuamu_cnt_read + +/* + * void cpuamu_cnt_write(unsigned int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + */ +func cpuamu_cnt_write + adr x2, 1f + add x2, x2, x0, lsl #3 /* each msr/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x2, x2, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x2 + +1: write CPUAMEVCNTR0_EL0 + write CPUAMEVCNTR1_EL0 + write CPUAMEVCNTR2_EL0 + write CPUAMEVCNTR3_EL0 + write CPUAMEVCNTR4_EL0 +endfunc cpuamu_cnt_write + +/* + * unsigned int cpuamu_read_cpuamcntenset_el0(void); + * + * Read the `CPUAMCNTENSET_EL0` CPU register and return + * it in `x0`. + */ +func cpuamu_read_cpuamcntenset_el0 + mrs x0, CPUAMCNTENSET_EL0 + ret +endfunc cpuamu_read_cpuamcntenset_el0 + +/* + * unsigned int cpuamu_read_cpuamcntenclr_el0(void); + * + * Read the `CPUAMCNTENCLR_EL0` CPU register and return + * it in `x0`. + */ +func cpuamu_read_cpuamcntenclr_el0 + mrs x0, CPUAMCNTENCLR_EL0 + ret +endfunc cpuamu_read_cpuamcntenclr_el0 + +/* + * void cpuamu_write_cpuamcntenset_el0(unsigned int mask); + * + * Write `mask` to the `CPUAMCNTENSET_EL0` CPU register. + */ +func cpuamu_write_cpuamcntenset_el0 + msr CPUAMCNTENSET_EL0, x0 + ret +endfunc cpuamu_write_cpuamcntenset_el0 + +/* + * void cpuamu_write_cpuamcntenclr_el0(unsigned int mask); + * + * Write `mask` to the `CPUAMCNTENCLR_EL0` CPU register. + */ +func cpuamu_write_cpuamcntenclr_el0 + msr CPUAMCNTENCLR_EL0, x0 + ret +endfunc cpuamu_write_cpuamcntenclr_el0 diff --git a/lib/cpus/aarch64/denver.S b/lib/cpus/aarch64/denver.S new file mode 100644 index 0000000..884281d --- /dev/null +++ b/lib/cpus/aarch64/denver.S @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2015-2023, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2020-2022, NVIDIA Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <context.h> +#include <denver.h> +#include <cpu_macros.S> +#include <plat_macros.S> + + /* ------------------------------------------------- + * CVE-2017-5715 mitigation + * + * Flush the indirect branch predictor and RSB on + * entry to EL3 by issuing a newly added instruction + * for Denver CPUs. + * + * To achieve this without performing any branch + * instruction, a per-cpu vbar is installed which + * executes the workaround and then branches off to + * the corresponding vector entry in the main vector + * table. + * ------------------------------------------------- + */ +vector_base workaround_bpflush_runtime_exceptions + + .macro apply_workaround + stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + + /* Disable cycle counter when event counting is prohibited */ + mrs x1, pmcr_el0 + orr x0, x1, #PMCR_EL0_DP_BIT + msr pmcr_el0, x0 + isb + + /* ------------------------------------------------- + * A new write-only system register where a write of + * 1 to bit 0 will cause the indirect branch predictor + * and RSB to be flushed. + * + * A write of 0 to bit 0 will be ignored. A write of + * 1 to any other bit will cause an MCA. + * ------------------------------------------------- + */ + mov x0, #1 + msr s3_0_c15_c0_6, x0 + isb + + ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + .endm + + /* --------------------------------------------------------------------- + * Current EL with SP_EL0 : 0x0 - 0x200 + * --------------------------------------------------------------------- + */ +vector_entry workaround_bpflush_sync_exception_sp_el0 + b sync_exception_sp_el0 +end_vector_entry workaround_bpflush_sync_exception_sp_el0 + +vector_entry workaround_bpflush_irq_sp_el0 + b irq_sp_el0 +end_vector_entry workaround_bpflush_irq_sp_el0 + +vector_entry workaround_bpflush_fiq_sp_el0 + b fiq_sp_el0 +end_vector_entry workaround_bpflush_fiq_sp_el0 + +vector_entry workaround_bpflush_serror_sp_el0 + b serror_sp_el0 +end_vector_entry workaround_bpflush_serror_sp_el0 + + /* --------------------------------------------------------------------- + * Current EL with SP_ELx: 0x200 - 0x400 + * --------------------------------------------------------------------- + */ +vector_entry workaround_bpflush_sync_exception_sp_elx + b sync_exception_sp_elx +end_vector_entry workaround_bpflush_sync_exception_sp_elx + +vector_entry workaround_bpflush_irq_sp_elx + b irq_sp_elx +end_vector_entry workaround_bpflush_irq_sp_elx + +vector_entry workaround_bpflush_fiq_sp_elx + b fiq_sp_elx +end_vector_entry workaround_bpflush_fiq_sp_elx + +vector_entry workaround_bpflush_serror_sp_elx + b serror_sp_elx +end_vector_entry workaround_bpflush_serror_sp_elx + + /* --------------------------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x600 + * --------------------------------------------------------------------- + */ +vector_entry workaround_bpflush_sync_exception_aarch64 + apply_workaround + b sync_exception_aarch64 +end_vector_entry workaround_bpflush_sync_exception_aarch64 + +vector_entry workaround_bpflush_irq_aarch64 + apply_workaround + b irq_aarch64 +end_vector_entry workaround_bpflush_irq_aarch64 + +vector_entry workaround_bpflush_fiq_aarch64 + apply_workaround + b fiq_aarch64 +end_vector_entry workaround_bpflush_fiq_aarch64 + +vector_entry workaround_bpflush_serror_aarch64 + apply_workaround + b serror_aarch64 +end_vector_entry workaround_bpflush_serror_aarch64 + + /* --------------------------------------------------------------------- + * Lower EL using AArch32 : 0x600 - 0x800 + * --------------------------------------------------------------------- + */ +vector_entry workaround_bpflush_sync_exception_aarch32 + apply_workaround + b sync_exception_aarch32 +end_vector_entry workaround_bpflush_sync_exception_aarch32 + +vector_entry workaround_bpflush_irq_aarch32 + apply_workaround + b irq_aarch32 +end_vector_entry workaround_bpflush_irq_aarch32 + +vector_entry workaround_bpflush_fiq_aarch32 + apply_workaround + b fiq_aarch32 +end_vector_entry workaround_bpflush_fiq_aarch32 + +vector_entry workaround_bpflush_serror_aarch32 + apply_workaround + b serror_aarch32 +end_vector_entry workaround_bpflush_serror_aarch32 + + .global denver_disable_dco + + /* --------------------------------------------- + * Disable debug interfaces + * --------------------------------------------- + */ +func denver_disable_ext_debug + mov x0, #1 + msr osdlr_el1, x0 + isb + dsb sy + ret +endfunc denver_disable_ext_debug + + /* ---------------------------------------------------- + * Enable dynamic code optimizer (DCO) + * ---------------------------------------------------- + */ +func denver_enable_dco + /* DCO is not supported on PN5 and later */ + mrs x1, midr_el1 + mov_imm x2, DENVER_MIDR_PN4 + cmp x1, x2 + b.hi 1f + + mov x18, x30 + bl plat_my_core_pos + mov x1, #1 + lsl x1, x1, x0 + msr s3_0_c15_c0_2, x1 + mov x30, x18 +1: ret +endfunc denver_enable_dco + + /* ---------------------------------------------------- + * Disable dynamic code optimizer (DCO) + * ---------------------------------------------------- + */ +func denver_disable_dco + /* DCO is not supported on PN5 and later */ + mrs x1, midr_el1 + mov_imm x2, DENVER_MIDR_PN4 + cmp x1, x2 + b.hi 2f + + /* turn off background work */ + mov x18, x30 + bl plat_my_core_pos + mov x1, #1 + lsl x1, x1, x0 + lsl x2, x1, #16 + msr s3_0_c15_c0_2, x2 + isb + + /* wait till the background work turns off */ +1: mrs x2, s3_0_c15_c0_2 + lsr x2, x2, #32 + and w2, w2, 0xFFFF + and x2, x2, x1 + cbnz x2, 1b + + mov x30, x18 +2: ret +endfunc denver_disable_dco + +workaround_reset_start denver, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 +#if IMAGE_BL31 + adr x1, workaround_bpflush_runtime_exceptions + msr vbar_el3, x1 +#endif +workaround_reset_end denver, CVE(2017, 5715) + +check_erratum_custom_start denver, CVE(2017, 5715) + mov x0, #ERRATA_MISSING +#if WORKAROUND_CVE_2017_5715 + /* + * Check if the CPU supports the special instruction + * required to flush the indirect branch predictor and + * RSB. Support for this operation can be determined by + * comparing bits 19:16 of ID_AFR0_EL1 with 0b0001. + */ + mrs x1, id_afr0_el1 + mov x2, #0x10000 + and x1, x1, x2 + cbz x1, 1f + mov x0, #ERRATA_APPLIES +1: +#endif + ret +check_erratum_custom_end denver, CVE(2017, 5715) + +workaround_reset_start denver, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + /* + * Denver CPUs with DENVER_MIDR_PN3 or earlier, use different + * bits in the ACTLR_EL3 register to disable speculative + * store buffer and memory disambiguation. + */ + mrs x0, midr_el1 + mov_imm x1, DENVER_MIDR_PN4 + cmp x0, x1 + mrs x0, actlr_el3 + mov x1, #(DENVER_CPU_DIS_MD_EL3 | DENVER_CPU_DIS_SSB_EL3) + mov x2, #(DENVER_PN4_CPU_DIS_MD_EL3 | DENVER_PN4_CPU_DIS_SSB_EL3) + csel x3, x1, x2, ne + orr x0, x0, x3 + msr actlr_el3, x0 + isb + dsb sy +workaround_reset_end denver, CVE(2018, 3639) + +check_erratum_chosen denver, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 + +cpu_reset_func_start denver + /* ---------------------------------------------------- + * Reset ACTLR.PMSTATE to C1 state + * ---------------------------------------------------- + */ + mrs x0, actlr_el1 + bic x0, x0, #DENVER_CPU_PMSTATE_MASK + orr x0, x0, #DENVER_CPU_PMSTATE_C1 + msr actlr_el1, x0 + + /* ---------------------------------------------------- + * Enable dynamic code optimizer (DCO) + * ---------------------------------------------------- + */ + bl denver_enable_dco +cpu_reset_func_end denver + + /* ---------------------------------------------------- + * The CPU Ops core power down function for Denver. + * ---------------------------------------------------- + */ +func denver_core_pwr_dwn + + mov x19, x30 + + /* --------------------------------------------- + * Force the debug interfaces to be quiescent + * --------------------------------------------- + */ + bl denver_disable_ext_debug + + ret x19 +endfunc denver_core_pwr_dwn + + /* ------------------------------------------------------- + * The CPU Ops cluster power down function for Denver. + * ------------------------------------------------------- + */ +func denver_cluster_pwr_dwn + ret +endfunc denver_cluster_pwr_dwn + +errata_report_shim denver + + /* --------------------------------------------- + * This function provides Denver specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.denver_regs, "aS" +denver_regs: /* The ascii list of register names to be reported */ + .asciz "actlr_el1", "" + +func denver_cpu_reg_dump + adr x6, denver_regs + mrs x8, ACTLR_EL1 + ret +endfunc denver_cpu_reg_dump + +/* macro to declare cpu_ops for Denver SKUs */ +.macro denver_cpu_ops_wa midr + declare_cpu_ops_wa denver, \midr, \ + denver_reset_func, \ + check_erratum_denver_5715, \ + CPU_NO_EXTRA2_FUNC, \ + CPU_NO_EXTRA3_FUNC, \ + denver_core_pwr_dwn, \ + denver_cluster_pwr_dwn +.endm + +denver_cpu_ops_wa DENVER_MIDR_PN0 +denver_cpu_ops_wa DENVER_MIDR_PN1 +denver_cpu_ops_wa DENVER_MIDR_PN2 +denver_cpu_ops_wa DENVER_MIDR_PN3 +denver_cpu_ops_wa DENVER_MIDR_PN4 +denver_cpu_ops_wa DENVER_MIDR_PN5 +denver_cpu_ops_wa DENVER_MIDR_PN6 +denver_cpu_ops_wa DENVER_MIDR_PN7 +denver_cpu_ops_wa DENVER_MIDR_PN8 +denver_cpu_ops_wa DENVER_MIDR_PN9 diff --git a/lib/cpus/aarch64/dsu_helpers.S b/lib/cpus/aarch64/dsu_helpers.S new file mode 100644 index 0000000..a34b9a6 --- /dev/null +++ b/lib/cpus/aarch64/dsu_helpers.S @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <dsu_def.h> +#include <lib/cpus/errata.h> + + /* ----------------------------------------------------------------------- + * DSU erratum 798953 check function + * Checks the DSU variant, revision and configuration to determine if + * the erratum applies. Erratum applies on all configurations of the + * DSU and if revision-variant is r0p0. + * + * The erratum was fixed in r0p1. + * + * This function is called from both assembly and C environment. So it + * follows AAPCS. + * + * Clobbers: x0-x3 + * ----------------------------------------------------------------------- + */ + .globl check_errata_dsu_798953 + .globl errata_dsu_798953_wa + +func check_errata_dsu_798953 + mov x2, #ERRATA_APPLIES + mov x3, #ERRATA_NOT_APPLIES + + /* Check if DSU is equal to r0p0 */ + mrs x1, CLUSTERIDR_EL1 + + /* DSU variant and revision bitfields in CLUSTERIDR are adjacent */ + ubfx x0, x1, #CLUSTERIDR_REV_SHIFT,\ + #(CLUSTERIDR_REV_BITS + CLUSTERIDR_VAR_BITS) + mov x1, #(0x0 << CLUSTERIDR_REV_SHIFT) + cmp x0, x1 + csel x0, x2, x3, EQ + ret +endfunc check_errata_dsu_798953 + + /* -------------------------------------------------- + * Errata Workaround for DSU erratum #798953. + * + * Can clobber only: x0-x8 + * -------------------------------------------------- + */ +func errata_dsu_798953_wa + mov x8, x30 + bl check_errata_dsu_798953 + cbz x0, 1f + + /* If erratum applies, disable high-level clock gating */ + mrs x0, CLUSTERACTLR_EL1 + orr x0, x0, #CLUSTERACTLR_EL1_DISABLE_CLOCK_GATING + msr CLUSTERACTLR_EL1, x0 + isb +1: + ret x8 +endfunc errata_dsu_798953_wa + + /* ----------------------------------------------------------------------- + * DSU erratum 936184 check function + * Checks the DSU variant, revision and configuration to determine if + * the erratum applies. Erratum applies if ACP interface is present + * in the DSU and revision-variant < r2p0. + * + * The erratum was fixed in r2p0. + * + * This function is called from both assembly and C environment. So it + * follows AAPCS. + * + * Clobbers: x0-x4 + * ----------------------------------------------------------------------- + */ + .globl check_errata_dsu_936184 + .globl errata_dsu_936184_wa + .weak is_scu_present_in_dsu + + /* -------------------------------------------------------------------- + * Default behaviour respresents SCU is always present with DSU. + * CPUs can override this definition if required. + * + * Can clobber only: x0-x3 + * -------------------------------------------------------------------- + */ +func is_scu_present_in_dsu + mov x0, #1 + ret +endfunc is_scu_present_in_dsu + +func check_errata_dsu_936184 + mov x4, x30 + bl is_scu_present_in_dsu + cmp x0, xzr + /* Default error status */ + mov x0, #ERRATA_NOT_APPLIES + + /* If SCU is not present, return without applying patch */ + b.eq 1f + + /* Erratum applies only if DSU has the ACP interface */ + mrs x1, CLUSTERCFR_EL1 + ubfx x1, x1, #CLUSTERCFR_ACP_SHIFT, #1 + cbz x1, 1f + + /* If ACP is present, check if DSU is older than r2p0 */ + mrs x1, CLUSTERIDR_EL1 + + /* DSU variant and revision bitfields in CLUSTERIDR are adjacent */ + ubfx x2, x1, #CLUSTERIDR_REV_SHIFT,\ + #(CLUSTERIDR_REV_BITS + CLUSTERIDR_VAR_BITS) + cmp x2, #(0x2 << CLUSTERIDR_VAR_SHIFT) + b.hs 1f + mov x0, #ERRATA_APPLIES +1: + ret x4 +endfunc check_errata_dsu_936184 + + /* -------------------------------------------------- + * Errata Workaround for DSU erratum #936184. + * + * Can clobber only: x0-x8 + * -------------------------------------------------- + */ +func errata_dsu_936184_wa + mov x8, x30 + bl check_errata_dsu_936184 + cbz x0, 1f + + /* If erratum applies, we set a mask to a DSU control register */ + mrs x0, CLUSTERACTLR_EL1 + ldr x1, =DSU_ERRATA_936184_MASK + orr x0, x0, x1 + msr CLUSTERACTLR_EL1, x0 + isb +1: + ret x8 +endfunc errata_dsu_936184_wa + + /* ----------------------------------------------------------------------- + * DSU erratum 2313941 check function + * Checks the DSU variant, revision and configuration to determine if + * the erratum applies. Erratum applies on all configurations of the + * DSU and if revision-variant is r0p0, r1p0, r2p0, r2p1, r3p0, r3p1. + * + * The erratum is still open. + * + * This function is called from both assembly and C environment. So it + * follows AAPCS. + * + * Clobbers: x0-x3 + * ----------------------------------------------------------------------- + */ + .globl check_errata_dsu_2313941 + .globl errata_dsu_2313941_wa + +func check_errata_dsu_2313941 + mov x2, #ERRATA_APPLIES + mov x3, #ERRATA_NOT_APPLIES + + /* Check if DSU version is less than or equal to r3p1 */ + mrs x1, CLUSTERIDR_EL1 + + /* DSU variant and revision bitfields in CLUSTERIDR are adjacent */ + ubfx x0, x1, #CLUSTERIDR_REV_SHIFT,\ + #(CLUSTERIDR_REV_BITS + CLUSTERIDR_VAR_BITS) + mov x1, #(0x31 << CLUSTERIDR_REV_SHIFT) + cmp x0, x1 + csel x0, x2, x3, LS + ret +endfunc check_errata_dsu_2313941 + + /* -------------------------------------------------- + * Errata Workaround for DSU erratum #2313941. + * + * Can clobber only: x0-x8 + * -------------------------------------------------- + */ +func errata_dsu_2313941_wa + mov x8, x30 + bl check_errata_dsu_2313941 + cbz x0, 1f + + /* If erratum applies, disable high-level clock gating */ + mrs x0, CLUSTERACTLR_EL1 + orr x0, x0, #CLUSTERACTLR_EL1_DISABLE_SCLK_GATING + msr CLUSTERACTLR_EL1, x0 + isb +1: + ret x8 +endfunc errata_dsu_2313941_wa diff --git a/lib/cpus/aarch64/generic.S b/lib/cpus/aarch64/generic.S new file mode 100644 index 0000000..ef1f048 --- /dev/null +++ b/lib/cpus/aarch64/generic.S @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2020, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <generic.h> +#include <cpu_macros.S> +#include <plat_macros.S> + + /* --------------------------------------------- + * Disable L1 data cache and unified L2 cache + * --------------------------------------------- + */ +func generic_disable_dcache + mrs x1, sctlr_el3 + bic x1, x1, #SCTLR_C_BIT + msr sctlr_el3, x1 + isb + ret +endfunc generic_disable_dcache + +func generic_core_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl generic_disable_dcache + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + ret x18 +endfunc generic_core_pwr_dwn + +func generic_cluster_pwr_dwn + mov x18, x30 + + /* --------------------------------------------- + * Turn off caches. + * --------------------------------------------- + */ + bl generic_disable_dcache + + /* --------------------------------------------- + * Flush L1 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level1 + + /* --------------------------------------------- + * Disable the optional ACP. + * --------------------------------------------- + */ + bl plat_disable_acp + + /* --------------------------------------------- + * Flush L2 caches. + * --------------------------------------------- + */ + mov x0, #DCCISW + bl dcsw_op_level2 + + ret x18 + +endfunc generic_cluster_pwr_dwn + +/* --------------------------------------------- + * Unimplemented functions. + * --------------------------------------------- + */ +.equ generic_errata_report, 0 +.equ generic_cpu_reg_dump, 0 +.equ generic_reset_func, 0 + +declare_cpu_ops generic, AARCH64_GENERIC_MIDR, \ + generic_reset_func, \ + generic_core_pwr_dwn, \ + generic_cluster_pwr_dwn diff --git a/lib/cpus/aarch64/neoverse_e1.S b/lib/cpus/aarch64/neoverse_e1.S new file mode 100644 index 0000000..45bd8d3 --- /dev/null +++ b/lib/cpus/aarch64/neoverse_e1.S @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2018-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <common/debug.h> +#include <neoverse_e1.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Neoverse E1 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Neoverse-E1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +/* + * ERRATA_DSU_936184: + * The errata is defined in dsu_helpers.S and applies to neoverse_e1. + * Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_neoverse_e1_936184, check_errata_dsu_936184 +.equ erratum_neoverse_e1_936184_wa, errata_dsu_936184_wa +add_erratum_entry neoverse_e1, ERRATUM(936184), ERRATA_DSU_936184, APPLY_AT_RESET + +cpu_reset_func_start neoverse_e1 +cpu_reset_func_end neoverse_e1 + +func neoverse_e1_cpu_pwr_dwn + mrs x0, NEOVERSE_E1_CPUPWRCTLR_EL1 + orr x0, x0, #NEOVERSE_E1_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + msr NEOVERSE_E1_CPUPWRCTLR_EL1, x0 + isb + ret +endfunc neoverse_e1_cpu_pwr_dwn + +errata_report_shim neoverse_e1 + +.section .rodata.neoverse_e1_regs, "aS" +neoverse_e1_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func neoverse_e1_cpu_reg_dump + adr x6, neoverse_e1_regs + mrs x8, NEOVERSE_E1_ECTLR_EL1 + ret +endfunc neoverse_e1_cpu_reg_dump + +declare_cpu_ops neoverse_e1, NEOVERSE_E1_MIDR, \ + neoverse_e1_reset_func, \ + neoverse_e1_cpu_pwr_dwn diff --git a/lib/cpus/aarch64/neoverse_hermes.S b/lib/cpus/aarch64/neoverse_hermes.S new file mode 100644 index 0000000..cb90b71 --- /dev/null +++ b/lib/cpus/aarch64/neoverse_hermes.S @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <neoverse_hermes.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Neoverse Hermes must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Neoverse Hermes supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +cpu_reset_func_start neoverse_hermes + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end neoverse_hermes + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func neoverse_hermes_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set NEOVERSE_HERMES_CPUPWRCTLR_EL1, NEOVERSE_HERMES_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + isb + ret +endfunc neoverse_hermes_core_pwr_dwn + +errata_report_shim neoverse_hermes + + /* --------------------------------------------- + * This function provides Neoverse Hermes specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.neoverse_hermes_regs, "aS" +neoverse_hermes_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func neoverse_hermes_cpu_reg_dump + adr x6, neoverse_hermes_regs + mrs x8, NEOVERSE_HERMES_CPUECTLR_EL1 + ret +endfunc neoverse_hermes_cpu_reg_dump + +declare_cpu_ops neoverse_hermes, NEOVERSE_HERMES_MIDR, \ + neoverse_hermes_reset_func, \ + neoverse_hermes_core_pwr_dwn diff --git a/lib/cpus/aarch64/neoverse_n1.S b/lib/cpus/aarch64/neoverse_n1.S new file mode 100644 index 0000000..36a7ee7 --- /dev/null +++ b/lib/cpus/aarch64/neoverse_n1.S @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <cpuamu.h> +#include <cpu_macros.S> +#include <neoverse_n1.h> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + + .global neoverse_n1_errata_ic_trap_handler + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table NEOVERSE_N1_BHB_LOOP_COUNT, neoverse_n1 +#endif /* WORKAROUND_CVE_2022_23960 */ + +/* + * ERRATA_DSU_936184: + * The errata is defined in dsu_helpers.S and applies to Neoverse N1. + * Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_neoverse_n1_936184, check_errata_dsu_936184 +.equ erratum_neoverse_n1_936184_wa, errata_dsu_936184_wa +add_erratum_entry neoverse_n1, ERRATUM(936184), ERRATA_DSU_936184, APPLY_AT_RESET + +workaround_reset_start neoverse_n1, ERRATUM(1043202), ERRATA_N1_1043202 + /* Apply instruction patching sequence */ + ldr x0, =0x0 + msr CPUPSELR_EL3, x0 + ldr x0, =0xF3BF8F2F + msr CPUPOR_EL3, x0 + ldr x0, =0xFFFFFFFF + msr CPUPMR_EL3, x0 + ldr x0, =0x800200071 + msr CPUPCR_EL3, x0 +workaround_reset_end neoverse_n1, ERRATUM(1043202) + +check_erratum_ls neoverse_n1, ERRATUM(1043202), CPU_REV(1, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1073348), ERRATA_N1_1073348 + sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 +workaround_reset_end neoverse_n1, ERRATUM(1073348) + +check_erratum_ls neoverse_n1, ERRATUM(1073348), CPU_REV(1, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1130799), ERRATA_N1_1130799 + sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 +workaround_reset_end neoverse_n1, ERRATUM(1130799) + +check_erratum_ls neoverse_n1, ERRATUM(1130799), CPU_REV(2, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1165347), ERRATA_N1_1165347 + sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 + sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 +workaround_reset_end neoverse_n1, ERRATUM(1165347) + +check_erratum_ls neoverse_n1, ERRATUM(1165347), CPU_REV(2, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1207823), ERRATA_N1_1207823 + sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 +workaround_reset_end neoverse_n1, ERRATUM(1207823) + +check_erratum_ls neoverse_n1, ERRATUM(1207823), CPU_REV(2, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1220197), ERRATA_N1_1220197 + sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_WS_THR_L2_MASK +workaround_reset_end neoverse_n1, ERRATUM(1220197) + +check_erratum_ls neoverse_n1, ERRATUM(1220197), CPU_REV(2, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1257314), ERRATA_N1_1257314 + sysreg_bit_set NEOVERSE_N1_CPUACTLR3_EL1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 +workaround_reset_end neoverse_n1, ERRATUM(1257314) + +check_erratum_ls neoverse_n1, ERRATUM(1257314), CPU_REV(3, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1262606), ERRATA_N1_1262606 + sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 +workaround_reset_end neoverse_n1, ERRATUM(1262606) + +check_erratum_ls neoverse_n1, ERRATUM(1262606), CPU_REV(3, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1262888), ERRATA_N1_1262888 + sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT +workaround_reset_end neoverse_n1, ERRATUM(1262888) + +check_erratum_ls neoverse_n1, ERRATUM(1262888), CPU_REV(3, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1275112), ERRATA_N1_1275112 + sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 +workaround_reset_end neoverse_n1, ERRATUM(1275112) + +check_erratum_ls neoverse_n1, ERRATUM(1275112), CPU_REV(3, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1315703), ERRATA_N1_1315703 + sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 +workaround_reset_end neoverse_n1, ERRATUM(1315703) + +check_erratum_ls neoverse_n1, ERRATUM(1315703), CPU_REV(3, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1542419), ERRATA_N1_1542419 + /* Apply instruction patching sequence */ + ldr x0, =0x0 + msr CPUPSELR_EL3, x0 + ldr x0, =0xEE670D35 + msr CPUPOR_EL3, x0 + ldr x0, =0xFFFF0FFF + msr CPUPMR_EL3, x0 + ldr x0, =0x08000020007D + msr CPUPCR_EL3, x0 + isb +workaround_reset_end neoverse_n1, ERRATUM(1542419) + +check_erratum_range neoverse_n1, ERRATUM(1542419), CPU_REV(3, 0), CPU_REV(4, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1868343), ERRATA_N1_1868343 + sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 +workaround_reset_end neoverse_n1, ERRATUM(1868343) + +check_erratum_ls neoverse_n1, ERRATUM(1868343), CPU_REV(4, 0) + +workaround_reset_start neoverse_n1, ERRATUM(1946160), ERRATA_N1_1946160 + mov x0, #3 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3900002 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 + mov x0, #4 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3800082 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 + mov x0, #5 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3800200 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF003E0 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 + isb +workaround_reset_end neoverse_n1, ERRATUM(1946160) + +check_erratum_range neoverse_n1, ERRATUM(1946160), CPU_REV(3, 0), CPU_REV(4, 1) + +workaround_runtime_start neoverse_n1, ERRATUM(2743102), ERRATA_N1_2743102 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end neoverse_n1, ERRATUM(2743102) + +check_erratum_ls neoverse_n1, ERRATUM(2743102), CPU_REV(4, 1) + +workaround_reset_start neoverse_n1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Neoverse-N1 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_neoverse_n1 +#endif /* IMAGE_BL31 */ +workaround_reset_end neoverse_n1, CVE(2022, 23960) + +check_erratum_chosen neoverse_n1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +/* -------------------------------------------------- + * Disable speculative loads if Neoverse N1 supports + * SSBS. + * + * Shall clobber: x0. + * -------------------------------------------------- + */ +func neoverse_n1_disable_speculative_loads + /* Check if the PE implements SSBS */ + mrs x0, id_aa64pfr1_el1 + tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) + b.eq 1f + + /* Disable speculative loads */ + msr SSBS, xzr + +1: + ret +endfunc neoverse_n1_disable_speculative_loads + +cpu_reset_func_start neoverse_n1 + bl neoverse_n1_disable_speculative_loads + + /* Forces all cacheable atomic instructions to be near */ + sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 + isb + +#if ENABLE_FEAT_AMU + /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ + sysreg_bit_set actlr_el3, NEOVERSE_N1_ACTLR_AMEN_BIT + /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ + sysreg_bit_set actlr_el2, NEOVERSE_N1_ACTLR_AMEN_BIT + /* Enable group0 counters */ + mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK + msr CPUAMCNTENSET_EL0, x0 +#endif + +#if NEOVERSE_Nx_EXTERNAL_LLC + /* Some system may have External LLC, core needs to be made aware */ + sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT +#endif +cpu_reset_func_end neoverse_n1 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func neoverse_n1_core_pwr_dwn + /* --------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------- + */ + sysreg_bit_set NEOVERSE_N1_CPUPWRCTLR_EL1, NEOVERSE_N1_CORE_PWRDN_EN_MASK + + apply_erratum neoverse_n1, ERRATUM(2743102), ERRATA_N1_2743102 + + isb + ret +endfunc neoverse_n1_core_pwr_dwn + +errata_report_shim neoverse_n1 + +/* + * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB + * inner-shareable invalidation to an arbitrary address followed by a DSB. + * + * x1: Exception Syndrome + */ +func neoverse_n1_errata_ic_trap_handler + cmp x1, #NEOVERSE_N1_EC_IC_TRAP + b.ne 1f + tlbi vae3is, xzr + dsb sy + + # Skip the IC instruction itself + mrs x3, elr_el3 + add x3, x3, #4 + msr elr_el3, x3 + + ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] + + /* + * Issue Error Synchronization Barrier to synchronize SErrors before + * exiting EL3. We're running with EAs unmasked, so any synchronized + * errors would be taken immediately; therefore no need to inspect + * DISR_EL1 register. + */ + esb + exception_return +1: + ret +endfunc neoverse_n1_errata_ic_trap_handler + + /* --------------------------------------------- + * This function provides neoverse_n1 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.neoverse_n1_regs, "aS" +neoverse_n1_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func neoverse_n1_cpu_reg_dump + adr x6, neoverse_n1_regs + mrs x8, NEOVERSE_N1_CPUECTLR_EL1 + ret +endfunc neoverse_n1_cpu_reg_dump + +declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ + neoverse_n1_reset_func, \ + neoverse_n1_errata_ic_trap_handler, \ + neoverse_n1_core_pwr_dwn diff --git a/lib/cpus/aarch64/neoverse_n1_pubsub.c b/lib/cpus/aarch64/neoverse_n1_pubsub.c new file mode 100644 index 0000000..ecfb501 --- /dev/null +++ b/lib/cpus/aarch64/neoverse_n1_pubsub.c @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <neoverse_n1.h> +#include <cpuamu.h> +#include <lib/el3_runtime/pubsub_events.h> + +static void *neoverse_n1_context_save(const void *arg) +{ + if (midr_match(NEOVERSE_N1_MIDR) != 0) + cpuamu_context_save(NEOVERSE_N1_AMU_NR_COUNTERS); + + return (void *)0; +} + +static void *neoverse_n1_context_restore(const void *arg) +{ + if (midr_match(NEOVERSE_N1_MIDR) != 0) + cpuamu_context_restore(NEOVERSE_N1_AMU_NR_COUNTERS); + + return (void *)0; +} + +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, neoverse_n1_context_save); +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, neoverse_n1_context_restore); diff --git a/lib/cpus/aarch64/neoverse_n2.S b/lib/cpus/aarch64/neoverse_n2.S new file mode 100644 index 0000000..477522f --- /dev/null +++ b/lib/cpus/aarch64/neoverse_n2.S @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2020-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <cpu_macros.S> +#include <neoverse_n2.h> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Neoverse N2 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Neoverse-N2 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table NEOVERSE_N2_BHB_LOOP_COUNT, neoverse_n2 +#endif /* WORKAROUND_CVE_2022_23960 */ + +/* + * ERRATA_DSU_2313941: + * The errata is defined in dsu_helpers.S and applies to Neoverse N2. + * Henceforth creating symbolic names to the already existing errata + * workaround functions to get them registered under the Errata Framework. + */ +.equ check_erratum_neoverse_n2_2313941, check_errata_dsu_2313941 +.equ erratum_neoverse_n2_2313941_wa, errata_dsu_2313941_wa +add_erratum_entry neoverse_n2, ERRATUM(2313941), ERRATA_DSU_2313941, APPLY_AT_RESET + +workaround_reset_start neoverse_n2, ERRATUM(2002655), ERRATA_N2_2002655 + /* Apply instruction patching sequence */ + ldr x0,=0x6 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xF3A08002 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFF0F7FE + msr S3_6_c15_c8_3,x0 + ldr x0,=0x40000001003ff + msr S3_6_c15_c8_1,x0 + ldr x0,=0x7 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xBF200000 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFEF0000 + msr S3_6_c15_c8_3,x0 + ldr x0,=0x40000001003f3 + msr S3_6_c15_c8_1,x0 +workaround_reset_end neoverse_n2, ERRATUM(2002655) + +check_erratum_ls neoverse_n2, ERRATUM(2002655), CPU_REV(0, 0) + +workaround_reset_start neoverse_n2, ERRATUM(2025414), ERRATA_N2_2025414 + sysreg_bit_set NEOVERSE_N2_CPUECTLR_EL1, NEOVERSE_N2_CPUECTLR_EL1_PFSTIDIS_BIT +workaround_reset_end neoverse_n2, ERRATUM(2025414) + +check_erratum_ls neoverse_n2, ERRATUM(2025414), CPU_REV(0, 0) + +workaround_reset_start neoverse_n2, ERRATUM(2067956), ERRATA_N2_2067956 + sysreg_bit_set NEOVERSE_N2_CPUACTLR_EL1, NEOVERSE_N2_CPUACTLR_EL1_BIT_46 +workaround_reset_end neoverse_n2, ERRATUM(2067956) + +check_erratum_ls neoverse_n2, ERRATUM(2067956), CPU_REV(0, 0) + +workaround_runtime_start neoverse_n2, ERRATUM(2009478), ERRATA_N2_2009478 + /* Stash ERRSELR_EL1 in x2 */ + mrs x2, ERRSELR_EL1 + + /* Select error record 0 and clear ED bit */ + msr ERRSELR_EL1, xzr + mrs x1, ERXCTLR_EL1 + bfi x1, xzr, #ERXCTLR_ED_SHIFT, #1 + msr ERXCTLR_EL1, x1 + + /* Restore ERRSELR_EL1 from x2 */ + msr ERRSELR_EL1, x2 +workaround_runtime_end neoverse_n2, ERRATUM(2009478), NO_ISB + +check_erratum_ls neoverse_n2, ERRATUM(2009478), CPU_REV(0, 0) + +workaround_reset_start neoverse_n2, ERRATUM(2138953), ERRATA_N2_2138953 + /* Apply instruction patching sequence */ + mrs x1, NEOVERSE_N2_CPUECTLR2_EL1 + mov x0, #NEOVERSE_N2_CPUECTLR2_EL1_PF_MODE_CNSRV + bfi x1, x0, #CPUECTLR2_EL1_PF_MODE_LSB, #CPUECTLR2_EL1_PF_MODE_WIDTH + msr NEOVERSE_N2_CPUECTLR2_EL1, x1 +workaround_reset_end neoverse_n2, ERRATUM(2138953) + +check_erratum_ls neoverse_n2, ERRATUM(2138953), CPU_REV(0, 3) + +workaround_reset_start neoverse_n2, ERRATUM(2138956), ERRATA_N2_2138956 + /* Apply instruction patching sequence */ + ldr x0,=0x3 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xF3A08002 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFF0F7FE + msr S3_6_c15_c8_3,x0 + ldr x0,=0x10002001003FF + msr S3_6_c15_c8_1,x0 + ldr x0,=0x4 + msr S3_6_c15_c8_0,x0 + ldr x0,=0xBF200000 + msr S3_6_c15_c8_2,x0 + ldr x0,=0xFFEF0000 + msr S3_6_c15_c8_3,x0 + ldr x0,=0x10002001003F3 + msr S3_6_c15_c8_1,x0 +workaround_reset_end neoverse_n2, ERRATUM(2138956) + +check_erratum_ls neoverse_n2, ERRATUM(2138956), CPU_REV(0, 0) + + +workaround_reset_start neoverse_n2, ERRATUM(2138958), ERRATA_N2_2138958 + /* Apply instruction patching sequence */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR5_EL1, NEOVERSE_N2_CPUACTLR5_EL1_BIT_13 +workaround_reset_end neoverse_n2, ERRATUM(2138958) + +check_erratum_ls neoverse_n2, ERRATUM(2138958), CPU_REV(0, 0) + +workaround_reset_start neoverse_n2, ERRATUM(2189731), ERRATA_N2_2189731 + sysreg_bit_set NEOVERSE_N2_CPUACTLR5_EL1, NEOVERSE_N2_CPUACTLR5_EL1_BIT_44 +workaround_reset_end neoverse_n2, ERRATUM(2189731) + +check_erratum_ls neoverse_n2, ERRATUM(2189731), CPU_REV(0, 0) + +workaround_reset_start neoverse_n2, ERRATUM(2242400), ERRATA_N2_2242400 + /* Apply instruction patching sequence */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR5_EL1, NEOVERSE_N2_CPUACTLR5_EL1_BIT_17 + ldr x0, =0x2 + msr S3_6_c15_c8_0, x0 + ldr x0, =0x10F600E000 + msr S3_6_c15_c8_2, x0 + ldr x0, =0x10FF80E000 + msr S3_6_c15_c8_3, x0 + ldr x0, =0x80000000003FF + msr S3_6_c15_c8_1, x0 +workaround_reset_end neoverse_n2, ERRATUM(2242400) + +check_erratum_ls neoverse_n2, ERRATUM(2242400), CPU_REV(0, 0) + +workaround_reset_start neoverse_n2, ERRATUM(2242415), ERRATA_N2_2242415 + sysreg_bit_set NEOVERSE_N2_CPUACTLR_EL1, NEOVERSE_N2_CPUACTLR_EL1_BIT_22 +workaround_reset_end neoverse_n2, ERRATUM(2242415) + +check_erratum_ls neoverse_n2, ERRATUM(2242415), CPU_REV(0, 0) + +workaround_reset_start neoverse_n2, ERRATUM(2280757), ERRATA_N2_2280757 + /* Apply instruction patching sequence */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR_EL1, NEOVERSE_N2_CPUACTLR_EL1_BIT_22 +workaround_reset_end neoverse_n2, ERRATUM(2280757) + +check_erratum_ls neoverse_n2, ERRATUM(2280757), CPU_REV(0, 0) + +workaround_runtime_start neoverse_n2, ERRATUM(2326639), ERRATA_N2_2326639 + /* Set bit 36 in ACTLR2_EL1 */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR2_EL1, NEOVERSE_N2_CPUACTLR2_EL1_BIT_36 +workaround_runtime_end neoverse_n2, ERRATUM(2326639) + +check_erratum_ls neoverse_n2, ERRATUM(2326639), CPU_REV(0, 0) + +workaround_runtime_start neoverse_n2, ERRATUM(2340933), ERRATA_N2_2340933 + /* Set bit 61 in CPUACTLR5_EL1 */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR5_EL1, BIT(61) +workaround_runtime_end neoverse_n2, ERRATUM(2340933) + +check_erratum_ls neoverse_n2, ERRATUM(2340933), CPU_REV(0, 0) + +workaround_runtime_start neoverse_n2, ERRATUM(2346952), ERRATA_N2_2346952 + /* Set TXREQ to STATIC and full L2 TQ size */ + mrs x1, NEOVERSE_N2_CPUECTLR2_EL1 + mov x0, #CPUECTLR2_EL1_TXREQ_STATIC_FULL + bfi x1, x0, #CPUECTLR2_EL1_TXREQ_LSB, #CPUECTLR2_EL1_TXREQ_WIDTH + msr NEOVERSE_N2_CPUECTLR2_EL1, x1 +workaround_runtime_end neoverse_n2, ERRATUM(2346952) + +check_erratum_ls neoverse_n2, ERRATUM(2346952), CPU_REV(0, 2) + +workaround_reset_start neoverse_n2, ERRATUM(2376738), ERRATA_N2_2376738 + /* Set CPUACTLR2_EL1[0] to 1 to force PLDW/PFRM + * ST to behave like PLD/PFRM LD and not cause + * invalidations to other PE caches. + */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR2_EL1, NEOVERSE_N2_CPUACTLR2_EL1_BIT_0 +workaround_reset_end neoverse_n2, ERRATUM(2376738) + +check_erratum_ls neoverse_n2, ERRATUM(2376738), CPU_REV(0, 3) + +workaround_reset_start neoverse_n2, ERRATUM(2388450), ERRATA_N2_2388450 + /*Set bit 40 in ACTLR2_EL1 */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR2_EL1, NEOVERSE_N2_CPUACTLR2_EL1_BIT_40 +workaround_reset_end neoverse_n2, ERRATUM(2388450) + +check_erratum_ls neoverse_n2, ERRATUM(2388450), CPU_REV(0, 0) + +workaround_reset_start neoverse_n2, ERRATUM(2743014), ERRATA_N2_2743014 + /* Set CPUACTLR5_EL1[56:55] to 2'b01 */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR5_EL1, NEOVERSE_N2_CPUACTLR5_EL1_BIT_55 + sysreg_bit_clear NEOVERSE_N2_CPUACTLR5_EL1, NEOVERSE_N2_CPUACTLR5_EL1_BIT_56 +workaround_reset_end neoverse_n2, ERRATUM(2743014) + +check_erratum_ls neoverse_n2, ERRATUM(2743014), CPU_REV(0, 2) + +workaround_runtime_start neoverse_n2, ERRATUM(2743089), ERRATA_N2_2743089 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end neoverse_n2, ERRATUM(2743089) + +check_erratum_ls neoverse_n2, ERRATUM(2743089), CPU_REV(0, 2) + +workaround_reset_start neoverse_n2, ERRATUM(2779511), ERRATA_N2_2779511 + /* Set bit 47 in ACTLR3_EL1 */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR3_EL1, NEOVERSE_N2_CPUACTLR3_EL1_BIT_47 +workaround_reset_end neoverse_n2, ERRATUM(2779511) + +check_erratum_ls neoverse_n2, ERRATUM(2779511), CPU_REV(0, 2) + +workaround_reset_start neoverse_n2, CVE(2022,23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Neoverse-N2 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_neoverse_n2 +#endif /* IMAGE_BL31 */ +workaround_reset_end neoverse_n2, CVE(2022,23960) + +check_erratum_chosen neoverse_n2, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + + /* ------------------------------------------- + * The CPU Ops reset function for Neoverse N2. + * ------------------------------------------- + */ +cpu_reset_func_start neoverse_n2 + + /* Check if the PE implements SSBS */ + mrs x0, id_aa64pfr1_el1 + tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) + b.eq 1f + + /* Disable speculative loads */ + msr SSBS, xzr +1: + /* Force all cacheable atomic instructions to be near */ + sysreg_bit_set NEOVERSE_N2_CPUACTLR2_EL1, NEOVERSE_N2_CPUACTLR2_EL1_BIT_2 + +#if ENABLE_FEAT_AMU + /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ + sysreg_bit_set cptr_el3, TAM_BIT + /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ + sysreg_bit_set cptr_el2, TAM_BIT + /* No need to enable the counters as this would be done at el3 exit */ +#endif + +#if NEOVERSE_Nx_EXTERNAL_LLC + /* Some systems may have External LLC, core needs to be made aware */ + sysreg_bit_set NEOVERSE_N2_CPUECTLR_EL1, NEOVERSE_N2_CPUECTLR_EL1_EXTLLC_BIT +#endif +cpu_reset_func_end neoverse_n2 + +func neoverse_n2_core_pwr_dwn + + apply_erratum neoverse_n2, ERRATUM(2009478), ERRATA_N2_2009478 + apply_erratum neoverse_n2, ERRATUM(2326639), ERRATA_N2_2326639, NO_GET_CPU_REV + + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * No need to do cache maintenance here. + * --------------------------------------------------- + */ + sysreg_bit_set NEOVERSE_N2_CPUPWRCTLR_EL1, NEOVERSE_N2_CORE_PWRDN_EN_BIT + + apply_erratum neoverse_n2, ERRATUM(2743089), ERRATA_N2_2743089 + + isb + ret +endfunc neoverse_n2_core_pwr_dwn + +errata_report_shim neoverse_n2 + + /* --------------------------------------------- + * This function provides Neoverse N2 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ASCII and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.neoverse_n2_regs, "aS" +neoverse_n2_regs: /* The ASCII list of register names to be reported */ + .asciz "cpupwrctlr_el1", "" + +func neoverse_n2_cpu_reg_dump + adr x6, neoverse_n2_regs + mrs x8, NEOVERSE_N2_CPUPWRCTLR_EL1 + ret +endfunc neoverse_n2_cpu_reg_dump + +declare_cpu_ops neoverse_n2, NEOVERSE_N2_MIDR, \ + neoverse_n2_reset_func, \ + neoverse_n2_core_pwr_dwn diff --git a/lib/cpus/aarch64/neoverse_n_common.S b/lib/cpus/aarch64/neoverse_n_common.S new file mode 100644 index 0000000..b816342 --- /dev/null +++ b/lib/cpus/aarch64/neoverse_n_common.S @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2020, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <neoverse_n_common.h> + + .global is_scu_present_in_dsu + +/* + * Check if the SCU L3 Unit is present on the DSU + * 1-> SCU present + * 0-> SCU not present + * + * This function is implemented as weak on dsu_helpers.S and must be + * overwritten for Neoverse Nx cores. + */ + +func is_scu_present_in_dsu + mrs x0, CPUCFR_EL1 + ubfx x0, x0, #SCU_SHIFT, #1 + eor x0, x0, #1 + ret +endfunc is_scu_present_in_dsu diff --git a/lib/cpus/aarch64/neoverse_poseidon.S b/lib/cpus/aarch64/neoverse_poseidon.S new file mode 100644 index 0000000..3b3245d --- /dev/null +++ b/lib/cpus/aarch64/neoverse_poseidon.S @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2022-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <neoverse_poseidon.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Neoverse Poseidon must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Neoverse Poseidon supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table NEOVERSE_POSEIDON_BHB_LOOP_COUNT, neoverse_poseidon +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start neoverse_poseidon, CVE(2022,23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Neoverse-poseidon generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_neoverse_poseidon + +#endif /* IMAGE_BL31 */ +workaround_reset_end neoverse_poseidon, CVE(2022,23960) + +check_erratum_chosen neoverse_poseidon, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func neoverse_poseidon_core_pwr_dwn + /* --------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------- + */ + sysreg_bit_set NEOVERSE_POSEIDON_CPUPWRCTLR_EL1, \ + NEOVERSE_POSEIDON_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + + isb + ret +endfunc neoverse_poseidon_core_pwr_dwn + +cpu_reset_func_start neoverse_poseidon + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end neoverse_poseidon + +errata_report_shim neoverse_poseidon + + /* --------------------------------------------- + * This function provides Neoverse-Poseidon specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.neoverse_poseidon_regs, "aS" +neoverse_poseidon_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func neoverse_poseidon_cpu_reg_dump + adr x6, neoverse_poseidon_regs + mrs x8, NEOVERSE_POSEIDON_CPUECTLR_EL1 + ret +endfunc neoverse_poseidon_cpu_reg_dump + +declare_cpu_ops neoverse_poseidon, NEOVERSE_POSEIDON_MIDR, \ + neoverse_poseidon_reset_func, \ + neoverse_poseidon_core_pwr_dwn diff --git a/lib/cpus/aarch64/neoverse_v1.S b/lib/cpus/aarch64/neoverse_v1.S new file mode 100644 index 0000000..2a49134 --- /dev/null +++ b/lib/cpus/aarch64/neoverse_v1.S @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2019-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <neoverse_v1.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Neoverse V1 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Neoverse-V1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table NEOVERSE_V1_BHB_LOOP_COUNT, neoverse_v1 +#endif /* WORKAROUND_CVE_2022_23960 */ + +workaround_reset_start neoverse_v1, ERRATUM(1618635), ERRATA_V1_1618635 + /* Inserts a DMB SY before and after MRS PAR_EL1 */ + ldr x0, =0x0 + msr NEOVERSE_V1_CPUPSELR_EL3, x0 + ldr x0, = 0xEE070F14 + msr NEOVERSE_V1_CPUPOR_EL3, x0 + ldr x0, = 0xFFFF0FFF + msr NEOVERSE_V1_CPUPMR_EL3, x0 + ldr x0, =0x4005027FF + msr NEOVERSE_V1_CPUPCR_EL3, x0 + + /* Inserts a DMB SY before STREX imm offset */ + ldr x0, =0x1 + msr NEOVERSE_V1_CPUPSELR_EL3, x0 + ldr x0, =0x00e8400000 + msr NEOVERSE_V1_CPUPOR_EL3, x0 + ldr x0, =0x00fff00000 + msr NEOVERSE_V1_CPUPMR_EL3, x0 + ldr x0, = 0x4001027FF + msr NEOVERSE_V1_CPUPCR_EL3, x0 + + /* Inserts a DMB SY before STREX[BHD}/STLEX* */ + ldr x0, =0x2 + msr NEOVERSE_V1_CPUPSELR_EL3, x0 + ldr x0, =0x00e8c00040 + msr NEOVERSE_V1_CPUPOR_EL3, x0 + ldr x0, =0x00fff00040 + msr NEOVERSE_V1_CPUPMR_EL3, x0 + ldr x0, = 0x4001027FF + msr NEOVERSE_V1_CPUPCR_EL3, x0 + + /* Inserts a DMB SY after STREX imm offset */ + ldr x0, =0x3 + msr NEOVERSE_V1_CPUPSELR_EL3, x0 + ldr x0, =0x00e8400000 + msr NEOVERSE_V1_CPUPOR_EL3, x0 + ldr x0, =0x00fff00000 + msr NEOVERSE_V1_CPUPMR_EL3, x0 + ldr x0, = 0x4004027FF + msr NEOVERSE_V1_CPUPCR_EL3, x0 + + /* Inserts a DMB SY after STREX[BHD}/STLEX* */ + ldr x0, =0x4 + msr NEOVERSE_V1_CPUPSELR_EL3, x0 + ldr x0, =0x00e8c00040 + msr NEOVERSE_V1_CPUPOR_EL3, x0 + ldr x0, =0x00fff00040 + msr NEOVERSE_V1_CPUPMR_EL3, x0 + ldr x0, = 0x4004027FF + msr NEOVERSE_V1_CPUPCR_EL3, x0 + +workaround_reset_end neoverse_v1, ERRATUM(1618635) + +check_erratum_ls neoverse_v1, ERRATUM(1618635), CPU_REV(0, 0) + +workaround_reset_start neoverse_v1, ERRATUM(1774420), ERRATA_V1_1774420 + /* Set bit 53 in CPUECTLR_EL1 */ + sysreg_bit_set NEOVERSE_V1_CPUECTLR_EL1, NEOVERSE_V1_CPUECTLR_EL1_BIT_53 +workaround_reset_end neoverse_v1, ERRATUM(1774420) + +check_erratum_ls neoverse_v1, ERRATUM(1774420), CPU_REV(1, 0) + +workaround_reset_start neoverse_v1, ERRATUM(1791573), ERRATA_V1_1791573 + /* Set bit 2 in ACTLR2_EL1 */ + sysreg_bit_set NEOVERSE_V1_ACTLR2_EL1, NEOVERSE_V1_ACTLR2_EL1_BIT_2 +workaround_reset_end neoverse_v1, ERRATUM(1791573) + +check_erratum_ls neoverse_v1, ERRATUM(1791573), CPU_REV(1, 0) + +workaround_reset_start neoverse_v1, ERRATUM(1852267), ERRATA_V1_1852267 + /* Set bit 28 in ACTLR2_EL1 */ + sysreg_bit_set NEOVERSE_V1_ACTLR2_EL1, NEOVERSE_V1_ACTLR2_EL1_BIT_28 +workaround_reset_end neoverse_v1, ERRATUM(1852267) + +check_erratum_ls neoverse_v1, ERRATUM(1852267), CPU_REV(1, 0) + +workaround_reset_start neoverse_v1, ERRATUM(1925756), ERRATA_V1_1925756 + /* Set bit 8 in CPUECTLR_EL1 */ + sysreg_bit_set NEOVERSE_V1_CPUECTLR_EL1, NEOVERSE_V1_CPUECTLR_EL1_BIT_8 +workaround_reset_end neoverse_v1, ERRATUM(1925756) + +check_erratum_ls neoverse_v1, ERRATUM(1925756), CPU_REV(1, 1) + +workaround_reset_start neoverse_v1, ERRATUM(1940577), ERRATA_V1_1940577 + mov x0, #0 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3900002 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 + + mov x0, #1 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3800082 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF00083 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 + + mov x0, #2 + msr S3_6_C15_C8_0, x0 + ldr x0, =0x10E3800200 + msr S3_6_C15_C8_2, x0 + ldr x0, =0x10FFF003E0 + msr S3_6_C15_C8_3, x0 + ldr x0, =0x2001003FF + msr S3_6_C15_C8_1, x0 + +workaround_reset_end neoverse_v1, ERRATUM(1940577) + +check_erratum_range neoverse_v1, ERRATUM(1940577), CPU_REV(1, 0), CPU_REV(1, 1) + +workaround_reset_start neoverse_v1, ERRATUM(1966096), ERRATA_V1_1966096 + mov x0, #0x3 + msr S3_6_C15_C8_0, x0 + ldr x0, =0xEE010F12 + msr S3_6_C15_C8_2, x0 + ldr x0, =0xFFFF0FFF + msr S3_6_C15_C8_3, x0 + ldr x0, =0x80000000003FF + msr S3_6_C15_C8_1, x0 +workaround_reset_end neoverse_v1, ERRATUM(1966096) + +check_erratum_range neoverse_v1, ERRATUM(1966096), CPU_REV(1, 0), CPU_REV(1, 1) + +workaround_reset_start neoverse_v1, ERRATUM(2108267), ERRATA_V1_2108267 + mrs x1, NEOVERSE_V1_CPUECTLR_EL1 + mov x0, #NEOVERSE_V1_CPUECTLR_EL1_PF_MODE_CNSRV + bfi x1, x0, #CPUECTLR_EL1_PF_MODE_LSB, #CPUECTLR_EL1_PF_MODE_WIDTH + msr NEOVERSE_V1_CPUECTLR_EL1, x1 +workaround_reset_end neoverse_v1, ERRATUM(2108267) + +check_erratum_ls neoverse_v1, ERRATUM(2108267), CPU_REV(1, 2) + +workaround_reset_start neoverse_v1, ERRATUM(2139242), ERRATA_V1_2139242 + mov x0, #0x3 + msr S3_6_C15_C8_0, x0 + ldr x0, =0xEE720F14 + msr S3_6_C15_C8_2, x0 + ldr x0, =0xFFFF0FDF + msr S3_6_C15_C8_3, x0 + ldr x0, =0x40000005003FF + msr S3_6_C15_C8_1, x0 +workaround_reset_end neoverse_v1, ERRATUM(2139242) + +check_erratum_ls neoverse_v1, ERRATUM(2139242), CPU_REV(1, 1) + +workaround_reset_start neoverse_v1, ERRATUM(2216392), ERRATA_V1_2216392 + ldr x0, =0x5 + msr S3_6_c15_c8_0, x0 /* CPUPSELR_EL3 */ + ldr x0, =0x10F600E000 + msr S3_6_c15_c8_2, x0 /* CPUPOR_EL3 */ + ldr x0, =0x10FF80E000 + msr S3_6_c15_c8_3, x0 /* CPUPMR_EL3 */ + ldr x0, =0x80000000003FF + msr S3_6_c15_c8_1, x0 /* CPUPCR_EL3 */ +workaround_reset_end neoverse_v1, ERRATUM(2216392) + +check_erratum_range neoverse_v1, ERRATUM(2216392), CPU_REV(1, 0), CPU_REV(1, 1) + +workaround_reset_start neoverse_v1, ERRATUM(2294912), ERRATA_V1_2294912 + /* Set bit 0 in ACTLR2_EL1 */ + sysreg_bit_set NEOVERSE_V1_ACTLR2_EL1, NEOVERSE_V1_ACTLR2_EL1_BIT_0 +workaround_reset_end neoverse_v1, ERRATUM(2294912) + +check_erratum_ls neoverse_v1, ERRATUM(2294912), CPU_REV(1, 2) + +workaround_reset_start neoverse_v1, ERRATUM(2372203), ERRATA_V1_2372203 + /* Set bit 40 in ACTLR2_EL1 */ + sysreg_bit_set NEOVERSE_V1_ACTLR2_EL1, NEOVERSE_V1_ACTLR2_EL1_BIT_40 +workaround_reset_end neoverse_v1, ERRATUM(2372203) + +check_erratum_ls neoverse_v1, ERRATUM(2372203), CPU_REV(1, 1) + +workaround_runtime_start neoverse_v1, ERRATUM(2743093), ERRATA_V1_2743093 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end neoverse_v1, ERRATUM(2743093) + +check_erratum_ls neoverse_v1, ERRATUM(2743093), CPU_REV(1, 2) + +workaround_reset_start neoverse_v1, ERRATUM(2743233), ERRATA_V1_2743233 + sysreg_bit_clear NEOVERSE_V1_ACTLR5_EL1, NEOVERSE_V1_ACTLR5_EL1_BIT_56 + sysreg_bit_set NEOVERSE_V1_ACTLR5_EL1, NEOVERSE_V1_ACTLR5_EL1_BIT_55 +workaround_reset_end neoverse_v1, ERRATUM(2743233) + +check_erratum_ls neoverse_v1, ERRATUM(2743233), CPU_REV(1, 2) + +workaround_reset_start neoverse_v1, ERRATUM(2779461), ERRATA_V1_2779461 + sysreg_bit_set NEOVERSE_V1_ACTLR3_EL1, NEOVERSE_V1_ACTLR3_EL1_BIT_47 +workaround_reset_end neoverse_v1, ERRATUM(2779461) + +check_erratum_ls neoverse_v1, ERRATUM(2779461), CPU_REV(1, 2) + + +workaround_reset_start neoverse_v1, CVE(2022,23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Neoverse-V1 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_neoverse_v1 +#endif /* IMAGE_BL31 */ +workaround_reset_end neoverse_v1, CVE(2022,23960) + +check_erratum_chosen neoverse_v1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func neoverse_v1_core_pwr_dwn + /* --------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------- + */ + sysreg_bit_set NEOVERSE_V1_CPUPWRCTLR_EL1, NEOVERSE_V1_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + apply_erratum neoverse_v1, ERRATUM(2743093), ERRATA_V1_2743093 + + isb + ret +endfunc neoverse_v1_core_pwr_dwn + +errata_report_shim neoverse_v1 + +cpu_reset_func_start neoverse_v1 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end neoverse_v1 + + /* --------------------------------------------- + * This function provides Neoverse-V1 specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.neoverse_v1_regs, "aS" +neoverse_v1_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func neoverse_v1_cpu_reg_dump + adr x6, neoverse_v1_regs + mrs x8, NEOVERSE_V1_CPUECTLR_EL1 + ret +endfunc neoverse_v1_cpu_reg_dump + +declare_cpu_ops neoverse_v1, NEOVERSE_V1_MIDR, \ + neoverse_v1_reset_func, \ + neoverse_v1_core_pwr_dwn diff --git a/lib/cpus/aarch64/neoverse_v2.S b/lib/cpus/aarch64/neoverse_v2.S new file mode 100644 index 0000000..bfd088d --- /dev/null +++ b/lib/cpus/aarch64/neoverse_v2.S @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <neoverse_v2.h> +#include <cpu_macros.S> +#include <plat_macros.S> +#include "wa_cve_2022_23960_bhb_vector.S" + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Neoverse V2 must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Neoverse V2 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +workaround_reset_start neoverse_v2, ERRATUM(2331132), ERRATA_V2_2331132 + sysreg_bitfield_insert NEOVERSE_V2_CPUECTLR2_EL1, NEOVERSE_V2_CPUECTLR2_EL1_PF_MODE_CNSRV, \ + NEOVERSE_V2_CPUECTLR2_EL1_PF_MODE_LSB, NEOVERSE_V2_CPUECTLR2_EL1_PF_MODE_WIDTH +workaround_reset_end neoverse_v2, ERRATUM(2331132) + +check_erratum_ls neoverse_v2, ERRATUM(2331132), CPU_REV(0, 2) + +workaround_reset_start neoverse_v2, ERRATUM(2719105), ERRATA_V2_2719105 + sysreg_bit_set NEOVERSE_V2_CPUACTLR2_EL1, NEOVERSE_V2_CPUACTLR2_EL1_BIT_0 +workaround_reset_end neoverse_v2, ERRATUM(2719105) + +check_erratum_ls neoverse_v2, ERRATUM(2719105), CPU_REV(0, 1) + +workaround_reset_start neoverse_v2, ERRATUM(2743011), ERRATA_V2_2743011 + sysreg_bit_set NEOVERSE_V2_CPUACTLR5_EL1, NEOVERSE_V2_CPUACTLR5_EL1_BIT_55 + sysreg_bit_clear NEOVERSE_V2_CPUACTLR5_EL1, NEOVERSE_V2_CPUACTLR5_EL1_BIT_56 +workaround_reset_end neoverse_v2, ERRATUM(2743011) + +check_erratum_ls neoverse_v2, ERRATUM(2743011), CPU_REV(0, 1) + +workaround_reset_start neoverse_v2, ERRATUM(2779510), ERRATA_V2_2779510 + sysreg_bit_set NEOVERSE_V2_CPUACTLR3_EL1, NEOVERSE_V2_CPUACTLR3_EL1_BIT_47 +workaround_reset_end neoverse_v2, ERRATUM(2779510) + +check_erratum_ls neoverse_v2, ERRATUM(2779510), CPU_REV(0, 1) + +workaround_runtime_start neoverse_v2, ERRATUM(2801372), ERRATA_V2_2801372 + /* dsb before isb of power down sequence */ + dsb sy +workaround_runtime_end neoverse_v2, ERRATUM(2801372), ERRATA_V2_2801372 + +check_erratum_ls neoverse_v2, ERRATUM(2801372), CPU_REV(0, 1) + +workaround_reset_start neoverse_v2, CVE(2022,23960), WORKAROUND_CVE_2022_23960 +#if IMAGE_BL31 + /* + * The Neoverse-V2 generic vectors are overridden to apply errata + * mitigation on exception entry from lower ELs. + */ + override_vector_table wa_cve_vbar_neoverse_v2 +#endif /* IMAGE_BL31 */ +workaround_reset_end neoverse_v2, CVE(2022,23960) + +check_erratum_chosen neoverse_v2, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 + +#if WORKAROUND_CVE_2022_23960 + wa_cve_2022_23960_bhb_vector_table NEOVERSE_V2_BHB_LOOP_COUNT, neoverse_v2 +#endif /* WORKAROUND_CVE_2022_23960 */ + + /* ---------------------------------------------------- + * HW will do the cache maintenance while powering down + * ---------------------------------------------------- + */ +func neoverse_v2_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set NEOVERSE_V2_CPUPWRCTLR_EL1, NEOVERSE_V2_CPUPWRCTLR_EL1_CORE_PWRDN_BIT + apply_erratum neoverse_v2, ERRATUM(2801372), ERRATA_V2_2801372 + + isb + ret +endfunc neoverse_v2_core_pwr_dwn + +cpu_reset_func_start neoverse_v2 + /* Disable speculative loads */ + msr SSBS, xzr +cpu_reset_func_end neoverse_v2 + +errata_report_shim neoverse_v2 + /* --------------------------------------------- + * This function provides Neoverse V2- + * specific register information for crash + * reporting. It needs to return with x6 + * pointing to a list of register names in ascii + * and x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.neoverse_v2_regs, "aS" +neoverse_v2_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func neoverse_v2_cpu_reg_dump + adr x6, neoverse_v2_regs + mrs x8, NEOVERSE_V2_CPUECTLR_EL1 + ret +endfunc neoverse_v2_cpu_reg_dump + +declare_cpu_ops neoverse_v2, NEOVERSE_V2_MIDR, \ + neoverse_v2_reset_func, \ + neoverse_v2_core_pwr_dwn diff --git a/lib/cpus/aarch64/nevis.S b/lib/cpus/aarch64/nevis.S new file mode 100644 index 0000000..36830a9 --- /dev/null +++ b/lib/cpus/aarch64/nevis.S @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <nevis.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Nevis must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Nevis supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +cpu_reset_func_start nevis + /* ---------------------------------------------------- + * Disable speculative loads + * ---------------------------------------------------- + */ + msr SSBS, xzr +cpu_reset_func_end nevis + +func nevis_core_pwr_dwn + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set NEVIS_IMP_CPUPWRCTLR_EL1, \ + NEVIS_IMP_CPUPWRCTLR_EL1_CORE_PWRDN_EN_BIT + isb + ret +endfunc nevis_core_pwr_dwn + +errata_report_shim nevis + +.section .rodata.nevis_regs, "aS" +nevis_regs: /* The ASCII list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func nevis_cpu_reg_dump + adr x6, nevis_regs + mrs x8, NEVIS_CPUECTLR_EL1 + ret +endfunc nevis_cpu_reg_dump + +declare_cpu_ops nevis, NEVIS_MIDR, \ + nevis_reset_func, \ + nevis_core_pwr_dwn diff --git a/lib/cpus/aarch64/qemu_max.S b/lib/cpus/aarch64/qemu_max.S new file mode 100644 index 0000000..00963bc --- /dev/null +++ b/lib/cpus/aarch64/qemu_max.S @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <arch.h> +#include <asm_macros.S> +#include <cpu_macros.S> +#include <qemu_max.h> + +func qemu_max_core_pwr_dwn + /* --------------------------------------------- + * Disable the Data Cache. + * --------------------------------------------- + */ + mrs x1, sctlr_el3 + bic x1, x1, #SCTLR_C_BIT + msr sctlr_el3, x1 + isb + + /* --------------------------------------------- + * Flush L1 cache to L2. + * --------------------------------------------- + */ + mov x18, lr + mov x0, #DCCISW + bl dcsw_op_level1 + mov lr, x18 + ret +endfunc qemu_max_core_pwr_dwn + +func qemu_max_cluster_pwr_dwn + /* --------------------------------------------- + * Disable the Data Cache. + * --------------------------------------------- + */ + mrs x1, sctlr_el3 + bic x1, x1, #SCTLR_C_BIT + msr sctlr_el3, x1 + isb + + /* --------------------------------------------- + * Flush all caches to PoC. + * --------------------------------------------- + */ + mov x0, #DCCISW + b dcsw_op_all +endfunc qemu_max_cluster_pwr_dwn + +errata_report_shim qemu_max + + /* --------------------------------------------- + * This function provides cpu specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.qemu_max_regs, "aS" +qemu_max_regs: /* The ascii list of register names to be reported */ + .asciz "" /* no registers to report */ + +func qemu_max_cpu_reg_dump + adr x6, qemu_max_regs + ret +endfunc qemu_max_cpu_reg_dump + + +/* cpu_ops for QEMU MAX */ +declare_cpu_ops qemu_max, QEMU_MAX_MIDR, CPU_NO_RESET_FUNC, \ + qemu_max_core_pwr_dwn, \ + qemu_max_cluster_pwr_dwn diff --git a/lib/cpus/aarch64/rainier.S b/lib/cpus/aarch64/rainier.S new file mode 100644 index 0000000..c770f54 --- /dev/null +++ b/lib/cpus/aarch64/rainier.S @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2020-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <context.h> +#include <cpu_macros.S> +#include <cpuamu.h> +#include <rainier.h> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Rainier CPU must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Rainier CPU supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +/* -------------------------------------------------- + * Disable speculative loads if Rainier supports + * SSBS. + * + * Shall clobber: x0. + * -------------------------------------------------- + */ +func rainier_disable_speculative_loads + /* Check if the PE implements SSBS */ + mrs x0, id_aa64pfr1_el1 + tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) + b.eq 1f + + /* Disable speculative loads */ + msr SSBS, xzr + +1: + ret +endfunc rainier_disable_speculative_loads + + /* Rainier R0P0 is based on Neoverse N1 R4P0. */ +workaround_reset_start rainier, ERRATUM(1868343), ERRATA_N1_1868343 + sysreg_bit_set RAINIER_CPUACTLR_EL1, RAINIER_CPUACTLR_EL1_BIT_13 +workaround_reset_end rainier, ERRATUM(1868343) + +check_erratum_ls rainier, ERRATUM(1868343), CPU_REV(0, 0) + +cpu_reset_func_start rainier + bl rainier_disable_speculative_loads + /* Forces all cacheable atomic instructions to be near */ + sysreg_bit_set RAINIER_CPUACTLR2_EL1, RAINIER_CPUACTLR2_EL1_BIT_2 + +#if ENABLE_FEAT_AMU + /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ + sysreg_bit_set actlr_el3, RAINIER_ACTLR_AMEN_BIT + + /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ + sysreg_bit_set actlr_el2, RAINIER_ACTLR_AMEN_BIT + + /* Enable group0 counters */ + mov x0, #RAINIER_AMU_GROUP0_MASK + msr CPUAMCNTENSET_EL0, x0 +#endif +cpu_reset_func_end rainier + + /* --------------------------------------------- + * HW will do the cache maintenance while powering down + * --------------------------------------------- + */ +func rainier_core_pwr_dwn + /* --------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------- + */ + sysreg_bit_set RAINIER_CPUPWRCTLR_EL1, RAINIER_CORE_PWRDN_EN_MASK + isb + ret +endfunc rainier_core_pwr_dwn + +errata_report_shim rainier + + /* --------------------------------------------- + * This function provides Rainier specific + * register information for crash reporting. + * It needs to return with x6 pointing to + * a list of register names in ascii and + * x8 - x15 having values of registers to be + * reported. + * --------------------------------------------- + */ +.section .rodata.rainier_regs, "aS" +rainier_regs: /* The ascii list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func rainier_cpu_reg_dump + adr x6, rainier_regs + mrs x8, RAINIER_CPUECTLR_EL1 + ret +endfunc rainier_cpu_reg_dump + +declare_cpu_ops rainier, RAINIER_MIDR, \ + rainier_reset_func, \ + rainier_core_pwr_dwn diff --git a/lib/cpus/aarch64/travis.S b/lib/cpus/aarch64/travis.S new file mode 100644 index 0000000..2abefe9 --- /dev/null +++ b/lib/cpus/aarch64/travis.S @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <common/bl_common.h> +#include <travis.h> +#include <cpu_macros.S> +#include <plat_macros.S> + +/* Hardware handled coherency */ +#if HW_ASSISTED_COHERENCY == 0 +#error "Travis must be compiled with HW_ASSISTED_COHERENCY enabled" +#endif + +/* 64-bit only core */ +#if CTX_INCLUDE_AARCH32_REGS == 1 +#error "Travis supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" +#endif + +cpu_reset_func_start travis + /* ---------------------------------------------------- + * Disable speculative loads + * ---------------------------------------------------- + */ + msr SSBS, xzr +cpu_reset_func_end travis + +func travis_core_pwr_dwn +#if ENABLE_SME_FOR_NS + /* --------------------------------------------------- + * Disable SME if enabled and supported + * --------------------------------------------------- + */ + mrs x0, ID_AA64PFR1_EL1 + ubfx x0, x0, #ID_AA64PFR1_EL1_SME_SHIFT, \ + #ID_AA64PFR1_EL1_SME_WIDTH + cmp x0, #ID_AA64PFR1_EL1_SME_NOT_SUPPORTED + b.eq 1f + msr TRAVIS_SVCRSM, xzr + msr TRAVIS_SVCRZA, xzr +1: +#endif + /* --------------------------------------------------- + * Enable CPU power down bit in power control register + * --------------------------------------------------- + */ + sysreg_bit_set TRAVIS_IMP_CPUPWRCTLR_EL1, \ + TRAVIS_IMP_CPUPWRCTLR_EL1_CORE_PWRDN_EN_BIT + isb + ret +endfunc travis_core_pwr_dwn + +errata_report_shim travis + +.section .rodata.travis_regs, "aS" +travis_regs: /* The ASCII list of register names to be reported */ + .asciz "cpuectlr_el1", "" + +func travis_cpu_reg_dump + adr x6, travis_regs + mrs x8, TRAVIS_IMP_CPUECTLR_EL1 + ret +endfunc travis_cpu_reg_dump + +declare_cpu_ops travis, TRAVIS_MIDR, \ + travis_reset_func, \ + travis_core_pwr_dwn diff --git a/lib/cpus/aarch64/wa_cve_2017_5715_bpiall.S b/lib/cpus/aarch64/wa_cve_2017_5715_bpiall.S new file mode 100644 index 0000000..8d9aa5e --- /dev/null +++ b/lib/cpus/aarch64/wa_cve_2017_5715_bpiall.S @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2017-2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <context.h> +#include <services/arm_arch_svc.h> + + .globl wa_cve_2017_5715_bpiall_vbar + +#define EMIT_BPIALL 0xee070fd5 +#define EMIT_SMC 0xe1600070 +#define ESR_EL3_A64_SMC0 0x5e000000 + + .macro apply_cve_2017_5715_wa _from_vector + /* + * Save register state to enable a call to AArch32 S-EL1 and return + * Identify the original calling vector in w2 (==_from_vector) + * Use w3-w6 for additional register state preservation while in S-EL1 + */ + + /* Save GP regs */ + stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + stp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] + stp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] + stp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10] + stp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12] + stp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14] + stp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16] + stp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18] + stp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20] + stp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22] + stp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24] + stp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26] + stp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28] + + /* Identify the original exception vector */ + mov w2, \_from_vector + + /* Preserve 32-bit system registers in GP registers through the workaround */ + mrs x3, esr_el3 + mrs x4, spsr_el3 + mrs x5, scr_el3 + mrs x6, sctlr_el1 + + /* + * Preserve LR and ELR_EL3 registers in the GP regs context. + * Temporarily use the CTX_GPREG_SP_EL0 slot to preserve ELR_EL3 + * through the workaround. This is OK because at this point the + * current state for this context's SP_EL0 is in the live system + * register, which is unmodified by the workaround. + */ + mrs x7, elr_el3 + stp x30, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] + + /* + * Load system registers for entry to S-EL1. + */ + + /* Mask all interrupts and set AArch32 Supervisor mode */ + movz w8, SPSR_MODE32(MODE32_svc, SPSR_T_ARM, SPSR_E_LITTLE, SPSR_AIF_MASK) + + /* Switch EL3 exception vectors while the workaround is executing. */ + adr x9, wa_cve_2017_5715_bpiall_ret_vbar + + /* Setup SCTLR_EL1 with MMU off and I$ on */ + ldr x10, stub_sel1_sctlr + + /* Land at the S-EL1 workaround stub */ + adr x11, aarch32_stub + + /* + * Setting SCR_EL3 to all zeroes means that the NS, RW + * and SMD bits are configured as expected. + */ + msr scr_el3, xzr + msr spsr_el3, x8 + msr vbar_el3, x9 + msr sctlr_el1, x10 + msr elr_el3, x11 + + eret + .endm + + /* --------------------------------------------------------------------- + * This vector table is used at runtime to enter the workaround at + * AArch32 S-EL1 for Sync/IRQ/FIQ/SError exceptions. If the workaround + * is not enabled, the existing runtime exception vector table is used. + * --------------------------------------------------------------------- + */ +vector_base wa_cve_2017_5715_bpiall_vbar + + /* --------------------------------------------------------------------- + * Current EL with SP_EL0 : 0x0 - 0x200 + * --------------------------------------------------------------------- + */ +vector_entry bpiall_sync_exception_sp_el0 + b sync_exception_sp_el0 + nop /* to force 8 byte alignment for the following stub */ + + /* + * Since each vector table entry is 128 bytes, we can store the + * stub context in the unused space to minimize memory footprint. + */ +stub_sel1_sctlr: + .quad SCTLR_AARCH32_EL1_RES1 | SCTLR_I_BIT + +aarch32_stub: + .word EMIT_BPIALL + .word EMIT_SMC + +end_vector_entry bpiall_sync_exception_sp_el0 + +vector_entry bpiall_irq_sp_el0 + b irq_sp_el0 +end_vector_entry bpiall_irq_sp_el0 + +vector_entry bpiall_fiq_sp_el0 + b fiq_sp_el0 +end_vector_entry bpiall_fiq_sp_el0 + +vector_entry bpiall_serror_sp_el0 + b serror_sp_el0 +end_vector_entry bpiall_serror_sp_el0 + + /* --------------------------------------------------------------------- + * Current EL with SP_ELx: 0x200 - 0x400 + * --------------------------------------------------------------------- + */ +vector_entry bpiall_sync_exception_sp_elx + b sync_exception_sp_elx +end_vector_entry bpiall_sync_exception_sp_elx + +vector_entry bpiall_irq_sp_elx + b irq_sp_elx +end_vector_entry bpiall_irq_sp_elx + +vector_entry bpiall_fiq_sp_elx + b fiq_sp_elx +end_vector_entry bpiall_fiq_sp_elx + +vector_entry bpiall_serror_sp_elx + b serror_sp_elx +end_vector_entry bpiall_serror_sp_elx + + /* --------------------------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x600 + * --------------------------------------------------------------------- + */ +vector_entry bpiall_sync_exception_aarch64 + apply_cve_2017_5715_wa 1 +end_vector_entry bpiall_sync_exception_aarch64 + +vector_entry bpiall_irq_aarch64 + apply_cve_2017_5715_wa 2 +end_vector_entry bpiall_irq_aarch64 + +vector_entry bpiall_fiq_aarch64 + apply_cve_2017_5715_wa 4 +end_vector_entry bpiall_fiq_aarch64 + +vector_entry bpiall_serror_aarch64 + apply_cve_2017_5715_wa 8 +end_vector_entry bpiall_serror_aarch64 + + /* --------------------------------------------------------------------- + * Lower EL using AArch32 : 0x600 - 0x800 + * --------------------------------------------------------------------- + */ +vector_entry bpiall_sync_exception_aarch32 + apply_cve_2017_5715_wa 1 +end_vector_entry bpiall_sync_exception_aarch32 + +vector_entry bpiall_irq_aarch32 + apply_cve_2017_5715_wa 2 +end_vector_entry bpiall_irq_aarch32 + +vector_entry bpiall_fiq_aarch32 + apply_cve_2017_5715_wa 4 +end_vector_entry bpiall_fiq_aarch32 + +vector_entry bpiall_serror_aarch32 + apply_cve_2017_5715_wa 8 +end_vector_entry bpiall_serror_aarch32 + + /* --------------------------------------------------------------------- + * This vector table is used while the workaround is executing. It + * installs a simple SMC handler to allow the Sync/IRQ/FIQ/SError + * workaround stubs to enter EL3 from S-EL1. It restores the previous + * EL3 state before proceeding with the normal runtime exception vector. + * --------------------------------------------------------------------- + */ +vector_base wa_cve_2017_5715_bpiall_ret_vbar + + /* --------------------------------------------------------------------- + * Current EL with SP_EL0 : 0x0 - 0x200 (UNUSED) + * --------------------------------------------------------------------- + */ +vector_entry bpiall_ret_sync_exception_sp_el0 + b report_unhandled_exception +end_vector_entry bpiall_ret_sync_exception_sp_el0 + +vector_entry bpiall_ret_irq_sp_el0 + b report_unhandled_interrupt +end_vector_entry bpiall_ret_irq_sp_el0 + +vector_entry bpiall_ret_fiq_sp_el0 + b report_unhandled_interrupt +end_vector_entry bpiall_ret_fiq_sp_el0 + +vector_entry bpiall_ret_serror_sp_el0 + b report_unhandled_exception +end_vector_entry bpiall_ret_serror_sp_el0 + + /* --------------------------------------------------------------------- + * Current EL with SP_ELx: 0x200 - 0x400 (UNUSED) + * --------------------------------------------------------------------- + */ +vector_entry bpiall_ret_sync_exception_sp_elx + b report_unhandled_exception +end_vector_entry bpiall_ret_sync_exception_sp_elx + +vector_entry bpiall_ret_irq_sp_elx + b report_unhandled_interrupt +end_vector_entry bpiall_ret_irq_sp_elx + +vector_entry bpiall_ret_fiq_sp_elx + b report_unhandled_interrupt +end_vector_entry bpiall_ret_fiq_sp_elx + +vector_entry bpiall_ret_serror_sp_elx + b report_unhandled_exception +end_vector_entry bpiall_ret_serror_sp_elx + + /* --------------------------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x600 (UNUSED) + * --------------------------------------------------------------------- + */ +vector_entry bpiall_ret_sync_exception_aarch64 + b report_unhandled_exception +end_vector_entry bpiall_ret_sync_exception_aarch64 + +vector_entry bpiall_ret_irq_aarch64 + b report_unhandled_interrupt +end_vector_entry bpiall_ret_irq_aarch64 + +vector_entry bpiall_ret_fiq_aarch64 + b report_unhandled_interrupt +end_vector_entry bpiall_ret_fiq_aarch64 + +vector_entry bpiall_ret_serror_aarch64 + b report_unhandled_exception +end_vector_entry bpiall_ret_serror_aarch64 + + /* --------------------------------------------------------------------- + * Lower EL using AArch32 : 0x600 - 0x800 + * --------------------------------------------------------------------- + */ +vector_entry bpiall_ret_sync_exception_aarch32 + /* + * w2 indicates which SEL1 stub was run and thus which original vector was used + * w3-w6 contain saved system register state (esr_el3 in w3) + * Restore LR and ELR_EL3 register state from the GP regs context + */ + ldp x30, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] + + /* Apply the restored system register state */ + msr esr_el3, x3 + msr spsr_el3, x4 + msr scr_el3, x5 + msr sctlr_el1, x6 + msr elr_el3, x7 + + /* + * Workaround is complete, so swap VBAR_EL3 to point + * to workaround entry table in preparation for subsequent + * Sync/IRQ/FIQ/SError exceptions. + */ + adr x0, wa_cve_2017_5715_bpiall_vbar + msr vbar_el3, x0 + + /* + * Restore all GP regs except x2 and x3 (esr). The value in x2 + * indicates the type of the original exception. + */ + ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + ldp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] + ldp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] + ldp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10] + ldp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12] + ldp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14] + ldp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16] + ldp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18] + ldp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20] + ldp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22] + ldp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24] + ldp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26] + ldp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28] + + /* Fast path Sync exceptions. Static predictor will fall through. */ + tbz w2, #0, workaround_not_sync + + /* + * Check if SMC is coming from A64 state on #0 + * with W0 = SMCCC_ARCH_WORKAROUND_1 or W0 = SMCCC_ARCH_WORKAROUND_3 + * + * This sequence evaluates as: + * (W0==SMCCC_ARCH_WORKAROUND_1) || (W0==SMCCC_ARCH_WORKAROUND_3) ? + * (ESR_EL3==SMC#0) : (NE) + * allowing use of a single branch operation + */ + orr w2, wzr, #SMCCC_ARCH_WORKAROUND_1 + cmp w0, w2 + orr w2, wzr, #SMCCC_ARCH_WORKAROUND_3 + ccmp w0, w2, #4, ne + mov_imm w2, ESR_EL3_A64_SMC0 + ccmp w3, w2, #0, eq + /* Static predictor will predict a fall through */ + bne 1f + eret +1: + /* restore x2 and x3 and continue sync exception handling */ + b bpiall_ret_sync_exception_aarch32_tail +end_vector_entry bpiall_ret_sync_exception_aarch32 + +vector_entry bpiall_ret_irq_aarch32 + b report_unhandled_interrupt + + /* + * Post-workaround fan-out for non-sync exceptions + */ +workaround_not_sync: + tbnz w2, #3, bpiall_ret_serror + tbnz w2, #2, bpiall_ret_fiq + /* IRQ */ + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + b irq_aarch64 + +bpiall_ret_fiq: + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + b fiq_aarch64 + +bpiall_ret_serror: + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + b serror_aarch64 +end_vector_entry bpiall_ret_irq_aarch32 + +vector_entry bpiall_ret_fiq_aarch32 + b report_unhandled_interrupt +end_vector_entry bpiall_ret_fiq_aarch32 + +vector_entry bpiall_ret_serror_aarch32 + b report_unhandled_exception +end_vector_entry bpiall_ret_serror_aarch32 + + /* + * Part of bpiall_ret_sync_exception_aarch32 to save vector space + */ +func bpiall_ret_sync_exception_aarch32_tail + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + b sync_exception_aarch64 +endfunc bpiall_ret_sync_exception_aarch32_tail diff --git a/lib/cpus/aarch64/wa_cve_2017_5715_mmu.S b/lib/cpus/aarch64/wa_cve_2017_5715_mmu.S new file mode 100644 index 0000000..823f849 --- /dev/null +++ b/lib/cpus/aarch64/wa_cve_2017_5715_mmu.S @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2017-2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <context.h> +#include <services/arm_arch_svc.h> + + .globl wa_cve_2017_5715_mmu_vbar + +#define ESR_EL3_A64_SMC0 0x5e000000 +#define ESR_EL3_A32_SMC0 0x4e000000 + +vector_base wa_cve_2017_5715_mmu_vbar + + .macro apply_cve_2017_5715_wa _is_sync_exception _esr_el3_val + stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + mrs x1, sctlr_el3 + /* Disable MMU */ + bic x1, x1, #SCTLR_M_BIT + msr sctlr_el3, x1 + isb + /* Enable MMU */ + orr x1, x1, #SCTLR_M_BIT + msr sctlr_el3, x1 + /* + * Defer ISB to avoid synchronizing twice in case we hit + * the workaround SMC call which will implicitly synchronize + * because of the ERET instruction. + */ + + /* + * Ensure SMC is coming from A64/A32 state on #0 + * with W0 = SMCCC_ARCH_WORKAROUND_1 or W0 = SMCCC_ARCH_WORKAROUND_3 + * + * This sequence evaluates as: + * (W0==SMCCC_ARCH_WORKAROUND_1) || (W0==SMCCC_ARCH_WORKAROUND_3) ? + * (ESR_EL3==SMC#0) : (NE) + * allowing use of a single branch operation + */ + .if \_is_sync_exception + orr w1, wzr, #SMCCC_ARCH_WORKAROUND_1 + cmp w0, w1 + orr w1, wzr, #SMCCC_ARCH_WORKAROUND_3 + ccmp w0, w1, #4, ne + mrs x0, esr_el3 + mov_imm w1, \_esr_el3_val + ccmp w0, w1, #0, eq + /* Static predictor will predict a fall through */ + bne 1f + exception_return +1: + .endif + + /* + * Synchronize now to enable the MMU. This is required + * to ensure the load pair below reads the data stored earlier. + */ + isb + ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + .endm + + /* --------------------------------------------------------------------- + * Current EL with SP_EL0 : 0x0 - 0x200 + * --------------------------------------------------------------------- + */ +vector_entry mmu_sync_exception_sp_el0 + b sync_exception_sp_el0 +end_vector_entry mmu_sync_exception_sp_el0 + +vector_entry mmu_irq_sp_el0 + b irq_sp_el0 +end_vector_entry mmu_irq_sp_el0 + +vector_entry mmu_fiq_sp_el0 + b fiq_sp_el0 +end_vector_entry mmu_fiq_sp_el0 + +vector_entry mmu_serror_sp_el0 + b serror_sp_el0 +end_vector_entry mmu_serror_sp_el0 + + /* --------------------------------------------------------------------- + * Current EL with SP_ELx: 0x200 - 0x400 + * --------------------------------------------------------------------- + */ +vector_entry mmu_sync_exception_sp_elx + b sync_exception_sp_elx +end_vector_entry mmu_sync_exception_sp_elx + +vector_entry mmu_irq_sp_elx + b irq_sp_elx +end_vector_entry mmu_irq_sp_elx + +vector_entry mmu_fiq_sp_elx + b fiq_sp_elx +end_vector_entry mmu_fiq_sp_elx + +vector_entry mmu_serror_sp_elx + b serror_sp_elx +end_vector_entry mmu_serror_sp_elx + + /* --------------------------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x600 + * --------------------------------------------------------------------- + */ +vector_entry mmu_sync_exception_aarch64 + apply_cve_2017_5715_wa _is_sync_exception=1 _esr_el3_val=ESR_EL3_A64_SMC0 + b sync_exception_aarch64 +end_vector_entry mmu_sync_exception_aarch64 + +vector_entry mmu_irq_aarch64 + apply_cve_2017_5715_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A64_SMC0 + b irq_aarch64 +end_vector_entry mmu_irq_aarch64 + +vector_entry mmu_fiq_aarch64 + apply_cve_2017_5715_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A64_SMC0 + b fiq_aarch64 +end_vector_entry mmu_fiq_aarch64 + +vector_entry mmu_serror_aarch64 + apply_cve_2017_5715_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A64_SMC0 + b serror_aarch64 +end_vector_entry mmu_serror_aarch64 + + /* --------------------------------------------------------------------- + * Lower EL using AArch32 : 0x600 - 0x800 + * --------------------------------------------------------------------- + */ +vector_entry mmu_sync_exception_aarch32 + apply_cve_2017_5715_wa _is_sync_exception=1 _esr_el3_val=ESR_EL3_A32_SMC0 + b sync_exception_aarch32 +end_vector_entry mmu_sync_exception_aarch32 + +vector_entry mmu_irq_aarch32 + apply_cve_2017_5715_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A32_SMC0 + b irq_aarch32 +end_vector_entry mmu_irq_aarch32 + +vector_entry mmu_fiq_aarch32 + apply_cve_2017_5715_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A32_SMC0 + b fiq_aarch32 +end_vector_entry mmu_fiq_aarch32 + +vector_entry mmu_serror_aarch32 + apply_cve_2017_5715_wa _is_sync_exception=0 _esr_el3_val=ESR_EL3_A32_SMC0 + b serror_aarch32 +end_vector_entry mmu_serror_aarch32 diff --git a/lib/cpus/aarch64/wa_cve_2022_23960_bhb.S b/lib/cpus/aarch64/wa_cve_2022_23960_bhb.S new file mode 100644 index 0000000..bf26658 --- /dev/null +++ b/lib/cpus/aarch64/wa_cve_2022_23960_bhb.S @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <context.h> + +#if WORKAROUND_CVE_2022_23960 + /* + * This macro applies the mitigation for CVE-2022-23960. + * The macro saves x2 to the CPU context. + * SP should point to the CPU context. + */ + .macro apply_cve_2022_23960_bhb_wa _bhb_loop_count + str x2, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + + /* CVE-BHB-NUM loop count */ + mov x2, \_bhb_loop_count + +1: + /* b pc+4 part of the workaround */ + b 2f +2: + subs x2, x2, #1 + bne 1b + speculation_barrier + ldr x2, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + .endm +#endif /* WORKAROUND_CVE_2022_23960 */ diff --git a/lib/cpus/aarch64/wa_cve_2022_23960_bhb_vector.S b/lib/cpus/aarch64/wa_cve_2022_23960_bhb_vector.S new file mode 100644 index 0000000..220fa11 --- /dev/null +++ b/lib/cpus/aarch64/wa_cve_2022_23960_bhb_vector.S @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2022, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <services/arm_arch_svc.h> +#include "wa_cve_2022_23960_bhb.S" + + /* + * This macro is used to isolate the vector table for relevant CPUs + * used in the mitigation for CVE_2022_23960. + */ + .macro wa_cve_2022_23960_bhb_vector_table _bhb_loop_count, _cpu + + .globl wa_cve_vbar_\_cpu + +vector_base wa_cve_vbar_\_cpu + /* --------------------------------------------------------------------- + * Current EL with SP_EL0 : 0x0 - 0x200 + * --------------------------------------------------------------------- + */ +vector_entry bhb_sync_exception_sp_el0_\_cpu + b sync_exception_sp_el0 +end_vector_entry bhb_sync_exception_sp_el0_\_cpu + +vector_entry bhb_irq_sp_el0_\_cpu + b irq_sp_el0 +end_vector_entry bhb_irq_sp_el0_\_cpu + +vector_entry bhb_fiq_sp_el0_\_cpu + b fiq_sp_el0 +end_vector_entry bhb_fiq_sp_el0_\_cpu + +vector_entry bhb_serror_sp_el0_\_cpu + b serror_sp_el0 +end_vector_entry bhb_serror_sp_el0_\_cpu + + /* --------------------------------------------------------------------- + * Current EL with SP_ELx: 0x200 - 0x400 + * --------------------------------------------------------------------- + */ +vector_entry bhb_sync_exception_sp_elx_\_cpu + b sync_exception_sp_elx +end_vector_entry bhb_sync_exception_sp_elx_\_cpu + +vector_entry bhb_irq_sp_elx_\_cpu + b irq_sp_elx +end_vector_entry bhb_irq_sp_elx_\_cpu + +vector_entry bhb_fiq_sp_elx_\_cpu + b fiq_sp_elx +end_vector_entry bhb_fiq_sp_elx_\_cpu + +vector_entry bhb_serror_sp_elx_\_cpu + b serror_sp_elx +end_vector_entry bhb_serror_sp_elx_\_cpu + + /* --------------------------------------------------------------------- + * Lower EL using AArch64 : 0x400 - 0x600 + * --------------------------------------------------------------------- + */ +vector_entry bhb_sync_exception_aarch64_\_cpu + apply_cve_2022_23960_bhb_wa \_bhb_loop_count + b sync_exception_aarch64 +end_vector_entry bhb_sync_exception_aarch64_\_cpu + +vector_entry bhb_irq_aarch64_\_cpu + apply_cve_2022_23960_bhb_wa \_bhb_loop_count + b irq_aarch64 +end_vector_entry bhb_irq_aarch64_\_cpu + +vector_entry bhb_fiq_aarch64_\_cpu + apply_cve_2022_23960_bhb_wa \_bhb_loop_count + b fiq_aarch64 +end_vector_entry bhb_fiq_aarch64_\_cpu + +vector_entry bhb_serror_aarch64_\_cpu + apply_cve_2022_23960_bhb_wa \_bhb_loop_count + b serror_aarch64 +end_vector_entry bhb_serror_aarch64_\_cpu + + /* --------------------------------------------------------------------- + * Lower EL using AArch32 : 0x600 - 0x800 + * --------------------------------------------------------------------- + */ +vector_entry bhb_sync_exception_aarch32_\_cpu + apply_cve_2022_23960_bhb_wa \_bhb_loop_count + b sync_exception_aarch32 +end_vector_entry bhb_sync_exception_aarch32_\_cpu + +vector_entry bhb_irq_aarch32_\_cpu + apply_cve_2022_23960_bhb_wa \_bhb_loop_count + b irq_aarch32 +end_vector_entry bhb_irq_aarch32_\_cpu + +vector_entry bhb_fiq_aarch32_\_cpu + apply_cve_2022_23960_bhb_wa \_bhb_loop_count + b fiq_aarch32 +end_vector_entry bhb_fiq_aarch32_\_cpu + +vector_entry bhb_serror_aarch32_\_cpu + apply_cve_2022_23960_bhb_wa \_bhb_loop_count + b serror_aarch32 +end_vector_entry bhb_serror_aarch32_\_cpu + .endm diff --git a/lib/cpus/cpu-ops.mk b/lib/cpus/cpu-ops.mk new file mode 100644 index 0000000..434ee08 --- /dev/null +++ b/lib/cpus/cpu-ops.mk @@ -0,0 +1,893 @@ +# +# Copyright (c) 2014-2023, Arm Limited and Contributors. All rights reserved. +# Copyright (c) 2020-2022, NVIDIA Corporation. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +include ${MAKE_HELPERS_DIRECTORY}$/build_macros.mk + +# Cortex A57 specific optimisation to skip L1 cache flush when +# cluster is powered down. +CPU_FLAG_LIST += SKIP_A57_L1_FLUSH_PWR_DWN + +# Flag to disable the cache non-temporal hint. +# It is enabled by default. +A53_DISABLE_NON_TEMPORAL_HINT ?=1 +CPU_FLAG_LIST += A53_DISABLE_NON_TEMPORAL_HINT + +# Flag to disable the cache non-temporal hint. +# It is enabled by default. +A57_DISABLE_NON_TEMPORAL_HINT ?=1 +CPU_FLAG_LIST += A57_DISABLE_NON_TEMPORAL_HINT + +# Flag to enable higher performance non-cacheable load forwarding. +# It is disabled by default. +CPU_FLAG_LIST += A57_ENABLE_NONCACHEABLE_LOAD_FWD + +WORKAROUND_CVE_2017_5715 ?=1 +CPU_FLAG_LIST += WORKAROUND_CVE_2017_5715 +WORKAROUND_CVE_2018_3639 ?=1 +CPU_FLAG_LIST += WORKAROUND_CVE_2018_3639 +CPU_FLAG_LIST += DYNAMIC_WORKAROUND_CVE_2018_3639 +WORKAROUND_CVE_2022_23960 ?=1 +CPU_FLAG_LIST += WORKAROUND_CVE_2022_23960 + +# Flags to indicate internal or external Last level cache +# By default internal +CPU_FLAG_LIST += NEOVERSE_Nx_EXTERNAL_LLC + +# CPU Errata Build flags. +# These should be enabled by the platform if the erratum workaround needs to be +# applied. + +# Flag to apply erratum 794073 workaround when disabling mmu. +CPU_FLAG_LIST += ERRATA_A9_794073 + +# Flag to apply erratum 816470 workaround during power down. This erratum +# applies only to revision >= r3p0 of the Cortex A15 cpu. +CPU_FLAG_LIST += ERRATA_A15_816470 + +# Flag to apply erratum 827671 workaround during reset. This erratum applies +# only to revision >= r3p0 of the Cortex A15 cpu. +CPU_FLAG_LIST += ERRATA_A15_827671 + +# Flag to apply erratum 852421 workaround during reset. This erratum applies +# only to revision <= r1p2 of the Cortex A17 cpu. +CPU_FLAG_LIST += ERRATA_A17_852421 + +# Flag to apply erratum 852423 workaround during reset. This erratum applies +# only to revision <= r1p2 of the Cortex A17 cpu. +CPU_FLAG_LIST += ERRATA_A17_852423 + +# Flag to apply erratum 855472 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A35 cpu. +CPU_FLAG_LIST += ERRATA_A35_855472 + +# Flag to apply erratum 819472 workaround during reset. This erratum applies +# only to revision <= r0p1 of the Cortex A53 cpu. +CPU_FLAG_LIST += ERRATA_A53_819472 + +# Flag to apply erratum 824069 workaround during reset. This erratum applies +# only to revision <= r0p2 of the Cortex A53 cpu. +CPU_FLAG_LIST += ERRATA_A53_824069 + +# Flag to apply erratum 826319 workaround during reset. This erratum applies +# only to revision <= r0p2 of the Cortex A53 cpu. +CPU_FLAG_LIST += ERRATA_A53_826319 + +# Flag to apply erratum 827319 workaround during reset. This erratum applies +# only to revision <= r0p2 of the Cortex A53 cpu. +CPU_FLAG_LIST += ERRATA_A53_827319 + +# Flag to apply erratum 835769 workaround at compile and link time. This +# erratum applies to revision <= r0p4 of the Cortex A53 cpu. Enabling this +# workaround can lead the linker to create "*.stub" sections. +CPU_FLAG_LIST += ERRATA_A53_835769 + +# Flag to apply erratum 836870 workaround during reset. This erratum applies +# only to revision <= r0p3 of the Cortex A53 cpu. From r0p4 and onwards, this +# erratum workaround is enabled by default in hardware. +CPU_FLAG_LIST += ERRATA_A53_836870 + +# Flag to apply erratum 843419 workaround at link time. +# This erratum applies to revision <= r0p4 of the Cortex A53 cpu. Enabling this +# workaround could lead the linker to emit "*.stub" sections which are 4kB +# aligned. +CPU_FLAG_LIST += ERRATA_A53_843419 + +# Flag to apply errata 855873 during reset. This errata applies to all +# revisions of the Cortex A53 CPU, but this firmware workaround only works +# for revisions r0p3 and higher. Earlier revisions are taken care +# of by the rich OS. +CPU_FLAG_LIST += ERRATA_A53_855873 + +# Flag to apply erratum 1530924 workaround during reset. This erratum applies +# to all revisions of Cortex A53 cpu. +CPU_FLAG_LIST += ERRATA_A53_1530924 + +# Flag to apply erratum 768277 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A55 cpu. +CPU_FLAG_LIST += ERRATA_A55_768277 + +# Flag to apply erratum 778703 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A55 cpu. +CPU_FLAG_LIST += ERRATA_A55_778703 + +# Flag to apply erratum 798797 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A55 cpu. +CPU_FLAG_LIST += ERRATA_A55_798797 + +# Flag to apply erratum 846532 workaround during reset. This erratum applies +# only to revision <= r0p1 of the Cortex A55 cpu. +CPU_FLAG_LIST += ERRATA_A55_846532 + +# Flag to apply erratum 903758 workaround during reset. This erratum applies +# only to revision <= r0p1 of the Cortex A55 cpu. +CPU_FLAG_LIST += ERRATA_A55_903758 + +# Flag to apply erratum 1221012 workaround during reset. This erratum applies +# only to revision <= r1p0 of the Cortex A55 cpu. +CPU_FLAG_LIST += ERRATA_A55_1221012 + +# Flag to apply erratum 1530923 workaround during reset. This erratum applies +# to all revisions of Cortex A55 cpu. +CPU_FLAG_LIST += ERRATA_A55_1530923 + +# Flag to apply erratum 806969 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_806969 + +# Flag to apply erratum 813419 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_813419 + +# Flag to apply erratum 813420 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_813420 + +# Flag to apply erratum 814670 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_814670 + +# Flag to apply erratum 817169 workaround during power down. This erratum +# applies only to revision <= r0p1 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_817169 + +# Flag to apply erratum 826974 workaround during reset. This erratum applies +# only to revision <= r1p1 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_826974 + +# Flag to apply erratum 826977 workaround during reset. This erratum applies +# only to revision <= r1p1 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_826977 + +# Flag to apply erratum 828024 workaround during reset. This erratum applies +# only to revision <= r1p1 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_828024 + +# Flag to apply erratum 829520 workaround during reset. This erratum applies +# only to revision <= r1p2 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_829520 + +# Flag to apply erratum 833471 workaround during reset. This erratum applies +# only to revision <= r1p2 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_833471 + +# Flag to apply erratum 855972 workaround during reset. This erratum applies +# only to revision <= r1p3 of the Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_859972 + +# Flag to apply erratum 1319537 workaround during reset. This erratum applies +# to all revisions of Cortex A57 cpu. +CPU_FLAG_LIST += ERRATA_A57_1319537 + +# Flag to apply erratum 855971 workaround during reset. This erratum applies +# only to revision <= r0p3 of the Cortex A72 cpu. +CPU_FLAG_LIST += ERRATA_A72_859971 + +# Flag to apply erratum 1319367 workaround during reset. This erratum applies +# to all revisions of Cortex A72 cpu. +CPU_FLAG_LIST += ERRATA_A72_1319367 + +# Flag to apply erratum 852427 workaround during reset. This erratum applies +# only to revision r0p0 of the Cortex A73 cpu. +CPU_FLAG_LIST += ERRATA_A73_852427 + +# Flag to apply erratum 855423 workaround during reset. This erratum applies +# only to revision <= r0p1 of the Cortex A73 cpu. +CPU_FLAG_LIST += ERRATA_A73_855423 + +# Flag to apply erratum 764081 workaround during reset. This erratum applies +# only to revision <= r0p0 of the Cortex A75 cpu. +CPU_FLAG_LIST += ERRATA_A75_764081 + +# Flag to apply erratum 790748 workaround during reset. This erratum applies +# only to revision <= r0p0 of the Cortex A75 cpu. +CPU_FLAG_LIST += ERRATA_A75_790748 + +# Flag to apply erratum 1073348 workaround during reset. This erratum applies +# only to revision <= r1p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1073348 + +# Flag to apply erratum 1130799 workaround during reset. This erratum applies +# only to revision <= r2p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1130799 + +# Flag to apply erratum 1220197 workaround during reset. This erratum applies +# only to revision <= r2p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1220197 + +# Flag to apply erratum 1257314 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1257314 + +# Flag to apply erratum 1262606 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1262606 + +# Flag to apply erratum 1262888 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1262888 + +# Flag to apply erratum 1275112 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1275112 + +# Flag to apply erratum 1286807 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1286807 + +# Flag to apply erratum 1791580 workaround during reset. This erratum applies +# only to revision <= r4p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1791580 + +# Flag to apply erratum 1165522 workaround during reset. This erratum applies +# to all revisions of Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1165522 + +# Flag to apply erratum 1868343 workaround during reset. This erratum applies +# only to revision <= r4p0 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1868343 + +# Flag to apply erratum 1946160 workaround during reset. This erratum applies +# only to revisions r3p0 - r4p1 of the Cortex A76 cpu. +CPU_FLAG_LIST += ERRATA_A76_1946160 + +# Flag to apply erratum 2743102 workaround during powerdown. This erratum +# applies to all revisions <= r4p1 of the Cortex A76 cpu and is still open. +CPU_FLAG_LIST += ERRATA_A76_2743102 + +# Flag to apply erratum 1508412 workaround during reset. This erratum applies +# only to revision <= r1p0 of the Cortex A77 cpu. +CPU_FLAG_LIST += ERRATA_A77_1508412 + +# Flag to apply erratum 1925769 workaround during reset. This erratum applies +# only to revision <= r1p1 of the Cortex A77 cpu. +CPU_FLAG_LIST += ERRATA_A77_1925769 + +# Flag to apply erratum 1946167 workaround during reset. This erratum applies +# only to revision <= r1p1 of the Cortex A77 cpu. +CPU_FLAG_LIST += ERRATA_A77_1946167 + +# Flag to apply erratum 1791578 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, and r1p1, it is still open. +CPU_FLAG_LIST += ERRATA_A77_1791578 + +# Flag to apply erratum 2356587 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, and r1p1, it is still open. +CPU_FLAG_LIST += ERRATA_A77_2356587 + +# Flag to apply erratum 1800714 workaround during reset. This erratum applies +# to revisions <= r1p1 of the Cortex A77 cpu. +CPU_FLAG_LIST += ERRATA_A77_1800714 + +# Flag to apply erratum 2743100 workaround during power down. This erratum +# applies to revisions r0p0, r1p0, and r1p1, it is still open. +CPU_FLAG_LIST += ERRATA_A77_2743100 + +# Flag to apply erratum 1688305 workaround during reset. This erratum applies +# to revisions r0p0 - r1p0 of the A78 cpu. +CPU_FLAG_LIST += ERRATA_A78_1688305 + +# Flag to apply erratum 1941498 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, and r1p1 of the A78 cpu. +CPU_FLAG_LIST += ERRATA_A78_1941498 + +# Flag to apply erratum 1951500 workaround during reset. This erratum applies +# to revisions r1p0 and r1p1 of the A78 cpu. The issue is present in r0p0 as +# well but there is no workaround for that revision. +CPU_FLAG_LIST += ERRATA_A78_1951500 + +# Flag to apply erratum 1821534 workaround during reset. This erratum applies +# to revisions r0p0 and r1p0 of the A78 cpu. +CPU_FLAG_LIST += ERRATA_A78_1821534 + +# Flag to apply erratum 1952683 workaround during reset. This erratum applies +# to revision r0p0 of the A78 cpu and was fixed in the revision r1p0. +CPU_FLAG_LIST += ERRATA_A78_1952683 + +# Flag to apply erratum 2132060 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, r1p1, and r1p2 of the A78 cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78_2132060 + +# Flag to apply erratum 2242635 workaround during reset. This erratum applies +# to revisions r1p0, r1p1, and r1p2 of the A78 cpu and is open. The issue is +# present in r0p0 as well but there is no workaround for that revision. +CPU_FLAG_LIST += ERRATA_A78_2242635 + +# Flag to apply erratum 2376745 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, r1p1, and r1p2 of the A78 cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78_2376745 + +# Flag to apply erratum 2395406 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, r1p1, and r1p2 of the A78 cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78_2395406 + +# Flag to apply erratum 2712571 workaround for non-arm interconnect ip. This +# erratum applies to revisions r0p0, r1p0, r1p1, and r1p2 of the A78 cpu. +# It is fixed in r1p2. +CPU_FLAG_LIST += ERRATA_A78_2712571 + +# Flag to apply erratum 2742426 workaround during reset. This erratum +# applies to revisions r0p0, r1p0, r1p1 and r1p2 of the A78 cpu. It is still +# open. +CPU_FLAG_LIST += ERRATA_A78_2742426 + +# Flag to apply erratum 2772019 workaround during powerdown. This erratum +# applies to revisions r0p0, r1p0, r1p1 and r1p2 of the A78 cpu. It is still +# open. +CPU_FLAG_LIST += ERRATA_A78_2772019 + +# Flag to apply erratum 2779479 workaround during reset. This erratum applies +# to revision r0p0, r1p0, r1p1 and r1p2 of the A78 cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78_2779479 + +# Flag to apply erratum 1941500 workaround during reset. This erratum applies +# to revisions r0p0 and r0p1 of the A78 AE cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78_AE_1941500 + +# Flag to apply erratum 1951502 workaround during reset. This erratum applies +# to revisions r0p0 and r0p1 of the A78 AE cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78_AE_1951502 + +# Flag to apply erratum 2376748 workaround during reset. This erratum applies +# to revisions r0p0, r0p1 and r0p2 of the A78 AE cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78_AE_2376748 + +# Flag to apply erratum 2395408 workaround during reset. This erratum applies +# to revisions r0p0 and r0p1 of the A78 AE cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78_AE_2395408 + +# Flag to apply erratum 1827430 workaround during reset. This erratum applies +# to revision r0p0 of the A78C cpu. It is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_A78C_1827430 + +# Flag to apply erratum 1827440 workaround during reset. This erratum applies +# to revision r0p0 of the A78C cpu. It is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_A78C_1827440 + +# Flag to apply erratum 2712574 workaround for non-arm interconnect ip. This +# erratum applies to revisions r0p0, r0p1 and r0p2 of the A78 AE cpu. +# It is still open. +CPU_FLAG_LIST += ERRATA_A78_AE_2712574 + +# Flag to apply erratum 2132064 workaround during reset. This erratum applies +# to revisions r0p1 and r0p2 of the A78C cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78C_2132064 + +# Flag to apply erratum 2242638 workaround during reset. This erratum applies +# to revisions r0p1 and r0p2 of the A78C cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78C_2242638 + +# Flag to apply erratum 2376749 workaround during reset. This erratum applies +# to revisions r0p1 and r0p2 of the A78C cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78C_2376749 + +# Flag to apply erratum 2395411 workaround during reset. This erratum applies +# to revisions r0p1 and r0p2 of the A78C cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78C_2395411 + +# Flag to apply erratum 2712575 workaround for non-arm interconnect ip. This +# erratum applies to revisions r0p1 and r0p2 of the A78C cpu. +# It is still open. +CPU_FLAG_LIST += ERRATA_A78C_2712575 + +# Flag to apply erratum 2772121 workaround during powerdown. This erratum +# applies to revisions r0p0, r0p1 and r0p2 of the A78C cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78C_2772121 + +# Flag to apply erratum 2779484 workaround during reset. This erratum +# applies to revisions r0p1 and r0p2 of the A78C cpu. It is still open. +CPU_FLAG_LIST += ERRATA_A78C_2779484 + +# Flag to apply erratum 1821534 workaround during reset. This erratum applies +# to revisions r0p0 - r1p0 of the X1 cpu and fixed in r1p1. +CPU_FLAG_LIST += ERRATA_X1_1821534 + +# Flag to apply erratum 1688305 workaround during reset. This erratum applies +# to revisions r0p0 - r1p0 of the X1 cpu and fixed in r1p1. +CPU_FLAG_LIST += ERRATA_X1_1688305 + +# Flag to apply erratum 1827429 workaround during reset. This erratum applies +# to revisions r0p0 - r1p0 of the X1 cpu and fixed in r1p1. +CPU_FLAG_LIST += ERRATA_X1_1827429 + +# Flag to apply T32 CLREX workaround during reset. This erratum applies +# only to r0p0 and r1p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1043202 + +# Flag to apply erratum 1073348 workaround during reset. This erratum applies +# only to revision r0p0 and r1p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1073348 + +# Flag to apply erratum 1130799 workaround during reset. This erratum applies +# only to revision <= r2p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1130799 + +# Flag to apply erratum 1165347 workaround during reset. This erratum applies +# only to revision <= r2p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1165347 + +# Flag to apply erratum 1207823 workaround during reset. This erratum applies +# only to revision <= r2p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1207823 + +# Flag to apply erratum 1220197 workaround during reset. This erratum applies +# only to revision <= r2p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1220197 + +# Flag to apply erratum 1257314 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1257314 + +# Flag to apply erratum 1262606 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1262606 + +# Flag to apply erratum 1262888 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1262888 + +# Flag to apply erratum 1275112 workaround during reset. This erratum applies +# only to revision <= r3p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1275112 + +# Flag to apply erratum 1315703 workaround during reset. This erratum applies +# to revisions before r3p1 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1315703 + +# Flag to apply erratum 1542419 workaround during reset. This erratum applies +# to revisions r3p0 - r4p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1542419 + +# Flag to apply erratum 1868343 workaround during reset. This erratum applies +# to revision <= r4p0 of the Neoverse N1 cpu. +CPU_FLAG_LIST += ERRATA_N1_1868343 + +# Flag to apply erratum 1946160 workaround during reset. This erratum applies +# to revisions r3p0, r3p1, r4p0, and r4p1 of the Neoverse N1 cpu. The issue +# exists in revisions r0p0, r1p0, and r2p0 as well but there is no workaround. +CPU_FLAG_LIST += ERRATA_N1_1946160 + +# Flag to apply erratum 2743102 workaround during powerdown. This erratum +# applies to all revisions <= r4p1 of the Neoverse N1 cpu and is still open. +CPU_FLAG_LIST += ERRATA_N1_2743102 + +# Flag to apply erratum 1618635 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse V1 cpu and was fixed in the revision r1p0. +CPU_FLAG_LIST += ERRATA_V1_1618635 + +# Flag to apply erratum 1774420 workaround during reset. This erratum applies +# to revisions r0p0 and r1p0 of the Neoverse V1 core, and was fixed in r1p1. +CPU_FLAG_LIST += ERRATA_V1_1774420 + +# Flag to apply erratum 1791573 workaround during reset. This erratum applies +# to revisions r0p0 and r1p0 of the Neoverse V1 core, and was fixed in r1p1. +CPU_FLAG_LIST += ERRATA_V1_1791573 + +# Flag to apply erratum 1852267 workaround during reset. This erratum applies +# to revisions r0p0 and r1p0 of the Neoverse V1 core, and was fixed in r1p1. +CPU_FLAG_LIST += ERRATA_V1_1852267 + +# Flag to apply erratum 1925756 workaround during reset. This needs to be +# enabled for r0p0, r1p0, and r1p1 of the Neoverse V1 core, it is still open. +CPU_FLAG_LIST += ERRATA_V1_1925756 + +# Flag to apply erratum 1940577 workaround during reset. This erratum applies +# to revisions r1p0 and r1p1 of the Neoverse V1 cpu. +CPU_FLAG_LIST += ERRATA_V1_1940577 + +# Flag to apply erratum 1966096 workaround during reset. This erratum applies +# to revisions r1p0 and r1p1 of the Neoverse V1 CPU and is open. This issue +# exists in r0p0 as well but there is no workaround for that revision. +CPU_FLAG_LIST += ERRATA_V1_1966096 + +# Flag to apply erratum 2139242 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, and r1p1 of the Neoverse V1 cpu and is still open. +CPU_FLAG_LIST += ERRATA_V1_2139242 + +# Flag to apply erratum 2108267 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, and r1p1 of the Neoverse V1 cpu and is still open. +CPU_FLAG_LIST += ERRATA_V1_2108267 + +# Flag to apply erratum 2216392 workaround during reset. This erratum applies +# to revisions r1p0 and r1p1 of the Neoverse V1 cpu and is still open. This +# issue exists in r0p0 as well but there is no workaround for that revision. +CPU_FLAG_LIST += ERRATA_V1_2216392 + +# Flag to apply erratum 2294912 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, and r1p1 and r1p2 of the Neoverse V1 cpu and is still open. +CPU_FLAG_LIST += ERRATA_V1_2294912 + +# Flag to apply erratum 2372203 workaround during reset. This erratum applies +# to revisions r0p0, r1p0 and r1p1 of the Neoverse V1 cpu and is still open. +CPU_FLAG_LIST += ERRATA_V1_2372203 + +# Flag to apply erratum 2701953 workaround to non-arm interconnect ip. This +# erratum applies to revisions r0p0, r1p0, r1p1 of the Neoverse V1 cpu, +# it is fixed in r1p2. +CPU_FLAG_LIST += ERRATA_V1_2701953 + +# Flag to apply erratum 2743093 workaround during powerdown. This erratum +# applies to revisions r0p0, r1p0, r1p1 and r1p2 of the Neoverse V1 cpu and is +# still open. +CPU_FLAG_LIST += ERRATA_V1_2743093 + +# Flag to apply erratum 2743233 workaround during powerdown. This erratum +# applies to revisions r0p0, r1p0, r1p1 and r1p2 of the Neoverse V1 cpu and is +# still open. +CPU_FLAG_LIST += ERRATA_V1_2743233 + +# Flag to apply erratum 2779461 workaround during powerdown. This erratum +# applies to revisions r0p0, r1p0, r1p1 and r1p2 of the Neoverse V1 cpu and is +# still open. +CPU_FLAG_LIST += ERRATA_V1_2779461 + +# Flag to apply erratum 1987031 workaround during reset. This erratum applies +# to revisions r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is still open. +CPU_FLAG_LIST += ERRATA_A710_1987031 + +# Flag to apply erratum 2081180 workaround during reset. This erratum applies +# to revisions r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is still open. +CPU_FLAG_LIST += ERRATA_A710_2081180 + +# Flag to apply erratum 2083908 workaround during reset. This erratum applies +# to revision r2p0 of the Cortex-A710 cpu and is still open. +CPU_FLAG_LIST += ERRATA_A710_2083908 + +# Flag to apply erratum 2058056 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, r2p0 and r2p1 of the Cortex-A710 cpu and is still +# open. +CPU_FLAG_LIST += ERRATA_A710_2058056 + +# Flag to apply erratum 2055002 workaround during reset. This erratum applies +# to revision r1p0, r2p0 of the Cortex-A710 cpu and is still open. +CPU_FLAG_LIST += ERRATA_A710_2055002 + +# Flag to apply erratum 2017096 workaround during reset. This erratum applies +# to revision r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is still open. +CPU_FLAG_LIST += ERRATA_A710_2017096 + +# Flag to apply erratum 2267065 workaround during reset. This erratum applies +# to revision r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is fixed in r2p1. +CPU_FLAG_LIST += ERRATA_A710_2267065 + +# Flag to apply erratum 2136059 workaround during reset. This erratum applies +# to revision r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is fixed in r2p1. +CPU_FLAG_LIST += ERRATA_A710_2136059 + +# Flag to apply erratum 2147715 workaround during reset. This erratum applies +# to revision r2p0 of the Cortex-A710 CPU and is fixed in revision r2p1. +CPU_FLAG_LIST += ERRATA_A710_2147715 + +# Flag to apply erratum 2216384 workaround during reset. This erratum applies +# to revision r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is fixed in r2p1. +CPU_FLAG_LIST += ERRATA_A710_2216384 + +# Flag to apply erratum 2282622 workaround during reset. This erratum applies +# to revision r0p0, r1p0, r2p0 and r2p1 of the Cortex-A710 cpu and is still +# open. +CPU_FLAG_LIST += ERRATA_A710_2282622 + +# Flag to apply erratum 2291219 workaround during reset. This erratum applies +# to revision r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is fixed in r2p1. +CPU_FLAG_LIST += ERRATA_A710_2291219 + +# Flag to apply erratum 2008768 workaround during reset. This erratum applies +# to revision r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is fixed in r2p1. +CPU_FLAG_LIST += ERRATA_A710_2008768 + +# Flag to apply erratum 2371105 workaround during reset. This erratum applies +# to revision r0p0, r1p0 and r2p0 of the Cortex-A710 cpu and is fixed in r2p1. +CPU_FLAG_LIST += ERRATA_A710_2371105 + +# Flag to apply erratum 2701952 workaround for non-arm interconnect ip. This +# erratum applies to revision r0p0, r1p0, r2p0, r2p1 of the Cortex-A710 cpu +# and is still open. +CPU_FLAG_LIST += ERRATA_A710_2701952 + +# Flag to apply erratum 2742423 workaround during reset. This erratum applies +# to revision r0p0, r1p0, r2p0 and r2p1 of the Cortex-A710 cpu and is still +# open. +CPU_FLAG_LIST += ERRATA_A710_2742423 + +# Flag to apply erratum 2768515 workaround during power down. This erratum +# applies to revision r0p0, r1p0, r2p0 and r2p1 of the Cortex-A710 cpu and is +# still open. +CPU_FLAG_LIST += ERRATA_A710_2768515 + +# Flag to apply erratum 2002655 workaround during reset. This erratum applies +# to revisions r0p0 of the Neoverse-N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2002655 + +# Flag to apply erratum 2009478 workaround during powerdown. This erratum +# applies to revision r0p0 of the Neoverse N2 cpu, it is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2009478 + +# Flag to apply erratum 2067956 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2067956 + +# Flag to apply erratum 2025414 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2025414 + +# Flag to apply erratum 2189731 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2189731 + +# Flag to apply erratum 2138956 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2138956 + +# Flag to apply erratum 2138953 workaround during reset. This erratum applies +# to revision r0p0, r0p1, r0p2, r0p3 of the Neoverse N2 cpu and is still open. +CPU_FLAG_LIST += ERRATA_N2_2138953 + +# Flag to apply erratum 2242415 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2242415 + +# Flag to apply erratum 2138958 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2138958 + +# Flag to apply erratum 2242400 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2242400 + +# Flag to apply erratum 2280757 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2280757 + +# Flag to apply erratum 2326639 workaroud during powerdown. This erratum +# applies to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2326639 + +# Flag to apply erratum 2340933 workaroud during reset. This erratum +# applies to revision r0p0 of the Neoverse N2 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2340933 + +# Flag to apply erratum 2346952 workaround during reset. This erratum applies +# to r0p0, r0p1, r0p2 of the Neoverse N2 cpu, it is fixed in r0p3. +CPU_FLAG_LIST += ERRATA_N2_2346952 + +# Flag to apply erratum 2376738 workaround during reset. This erratum applies +# to revision r0p0, r0p1, r0p2, r0p3 of the Neoverse N2 cpu and is still open. +CPU_FLAG_LIST += ERRATA_N2_2376738 + +# Flag to apply erratum 2388450 workaround during reset. This erratum applies +# to revision r0p0 of the Neoverse N2 cpu, it is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_N2_2388450 + +# Flag to apply erratum 2728475 workaround for non-arm interconnect ip. This +# erratum applies to r0p0, r0p1, r0p2 of the Neoverse N2 cpu, it is fixed in +# r0p3. +CPU_FLAG_LIST += ERRATA_N2_2728475 + +# Flag to apply erratum 2743014 workaround during reset. This erratum applies +# to r0p0, r0p1, r0p2 of the Neoverse N2 cpu, it is fixed in r0p3. +CPU_FLAG_LIST += ERRATA_N2_2743014 + +# Flag to apply erratum 2743089 workaround during during powerdown. This erratum +# applies to all revisions <= r0p2 of the Neoverse N2 cpu, it is fixed in r0p3. +CPU_FLAG_LIST += ERRATA_N2_2743089 + +# Flag to apply erratum 2779511 workaround during reset. This erratum applies +# to r0p0, r0p1, r0p2 of the Neoverse N2 cpu, it is fixed in r0p3. +CPU_FLAG_LIST += ERRATA_N2_2779511 + +# Flag to apply erratum 2002765 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, and r2p0 of the Cortex-X2 cpu and is still open. +CPU_FLAG_LIST += ERRATA_X2_2002765 + +# Flag to apply erratum 2058056 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, r2p0 and r2p1 of the Cortex-X2 cpu and is still open. +CPU_FLAG_LIST += ERRATA_X2_2058056 + +# Flag to apply erratum 2083908 workaround during reset. This erratum applies +# to revision r2p0 of the Cortex-X2 cpu and is still open. +CPU_FLAG_LIST += ERRATA_X2_2083908 + +# Flag to apply erratum 2017096 workaround during reset. This erratum applies +# only to revisions r0p0, r1p0 and r2p0 of the Cortex-X2 cpu, it is fixed in +# r2p1. +CPU_FLAG_LIST += ERRATA_X2_2017096 + +# Flag to apply erratum 2081180 workaround during reset. This erratum applies +# only to revisions r0p0, r1p0 and r2p0 of the Cortex-X2 cpu, it is fixed in +# r2p1. +CPU_FLAG_LIST += ERRATA_X2_2081180 + +# Flag to apply erratum 2216384 workaround during reset. This erratum applies +# only to revisions r0p0, r1p0 and r2p0 of the Cortex-X2 cpu, it is fixed in +# r2p1. +CPU_FLAG_LIST += ERRATA_X2_2216384 + +# Flag to apply erratum 2147715 workaround during reset. This erratum applies +# only to revision r2p0 of the Cortex-X2 cpu, it is fixed in r2p1. +CPU_FLAG_LIST += ERRATA_X2_2147715 + +# Flag to apply erratum 2282622 workaround during reset. This erratum applies +# to revision r0p0, r1p0, r2p0 and r2p1 of the Cortex-X2 cpu and is still +# open. +CPU_FLAG_LIST += ERRATA_X2_2282622 + +# Flag to apply erratum 2371105 workaround during reset. This erratum applies +# to revision r0p0, r1p0 and r2p0 of the Cortex-X2 cpu and is fixed in r2p1. +CPU_FLAG_LIST += ERRATA_X2_2371105 + +# Flag to apply erratum 2701952 workaround for non-arm interconnect ip. This +# erratum applies to revisions r0p0, r1p0, r2p0, r2p1 of the Cortex-x2 cpu +# and is still open. +CPU_FLAG_LIST += ERRATA_X2_2701952 + +# Flag to apply erratum 2742423 workaround during reset. This erratum applies +# to revisions r0p0, r1p0, r2p0 and r2p1 of the Cortex-X2 cpu and is still open. +CPU_FLAG_LIST += ERRATA_X2_2742423 + +# Flag to apply erratum 2768515 workaround during power down. This erratum +# applies to revision r0p0, r1p0, r2p0 and r2p1 of the Cortex-X2 cpu and is +# still open. +CPU_FLAG_LIST += ERRATA_X2_2768515 + +# Flag to apply erratum 2070301 workaround on reset. This erratum applies +# to revisions r0p0, r1p0, r1p1 and r1p2 of the Cortex-X3 cpu and is +# still open. +CPU_FLAG_LIST += ERRATA_X3_2070301 + +# Flag to apply erratum 2313909 workaround on powerdown. This erratum applies +# to revisions r0p0 and r1p0 of the Cortex-X3 cpu, it is fixed in r1p1. +CPU_FLAG_LIST += ERRATA_X3_2313909 + +# Flag to apply erratum 2615812 workaround on powerdown. This erratum applies +# to revisions r0p0, r1p0, r1p1 of the Cortex-X3 cpu, it is still open. +CPU_FLAG_LIST += ERRATA_X3_2615812 + +# Flag to apply erratum 2742421 workaround on reset. This erratum applies +# to revisions r0p0, r1p0 and r1p1 of the Cortex-X3 cpu, it is fixed in r1p2. +CPU_FLAG_LIST += ERRATA_X3_2742421 + +# Flag to apply erratum 1922240 workaround during reset. This erratum applies +# to revision r0p0 of the Cortex-A510 cpu and is fixed in r0p1. +CPU_FLAG_LIST += ERRATA_A510_1922240 + +# Flag to apply erratum 2288014 workaround during reset. This erratum applies +# to revisions r0p0, r0p1, r0p2, r0p3 and r1p0 of the Cortex-A510 cpu and is +# fixed in r1p1. +CPU_FLAG_LIST += ERRATA_A510_2288014 + +# Flag to apply erratum 2042739 workaround during reset. This erratum applies +# to revisions r0p0, r0p1 and r0p2 of the Cortex-A510 cpu and is fixed in r0p3. +CPU_FLAG_LIST += ERRATA_A510_2042739 + +# Flag to apply erratum 2041909 workaround during reset. This erratum applies +# to revision r0p2 of the Cortex-A510 cpu and is fixed in r0p3. The issue is +# present in r0p0 and r0p1 but there is no workaround for those revisions. +CPU_FLAG_LIST += ERRATA_A510_2041909 + +# Flag to aply erratum 2080326 workaround during reset. This erratum applies +# to revision r0p2 of the Cortex-A510 cpu and is fixed in r0p3. The issue is +# also present in r0p0 and r0p1 but there is no workaround for those revisions. +CPU_FLAG_LIST += ERRATA_A510_2080326 + +# Flag to apply erratum 2250311 workaround during reset. This erratum applies +# to revisions r0p0, r0p1, r0p2, r0p3 and r1p0, and is fixed in r1p1. +CPU_FLAG_LIST += ERRATA_A510_2250311 + +# Flag to apply erratum 2218950 workaround during reset. This erratum applies +# to revisions r0p0, r0p1, r0p2, r0p3 and r1p0, and is fixed in r1p1. +CPU_FLAG_LIST += ERRATA_A510_2218950 + +# Flag to apply erratum 2172148 workaround during reset. This erratum applies +# to revisions r0p0, r0p1, r0p2, r0p3 and r1p0, and is fixed in r1p1. +CPU_FLAG_LIST += ERRATA_A510_2172148 + +# Flag to apply erratum 2347730 workaround during reset. This erratum applies +# to revisions r0p0, r0p1, r0p2, r0p3, r1p0 and r1p1 of the Cortex-A510 CPU, +# and is fixed in r1p2. +CPU_FLAG_LIST += ERRATA_A510_2347730 + +# Flag to apply erratum 2371937 workaround during reset. This erratum applies +# to revisions r0p0, r0p1, r0p2, r0p3, r1p0, and r1p1. It is fixed in r1p2. +CPU_FLAG_LIST += ERRATA_A510_2371937 + +# Flag to apply erratum 2666669 workaround during reset. This erratum applies +# to revisions r0p0, r0p1, r0p2, r0p3, r1p0, and r1p1. It is fixed in r1p2. +CPU_FLAG_LIST += ERRATA_A510_2666669 + +# Flag to apply erratum 2684597 workaround during powerdown. This erratum +# applies to revision r0p0, r0p1, r0p2, r0p3, r1p0, r1p1 and r1p2 of the +# Cortex-A510 cpu and is fixed in r1p3. +CPU_FLAG_LIST += ERRATA_A510_2684597 + +# Flag to apply erratum 2331132 workaround during reset. This erratum applies +# to revisions r0p0, r0p1 and r0p2. It is still open. +CPU_FLAG_LIST += ERRATA_V2_2331132 + +# Flag to apply erratum 2719103 workaround for non-arm interconnect ip. This +# erratum applies to revisions r0p0, rop1. Fixed in r0p2. +CPU_FLAG_LIST += ERRATA_V2_2719103 + +# Flag to apply erratum 2719105 workaround during reset. This erratum applies +# to revisions r0p0 and r0p1. It is fixed in r0p2. +CPU_FLAG_LIST += ERRATA_V2_2719105 + +# Flag to apply erratum 2743011 workaround during reset. This erratum applies +# to revisions r0p0 and r0p1. It is fixed in r0p2. +CPU_FLAG_LIST += ERRATA_V2_2743011 + +# Flag to apply erratum 2779510 workaround during reset. This erratum applies +# to revisions r0p0 and r0p1. It is fixed in r0p2. +CPU_FLAG_LIST += ERRATA_V2_2779510 + +# Flag to apply erratum 2801372 workaround for all configurations. +# This erratum applies to revisions r0p0, r0p1. Fixed in r0p2. +CPU_FLAG_LIST += ERRATA_V2_2801372 + +# Flag to apply erratum 2701951 workaround for non-arm interconnect ip. +# This erratum applies to revisions r0p0, r1p0, and r1p1. Its is fixed in r1p2. +CPU_FLAG_LIST += ERRATA_A715_2701951 + +# Flag to apply DSU erratum 798953. This erratum applies to DSUs revision r0p0. +# Applying the workaround results in higher DSU power consumption on idle. +CPU_FLAG_LIST += ERRATA_DSU_798953 + +# Flag to apply DSU erratum 936184. This erratum applies to DSUs containing +# the ACP interface and revision < r2p0. Applying the workaround results in +# higher DSU power consumption on idle. +CPU_FLAG_LIST += ERRATA_DSU_936184 + +# Flag to apply DSU erratum 2313941. This erratum applies to DSUs revisions +# r0p0, r1p0, r2p0, r2p1, r3p0, r3p1 and is still open. Applying the workaround +# results in higher DSU power consumption on idle. +CPU_FLAG_LIST += ERRATA_DSU_2313941 + +ifneq (${DYNAMIC_WORKAROUND_CVE_2018_3639},0) + ifeq (${WORKAROUND_CVE_2018_3639},0) + $(error "Error: WORKAROUND_CVE_2018_3639 must be 1 if DYNAMIC_WORKAROUND_CVE_2018_3639 is 1") + endif +endif + +# process all flags +$(eval $(call default_zeros, $(CPU_FLAG_LIST))) +$(eval $(call add_defines, $(CPU_FLAG_LIST))) +$(eval $(call assert_booleans, $(CPU_FLAG_LIST))) + +# Errata build flags +ifneq (${ERRATA_A53_843419},0) +TF_LDFLAGS_aarch64 += --fix-cortex-a53-843419 +endif + +ifneq (${ERRATA_A53_835769},0) +TF_CFLAGS_aarch64 += -mfix-cortex-a53-835769 +TF_LDFLAGS_aarch64 += --fix-cortex-a53-835769 +endif + +ifneq ($(filter 1,${ERRATA_A53_1530924} ${ERRATA_A55_1530923} \ + ${ERRATA_A57_1319537} ${ERRATA_A72_1319367} ${ERRATA_A76_1165522}),) +ERRATA_SPECULATIVE_AT := 1 +else +ERRATA_SPECULATIVE_AT := 0 +endif diff --git a/lib/cpus/errata_report.c b/lib/cpus/errata_report.c new file mode 100644 index 0000000..4e9bdfc --- /dev/null +++ b/lib/cpus/errata_report.c @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* Runtime firmware routines to report errata status for the current CPU. */ + +#include <assert.h> +#include <stdbool.h> + +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/cpus/cpu_ops.h> +#include <lib/cpus/errata.h> +#include <lib/el3_runtime/cpu_data.h> +#include <lib/spinlock.h> + +#ifdef IMAGE_BL1 +# define BL_STRING "BL1" +#elif defined(__aarch64__) && defined(IMAGE_BL31) +# define BL_STRING "BL31" +#elif !defined(__aarch64__) && defined(IMAGE_BL32) +# define BL_STRING "BL32" +#elif defined(IMAGE_BL2) && RESET_TO_BL2 +# define BL_STRING "BL2" +#else +# error This image should not be printing errata status +#endif + +/* Errata format: BL stage, CPU, errata ID, message */ +#define ERRATA_FORMAT "%s: %s: CPU workaround for %s was %s\n" + +#define CVE_FORMAT "%s: %s: CPU workaround for CVE %u_%u was %s\n" +#define ERRATUM_FORMAT "%s: %s: CPU workaround for erratum %u was %s\n" + + +static __unused void print_status(int status, char *cpu_str, uint16_t cve, uint32_t id) +{ + if (status == ERRATA_MISSING) { + if (cve) { + WARN(CVE_FORMAT, BL_STRING, cpu_str, cve, id, "missing!"); + } else { + WARN(ERRATUM_FORMAT, BL_STRING, cpu_str, id, "missing!"); + } + } else if (status == ERRATA_APPLIES) { + if (cve) { + INFO(CVE_FORMAT, BL_STRING, cpu_str, cve, id, "applied"); + } else { + INFO(ERRATUM_FORMAT, BL_STRING, cpu_str, id, "applied"); + } + } else { + if (cve) { + VERBOSE(CVE_FORMAT, BL_STRING, cpu_str, cve, id, "not applied"); + } else { + VERBOSE(ERRATUM_FORMAT, BL_STRING, cpu_str, id, "not applied"); + } + } +} + +#if !REPORT_ERRATA +void print_errata_status(void) {} +#else /* !REPORT_ERRATA */ +/* + * New errata status message printer + * The order checking function is hidden behind the FEATURE_DETECTION flag to + * save space. This functionality is only useful on development and platform + * bringup builds, when FEATURE_DETECTION should be used anyway + */ +void __unused generic_errata_report(void) +{ + struct cpu_ops *cpu_ops = get_cpu_ops_ptr(); + struct erratum_entry *entry = cpu_ops->errata_list_start; + struct erratum_entry *end = cpu_ops->errata_list_end; + long rev_var = cpu_get_rev_var(); +#if FEATURE_DETECTION + uint32_t last_erratum_id = 0; + uint16_t last_cve_yr = 0; + bool check_cve = false; + bool failed = false; +#endif /* FEATURE_DETECTION */ + + for (; entry != end; entry += 1) { + uint64_t status = entry->check_func(rev_var); + + assert(entry->id != 0); + + /* + * Errata workaround has not been compiled in. If the errata + * would have applied had it been compiled in, print its status + * as missing. + */ + if (status == ERRATA_APPLIES && entry->chosen == 0) { + status = ERRATA_MISSING; + } + + print_status(status, cpu_ops->cpu_str, entry->cve, entry->id); + +#if FEATURE_DETECTION + if (entry->cve) { + if (last_cve_yr > entry->cve || + (last_cve_yr == entry->cve && last_erratum_id >= entry->id)) { + ERROR("CVE %u_%u was out of order!\n", + entry->cve, entry->id); + failed = true; + } + check_cve = true; + last_cve_yr = entry->cve; + } else { + if (last_erratum_id >= entry->id || check_cve) { + ERROR("Erratum %u was out of order!\n", + entry->id); + failed = true; + } + } + last_erratum_id = entry->id; +#endif /* FEATURE_DETECTION */ + } + +#if FEATURE_DETECTION + /* + * enforce errata and CVEs are in ascending order and that CVEs are + * after errata + */ + assert(!failed); +#endif /* FEATURE_DETECTION */ +} + +/* + * Returns whether errata needs to be reported. Passed arguments are private to + * a CPU type. + */ +static __unused int errata_needs_reporting(spinlock_t *lock, uint32_t *reported) +{ + bool report_now; + + /* If already reported, return false. */ + if (*reported != 0U) + return 0; + + /* + * Acquire lock. Determine whether status needs reporting, and then mark + * report status to true. + */ + spin_lock(lock); + report_now = (*reported == 0U); + if (report_now) + *reported = 1; + spin_unlock(lock); + + return report_now; +} + +/* + * Function to print errata status for the calling CPU (and others of the same + * type). Must be called only: + * - when MMU and data caches are enabled; + * - after cpu_ops have been initialized in per-CPU data. + */ +void print_errata_status(void) +{ + struct cpu_ops *cpu_ops; +#ifdef IMAGE_BL1 + /* + * BL1 doesn't have per-CPU data. So retrieve the CPU operations + * directly. + */ + cpu_ops = get_cpu_ops_ptr(); + + if (cpu_ops->errata_func != NULL) { + cpu_ops->errata_func(); + } +#else /* IMAGE_BL1 */ + cpu_ops = (void *) get_cpu_data(cpu_ops_ptr); + + assert(cpu_ops != NULL); + + if (cpu_ops->errata_func == NULL) { + return; + } + + if (errata_needs_reporting(cpu_ops->errata_lock, cpu_ops->errata_reported)) { + cpu_ops->errata_func(); + } +#endif /* IMAGE_BL1 */ +} + +/* + * Old errata status message printer + * TODO: remove once all cpus have been converted to the new printing method + */ +void __unused errata_print_msg(unsigned int status, const char *cpu, const char *id) +{ + /* Errata status strings */ + static const char *const errata_status_str[] = { + [ERRATA_NOT_APPLIES] = "not applied", + [ERRATA_APPLIES] = "applied", + [ERRATA_MISSING] = "missing!" + }; + static const char *const __unused bl_str = BL_STRING; + const char *msg __unused; + + + assert(status < ARRAY_SIZE(errata_status_str)); + assert(cpu != NULL); + assert(id != NULL); + + msg = errata_status_str[status]; + + switch (status) { + case ERRATA_NOT_APPLIES: + VERBOSE(ERRATA_FORMAT, bl_str, cpu, id, msg); + break; + + case ERRATA_APPLIES: + INFO(ERRATA_FORMAT, bl_str, cpu, id, msg); + break; + + case ERRATA_MISSING: + WARN(ERRATA_FORMAT, bl_str, cpu, id, msg); + break; + + default: + WARN(ERRATA_FORMAT, bl_str, cpu, id, "unknown"); + break; + } +} +#endif /* !REPORT_ERRATA */ diff --git a/lib/debugfs/blobs.h b/lib/debugfs/blobs.h new file mode 100644 index 0000000..54ca9f7 --- /dev/null +++ b/lib/debugfs/blobs.h @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "dev.h" + +static const dirtab_t blobtab[] = { + {"ctl", DEV_ROOT_QBLOBCTL, 0, O_READ}, + {"fip.bin", DEV_ROOT_QBLOBCTL + 1, 0x100000, O_READ, (void *)0x8000000} +}; diff --git a/lib/debugfs/debugfs.mk b/lib/debugfs/debugfs.mk new file mode 100644 index 0000000..caa9fcf --- /dev/null +++ b/lib/debugfs/debugfs.mk @@ -0,0 +1,13 @@ +# +# Copyright (c) 2019, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +DEBUGFS_SRCS := $(addprefix lib/debugfs/, \ + dev.c \ + devc.c \ + devroot.c \ + devfip.c) + +DEBUGFS_SRCS += lib/debugfs/debugfs_smc.c diff --git a/lib/debugfs/debugfs_smc.c b/lib/debugfs/debugfs_smc.c new file mode 100644 index 0000000..64c64f8 --- /dev/null +++ b/lib/debugfs/debugfs_smc.c @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdint.h> +#include <stdbool.h> +#include <string.h> + +#include <lib/debugfs.h> +#include <lib/smccc.h> +#include <lib/spinlock.h> +#include <lib/xlat_tables/xlat_tables_v2.h> +#include <smccc_helpers.h> + +#define MAX_PATH_LEN 256 + +#define MOUNT 0 +#define CREATE 1 +#define OPEN 2 +#define CLOSE 3 +#define READ 4 +#define WRITE 5 +#define SEEK 6 +#define BIND 7 +#define STAT 8 +#define INIT 10 +#define VERSION 11 + +/* This is the virtual address to which we map the NS shared buffer */ +#define DEBUGFS_SHARED_BUF_VIRT ((void *)0x81000000U) + +static union debugfs_parms { + struct { + char fname[MAX_PATH_LEN]; + } open; + + struct { + char srv[MAX_PATH_LEN]; + char where[MAX_PATH_LEN]; + char spec[MAX_PATH_LEN]; + } mount; + + struct { + char path[MAX_PATH_LEN]; + dir_t dir; + } stat; + + struct { + char oldpath[MAX_PATH_LEN]; + char newpath[MAX_PATH_LEN]; + } bind; +} parms; + +/* debugfs_access_lock protects shared buffer and internal */ +/* FS functions from concurrent accesses. */ +static spinlock_t debugfs_access_lock; + +static bool debugfs_initialized; + +uintptr_t debugfs_smc_handler(unsigned int smc_fid, + u_register_t cmd, + u_register_t arg2, + u_register_t arg3, + u_register_t arg4, + void *cookie, + void *handle, + u_register_t flags) +{ + int64_t smc_ret = DEBUGFS_E_INVALID_PARAMS, smc_resp = 0; + int ret; + + /* Allow calls from non-secure only */ + if (is_caller_secure(flags)) { + SMC_RET1(handle, DEBUGFS_E_DENIED); + } + + /* Expect a SiP service fast call */ + if ((GET_SMC_TYPE(smc_fid) != SMC_TYPE_FAST) || + (GET_SMC_OEN(smc_fid) != OEN_SIP_START)) { + SMC_RET1(handle, SMC_UNK); + } + + /* Truncate parameters if 32b SMC convention call */ + if (GET_SMC_CC(smc_fid) == SMC_32) { + arg2 &= 0xffffffff; + arg3 &= 0xffffffff; + arg4 &= 0xffffffff; + } + + spin_lock(&debugfs_access_lock); + + if (debugfs_initialized == true) { + /* Copy NS shared buffer to internal secure location */ + memcpy(&parms, (void *)DEBUGFS_SHARED_BUF_VIRT, + sizeof(union debugfs_parms)); + } + + switch (cmd) { + case INIT: + if (debugfs_initialized == false) { + /* TODO: check PA validity e.g. whether */ + /* it is an NS region. */ + ret = mmap_add_dynamic_region(arg2, + (uintptr_t)DEBUGFS_SHARED_BUF_VIRT, + PAGE_SIZE_4KB, + MT_MEMORY | MT_RW | MT_NS); + if (ret == 0) { + debugfs_initialized = true; + smc_ret = SMC_OK; + smc_resp = 0; + } + } + break; + + case VERSION: + smc_ret = SMC_OK; + smc_resp = DEBUGFS_VERSION; + break; + + case MOUNT: + ret = mount(parms.mount.srv, + parms.mount.where, + parms.mount.spec); + if (ret == 0) { + smc_ret = SMC_OK; + smc_resp = 0; + } + break; + + case OPEN: + ret = open(parms.open.fname, arg2); + if (ret >= 0) { + smc_ret = SMC_OK; + smc_resp = ret; + } + break; + + case CLOSE: + ret = close(arg2); + if (ret == 0) { + smc_ret = SMC_OK; + smc_resp = 0; + } + break; + + case READ: + ret = read(arg2, DEBUGFS_SHARED_BUF_VIRT, arg3); + if (ret >= 0) { + smc_ret = SMC_OK; + smc_resp = ret; + } + break; + + case SEEK: + ret = seek(arg2, arg3, arg4); + if (ret == 0) { + smc_ret = SMC_OK; + smc_resp = 0; + } + break; + + case BIND: + ret = bind(parms.bind.oldpath, parms.bind.newpath); + if (ret == 0) { + smc_ret = SMC_OK; + smc_resp = 0; + } + break; + + case STAT: + ret = stat(parms.stat.path, &parms.stat.dir); + if (ret == 0) { + memcpy((void *)DEBUGFS_SHARED_BUF_VIRT, &parms, + sizeof(union debugfs_parms)); + smc_ret = SMC_OK; + smc_resp = 0; + } + break; + + /* Not implemented */ + case CREATE: + /* Intentional fall-through */ + + /* Not implemented */ + case WRITE: + /* Intentional fall-through */ + + default: + smc_ret = SMC_UNK; + smc_resp = 0; + } + + spin_unlock(&debugfs_access_lock); + + SMC_RET2(handle, smc_ret, smc_resp); + + /* Not reached */ + return smc_ret; +} + +int debugfs_smc_setup(void) +{ + debugfs_initialized = false; + debugfs_access_lock.lock = 0; + + return 0; +} diff --git a/lib/debugfs/dev.c b/lib/debugfs/dev.c new file mode 100644 index 0000000..2fc1d40 --- /dev/null +++ b/lib/debugfs/dev.c @@ -0,0 +1,853 @@ +/* + * Copyright (c) 2019-2021, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <cdefs.h> +#include <common/debug.h> +#include <lib/debugfs.h> +#include <string.h> + +#include "dev.h" + +#define NR_MOUNT_POINTS 4 + +struct mount_point { + chan_t *new; + chan_t *old; +}; + +/* This array contains all the available channels of the filesystem. + * A file descriptor is the index of a specific channel in this array. + */ +static chan_t fdset[NR_CHANS]; + +/* This array contains all the available mount points of the filesystem. */ +static struct mount_point mount_points[NR_MOUNT_POINTS]; + +/* This variable stores the channel associated to the root directory. */ +static chan_t slash_channel; + +/* This function creates a channel from a device index and registers + * it to fdset. + */ +static chan_t *create_new_channel(unsigned char index) +{ + chan_t *channel = NULL; + int i; + + for (i = 0; i < NR_CHANS; i++) { + if (fdset[i].index == NODEV) { + channel = &fdset[i]; + channel->index = index; + break; + } + } + + return channel; +} + +/******************************************************************************* + * This function returns a pointer to an existing channel in fdset from a file + * descriptor. + ******************************************************************************/ +static chan_t *fd_to_channel(int fd) +{ + if ((fd < 0) || (fd >= NR_CHANS) || (fdset[fd].index == NODEV)) { + return NULL; + } + + return &fdset[fd]; +} + +/******************************************************************************* + * This function returns a file descriptor from a channel. + * The caller must be sure that the channel is registered in fdset. + ******************************************************************************/ +static int channel_to_fd(chan_t *channel) +{ + return (channel == NULL) ? -1 : (channel - fdset); +} + +/******************************************************************************* + * This function checks the validity of a mode. + ******************************************************************************/ +static bool is_valid_mode(int mode) +{ + if ((mode & O_READ) && (mode & (O_WRITE | O_RDWR))) { + return false; + } + if ((mode & O_WRITE) && (mode & (O_READ | O_RDWR))) { + return false; + } + if ((mode & O_RDWR) && (mode & (O_READ | O_WRITE))) { + return false; + } + + return true; +} + +/******************************************************************************* + * This function extracts the next part of the given path contained and puts it + * in token. It returns a pointer to the remainder of the path. + ******************************************************************************/ +static const char *next(const char *path, char *token) +{ + int index; + const char *cursor; + + while (*path == '/') { + ++path; + } + + index = 0; + cursor = path; + if (*path != '\0') { + while (*cursor != '/' && *cursor != '\0') { + if (index == NAMELEN) { + return NULL; + } + token[index++] = *cursor++; + } + } + token[index] = '\0'; + + return cursor; +} + +/******************************************************************************* + * This function returns the driver index in devtab of the driver + * identified by id. + ******************************************************************************/ +static int get_device_index(int id) +{ + int index; + dev_t * const *dp; + + for (index = 0, dp = devtab; *dp && (*dp)->id != id; ++dp) { + index++; + } + + if (*dp == NULL) { + return -1; + } + + return index; +} + +/******************************************************************************* + * This function clears a given channel fields + ******************************************************************************/ +static void channel_clear(chan_t *channel) +{ + channel->offset = 0; + channel->qid = 0; + channel->index = NODEV; + channel->dev = 0; + channel->mode = 0; +} + +/******************************************************************************* + * This function closes the channel pointed to by c. + ******************************************************************************/ +void channel_close(chan_t *channel) +{ + if (channel != NULL) { + channel_clear(channel); + } +} + +/******************************************************************************* + * This function copies data from src to dst after applying the offset of the + * channel c. nbytes bytes are expected to be copied unless the data goes over + * dst + len. + * It returns the actual number of bytes that were copied. + ******************************************************************************/ +int buf_to_channel(chan_t *channel, void *dst, void *src, int nbytes, long len) +{ + const char *addr = src; + + if ((channel == NULL) || (dst == NULL) || (src == NULL)) { + return 0; + } + + if (channel->offset >= len) { + return 0; + } + + if ((channel->offset + nbytes) > len) { + nbytes = len - channel->offset; + } + + memcpy(dst, addr + channel->offset, nbytes); + + channel->offset += nbytes; + + return nbytes; +} + +/******************************************************************************* + * This function checks whether a channel (identified by its device index and + * qid) is registered as a mount point. + * Returns a pointer to the channel it is mounted to when found, NULL otherwise. + ******************************************************************************/ +static chan_t *mount_point_to_channel(int index, qid_t qid) +{ + chan_t *channel; + struct mount_point *mp; + + for (mp = mount_points; mp < &mount_points[NR_MOUNT_POINTS]; mp++) { + channel = mp->new; + if (channel == NULL) { + continue; + } + + if ((channel->index == index) && (channel->qid == qid)) { + return mp->old; + } + } + + return NULL; +} + +/******************************************************************************* + * This function calls the attach function of the driver identified by id. + ******************************************************************************/ +chan_t *attach(int id, int dev) +{ + /* Get the devtab index for the driver identified by id */ + int index = get_device_index(id); + + if (index < 0) { + return NULL; + } + + return devtab[index]->attach(id, dev); +} + +/******************************************************************************* + * This function is the default implementation of the driver attach function. + * It creates a new channel and returns a pointer to it. + ******************************************************************************/ +chan_t *devattach(int id, int dev) +{ + chan_t *channel; + int index; + + index = get_device_index(id); + if (index < 0) { + return NULL; + } + + channel = create_new_channel(index); + if (channel == NULL) { + return NULL; + } + + channel->dev = dev; + channel->qid = CHDIR; + + return channel; +} + +/******************************************************************************* + * This function returns a channel given a path. + * It goes through the filesystem, from the root namespace ('/') or from a + * device namespace ('#'), switching channel on mount points. + ******************************************************************************/ +chan_t *path_to_channel(const char *path, int mode) +{ + int i, n; + const char *path_next; + chan_t *mnt, *channel; + char elem[NAMELEN]; + + if (path == NULL) { + return NULL; + } + + switch (path[0]) { + case '/': + channel = clone(&slash_channel, NULL); + path_next = path; + break; + case '#': + path_next = next(path + 1, elem); + if (path_next == NULL) { + goto noent; + } + + n = 0; + for (i = 1; (elem[i] >= '0') && (elem[i] <= '9'); i++) { + n += elem[i] - '0'; + } + + if (elem[i] != '\0') { + goto noent; + } + + channel = attach(elem[0], n); + break; + default: + return NULL; + } + + if (channel == NULL) { + return NULL; + } + + for (path_next = next(path_next, elem); *elem; + path_next = next(path_next, elem)) { + if ((channel->qid & CHDIR) == 0) { + goto notfound; + } + + if (devtab[channel->index]->walk(channel, elem) < 0) { + channel_close(channel); + goto notfound; + } + + mnt = mount_point_to_channel(channel->index, channel->qid); + if (mnt != NULL) { + clone(mnt, channel); + } + } + + if (path_next == NULL) { + goto notfound; + } + + /* TODO: check mode */ + return channel; + +notfound: + channel_close(channel); +noent: + return NULL; +} + +/******************************************************************************* + * This function calls the clone function of the driver associated to the + * channel c. + ******************************************************************************/ +chan_t *clone(chan_t *c, chan_t *nc) +{ + if (c->index == NODEV) { + return NULL; + } + + return devtab[c->index]->clone(c, nc); +} + +/******************************************************************************* + * This function is the default implementation of the driver clone function. + * It creates a new channel and returns a pointer to it. + * It clones channel into new_channel. + ******************************************************************************/ +chan_t *devclone(chan_t *channel, chan_t *new_channel) +{ + if (channel == NULL) { + return NULL; + } + + if (new_channel == NULL) { + new_channel = create_new_channel(channel->index); + if (new_channel == NULL) { + return NULL; + } + } + + new_channel->qid = channel->qid; + new_channel->dev = channel->dev; + new_channel->mode = channel->mode; + new_channel->offset = channel->offset; + new_channel->index = channel->index; + + return new_channel; +} + +/******************************************************************************* + * This function is the default implementation of the driver walk function. + * It goes through all the elements of tab using the gen function until a match + * is found with name. + * If a match is found, it copies the qid of the new directory. + ******************************************************************************/ +int devwalk(chan_t *channel, const char *name, const dirtab_t *tab, + int ntab, devgen_t *gen) +{ + int i; + dir_t dir; + + if ((channel == NULL) || (name == NULL) || (gen == NULL)) { + return -1; + } + + if ((name[0] == '.') && (name[1] == '\0')) { + return 1; + } + + for (i = 0; ; i++) { + switch ((*gen)(channel, tab, ntab, i, &dir)) { + case 0: + /* Intentional fall-through */ + case -1: + return -1; + case 1: + if (strncmp(name, dir.name, NAMELEN) != 0) { + continue; + } + channel->qid = dir.qid; + return 1; + } + } +} + +/******************************************************************************* + * This is a helper function which exposes the content of a directory, element + * by element. It is meant to be called until the end of the directory is + * reached or an error occurs. + * It returns -1 on error, 0 on end of directory and 1 when a new file is found. + ******************************************************************************/ +int dirread(chan_t *channel, dir_t *dir, const dirtab_t *tab, + int ntab, devgen_t *gen) +{ + int i, ret; + + if ((channel == NULL) || (dir == NULL) || (gen == NULL)) { + return -1; + } + + i = channel->offset/sizeof(dir_t); + ret = (*gen)(channel, tab, ntab, i, dir); + if (ret == 1) { + channel->offset += sizeof(dir_t); + } + + return ret; +} + +/******************************************************************************* + * This function sets the elements of dir. + ******************************************************************************/ +void make_dir_entry(chan_t *channel, dir_t *dir, + const char *name, long length, qid_t qid, unsigned int mode) +{ + if ((channel == NULL) || (dir == NULL) || (name == NULL)) { + return; + } + + strlcpy(dir->name, name, sizeof(dir->name)); + dir->length = length; + dir->qid = qid; + dir->mode = mode; + + if ((qid & CHDIR) != 0) { + dir->mode |= O_DIR; + } + + dir->index = channel->index; + dir->dev = channel->dev; +} + +/******************************************************************************* + * This function is the default implementation of the internal driver gen + * function. + * It copies and formats the information of the nth element of tab into dir. + ******************************************************************************/ +int devgen(chan_t *channel, const dirtab_t *tab, int ntab, int n, dir_t *dir) +{ + const dirtab_t *dp; + + if ((channel == NULL) || (dir == NULL) || (tab == NULL) || + (n >= ntab)) { + return 0; + } + + dp = &tab[n]; + make_dir_entry(channel, dir, dp->name, dp->length, dp->qid, dp->perm); + return 1; +} + +/******************************************************************************* + * This function returns a file descriptor identifying the channel associated to + * the given path. + ******************************************************************************/ +int open(const char *path, int mode) +{ + chan_t *channel; + + if (path == NULL) { + return -1; + } + + if (is_valid_mode(mode) == false) { + return -1; + } + + channel = path_to_channel(path, mode); + + return channel_to_fd(channel); +} + +/******************************************************************************* + * This function closes the channel identified by the file descriptor fd. + ******************************************************************************/ +int close(int fd) +{ + chan_t *channel; + + channel = fd_to_channel(fd); + if (channel == NULL) { + return -1; + } + + channel_close(channel); + return 0; +} + +/******************************************************************************* + * This function is the default implementation of the driver stat function. + * It goes through all the elements of tab using the gen function until a match + * is found with file. + * If a match is found, dir contains the information file. + ******************************************************************************/ +int devstat(chan_t *dirc, const char *file, dir_t *dir, + const dirtab_t *tab, int ntab, devgen_t *gen) +{ + int i, r = 0; + chan_t *c, *mnt; + + if ((dirc == NULL) || (dir == NULL) || (gen == NULL)) { + return -1; + } + + c = path_to_channel(file, O_STAT); + if (c == NULL) { + return -1; + } + + for (i = 0; ; i++) { + switch ((*gen)(dirc, tab, ntab, i, dir)) { + case 0: + /* Intentional fall-through */ + case -1: + r = -1; + goto leave; + case 1: + mnt = mount_point_to_channel(dir->index, dir->qid); + if (mnt != NULL) { + dir->qid = mnt->qid; + dir->index = mnt->index; + } + + if ((dir->qid != c->qid) || (dir->index != c->index)) { + continue; + } + + goto leave; + } + } + +leave: + channel_close(c); + return r; +} + +/******************************************************************************* + * This function calls the stat function of the driver associated to the parent + * directory of the file in path. + * The result is stored in dir. + ******************************************************************************/ +int stat(const char *path, dir_t *dir) +{ + int r; + size_t len; + chan_t *channel; + char *p, dirname[PATHLEN]; + + if ((path == NULL) || (dir == NULL)) { + return -1; + } + + len = strlen(path); + if ((len + 1) > sizeof(dirname)) { + return -1; + } + + memcpy(dirname, path, len); + for (p = dirname + len; p > dirname; --p) { + if (*p != '/') { + break; + } + } + + p = memrchr(dirname, '/', p - dirname); + if (p == NULL) { + return -1; + } + + dirname[p - dirname + 1] = '\0'; + + channel = path_to_channel(dirname, O_STAT); + if (channel == NULL) { + return -1; + } + + r = devtab[channel->index]->stat(channel, path, dir); + channel_close(channel); + + return r; +} + +/******************************************************************************* + * This function calls the read function of the driver associated to fd. + * It fills buf with at most n bytes. + * It returns the number of bytes that were actually read. + ******************************************************************************/ +int read(int fd, void *buf, int n) +{ + chan_t *channel; + + if (buf == NULL) { + return -1; + } + + channel = fd_to_channel(fd); + if (channel == NULL) { + return -1; + } + + if (((channel->qid & CHDIR) != 0) && (n < sizeof(dir_t))) { + return -1; + } + + return devtab[channel->index]->read(channel, buf, n); +} + +/******************************************************************************* + * This function calls the write function of the driver associated to fd. + * It writes at most n bytes of buf. + * It returns the number of bytes that were actually written. + ******************************************************************************/ +int write(int fd, void *buf, int n) +{ + chan_t *channel; + + if (buf == NULL) { + return -1; + } + + channel = fd_to_channel(fd); + if (channel == NULL) { + return -1; + } + + if ((channel->qid & CHDIR) != 0) { + return -1; + } + + return devtab[channel->index]->write(channel, buf, n); +} + +/******************************************************************************* + * This function calls the seek function of the driver associated to fd. + * It applies the offset off according to the strategy whence. + ******************************************************************************/ +int seek(int fd, long off, int whence) +{ + chan_t *channel; + + channel = fd_to_channel(fd); + if (channel == NULL) { + return -1; + } + + if ((channel->qid & CHDIR) != 0) { + return -1; + } + + return devtab[channel->index]->seek(channel, off, whence); +} + +/******************************************************************************* + * This function is the default error implementation of the driver mount + * function. + ******************************************************************************/ +chan_t *deverrmount(chan_t *channel, const char *spec) +{ + return NULL; +} + +/******************************************************************************* + * This function is the default error implementation of the driver write + * function. + ******************************************************************************/ +int deverrwrite(chan_t *channel, void *buf, int n) +{ + return -1; +} + +/******************************************************************************* + * This function is the default error implementation of the driver seek + * function. + ******************************************************************************/ +int deverrseek(chan_t *channel, long off, int whence) +{ + return -1; +} + +/******************************************************************************* + * This function is the default implementation of the driver seek function. + * It applies the offset off according to the strategy whence to the channel c. + ******************************************************************************/ +int devseek(chan_t *channel, long off, int whence) +{ + switch (whence) { + case KSEEK_SET: + channel->offset = off; + break; + case KSEEK_CUR: + channel->offset += off; + break; + case KSEEK_END: + /* Not implemented */ + return -1; + } + + return 0; +} + +/******************************************************************************* + * This function registers the channel associated to the path new as a mount + * point for the channel c. + ******************************************************************************/ +static int add_mount_point(chan_t *channel, const char *new) +{ + int i; + chan_t *cn; + struct mount_point *mp; + + if (new == NULL) { + goto err0; + } + + cn = path_to_channel(new, O_READ); + if (cn == NULL) { + goto err0; + } + + if ((cn->qid & CHDIR) == 0) { + goto err1; + } + + for (i = NR_MOUNT_POINTS - 1; i >= 0; i--) { + mp = &mount_points[i]; + if (mp->new == NULL) { + break; + } + } + + if (i < 0) { + goto err1; + } + + mp->new = cn; + mp->old = channel; + + return 0; + +err1: + channel_close(cn); +err0: + return -1; +} + +/******************************************************************************* + * This function registers the path new as a mount point for the path old. + ******************************************************************************/ +int bind(const char *old, const char *new) +{ + chan_t *channel; + + channel = path_to_channel(old, O_BIND); + if (channel == NULL) { + return -1; + } + + if (add_mount_point(channel, new) < 0) { + channel_close(channel); + return -1; + } + + return 0; +} + +/******************************************************************************* + * This function calls the mount function of the driver associated to the path + * srv. + * It mounts the path srv on the path where. + ******************************************************************************/ +int mount(const char *srv, const char *where, const char *spec) +{ + chan_t *channel, *mount_point_chan; + int ret; + + channel = path_to_channel(srv, O_RDWR); + if (channel == NULL) { + goto err0; + } + + mount_point_chan = devtab[channel->index]->mount(channel, spec); + if (mount_point_chan == NULL) { + goto err1; + } + + ret = add_mount_point(mount_point_chan, where); + if (ret < 0) { + goto err2; + } + + channel_close(channel); + + return 0; + +err2: + channel_close(mount_point_chan); +err1: + channel_close(channel); +err0: + return -1; +} + +/******************************************************************************* + * This function initializes the device environment. + * It creates the '/' channel. + * It links the device drivers to the physical drivers. + ******************************************************************************/ +void debugfs_init(void) +{ + chan_t *channel, *cloned_channel; + + for (channel = fdset; channel < &fdset[NR_CHANS]; channel++) { + channel_clear(channel); + } + + channel = devattach('/', 0); + if (channel == NULL) { + panic(); + } + + cloned_channel = clone(channel, &slash_channel); + if (cloned_channel == NULL) { + panic(); + } + + channel_close(channel); + devlink(); +} + +__dead2 void devpanic(const char *cause) +{ + panic(); +} diff --git a/lib/debugfs/dev.h b/lib/debugfs/dev.h new file mode 100644 index 0000000..c142651 --- /dev/null +++ b/lib/debugfs/dev.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef DEV_H +#define DEV_H + +#include <cdefs.h> +#include <lib/debugfs.h> +#include <stddef.h> + +/* FIXME: need configurability */ +#define NR_CHANS 10 +#define NR_CONSS 1 +#define NR_BINDS 4 +#define NR_FILES 18 + +#define NODEV 255 +#define CHDIR (1 << 15) + +#define SYNCDEV 0 +#define SYNCALL 1 + +typedef struct dev dev_t; +typedef struct chan chan_t; +typedef struct dirtab dirtab_t; +typedef int devgen_t(chan_t *, const dirtab_t *, int, int, dir_t *); +typedef struct attr attr_t; + +enum { + DEV_ROOT_QROOT, + DEV_ROOT_QDEV, + DEV_ROOT_QFIP, + DEV_ROOT_QBLOBS, + DEV_ROOT_QBLOBCTL, + DEV_ROOT_QPSCI +}; + +/******************************************************************************* + * This structure contains the necessary information to represent a directory + * of the filesystem. + ******************************************************************************/ +struct dirtab { + char name[NAMELEN]; + qid_t qid; + long length; + unsigned char perm; + void *data; +}; + +/******************************************************************************* + * This structure defines the interface of device drivers. + * Each driver must implement a subset of those functions. + * It is possible to redirect to default implementations defined in dev.c. + ******************************************************************************/ +/* FIXME: comments for the callbacks */ +struct dev { + char id; + int (*stat)(chan_t *c, const char *file, dir_t *dir); + int (*walk)(chan_t *c, const char *name); + int (*read)(chan_t *c, void *buf, int n); + int (*write)(chan_t *c, void *buf, int n); + int (*seek)(chan_t *c, long off, int whence); + chan_t *(*clone)(chan_t *c, chan_t *nc); + chan_t *(*attach)(int id, int dev); + chan_t *(*mount)(chan_t *c, const char *spec); +}; + +/******************************************************************************* + * This structure defines the channel structure. + * A channel is a handle on an element of the filesystem. + ******************************************************************************/ +struct chan { + long offset; + qid_t qid; + unsigned char index; /* device index in devtab */ + unsigned char dev; + unsigned char mode; +}; + +/******************************************************************************* + * This structure defines an abstract argument passed to physical drivers from + * the configuration file. + ******************************************************************************/ +struct attr { + char *key; + char *value; +}; + +chan_t *path_to_channel(const char *path, int mode); +chan_t *clone(chan_t *c, chan_t *nc); +chan_t *attach(int id, int dev); +void channel_close(chan_t *c); +int buf_to_channel(chan_t *c, void *dst, void *src, int nbytes, long len); +int dirread(chan_t *c, dir_t *dir, const dirtab_t *tab, + int ntab, devgen_t *gen); +void make_dir_entry(chan_t *c, dir_t *dir, const char *name, long length, + qid_t qid, unsigned int mode); +void devlink(void); + +chan_t *devattach(int id, int dev); +int devseek(chan_t *c, long off, int whence); +chan_t *devclone(chan_t *c, chan_t *nc); +int devgen(chan_t *c, const dirtab_t *tab, int ntab, int n, dir_t *dir); +int devwalk(chan_t *c, const char *name, const dirtab_t *tab, int ntab, + devgen_t *gen); +int devstat(chan_t *dirc, const char *file, dir_t *dir, + const dirtab_t *tab, int ntab, devgen_t *gen); + +chan_t *deverrmount(chan_t *c, const char *spec); +int deverrwrite(chan_t *c, void *buf, int n); +int deverrseek(chan_t *c, long off, int whence); + +extern dev_t *const devtab[]; + +void __dead2 devpanic(const char *cause); + +#endif /* DEV_H */ diff --git a/lib/debugfs/devc.c b/lib/debugfs/devc.c new file mode 100644 index 0000000..1099a85 --- /dev/null +++ b/lib/debugfs/devc.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +typedef struct dev dev_t; + +extern dev_t rootdevtab; +extern dev_t fipdevtab; + +dev_t *const devtab[] = { + &rootdevtab, + &fipdevtab, + 0 +}; + +void devlink(void) +{ +} diff --git a/lib/debugfs/devfip.c b/lib/debugfs/devfip.c new file mode 100644 index 0000000..85e6403 --- /dev/null +++ b/lib/debugfs/devfip.c @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2019-2021, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <lib/debugfs.h> +#include <limits.h> +#include <plat/arm/common/plat_arm.h> +#include <stdlib.h> +#include <string.h> +#include <tools_share/firmware_image_package.h> + +#include "dev.h" + +#define NR_FIPS 1 +#define STOC_HEADER (sizeof(fip_toc_header_t)) +#define STOC_ENTRY (sizeof(fip_toc_entry_t)) + +struct fipfile { + chan_t *c; + long offset[NR_FILES]; + long size[NR_FILES]; +}; + +struct fip_entry { + uuid_t uuid; + long long offset_address; + long long size; + long long flags; +}; + +struct uuidnames { + const char name[NAMELEN]; + const uuid_t uuid; +}; + +/******************************************************************************* + * This array links the FIP file names to their UUID. + * The elements are ordered according to the image number stored in + * tbbr_img_def.h, starting at index 1. + * + * TODO: this name to uuid binding will preferably be done using + * the coming Property Access Layer / Firmware CONFiguration feature. + ******************************************************************************/ +static const struct uuidnames uuidnames[] = { + {"", { {0}, {0}, {0}, 0, 0, {0} } }, + {"bl2.bin", UUID_TRUSTED_BOOT_FIRMWARE_BL2}, + {"scp-bl2.bin", UUID_SCP_FIRMWARE_SCP_BL2}, + {"bl31.bin", UUID_EL3_RUNTIME_FIRMWARE_BL31}, + {"bl32.bin", UUID_SECURE_PAYLOAD_BL32}, + {"bl33.bin", UUID_NON_TRUSTED_FIRMWARE_BL33}, + {"tb-fw.crt", UUID_TRUSTED_BOOT_FW_CERT}, + {"trstd-k.crt", UUID_TRUSTED_KEY_CERT}, + {"scp-fw-k.crt", UUID_SCP_FW_KEY_CERT}, + {"soc-fw-k.crt", UUID_SOC_FW_KEY_CERT}, + {"tos-fw-k.crt", UUID_TRUSTED_OS_FW_KEY_CERT}, + {"nt-fw-k.crt", UUID_NON_TRUSTED_FW_KEY_CERT}, + {"scp-fw-c.crt", UUID_SCP_FW_CONTENT_CERT}, + {"soc-fw-c.crt", UUID_SOC_FW_CONTENT_CERT}, + {"tos-fw-c.crt", UUID_TRUSTED_OS_FW_CONTENT_CERT}, + {"nt-fw-c.crt", UUID_NON_TRUSTED_FW_CONTENT_CERT}, + { }, + {"fwu.crt", UUID_TRUSTED_FWU_CERT}, + {"scp-bl2u.bin", UUID_TRUSTED_UPDATE_FIRMWARE_SCP_BL2U}, + {"bl2u.bin", UUID_TRUSTED_UPDATE_FIRMWARE_BL2U}, + {"ns-bl2u.bin", UUID_TRUSTED_UPDATE_FIRMWARE_NS_BL2U}, + {"bl32-xtr1.bin", UUID_SECURE_PAYLOAD_BL32_EXTRA1}, + {"bl32-xtr2.bin", UUID_SECURE_PAYLOAD_BL32_EXTRA2}, + {"hw.cfg", UUID_HW_CONFIG}, + {"tb-fw.cfg", UUID_TB_FW_CONFIG}, + {"soc-fw.cfg", UUID_SOC_FW_CONFIG}, + {"tos-fw.cfg", UUID_TOS_FW_CONFIG}, + {"nt-fw.cfg", UUID_NT_FW_CONFIG}, + {"fw.cfg", UUID_FW_CONFIG}, + {"rot-k.crt", UUID_ROT_KEY_CERT}, + {"nt-k.crt", UUID_NON_TRUSTED_WORLD_KEY_CERT}, + {"sip-sp.crt", UUID_SIP_SECURE_PARTITION_CONTENT_CERT}, + {"plat-sp.crt", UUID_PLAT_SECURE_PARTITION_CONTENT_CERT} +}; + +/******************************************************************************* + * This array contains all the available FIP files. + ******************************************************************************/ +static struct fipfile archives[NR_FIPS]; + +/******************************************************************************* + * This variable stores the current number of registered FIP files. + ******************************************************************************/ +static int nfips; + +/******************************************************************************* + * This function parses the ToC of the FIP. + ******************************************************************************/ +static int get_entry(chan_t *c, struct fip_entry *entry) +{ + int n; + + n = devtab[c->index]->read(c, entry, sizeof(struct fip_entry)); + if (n <= 0) { + return n; + } + + if (n != sizeof(struct fip_entry)) { + return -1; + } + + if (entry->size == 0) { + return 0; + } + + return 1; +} + +/******************************************************************************* + * This function exposes the FIP images as files. + ******************************************************************************/ +static int fipgen(chan_t *c, const dirtab_t *tab, int ntab, int n, dir_t *dir) +{ + int i, r; + long off; + chan_t nc; + struct fip_entry entry; + struct fipfile *fip; + static const char unk[] = "unknown"; + + if (c->dev >= nfips) { + panic(); + } + + if (clone(archives[c->dev].c, &nc) == NULL) { + panic(); + } + + fip = &archives[nc.dev]; + + off = STOC_HEADER; + for (i = 0; i <= n; i++) { + if (fip->offset[i] == -1) { + return 0; + } + + if (devtab[nc.index]->seek(&nc, off, KSEEK_SET) < 0) { + return -1; + } + + r = get_entry(&nc, &entry); + if (r <= 0) { + return r; + } + + off += sizeof(entry); + } + + for (i = 1; i < NELEM(uuidnames); i++) { + if (memcmp(&uuidnames[i].uuid, + &entry.uuid, sizeof(uuid_t)) == 0) { + break; + } + } + + if (i < NELEM(uuidnames)) { + make_dir_entry(c, dir, uuidnames[i].name, + entry.size, n, O_READ); + } else { + // TODO: set name depending on uuid node value + make_dir_entry(c, dir, unk, entry.size, n, O_READ); + } + + return 1; +} + +static int fipwalk(chan_t *c, const char *name) +{ + return devwalk(c, name, NULL, 0, fipgen); +} + +static int fipstat(chan_t *c, const char *file, dir_t *dir) +{ + return devstat(c, file, dir, NULL, 0, fipgen); +} + +/******************************************************************************* + * This function copies at most n bytes of the FIP image referred by c into + * buf. + ******************************************************************************/ +static int fipread(chan_t *c, void *buf, int n) +{ + long off; + chan_t cs; + struct fipfile *fip; + long size; + + /* Only makes sense when using debug language */ + assert(c->qid != CHDIR); + + if ((c->dev >= nfips) || ((c->qid & CHDIR) != 0)) { + panic(); + } + + fip = &archives[c->dev]; + + if ((c->qid >= NR_FILES) || (fip->offset[c->qid] < 0)) { + panic(); + } + + if (clone(fip->c, &cs) == NULL) { + panic(); + } + + size = fip->size[c->qid]; + if (c->offset >= size) { + return 0; + } + + if (n < 0) { + return -1; + } + + if (n > (size - c->offset)) { + n = size - c->offset; + } + + off = fip->offset[c->qid] + c->offset; + if (devtab[cs.index]->seek(&cs, off, KSEEK_SET) < 0) { + return -1; + } + + n = devtab[cs.index]->read(&cs, buf, n); + if (n > 0) { + c->offset += n; + } + + return n; +} + +/******************************************************************************* + * This function parses the FIP spec and registers its images in order to + * expose them as files in the driver namespace. + * It acts as an initialization function for the FIP driver. + * It returns a pointer to the newly created channel. + ******************************************************************************/ +static chan_t *fipmount(chan_t *c, const char *spec) +{ + int r, n, t; + chan_t *cspec; + uint32_t hname; + struct fip_entry entry; + struct fipfile *fip; + dir_t dir; + + if (nfips == NR_FIPS) { + return NULL; + } + + fip = &archives[nfips]; + + for (n = 0; n < NR_FILES; n++) { + fip->offset[n] = -1; + } + + cspec = path_to_channel(spec, O_READ); + if (cspec == NULL) { + return NULL; + } + + fip->c = cspec; + + r = devtab[cspec->index]->read(cspec, &hname, sizeof(hname)); + if (r < 0) { + goto err; + } + + if ((r != sizeof(hname)) || (hname != TOC_HEADER_NAME)) { + goto err; + } + + if (stat(spec, &dir) < 0) { + goto err; + } + + t = cspec->index; + if (devtab[t]->seek(cspec, STOC_HEADER, KSEEK_SET) < 0) { + goto err; + } + + for (n = 0; n < NR_FILES; n++) { + switch (get_entry(cspec, &entry)) { + case 0: + return attach('F', nfips++); + case -1: + goto err; + default: + if ((entry.offset_address + entry.size) > dir.length) { + goto err; + } + + fip->offset[n] = entry.offset_address; + fip->size[n] = entry.size; + break; + } + } + +err: + channel_close(cspec); + return NULL; +} + +const dev_t fipdevtab = { + .id = 'F', + .stat = fipstat, + .clone = devclone, + .attach = devattach, + .walk = fipwalk, + .read = fipread, + .write = deverrwrite, + .mount = fipmount, + .seek = devseek +}; + diff --git a/lib/debugfs/devroot.c b/lib/debugfs/devroot.c new file mode 100644 index 0000000..9dd6c92 --- /dev/null +++ b/lib/debugfs/devroot.c @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <common/debug.h> +#include <lib/debugfs.h> + +#include "blobs.h" +#include "dev.h" + +/******************************************************************************* + * This array contains the directories available from the root directory. + ******************************************************************************/ +static const dirtab_t dirtab[] = { + {"dev", CHDIR | DEV_ROOT_QDEV, 0, O_READ}, + {"blobs", CHDIR | DEV_ROOT_QBLOBS, 0, O_READ}, + {"fip", CHDIR | DEV_ROOT_QFIP, 0, O_READ} +}; + +static const dirtab_t devfstab[] = { +}; + +/******************************************************************************* + * This function exposes the elements of the root directory. + * It also exposes the content of the dev and blobs directories. + ******************************************************************************/ +static int rootgen(chan_t *channel, const dirtab_t *tab, int ntab, + int n, dir_t *dir) +{ + switch (channel->qid & ~CHDIR) { + case DEV_ROOT_QROOT: + tab = dirtab; + ntab = NELEM(dirtab); + break; + case DEV_ROOT_QDEV: + tab = devfstab; + ntab = NELEM(devfstab); + break; + case DEV_ROOT_QBLOBS: + tab = blobtab; + ntab = NELEM(blobtab); + break; + default: + return 0; + } + + return devgen(channel, tab, ntab, n, dir); +} + +static int rootwalk(chan_t *channel, const char *name) +{ + return devwalk(channel, name, NULL, 0, rootgen); +} + +/******************************************************************************* + * This function copies at most n bytes from the element referred by c into buf. + ******************************************************************************/ +static int rootread(chan_t *channel, void *buf, int size) +{ + const dirtab_t *dp; + dir_t *dir; + + if ((channel->qid & CHDIR) != 0) { + if (size < sizeof(dir_t)) { + return -1; + } + + dir = buf; + return dirread(channel, dir, NULL, 0, rootgen); + } + + /* Only makes sense when using debug language */ + assert(channel->qid != DEV_ROOT_QBLOBCTL); + + dp = &blobtab[channel->qid - DEV_ROOT_QBLOBCTL]; + return buf_to_channel(channel, buf, dp->data, size, dp->length); +} + +static int rootstat(chan_t *channel, const char *file, dir_t *dir) +{ + return devstat(channel, file, dir, NULL, 0, rootgen); +} + +const dev_t rootdevtab = { + .id = '/', + .stat = rootstat, + .clone = devclone, + .attach = devattach, + .walk = rootwalk, + .read = rootread, + .write = deverrwrite, + .mount = deverrmount, + .seek = devseek +}; diff --git a/lib/el3_runtime/aarch32/context_mgmt.c b/lib/el3_runtime/aarch32/context_mgmt.c new file mode 100644 index 0000000..b60b8e0 --- /dev/null +++ b/lib/el3_runtime/aarch32/context_mgmt.c @@ -0,0 +1,351 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdbool.h> +#include <string.h> + +#include <platform_def.h> + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <common/bl_common.h> +#include <context.h> +#include <lib/el3_runtime/context_mgmt.h> +#include <lib/extensions/amu.h> +#include <lib/extensions/pmuv3.h> +#include <lib/extensions/sys_reg_trace.h> +#include <lib/extensions/trf.h> +#include <lib/utils.h> + +/******************************************************************************* + * Context management library initialisation routine. This library is used by + * runtime services to share pointers to 'cpu_context' structures for the secure + * and non-secure states. Management of the structures and their associated + * memory is not done by the context management library e.g. the PSCI service + * manages the cpu context used for entry from and exit to the non-secure state. + * The Secure payload manages the context(s) corresponding to the secure state. + * It also uses this library to get access to the non-secure + * state cpu context pointers. + ******************************************************************************/ +void cm_init(void) +{ + /* + * The context management library has only global data to initialize, but + * that will be done when the BSS is zeroed out + */ +} + +/******************************************************************************* + * The following function initializes the cpu_context 'ctx' for + * first use, and sets the initial entrypoint state as specified by the + * entry_point_info structure. + * + * The security state to initialize is determined by the SECURE attribute + * of the entry_point_info. + * + * The EE and ST attributes are used to configure the endianness and secure + * timer availability for the new execution context. + * + * To prepare the register state for entry call cm_prepare_el3_exit() and + * el3_exit(). For Secure-EL1 cm_prepare_el3_exit() is equivalent to + * cm_el1_sysregs_context_restore(). + ******************************************************************************/ +void cm_setup_context(cpu_context_t *ctx, const entry_point_info_t *ep) +{ + unsigned int security_state; + uint32_t scr, sctlr; + regs_t *reg_ctx; + + assert(ctx != NULL); + + security_state = GET_SECURITY_STATE(ep->h.attr); + + /* Clear any residual register values from the context */ + zeromem(ctx, sizeof(*ctx)); + + reg_ctx = get_regs_ctx(ctx); + + /* + * Base the context SCR on the current value, adjust for entry point + * specific requirements + */ + scr = read_scr(); + scr &= ~(SCR_NS_BIT | SCR_HCE_BIT); + + if (security_state != SECURE) + scr |= SCR_NS_BIT; + + if (security_state != SECURE) { + /* + * Set up SCTLR for the Non-secure context. + * + * SCTLR.EE: Endianness is taken from the entrypoint attributes. + * + * SCTLR.M, SCTLR.C and SCTLR.I: These fields must be zero (as + * required by PSCI specification) + * + * Set remaining SCTLR fields to their architecturally defined + * values. Some fields reset to an IMPLEMENTATION DEFINED value: + * + * SCTLR.TE: Set to zero so that exceptions to an Exception + * Level executing at PL1 are taken to A32 state. + * + * SCTLR.V: Set to zero to select the normal exception vectors + * with base address held in VBAR. + */ + assert(((ep->spsr >> SPSR_E_SHIFT) & SPSR_E_MASK) == + (EP_GET_EE(ep->h.attr) >> EP_EE_SHIFT)); + + sctlr = (EP_GET_EE(ep->h.attr) != 0U) ? SCTLR_EE_BIT : 0U; + sctlr |= (SCTLR_RESET_VAL & ~(SCTLR_TE_BIT | SCTLR_V_BIT)); + write_ctx_reg(reg_ctx, CTX_NS_SCTLR, sctlr); + } + + /* + * The target exception level is based on the spsr mode requested. If + * execution is requested to hyp mode, HVC is enabled via SCR.HCE. + */ + if (GET_M32(ep->spsr) == MODE32_hyp) + scr |= SCR_HCE_BIT; + + /* + * Store the initialised values for SCTLR and SCR in the cpu_context. + * The Hyp mode registers are not part of the saved context and are + * set-up in cm_prepare_el3_exit(). + */ + write_ctx_reg(reg_ctx, CTX_SCR, scr); + write_ctx_reg(reg_ctx, CTX_LR, ep->pc); + write_ctx_reg(reg_ctx, CTX_SPSR, ep->spsr); + + /* + * Store the r0-r3 value from the entrypoint into the context + * Use memcpy as we are in control of the layout of the structures + */ + memcpy((void *)reg_ctx, (void *)&ep->args, sizeof(aapcs32_params_t)); +} + +/******************************************************************************* + * Enable architecture extensions on first entry to Non-secure world. + * When EL2 is implemented but unused `el2_unused` is non-zero, otherwise + * it is zero. + ******************************************************************************/ +static void enable_extensions_nonsecure(bool el2_unused) +{ +#if IMAGE_BL32 + if (is_feat_amu_supported()) { + amu_enable(el2_unused); + } + + if (is_feat_sys_reg_trace_supported()) { + sys_reg_trace_init_el3(); + } + + if (is_feat_trf_supported()) { + trf_init_el3(); + } + + /* + * Also applies to PMU < v3. The PMU is only disabled for EL3 and Secure + * state execution. This does not affect lower NS ELs. + */ + pmuv3_init_el3(); +#endif /* IMAGE_BL32 */ +} + +/******************************************************************************* + * The following function initializes the cpu_context for a CPU specified by + * its `cpu_idx` for first use, and sets the initial entrypoint state as + * specified by the entry_point_info structure. + ******************************************************************************/ +void cm_init_context_by_index(unsigned int cpu_idx, + const entry_point_info_t *ep) +{ + cpu_context_t *ctx; + ctx = cm_get_context_by_index(cpu_idx, GET_SECURITY_STATE(ep->h.attr)); + cm_setup_context(ctx, ep); +} + +/******************************************************************************* + * The following function initializes the cpu_context for the current CPU + * for first use, and sets the initial entrypoint state as specified by the + * entry_point_info structure. + ******************************************************************************/ +void cm_init_my_context(const entry_point_info_t *ep) +{ + cpu_context_t *ctx; + ctx = cm_get_context(GET_SECURITY_STATE(ep->h.attr)); + cm_setup_context(ctx, ep); +} + +/******************************************************************************* + * Prepare the CPU system registers for first entry into secure or normal world + * + * If execution is requested to hyp mode, HSCTLR is initialized + * If execution is requested to non-secure PL1, and the CPU supports + * HYP mode then HYP mode is disabled by configuring all necessary HYP mode + * registers. + ******************************************************************************/ +void cm_prepare_el3_exit(uint32_t security_state) +{ + uint32_t hsctlr, scr; + cpu_context_t *ctx = cm_get_context(security_state); + bool el2_unused = false; + + assert(ctx != NULL); + + if (security_state == NON_SECURE) { + scr = read_ctx_reg(get_regs_ctx(ctx), CTX_SCR); + if ((scr & SCR_HCE_BIT) != 0U) { + /* Use SCTLR value to initialize HSCTLR */ + hsctlr = read_ctx_reg(get_regs_ctx(ctx), + CTX_NS_SCTLR); + hsctlr |= HSCTLR_RES1; + /* Temporarily set the NS bit to access HSCTLR */ + write_scr(read_scr() | SCR_NS_BIT); + /* + * Make sure the write to SCR is complete so that + * we can access HSCTLR + */ + isb(); + write_hsctlr(hsctlr); + isb(); + + write_scr(read_scr() & ~SCR_NS_BIT); + isb(); + } else if ((read_id_pfr1() & + (ID_PFR1_VIRTEXT_MASK << ID_PFR1_VIRTEXT_SHIFT)) != 0U) { + el2_unused = true; + + /* + * Set the NS bit to access NS copies of certain banked + * registers + */ + write_scr(read_scr() | SCR_NS_BIT); + isb(); + + /* + * Hyp / PL2 present but unused, need to disable safely. + * HSCTLR can be ignored in this case. + * + * Set HCR to its architectural reset value so that + * Non-secure operations do not trap to Hyp mode. + */ + write_hcr(HCR_RESET_VAL); + + /* + * Set HCPTR to its architectural reset value so that + * Non-secure access from EL1 or EL0 to trace and to + * Advanced SIMD and floating point functionality does + * not trap to Hyp mode. + */ + write_hcptr(HCPTR_RESET_VAL); + + /* + * Initialise CNTHCTL. All fields are architecturally + * UNKNOWN on reset and are set to zero except for + * field(s) listed below. + * + * CNTHCTL.PL1PCEN: Disable traps to Hyp mode of + * Non-secure EL0 and EL1 accessed to the physical + * timer registers. + * + * CNTHCTL.PL1PCTEN: Disable traps to Hyp mode of + * Non-secure EL0 and EL1 accessed to the physical + * counter registers. + */ + write_cnthctl(CNTHCTL_RESET_VAL | + PL1PCEN_BIT | PL1PCTEN_BIT); + + /* + * Initialise CNTVOFF to zero as it resets to an + * IMPLEMENTATION DEFINED value. + */ + write64_cntvoff(0); + + /* + * Set VPIDR and VMPIDR to match MIDR_EL1 and MPIDR + * respectively. + */ + write_vpidr(read_midr()); + write_vmpidr(read_mpidr()); + + /* + * Initialise VTTBR, setting all fields rather than + * relying on the hw. Some fields are architecturally + * UNKNOWN at reset. + * + * VTTBR.VMID: Set to zero which is the architecturally + * defined reset value. Even though EL1&0 stage 2 + * address translation is disabled, cache maintenance + * operations depend on the VMID. + * + * VTTBR.BADDR: Set to zero as EL1&0 stage 2 address + * translation is disabled. + */ + write64_vttbr(VTTBR_RESET_VAL & + ~((VTTBR_VMID_MASK << VTTBR_VMID_SHIFT) + | (VTTBR_BADDR_MASK << VTTBR_BADDR_SHIFT))); + + /* + * Initialise HDCR, setting all the fields rather than + * relying on hw. + * + * HDCR.HPMN: Set to value of PMCR.N which is the + * architecturally-defined reset value. + * + * HDCR.HLP: Set to one so that event counter + * overflow, that is recorded in PMOVSCLR[0-30], + * occurs on the increment that changes + * PMEVCNTR<n>[63] from 1 to 0, when ARMv8.5-PMU is + * implemented. This bit is RES0 in versions of the + * architecture earlier than ARMv8.5, setting it to 1 + * doesn't have any effect on them. + * This bit is Reserved, UNK/SBZP in ARMv7. + * + * HDCR.HPME: Set to zero to disable EL2 Event + * counters. + */ +#if (ARM_ARCH_MAJOR > 7) + write_hdcr((HDCR_RESET_VAL | HDCR_HLP_BIT | + ((read_pmcr() & PMCR_N_BITS) >> + PMCR_N_SHIFT)) & ~HDCR_HPME_BIT); +#else + write_hdcr((HDCR_RESET_VAL | + ((read_pmcr() & PMCR_N_BITS) >> + PMCR_N_SHIFT)) & ~HDCR_HPME_BIT); +#endif + /* + * Set HSTR to its architectural reset value so that + * access to system registers in the cproc=1111 + * encoding space do not trap to Hyp mode. + */ + write_hstr(HSTR_RESET_VAL); + /* + * Set CNTHP_CTL to its architectural reset value to + * disable the EL2 physical timer and prevent timer + * interrupts. Some fields are architecturally UNKNOWN + * on reset and are set to zero. + */ + write_cnthp_ctl(CNTHP_CTL_RESET_VAL); + isb(); + + write_scr(read_scr() & ~SCR_NS_BIT); + isb(); + } + enable_extensions_nonsecure(el2_unused); + } +} + +/******************************************************************************* + * This function is used to exit to Non-secure world. It simply calls the + * cm_prepare_el3_exit function for AArch32. + ******************************************************************************/ +void cm_prepare_el3_exit_ns(void) +{ + cm_prepare_el3_exit(NON_SECURE); +} diff --git a/lib/el3_runtime/aarch32/cpu_data.S b/lib/el3_runtime/aarch32/cpu_data.S new file mode 100644 index 0000000..e59b7fd --- /dev/null +++ b/lib/el3_runtime/aarch32/cpu_data.S @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <lib/el3_runtime/cpu_data.h> + + .globl _cpu_data + .globl _cpu_data_by_index + +/* ----------------------------------------------------------------- + * cpu_data_t *_cpu_data(void) + * + * Return the cpu_data structure for the current CPU. + * ----------------------------------------------------------------- + */ +func _cpu_data + /* r12 is pushed to meet the 8 byte stack alignment requirement */ + push {r12, lr} + bl plat_my_core_pos + pop {r12, lr} + b _cpu_data_by_index +endfunc _cpu_data + +/* ----------------------------------------------------------------- + * cpu_data_t *_cpu_data_by_index(uint32_t cpu_index) + * + * Return the cpu_data structure for the CPU with given linear index + * + * This can be called without a valid stack. + * clobbers: r0, r1 + * ----------------------------------------------------------------- + */ +func _cpu_data_by_index + mov_imm r1, CPU_DATA_SIZE + mul r0, r0, r1 + ldr r1, =percpu_data + add r0, r0, r1 + bx lr +endfunc _cpu_data_by_index diff --git a/lib/el3_runtime/aarch64/context.S b/lib/el3_runtime/aarch64/context.S new file mode 100644 index 0000000..631094f --- /dev/null +++ b/lib/el3_runtime/aarch64/context.S @@ -0,0 +1,684 @@ +/* + * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> +#include <context.h> +#include <el3_common_macros.S> + + .global el1_sysregs_context_save + .global el1_sysregs_context_restore +#if CTX_INCLUDE_FPREGS + .global fpregs_context_save + .global fpregs_context_restore +#endif /* CTX_INCLUDE_FPREGS */ + .global prepare_el3_entry + .global restore_gp_pmcr_pauth_regs + .global save_and_update_ptw_el1_sys_regs + .global el3_exit + + +/* ------------------------------------------------------------------ + * The following function strictly follows the AArch64 PCS to use + * x9-x17 (temporary caller-saved registers) to save EL1 system + * register context. It assumes that 'x0' is pointing to a + * 'el1_sys_regs' structure where the register context will be saved. + * ------------------------------------------------------------------ + */ +func el1_sysregs_context_save + + mrs x9, spsr_el1 + mrs x10, elr_el1 + stp x9, x10, [x0, #CTX_SPSR_EL1] + +#if !ERRATA_SPECULATIVE_AT + mrs x15, sctlr_el1 + mrs x16, tcr_el1 + stp x15, x16, [x0, #CTX_SCTLR_EL1] +#endif /* ERRATA_SPECULATIVE_AT */ + + mrs x17, cpacr_el1 + mrs x9, csselr_el1 + stp x17, x9, [x0, #CTX_CPACR_EL1] + + mrs x10, sp_el1 + mrs x11, esr_el1 + stp x10, x11, [x0, #CTX_SP_EL1] + + mrs x12, ttbr0_el1 + mrs x13, ttbr1_el1 + stp x12, x13, [x0, #CTX_TTBR0_EL1] + + mrs x14, mair_el1 + mrs x15, amair_el1 + stp x14, x15, [x0, #CTX_MAIR_EL1] + + mrs x16, actlr_el1 + mrs x17, tpidr_el1 + stp x16, x17, [x0, #CTX_ACTLR_EL1] + + mrs x9, tpidr_el0 + mrs x10, tpidrro_el0 + stp x9, x10, [x0, #CTX_TPIDR_EL0] + + mrs x13, par_el1 + mrs x14, far_el1 + stp x13, x14, [x0, #CTX_PAR_EL1] + + mrs x15, afsr0_el1 + mrs x16, afsr1_el1 + stp x15, x16, [x0, #CTX_AFSR0_EL1] + + mrs x17, contextidr_el1 + mrs x9, vbar_el1 + stp x17, x9, [x0, #CTX_CONTEXTIDR_EL1] + + /* Save AArch32 system registers if the build has instructed so */ +#if CTX_INCLUDE_AARCH32_REGS + mrs x11, spsr_abt + mrs x12, spsr_und + stp x11, x12, [x0, #CTX_SPSR_ABT] + + mrs x13, spsr_irq + mrs x14, spsr_fiq + stp x13, x14, [x0, #CTX_SPSR_IRQ] + + mrs x15, dacr32_el2 + mrs x16, ifsr32_el2 + stp x15, x16, [x0, #CTX_DACR32_EL2] +#endif /* CTX_INCLUDE_AARCH32_REGS */ + + /* Save NS timer registers if the build has instructed so */ +#if NS_TIMER_SWITCH + mrs x10, cntp_ctl_el0 + mrs x11, cntp_cval_el0 + stp x10, x11, [x0, #CTX_CNTP_CTL_EL0] + + mrs x12, cntv_ctl_el0 + mrs x13, cntv_cval_el0 + stp x12, x13, [x0, #CTX_CNTV_CTL_EL0] + + mrs x14, cntkctl_el1 + str x14, [x0, #CTX_CNTKCTL_EL1] +#endif /* NS_TIMER_SWITCH */ + + /* Save MTE system registers if the build has instructed so */ +#if CTX_INCLUDE_MTE_REGS + mrs x15, TFSRE0_EL1 + mrs x16, TFSR_EL1 + stp x15, x16, [x0, #CTX_TFSRE0_EL1] + + mrs x9, RGSR_EL1 + mrs x10, GCR_EL1 + stp x9, x10, [x0, #CTX_RGSR_EL1] +#endif /* CTX_INCLUDE_MTE_REGS */ + + ret +endfunc el1_sysregs_context_save + +/* ------------------------------------------------------------------ + * The following function strictly follows the AArch64 PCS to use + * x9-x17 (temporary caller-saved registers) to restore EL1 system + * register context. It assumes that 'x0' is pointing to a + * 'el1_sys_regs' structure from where the register context will be + * restored + * ------------------------------------------------------------------ + */ +func el1_sysregs_context_restore + + ldp x9, x10, [x0, #CTX_SPSR_EL1] + msr spsr_el1, x9 + msr elr_el1, x10 + +#if !ERRATA_SPECULATIVE_AT + ldp x15, x16, [x0, #CTX_SCTLR_EL1] + msr sctlr_el1, x15 + msr tcr_el1, x16 +#endif /* ERRATA_SPECULATIVE_AT */ + + ldp x17, x9, [x0, #CTX_CPACR_EL1] + msr cpacr_el1, x17 + msr csselr_el1, x9 + + ldp x10, x11, [x0, #CTX_SP_EL1] + msr sp_el1, x10 + msr esr_el1, x11 + + ldp x12, x13, [x0, #CTX_TTBR0_EL1] + msr ttbr0_el1, x12 + msr ttbr1_el1, x13 + + ldp x14, x15, [x0, #CTX_MAIR_EL1] + msr mair_el1, x14 + msr amair_el1, x15 + + ldp x16, x17, [x0, #CTX_ACTLR_EL1] + msr actlr_el1, x16 + msr tpidr_el1, x17 + + ldp x9, x10, [x0, #CTX_TPIDR_EL0] + msr tpidr_el0, x9 + msr tpidrro_el0, x10 + + ldp x13, x14, [x0, #CTX_PAR_EL1] + msr par_el1, x13 + msr far_el1, x14 + + ldp x15, x16, [x0, #CTX_AFSR0_EL1] + msr afsr0_el1, x15 + msr afsr1_el1, x16 + + ldp x17, x9, [x0, #CTX_CONTEXTIDR_EL1] + msr contextidr_el1, x17 + msr vbar_el1, x9 + + /* Restore AArch32 system registers if the build has instructed so */ +#if CTX_INCLUDE_AARCH32_REGS + ldp x11, x12, [x0, #CTX_SPSR_ABT] + msr spsr_abt, x11 + msr spsr_und, x12 + + ldp x13, x14, [x0, #CTX_SPSR_IRQ] + msr spsr_irq, x13 + msr spsr_fiq, x14 + + ldp x15, x16, [x0, #CTX_DACR32_EL2] + msr dacr32_el2, x15 + msr ifsr32_el2, x16 +#endif /* CTX_INCLUDE_AARCH32_REGS */ + + /* Restore NS timer registers if the build has instructed so */ +#if NS_TIMER_SWITCH + ldp x10, x11, [x0, #CTX_CNTP_CTL_EL0] + msr cntp_ctl_el0, x10 + msr cntp_cval_el0, x11 + + ldp x12, x13, [x0, #CTX_CNTV_CTL_EL0] + msr cntv_ctl_el0, x12 + msr cntv_cval_el0, x13 + + ldr x14, [x0, #CTX_CNTKCTL_EL1] + msr cntkctl_el1, x14 +#endif /* NS_TIMER_SWITCH */ + + /* Restore MTE system registers if the build has instructed so */ +#if CTX_INCLUDE_MTE_REGS + ldp x11, x12, [x0, #CTX_TFSRE0_EL1] + msr TFSRE0_EL1, x11 + msr TFSR_EL1, x12 + + ldp x13, x14, [x0, #CTX_RGSR_EL1] + msr RGSR_EL1, x13 + msr GCR_EL1, x14 +#endif /* CTX_INCLUDE_MTE_REGS */ + + /* No explict ISB required here as ERET covers it */ + ret +endfunc el1_sysregs_context_restore + +/* ------------------------------------------------------------------ + * The following function follows the aapcs_64 strictly to use + * x9-x17 (temporary caller-saved registers according to AArch64 PCS) + * to save floating point register context. It assumes that 'x0' is + * pointing to a 'fp_regs' structure where the register context will + * be saved. + * + * Access to VFP registers will trap if CPTR_EL3.TFP is set. + * However currently we don't use VFP registers nor set traps in + * Trusted Firmware, and assume it's cleared. + * + * TODO: Revisit when VFP is used in secure world + * ------------------------------------------------------------------ + */ +#if CTX_INCLUDE_FPREGS +func fpregs_context_save + stp q0, q1, [x0, #CTX_FP_Q0] + stp q2, q3, [x0, #CTX_FP_Q2] + stp q4, q5, [x0, #CTX_FP_Q4] + stp q6, q7, [x0, #CTX_FP_Q6] + stp q8, q9, [x0, #CTX_FP_Q8] + stp q10, q11, [x0, #CTX_FP_Q10] + stp q12, q13, [x0, #CTX_FP_Q12] + stp q14, q15, [x0, #CTX_FP_Q14] + stp q16, q17, [x0, #CTX_FP_Q16] + stp q18, q19, [x0, #CTX_FP_Q18] + stp q20, q21, [x0, #CTX_FP_Q20] + stp q22, q23, [x0, #CTX_FP_Q22] + stp q24, q25, [x0, #CTX_FP_Q24] + stp q26, q27, [x0, #CTX_FP_Q26] + stp q28, q29, [x0, #CTX_FP_Q28] + stp q30, q31, [x0, #CTX_FP_Q30] + + mrs x9, fpsr + str x9, [x0, #CTX_FP_FPSR] + + mrs x10, fpcr + str x10, [x0, #CTX_FP_FPCR] + +#if CTX_INCLUDE_AARCH32_REGS + mrs x11, fpexc32_el2 + str x11, [x0, #CTX_FP_FPEXC32_EL2] +#endif /* CTX_INCLUDE_AARCH32_REGS */ + ret +endfunc fpregs_context_save + +/* ------------------------------------------------------------------ + * The following function follows the aapcs_64 strictly to use x9-x17 + * (temporary caller-saved registers according to AArch64 PCS) to + * restore floating point register context. It assumes that 'x0' is + * pointing to a 'fp_regs' structure from where the register context + * will be restored. + * + * Access to VFP registers will trap if CPTR_EL3.TFP is set. + * However currently we don't use VFP registers nor set traps in + * Trusted Firmware, and assume it's cleared. + * + * TODO: Revisit when VFP is used in secure world + * ------------------------------------------------------------------ + */ +func fpregs_context_restore + ldp q0, q1, [x0, #CTX_FP_Q0] + ldp q2, q3, [x0, #CTX_FP_Q2] + ldp q4, q5, [x0, #CTX_FP_Q4] + ldp q6, q7, [x0, #CTX_FP_Q6] + ldp q8, q9, [x0, #CTX_FP_Q8] + ldp q10, q11, [x0, #CTX_FP_Q10] + ldp q12, q13, [x0, #CTX_FP_Q12] + ldp q14, q15, [x0, #CTX_FP_Q14] + ldp q16, q17, [x0, #CTX_FP_Q16] + ldp q18, q19, [x0, #CTX_FP_Q18] + ldp q20, q21, [x0, #CTX_FP_Q20] + ldp q22, q23, [x0, #CTX_FP_Q22] + ldp q24, q25, [x0, #CTX_FP_Q24] + ldp q26, q27, [x0, #CTX_FP_Q26] + ldp q28, q29, [x0, #CTX_FP_Q28] + ldp q30, q31, [x0, #CTX_FP_Q30] + + ldr x9, [x0, #CTX_FP_FPSR] + msr fpsr, x9 + + ldr x10, [x0, #CTX_FP_FPCR] + msr fpcr, x10 + +#if CTX_INCLUDE_AARCH32_REGS + ldr x11, [x0, #CTX_FP_FPEXC32_EL2] + msr fpexc32_el2, x11 +#endif /* CTX_INCLUDE_AARCH32_REGS */ + + /* + * No explict ISB required here as ERET to + * switch to secure EL1 or non-secure world + * covers it + */ + + ret +endfunc fpregs_context_restore +#endif /* CTX_INCLUDE_FPREGS */ + + /* + * Set SCR_EL3.EA bit to enable SErrors at EL3 + */ + .macro enable_serror_at_el3 + mrs x8, scr_el3 + orr x8, x8, #SCR_EA_BIT + msr scr_el3, x8 + .endm + + /* + * Set the PSTATE bits not set when the exception was taken as + * described in the AArch64.TakeException() pseudocode function + * in ARM DDI 0487F.c page J1-7635 to a default value. + */ + .macro set_unset_pstate_bits + /* + * If Data Independent Timing (DIT) functionality is implemented, + * always enable DIT in EL3 + */ +#if ENABLE_FEAT_DIT +#if ENABLE_FEAT_DIT == 2 + mrs x8, id_aa64pfr0_el1 + and x8, x8, #(ID_AA64PFR0_DIT_MASK << ID_AA64PFR0_DIT_SHIFT) + cbz x8, 1f +#endif + mov x8, #DIT_BIT + msr DIT, x8 +1: +#endif /* ENABLE_FEAT_DIT */ + .endm /* set_unset_pstate_bits */ + +/*------------------------------------------------------------------------- + * This macro checks the ENABLE_FEAT_MPAM state, performs ID register + * check to see if the platform supports MPAM extension and restores MPAM3 + * register value if it is FEAT_STATE_ENABLED/FEAT_STATE_CHECKED. + * + * This is particularly more complicated because we can't check + * if the platform supports MPAM by looking for status of a particular bit + * in the MDCR_EL3 or CPTR_EL3 register like other extensions. + * ------------------------------------------------------------------------ + */ + + .macro restore_mpam3_el3 +#if ENABLE_FEAT_MPAM +#if ENABLE_FEAT_MPAM == 2 + + mrs x8, id_aa64pfr0_el1 + lsr x8, x8, #(ID_AA64PFR0_MPAM_SHIFT) + and x8, x8, #(ID_AA64PFR0_MPAM_MASK) + mrs x7, id_aa64pfr1_el1 + lsr x7, x7, #(ID_AA64PFR1_MPAM_FRAC_SHIFT) + and x7, x7, #(ID_AA64PFR1_MPAM_FRAC_MASK) + orr x7, x7, x8 + cbz x7, no_mpam +#endif + /* ----------------------------------------------------------- + * Restore MPAM3_EL3 register as per context state + * Currently we only enable MPAM for NS world and trap to EL3 + * for MPAM access in lower ELs of Secure and Realm world + * ----------------------------------------------------------- + */ + ldr x17, [sp, #CTX_EL3STATE_OFFSET + CTX_MPAM3_EL3] + msr S3_6_C10_C5_0, x17 /* mpam3_el3 */ + +no_mpam: +#endif + .endm /* restore_mpam3_el3 */ + +/* ------------------------------------------------------------------ + * The following macro is used to save and restore all the general + * purpose and ARMv8.3-PAuth (if enabled) registers. + * It also checks if the Secure Cycle Counter (PMCCNTR_EL0) + * is disabled in EL3/Secure (ARMv8.5-PMU), wherein PMCCNTR_EL0 + * needs not to be saved/restored during world switch. + * + * Ideally we would only save and restore the callee saved registers + * when a world switch occurs but that type of implementation is more + * complex. So currently we will always save and restore these + * registers on entry and exit of EL3. + * clobbers: x18 + * ------------------------------------------------------------------ + */ + .macro save_gp_pmcr_pauth_regs + stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + stp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] + stp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] + stp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10] + stp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12] + stp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14] + stp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16] + stp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18] + stp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20] + stp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22] + stp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24] + stp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26] + stp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28] + mrs x18, sp_el0 + str x18, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_SP_EL0] + + /* PMUv3 is presumed to be always present */ + mrs x9, pmcr_el0 + str x9, [sp, #CTX_EL3STATE_OFFSET + CTX_PMCR_EL0] + /* Disable cycle counter when event counting is prohibited */ + orr x9, x9, #PMCR_EL0_DP_BIT + msr pmcr_el0, x9 + isb +#if CTX_INCLUDE_PAUTH_REGS + /* ---------------------------------------------------------- + * Save the ARMv8.3-PAuth keys as they are not banked + * by exception level + * ---------------------------------------------------------- + */ + add x19, sp, #CTX_PAUTH_REGS_OFFSET + + mrs x20, APIAKeyLo_EL1 /* x21:x20 = APIAKey */ + mrs x21, APIAKeyHi_EL1 + mrs x22, APIBKeyLo_EL1 /* x23:x22 = APIBKey */ + mrs x23, APIBKeyHi_EL1 + mrs x24, APDAKeyLo_EL1 /* x25:x24 = APDAKey */ + mrs x25, APDAKeyHi_EL1 + mrs x26, APDBKeyLo_EL1 /* x27:x26 = APDBKey */ + mrs x27, APDBKeyHi_EL1 + mrs x28, APGAKeyLo_EL1 /* x29:x28 = APGAKey */ + mrs x29, APGAKeyHi_EL1 + + stp x20, x21, [x19, #CTX_PACIAKEY_LO] + stp x22, x23, [x19, #CTX_PACIBKEY_LO] + stp x24, x25, [x19, #CTX_PACDAKEY_LO] + stp x26, x27, [x19, #CTX_PACDBKEY_LO] + stp x28, x29, [x19, #CTX_PACGAKEY_LO] +#endif /* CTX_INCLUDE_PAUTH_REGS */ + .endm /* save_gp_pmcr_pauth_regs */ + +/* ----------------------------------------------------------------- + * This function saves the context and sets the PSTATE to a known + * state, preparing entry to el3. + * Save all the general purpose and ARMv8.3-PAuth (if enabled) + * registers. + * Then set any of the PSTATE bits that are not set by hardware + * according to the Aarch64.TakeException pseudocode in the Arm + * Architecture Reference Manual to a default value for EL3. + * clobbers: x17 + * ----------------------------------------------------------------- + */ +func prepare_el3_entry + save_gp_pmcr_pauth_regs + enable_serror_at_el3 + /* + * Set the PSTATE bits not described in the Aarch64.TakeException + * pseudocode to their default values. + */ + set_unset_pstate_bits + ret +endfunc prepare_el3_entry + +/* ------------------------------------------------------------------ + * This function restores ARMv8.3-PAuth (if enabled) and all general + * purpose registers except x30 from the CPU context. + * x30 register must be explicitly restored by the caller. + * ------------------------------------------------------------------ + */ +func restore_gp_pmcr_pauth_regs +#if CTX_INCLUDE_PAUTH_REGS + /* Restore the ARMv8.3 PAuth keys */ + add x10, sp, #CTX_PAUTH_REGS_OFFSET + + ldp x0, x1, [x10, #CTX_PACIAKEY_LO] /* x1:x0 = APIAKey */ + ldp x2, x3, [x10, #CTX_PACIBKEY_LO] /* x3:x2 = APIBKey */ + ldp x4, x5, [x10, #CTX_PACDAKEY_LO] /* x5:x4 = APDAKey */ + ldp x6, x7, [x10, #CTX_PACDBKEY_LO] /* x7:x6 = APDBKey */ + ldp x8, x9, [x10, #CTX_PACGAKEY_LO] /* x9:x8 = APGAKey */ + + msr APIAKeyLo_EL1, x0 + msr APIAKeyHi_EL1, x1 + msr APIBKeyLo_EL1, x2 + msr APIBKeyHi_EL1, x3 + msr APDAKeyLo_EL1, x4 + msr APDAKeyHi_EL1, x5 + msr APDBKeyLo_EL1, x6 + msr APDBKeyHi_EL1, x7 + msr APGAKeyLo_EL1, x8 + msr APGAKeyHi_EL1, x9 +#endif /* CTX_INCLUDE_PAUTH_REGS */ + + /* PMUv3 is presumed to be always present */ + ldr x0, [sp, #CTX_EL3STATE_OFFSET + CTX_PMCR_EL0] + msr pmcr_el0, x0 + ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + ldp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] + ldp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] + ldp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10] + ldp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12] + ldp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14] + ldp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16] + ldp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18] + ldp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20] + ldp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22] + ldp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24] + ldp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26] + ldr x28, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_SP_EL0] + msr sp_el0, x28 + ldp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28] + ret +endfunc restore_gp_pmcr_pauth_regs + +/* + * In case of ERRATA_SPECULATIVE_AT, save SCTLR_EL1 and TCR_EL1 + * registers and update EL1 registers to disable stage1 and stage2 + * page table walk + */ +func save_and_update_ptw_el1_sys_regs + /* ---------------------------------------------------------- + * Save only sctlr_el1 and tcr_el1 registers + * ---------------------------------------------------------- + */ + mrs x29, sctlr_el1 + str x29, [sp, #(CTX_EL1_SYSREGS_OFFSET + CTX_SCTLR_EL1)] + mrs x29, tcr_el1 + str x29, [sp, #(CTX_EL1_SYSREGS_OFFSET + CTX_TCR_EL1)] + + /* ------------------------------------------------------------ + * Must follow below order in order to disable page table + * walk for lower ELs (EL1 and EL0). First step ensures that + * page table walk is disabled for stage1 and second step + * ensures that page table walker should use TCR_EL1.EPDx + * bits to perform address translation. ISB ensures that CPU + * does these 2 steps in order. + * + * 1. Update TCR_EL1.EPDx bits to disable page table walk by + * stage1. + * 2. Enable MMU bit to avoid identity mapping via stage2 + * and force TCR_EL1.EPDx to be used by the page table + * walker. + * ------------------------------------------------------------ + */ + orr x29, x29, #(TCR_EPD0_BIT) + orr x29, x29, #(TCR_EPD1_BIT) + msr tcr_el1, x29 + isb + mrs x29, sctlr_el1 + orr x29, x29, #SCTLR_M_BIT + msr sctlr_el1, x29 + isb + + ret +endfunc save_and_update_ptw_el1_sys_regs + +/* ----------------------------------------------------------------- +* The below macro returns the address of the per_world context for +* the security state, retrieved through "get_security_state" macro. +* The per_world context address is returned in the register argument. +* Clobbers: x9, x10 +* ------------------------------------------------------------------ +*/ + +.macro get_per_world_context _reg:req + ldr x10, [sp, #CTX_EL3STATE_OFFSET + CTX_SCR_EL3] + get_security_state x9, x10 + mov_imm x10, (CTX_GLOBAL_EL3STATE_END - CTX_CPTR_EL3) + mul x9, x9, x10 + adrp x10, per_world_context + add x10, x10, :lo12:per_world_context + add x9, x9, x10 + mov \_reg, x9 +.endm + +/* ------------------------------------------------------------------ + * This routine assumes that the SP_EL3 is pointing to a valid + * context structure from where the gp regs and other special + * registers can be retrieved. + * ------------------------------------------------------------------ + */ +func el3_exit +#if ENABLE_ASSERTIONS + /* el3_exit assumes SP_EL0 on entry */ + mrs x17, spsel + cmp x17, #MODE_SP_EL0 + ASM_ASSERT(eq) +#endif /* ENABLE_ASSERTIONS */ + + /* ---------------------------------------------------------- + * Save the current SP_EL0 i.e. the EL3 runtime stack which + * will be used for handling the next SMC. + * Then switch to SP_EL3. + * ---------------------------------------------------------- + */ + mov x17, sp + msr spsel, #MODE_SP_ELX + str x17, [sp, #CTX_EL3STATE_OFFSET + CTX_RUNTIME_SP] + + /* ---------------------------------------------------------- + * Restore CPTR_EL3. + * ZCR is only restored if SVE is supported and enabled. + * Synchronization is required before zcr_el3 is addressed. + * ---------------------------------------------------------- + */ + + /* The address of the per_world context is stored in x9 */ + get_per_world_context x9 + + ldp x19, x20, [x9, #CTX_CPTR_EL3] + msr cptr_el3, x19 + +#if IMAGE_BL31 + ands x19, x19, #CPTR_EZ_BIT + beq sve_not_enabled + + isb + msr S3_6_C1_C2_0, x20 /* zcr_el3 */ +sve_not_enabled: + + restore_mpam3_el3 + +#endif /* IMAGE_BL31 */ + +#if IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639 + /* ---------------------------------------------------------- + * Restore mitigation state as it was on entry to EL3 + * ---------------------------------------------------------- + */ + ldr x17, [sp, #CTX_CVE_2018_3639_OFFSET + CTX_CVE_2018_3639_DISABLE] + cbz x17, 1f + blr x17 +1: +#endif /* IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639 */ + +#if IMAGE_BL31 + synchronize_errors +#endif /* IMAGE_BL31 */ + + /* ---------------------------------------------------------- + * Restore SPSR_EL3, ELR_EL3 and SCR_EL3 prior to ERET + * ---------------------------------------------------------- + */ + ldr x18, [sp, #CTX_EL3STATE_OFFSET + CTX_SCR_EL3] + ldp x16, x17, [sp, #CTX_EL3STATE_OFFSET + CTX_SPSR_EL3] + msr scr_el3, x18 + msr spsr_el3, x16 + msr elr_el3, x17 + + restore_ptw_el1_sys_regs + + /* ---------------------------------------------------------- + * Restore general purpose (including x30), PMCR_EL0 and + * ARMv8.3-PAuth registers. + * Exit EL3 via ERET to a lower exception level. + * ---------------------------------------------------------- + */ + bl restore_gp_pmcr_pauth_regs + ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] + +#ifdef IMAGE_BL31 + /* Clear the EL3 flag as we are exiting el3 */ + str xzr, [sp, #CTX_EL3STATE_OFFSET + CTX_NESTED_EA_FLAG] +#endif /* IMAGE_BL31 */ + + exception_return + +endfunc el3_exit diff --git a/lib/el3_runtime/aarch64/context_mgmt.c b/lib/el3_runtime/aarch64/context_mgmt.c new file mode 100644 index 0000000..fdd1388 --- /dev/null +++ b/lib/el3_runtime/aarch64/context_mgmt.c @@ -0,0 +1,1531 @@ +/* + * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2022, NVIDIA Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdbool.h> +#include <string.h> + +#include <platform_def.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <arch_features.h> +#include <bl31/interrupt_mgmt.h> +#include <common/bl_common.h> +#include <common/debug.h> +#include <context.h> +#include <drivers/arm/gicv3.h> +#include <lib/el3_runtime/context_mgmt.h> +#include <lib/el3_runtime/cpu_data.h> +#include <lib/el3_runtime/pubsub_events.h> +#include <lib/extensions/amu.h> +#include <lib/extensions/brbe.h> +#include <lib/extensions/mpam.h> +#include <lib/extensions/pmuv3.h> +#include <lib/extensions/sme.h> +#include <lib/extensions/spe.h> +#include <lib/extensions/sve.h> +#include <lib/extensions/sys_reg_trace.h> +#include <lib/extensions/trbe.h> +#include <lib/extensions/trf.h> +#include <lib/utils.h> + +#if ENABLE_FEAT_TWED +/* Make sure delay value fits within the range(0-15) */ +CASSERT(((TWED_DELAY & ~SCR_TWEDEL_MASK) == 0U), assert_twed_delay_value_check); +#endif /* ENABLE_FEAT_TWED */ + +per_world_context_t per_world_context[CPU_DATA_CONTEXT_NUM]; +static bool has_secure_perworld_init; + +static void manage_extensions_nonsecure(cpu_context_t *ctx); +static void manage_extensions_secure(cpu_context_t *ctx); +static void manage_extensions_secure_per_world(void); + +static void setup_el1_context(cpu_context_t *ctx, const struct entry_point_info *ep) +{ + u_register_t sctlr_elx, actlr_elx; + + /* + * Initialise SCTLR_EL1 to the reset value corresponding to the target + * execution state setting all fields rather than relying on the hw. + * Some fields have architecturally UNKNOWN reset values and these are + * set to zero. + * + * SCTLR.EE: Endianness is taken from the entrypoint attributes. + * + * SCTLR.M, SCTLR.C and SCTLR.I: These fields must be zero (as + * required by PSCI specification) + */ + sctlr_elx = (EP_GET_EE(ep->h.attr) != 0U) ? SCTLR_EE_BIT : 0UL; + if (GET_RW(ep->spsr) == MODE_RW_64) { + sctlr_elx |= SCTLR_EL1_RES1; + } else { + /* + * If the target execution state is AArch32 then the following + * fields need to be set. + * + * SCTRL_EL1.nTWE: Set to one so that EL0 execution of WFE + * instructions are not trapped to EL1. + * + * SCTLR_EL1.nTWI: Set to one so that EL0 execution of WFI + * instructions are not trapped to EL1. + * + * SCTLR_EL1.CP15BEN: Set to one to enable EL0 execution of the + * CP15DMB, CP15DSB, and CP15ISB instructions. + */ + sctlr_elx |= SCTLR_AARCH32_EL1_RES1 | SCTLR_CP15BEN_BIT + | SCTLR_NTWI_BIT | SCTLR_NTWE_BIT; + } + +#if ERRATA_A75_764081 + /* + * If workaround of errata 764081 for Cortex-A75 is used then set + * SCTLR_EL1.IESB to enable Implicit Error Synchronization Barrier. + */ + sctlr_elx |= SCTLR_IESB_BIT; +#endif + /* Store the initialised SCTLR_EL1 value in the cpu_context */ + write_ctx_reg(get_el1_sysregs_ctx(ctx), CTX_SCTLR_EL1, sctlr_elx); + + /* + * Base the context ACTLR_EL1 on the current value, as it is + * implementation defined. The context restore process will write + * the value from the context to the actual register and can cause + * problems for processor cores that don't expect certain bits to + * be zero. + */ + actlr_elx = read_actlr_el1(); + write_ctx_reg((get_el1_sysregs_ctx(ctx)), (CTX_ACTLR_EL1), (actlr_elx)); +} + +/****************************************************************************** + * This function performs initializations that are specific to SECURE state + * and updates the cpu context specified by 'ctx'. + *****************************************************************************/ +static void setup_secure_context(cpu_context_t *ctx, const struct entry_point_info *ep) +{ + u_register_t scr_el3; + el3_state_t *state; + + state = get_el3state_ctx(ctx); + scr_el3 = read_ctx_reg(state, CTX_SCR_EL3); + +#if defined(IMAGE_BL31) && !defined(SPD_spmd) + /* + * SCR_EL3.IRQ, SCR_EL3.FIQ: Enable the physical FIQ and IRQ routing as + * indicated by the interrupt routing model for BL31. + */ + scr_el3 |= get_scr_el3_from_routing_model(SECURE); +#endif + +#if !CTX_INCLUDE_MTE_REGS || ENABLE_ASSERTIONS + /* Get Memory Tagging Extension support level */ + unsigned int mte = get_armv8_5_mte_support(); +#endif + /* + * Allow access to Allocation Tags when CTX_INCLUDE_MTE_REGS + * is set, or when MTE is only implemented at EL0. + */ +#if CTX_INCLUDE_MTE_REGS + assert((mte == MTE_IMPLEMENTED_ELX) || (mte == MTE_IMPLEMENTED_ASY)); + scr_el3 |= SCR_ATA_BIT; +#else + if (mte == MTE_IMPLEMENTED_EL0) { + scr_el3 |= SCR_ATA_BIT; + } +#endif /* CTX_INCLUDE_MTE_REGS */ + + write_ctx_reg(state, CTX_SCR_EL3, scr_el3); + + /* + * Initialize EL1 context registers unless SPMC is running + * at S-EL2. + */ +#if !SPMD_SPM_AT_SEL2 + setup_el1_context(ctx, ep); +#endif + + manage_extensions_secure(ctx); + + /** + * manage_extensions_secure_per_world api has to be executed once, + * as the registers getting initialised, maintain constant value across + * all the cpus for the secure world. + * Henceforth, this check ensures that the registers are initialised once + * and avoids re-initialization from multiple cores. + */ + if (!has_secure_perworld_init) { + manage_extensions_secure_per_world(); + } + +} + +#if ENABLE_RME +/****************************************************************************** + * This function performs initializations that are specific to REALM state + * and updates the cpu context specified by 'ctx'. + *****************************************************************************/ +static void setup_realm_context(cpu_context_t *ctx, const struct entry_point_info *ep) +{ + u_register_t scr_el3; + el3_state_t *state; + + state = get_el3state_ctx(ctx); + scr_el3 = read_ctx_reg(state, CTX_SCR_EL3); + + scr_el3 |= SCR_NS_BIT | SCR_NSE_BIT; + + if (is_feat_csv2_2_supported()) { + /* Enable access to the SCXTNUM_ELx registers. */ + scr_el3 |= SCR_EnSCXT_BIT; + } + + write_ctx_reg(state, CTX_SCR_EL3, scr_el3); +} +#endif /* ENABLE_RME */ + +/****************************************************************************** + * This function performs initializations that are specific to NON-SECURE state + * and updates the cpu context specified by 'ctx'. + *****************************************************************************/ +static void setup_ns_context(cpu_context_t *ctx, const struct entry_point_info *ep) +{ + u_register_t scr_el3; + el3_state_t *state; + + state = get_el3state_ctx(ctx); + scr_el3 = read_ctx_reg(state, CTX_SCR_EL3); + + /* SCR_NS: Set the NS bit */ + scr_el3 |= SCR_NS_BIT; + + /* Allow access to Allocation Tags when MTE is implemented. */ + scr_el3 |= SCR_ATA_BIT; + +#if !CTX_INCLUDE_PAUTH_REGS + /* + * Pointer Authentication feature, if present, is always enabled by default + * for Non secure lower exception levels. We do not have an explicit + * flag to set it. + * CTX_INCLUDE_PAUTH_REGS flag, is explicitly used to enable for lower + * exception levels of secure and realm worlds. + * + * To prevent the leakage between the worlds during world switch, + * we enable it only for the non-secure world. + * + * If the Secure/realm world wants to use pointer authentication, + * CTX_INCLUDE_PAUTH_REGS must be explicitly set to 1, in which case + * it will be enabled globally for all the contexts. + * + * SCR_EL3.API: Set to one to not trap any PAuth instructions at ELs + * other than EL3 + * + * SCR_EL3.APK: Set to one to not trap any PAuth key values at ELs other + * than EL3 + */ + scr_el3 |= SCR_API_BIT | SCR_APK_BIT; + +#endif /* CTX_INCLUDE_PAUTH_REGS */ + +#if HANDLE_EA_EL3_FIRST_NS + /* SCR_EL3.EA: Route External Abort and SError Interrupt to EL3. */ + scr_el3 |= SCR_EA_BIT; +#endif + +#if RAS_TRAP_NS_ERR_REC_ACCESS + /* + * SCR_EL3.TERR: Trap Error record accesses. Accesses to the RAS ERR + * and RAS ERX registers from EL1 and EL2(from any security state) + * are trapped to EL3. + * Set here to trap only for NS EL1/EL2 + * + */ + scr_el3 |= SCR_TERR_BIT; +#endif + + if (is_feat_csv2_2_supported()) { + /* Enable access to the SCXTNUM_ELx registers. */ + scr_el3 |= SCR_EnSCXT_BIT; + } + +#ifdef IMAGE_BL31 + /* + * SCR_EL3.IRQ, SCR_EL3.FIQ: Enable the physical FIQ and IRQ routing as + * indicated by the interrupt routing model for BL31. + */ + scr_el3 |= get_scr_el3_from_routing_model(NON_SECURE); +#endif + write_ctx_reg(state, CTX_SCR_EL3, scr_el3); + + /* Initialize EL1 context registers */ + setup_el1_context(ctx, ep); + + /* Initialize EL2 context registers */ +#if CTX_INCLUDE_EL2_REGS + + /* + * Initialize SCTLR_EL2 context register using Endianness value + * taken from the entrypoint attribute. + */ + u_register_t sctlr_el2 = (EP_GET_EE(ep->h.attr) != 0U) ? SCTLR_EE_BIT : 0UL; + sctlr_el2 |= SCTLR_EL2_RES1; + write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_SCTLR_EL2, + sctlr_el2); + + if (is_feat_hcx_supported()) { + /* + * Initialize register HCRX_EL2 with its init value. + * As the value of HCRX_EL2 is UNKNOWN on reset, there is a + * chance that this can lead to unexpected behavior in lower + * ELs that have not been updated since the introduction of + * this feature if not properly initialized, especially when + * it comes to those bits that enable/disable traps. + */ + write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_HCRX_EL2, + HCRX_EL2_INIT_VAL); + } + + if (is_feat_fgt_supported()) { + /* + * Initialize HFG*_EL2 registers with a default value so legacy + * systems unaware of FEAT_FGT do not get trapped due to their lack + * of initialization for this feature. + */ + write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_HFGITR_EL2, + HFGITR_EL2_INIT_VAL); + write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_HFGRTR_EL2, + HFGRTR_EL2_INIT_VAL); + write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_HFGWTR_EL2, + HFGWTR_EL2_INIT_VAL); + } +#endif /* CTX_INCLUDE_EL2_REGS */ + + manage_extensions_nonsecure(ctx); +} + +/******************************************************************************* + * The following function performs initialization of the cpu_context 'ctx' + * for first use that is common to all security states, and sets the + * initial entrypoint state as specified by the entry_point_info structure. + * + * The EE and ST attributes are used to configure the endianness and secure + * timer availability for the new execution context. + ******************************************************************************/ +static void setup_context_common(cpu_context_t *ctx, const entry_point_info_t *ep) +{ + u_register_t scr_el3; + el3_state_t *state; + gp_regs_t *gp_regs; + + state = get_el3state_ctx(ctx); + + /* Clear any residual register values from the context */ + zeromem(ctx, sizeof(*ctx)); + + /* + * The lower-EL context is zeroed so that no stale values leak to a world. + * It is assumed that an all-zero lower-EL context is good enough for it + * to boot correctly. However, there are very few registers where this + * is not true and some values need to be recreated. + */ +#if CTX_INCLUDE_EL2_REGS + el2_sysregs_t *el2_ctx = get_el2_sysregs_ctx(ctx); + + /* + * These bits are set in the gicv3 driver. Losing them (especially the + * SRE bit) is problematic for all worlds. Henceforth recreate them. + */ + u_register_t icc_sre_el2 = ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT | + ICC_SRE_EN_BIT | ICC_SRE_SRE_BIT; + write_ctx_reg(el2_ctx, CTX_ICC_SRE_EL2, icc_sre_el2); +#endif /* CTX_INCLUDE_EL2_REGS */ + + /* Start with a clean SCR_EL3 copy as all relevant values are set */ + scr_el3 = SCR_RESET_VAL; + + /* + * SCR_EL3.TWE: Set to zero so that execution of WFE instructions at + * EL2, EL1 and EL0 are not trapped to EL3. + * + * SCR_EL3.TWI: Set to zero so that execution of WFI instructions at + * EL2, EL1 and EL0 are not trapped to EL3. + * + * SCR_EL3.SMD: Set to zero to enable SMC calls at EL1 and above, from + * both Security states and both Execution states. + * + * SCR_EL3.SIF: Set to one to disable secure instruction execution from + * Non-secure memory. + */ + scr_el3 &= ~(SCR_TWE_BIT | SCR_TWI_BIT | SCR_SMD_BIT); + + scr_el3 |= SCR_SIF_BIT; + + /* + * SCR_EL3.RW: Set the execution state, AArch32 or AArch64, for next + * Exception level as specified by SPSR. + */ + if (GET_RW(ep->spsr) == MODE_RW_64) { + scr_el3 |= SCR_RW_BIT; + } + + /* + * SCR_EL3.ST: Traps Secure EL1 accesses to the Counter-timer Physical + * Secure timer registers to EL3, from AArch64 state only, if specified + * by the entrypoint attributes. If SEL2 is present and enabled, the ST + * bit always behaves as 1 (i.e. secure physical timer register access + * is not trapped) + */ + if (EP_GET_ST(ep->h.attr) != 0U) { + scr_el3 |= SCR_ST_BIT; + } + + /* + * If FEAT_HCX is enabled, enable access to HCRX_EL2 by setting + * SCR_EL3.HXEn. + */ + if (is_feat_hcx_supported()) { + scr_el3 |= SCR_HXEn_BIT; + } + + /* + * If FEAT_RNG_TRAP is enabled, all reads of the RNDR and RNDRRS + * registers are trapped to EL3. + */ +#if ENABLE_FEAT_RNG_TRAP + scr_el3 |= SCR_TRNDR_BIT; +#endif + +#if FAULT_INJECTION_SUPPORT + /* Enable fault injection from lower ELs */ + scr_el3 |= SCR_FIEN_BIT; +#endif + +#if CTX_INCLUDE_PAUTH_REGS + /* + * Enable Pointer Authentication globally for all the worlds. + * + * SCR_EL3.API: Set to one to not trap any PAuth instructions at ELs + * other than EL3 + * + * SCR_EL3.APK: Set to one to not trap any PAuth key values at ELs other + * than EL3 + */ + scr_el3 |= SCR_API_BIT | SCR_APK_BIT; +#endif /* CTX_INCLUDE_PAUTH_REGS */ + + /* + * SCR_EL3.TCR2EN: Enable access to TCR2_ELx for AArch64 if present. + */ + if (is_feat_tcr2_supported() && (GET_RW(ep->spsr) == MODE_RW_64)) { + scr_el3 |= SCR_TCR2EN_BIT; + } + + /* + * SCR_EL3.PIEN: Enable permission indirection and overlay + * registers for AArch64 if present. + */ + if (is_feat_sxpie_supported() || is_feat_sxpoe_supported()) { + scr_el3 |= SCR_PIEN_BIT; + } + + /* + * SCR_EL3.GCSEn: Enable GCS registers for AArch64 if present. + */ + if ((is_feat_gcs_supported()) && (GET_RW(ep->spsr) == MODE_RW_64)) { + scr_el3 |= SCR_GCSEn_BIT; + } + + /* + * SCR_EL3.HCE: Enable HVC instructions if next execution state is + * AArch64 and next EL is EL2, or if next execution state is AArch32 and + * next mode is Hyp. + * SCR_EL3.FGTEn: Enable Fine Grained Virtualization Traps under the + * same conditions as HVC instructions and when the processor supports + * ARMv8.6-FGT. + * SCR_EL3.ECVEn: Enable Enhanced Counter Virtualization (ECV) + * CNTPOFF_EL2 register under the same conditions as HVC instructions + * and when the processor supports ECV. + */ + if (((GET_RW(ep->spsr) == MODE_RW_64) && (GET_EL(ep->spsr) == MODE_EL2)) + || ((GET_RW(ep->spsr) != MODE_RW_64) + && (GET_M32(ep->spsr) == MODE32_hyp))) { + scr_el3 |= SCR_HCE_BIT; + + if (is_feat_fgt_supported()) { + scr_el3 |= SCR_FGTEN_BIT; + } + + if (is_feat_ecv_supported()) { + scr_el3 |= SCR_ECVEN_BIT; + } + } + + /* Enable WFE trap delay in SCR_EL3 if supported and configured */ + if (is_feat_twed_supported()) { + /* Set delay in SCR_EL3 */ + scr_el3 &= ~(SCR_TWEDEL_MASK << SCR_TWEDEL_SHIFT); + scr_el3 |= ((TWED_DELAY & SCR_TWEDEL_MASK) + << SCR_TWEDEL_SHIFT); + + /* Enable WFE delay */ + scr_el3 |= SCR_TWEDEn_BIT; + } + +#if IMAGE_BL31 && defined(SPD_spmd) && SPMD_SPM_AT_SEL2 + /* Enable S-EL2 if FEAT_SEL2 is implemented for all the contexts. */ + if (is_feat_sel2_supported()) { + scr_el3 |= SCR_EEL2_BIT; + } +#endif /* (IMAGE_BL31 && defined(SPD_spmd) && SPMD_SPM_AT_SEL2) */ + + if (is_feat_mpam_supported()) { + write_ctx_reg(get_el3state_ctx(ctx), CTX_MPAM3_EL3, \ + MPAM3_EL3_RESET_VAL); + } + + /* + * Populate EL3 state so that we've the right context + * before doing ERET + */ + write_ctx_reg(state, CTX_SCR_EL3, scr_el3); + write_ctx_reg(state, CTX_ELR_EL3, ep->pc); + write_ctx_reg(state, CTX_SPSR_EL3, ep->spsr); + + /* + * Store the X0-X7 value from the entrypoint into the context + * Use memcpy as we are in control of the layout of the structures + */ + gp_regs = get_gpregs_ctx(ctx); + memcpy(gp_regs, (void *)&ep->args, sizeof(aapcs64_params_t)); +} + +/******************************************************************************* + * Context management library initialization routine. This library is used by + * runtime services to share pointers to 'cpu_context' structures for secure + * non-secure and realm states. Management of the structures and their associated + * memory is not done by the context management library e.g. the PSCI service + * manages the cpu context used for entry from and exit to the non-secure state. + * The Secure payload dispatcher service manages the context(s) corresponding to + * the secure state. It also uses this library to get access to the non-secure + * state cpu context pointers. + * Lastly, this library provides the API to make SP_EL3 point to the cpu context + * which will be used for programming an entry into a lower EL. The same context + * will be used to save state upon exception entry from that EL. + ******************************************************************************/ +void __init cm_init(void) +{ + /* + * The context management library has only global data to initialize, but + * that will be done when the BSS is zeroed out. + */ +} + +/******************************************************************************* + * This is the high-level function used to initialize the cpu_context 'ctx' for + * first use. It performs initializations that are common to all security states + * and initializations specific to the security state specified in 'ep' + ******************************************************************************/ +void cm_setup_context(cpu_context_t *ctx, const entry_point_info_t *ep) +{ + unsigned int security_state; + + assert(ctx != NULL); + + /* + * Perform initializations that are common + * to all security states + */ + setup_context_common(ctx, ep); + + security_state = GET_SECURITY_STATE(ep->h.attr); + + /* Perform security state specific initializations */ + switch (security_state) { + case SECURE: + setup_secure_context(ctx, ep); + break; +#if ENABLE_RME + case REALM: + setup_realm_context(ctx, ep); + break; +#endif + case NON_SECURE: + setup_ns_context(ctx, ep); + break; + default: + ERROR("Invalid security state\n"); + panic(); + break; + } +} + +/******************************************************************************* + * Enable architecture extensions for EL3 execution. This function only updates + * registers in-place which are expected to either never change or be + * overwritten by el3_exit. + ******************************************************************************/ +#if IMAGE_BL31 +void cm_manage_extensions_el3(void) +{ + if (is_feat_spe_supported()) { + spe_init_el3(); + } + + if (is_feat_amu_supported()) { + amu_init_el3(); + } + + if (is_feat_sme_supported()) { + sme_init_el3(); + } + + if (is_feat_trbe_supported()) { + trbe_init_el3(); + } + + if (is_feat_brbe_supported()) { + brbe_init_el3(); + } + + if (is_feat_trf_supported()) { + trf_init_el3(); + } + + pmuv3_init_el3(); +} +#endif /* IMAGE_BL31 */ + +/******************************************************************************* + * Initialise per_world_context for Non-Secure world. + * This function enables the architecture extensions, which have same value + * across the cores for the non-secure world. + ******************************************************************************/ +#if IMAGE_BL31 +void manage_extensions_nonsecure_per_world(void) +{ + if (is_feat_sme_supported()) { + sme_enable_per_world(&per_world_context[CPU_CONTEXT_NS]); + } + + if (is_feat_sve_supported()) { + sve_enable_per_world(&per_world_context[CPU_CONTEXT_NS]); + } + + if (is_feat_amu_supported()) { + amu_enable_per_world(&per_world_context[CPU_CONTEXT_NS]); + } + + if (is_feat_sys_reg_trace_supported()) { + sys_reg_trace_enable_per_world(&per_world_context[CPU_CONTEXT_NS]); + } +} +#endif /* IMAGE_BL31 */ + +/******************************************************************************* + * Initialise per_world_context for Secure world. + * This function enables the architecture extensions, which have same value + * across the cores for the secure world. + ******************************************************************************/ + +static void manage_extensions_secure_per_world(void) +{ +#if IMAGE_BL31 + if (is_feat_sme_supported()) { + + if (ENABLE_SME_FOR_SWD) { + /* + * Enable SME, SVE, FPU/SIMD in secure context, SPM must ensure + * SME, SVE, and FPU/SIMD context properly managed. + */ + sme_enable_per_world(&per_world_context[CPU_CONTEXT_SECURE]); + } else { + /* + * Disable SME, SVE, FPU/SIMD in secure context so non-secure + * world can safely use the associated registers. + */ + sme_disable_per_world(&per_world_context[CPU_CONTEXT_SECURE]); + } + } + if (is_feat_sve_supported()) { + if (ENABLE_SVE_FOR_SWD) { + /* + * Enable SVE and FPU in secure context, SPM must ensure + * that the SVE and FPU register contexts are properly managed. + */ + sve_enable_per_world(&per_world_context[CPU_CONTEXT_SECURE]); + } else { + /* + * Disable SVE and FPU in secure context so non-secure world + * can safely use them. + */ + sve_disable_per_world(&per_world_context[CPU_CONTEXT_SECURE]); + } + } + + /* NS can access this but Secure shouldn't */ + if (is_feat_sys_reg_trace_supported()) { + sys_reg_trace_disable_per_world(&per_world_context[CPU_CONTEXT_SECURE]); + } + + has_secure_perworld_init = true; +#endif /* IMAGE_BL31 */ +} + +/******************************************************************************* + * Enable architecture extensions on first entry to Non-secure world. + ******************************************************************************/ +static void manage_extensions_nonsecure(cpu_context_t *ctx) +{ +#if IMAGE_BL31 + if (is_feat_amu_supported()) { + amu_enable(ctx); + } + + if (is_feat_sme_supported()) { + sme_enable(ctx); + } + + if (is_feat_mpam_supported()) { + mpam_enable(ctx); + } + pmuv3_enable(ctx); +#endif /* IMAGE_BL31 */ +} + +/* TODO: move to lib/extensions/pauth when it has been ported to FEAT_STATE */ +static __unused void enable_pauth_el2(void) +{ + u_register_t hcr_el2 = read_hcr_el2(); + /* + * For Armv8.3 pointer authentication feature, disable traps to EL2 when + * accessing key registers or using pointer authentication instructions + * from lower ELs. + */ + hcr_el2 |= (HCR_API_BIT | HCR_APK_BIT); + + write_hcr_el2(hcr_el2); +} + +#if INIT_UNUSED_NS_EL2 +/******************************************************************************* + * Enable architecture extensions in-place at EL2 on first entry to Non-secure + * world when EL2 is empty and unused. + ******************************************************************************/ +static void manage_extensions_nonsecure_el2_unused(void) +{ +#if IMAGE_BL31 + if (is_feat_spe_supported()) { + spe_init_el2_unused(); + } + + if (is_feat_amu_supported()) { + amu_init_el2_unused(); + } + + if (is_feat_mpam_supported()) { + mpam_init_el2_unused(); + } + + if (is_feat_trbe_supported()) { + trbe_init_el2_unused(); + } + + if (is_feat_sys_reg_trace_supported()) { + sys_reg_trace_init_el2_unused(); + } + + if (is_feat_trf_supported()) { + trf_init_el2_unused(); + } + + pmuv3_init_el2_unused(); + + if (is_feat_sve_supported()) { + sve_init_el2_unused(); + } + + if (is_feat_sme_supported()) { + sme_init_el2_unused(); + } + +#if ENABLE_PAUTH + enable_pauth_el2(); +#endif /* ENABLE_PAUTH */ +#endif /* IMAGE_BL31 */ +} +#endif /* INIT_UNUSED_NS_EL2 */ + +/******************************************************************************* + * Enable architecture extensions on first entry to Secure world. + ******************************************************************************/ +static void manage_extensions_secure(cpu_context_t *ctx) +{ +#if IMAGE_BL31 + if (is_feat_sme_supported()) { + if (ENABLE_SME_FOR_SWD) { + /* + * Enable SME, SVE, FPU/SIMD in secure context, secure manager + * must ensure SME, SVE, and FPU/SIMD context properly managed. + */ + sme_init_el3(); + sme_enable(ctx); + } else { + /* + * Disable SME, SVE, FPU/SIMD in secure context so non-secure + * world can safely use the associated registers. + */ + sme_disable(ctx); + } + } +#endif /* IMAGE_BL31 */ +} + +/******************************************************************************* + * The following function initializes the cpu_context for a CPU specified by + * its `cpu_idx` for first use, and sets the initial entrypoint state as + * specified by the entry_point_info structure. + ******************************************************************************/ +void cm_init_context_by_index(unsigned int cpu_idx, + const entry_point_info_t *ep) +{ + cpu_context_t *ctx; + ctx = cm_get_context_by_index(cpu_idx, GET_SECURITY_STATE(ep->h.attr)); + cm_setup_context(ctx, ep); +} + +/******************************************************************************* + * The following function initializes the cpu_context for the current CPU + * for first use, and sets the initial entrypoint state as specified by the + * entry_point_info structure. + ******************************************************************************/ +void cm_init_my_context(const entry_point_info_t *ep) +{ + cpu_context_t *ctx; + ctx = cm_get_context(GET_SECURITY_STATE(ep->h.attr)); + cm_setup_context(ctx, ep); +} + +/* EL2 present but unused, need to disable safely. SCTLR_EL2 can be ignored */ +static void init_nonsecure_el2_unused(cpu_context_t *ctx) +{ +#if INIT_UNUSED_NS_EL2 + u_register_t hcr_el2 = HCR_RESET_VAL; + u_register_t mdcr_el2; + u_register_t scr_el3; + + scr_el3 = read_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3); + + /* Set EL2 register width: Set HCR_EL2.RW to match SCR_EL3.RW */ + if ((scr_el3 & SCR_RW_BIT) != 0U) { + hcr_el2 |= HCR_RW_BIT; + } + + write_hcr_el2(hcr_el2); + + /* + * Initialise CPTR_EL2 setting all fields rather than relying on the hw. + * All fields have architecturally UNKNOWN reset values. + */ + write_cptr_el2(CPTR_EL2_RESET_VAL); + + /* + * Initialise CNTHCTL_EL2. All fields are architecturally UNKNOWN on + * reset and are set to zero except for field(s) listed below. + * + * CNTHCTL_EL2.EL1PTEN: Set to one to disable traps to Hyp mode of + * Non-secure EL0 and EL1 accesses to the physical timer registers. + * + * CNTHCTL_EL2.EL1PCTEN: Set to one to disable traps to Hyp mode of + * Non-secure EL0 and EL1 accesses to the physical counter registers. + */ + write_cnthctl_el2(CNTHCTL_RESET_VAL | EL1PCEN_BIT | EL1PCTEN_BIT); + + /* + * Initialise CNTVOFF_EL2 to zero as it resets to an architecturally + * UNKNOWN value. + */ + write_cntvoff_el2(0); + + /* + * Set VPIDR_EL2 and VMPIDR_EL2 to match MIDR_EL1 and MPIDR_EL1 + * respectively. + */ + write_vpidr_el2(read_midr_el1()); + write_vmpidr_el2(read_mpidr_el1()); + + /* + * Initialise VTTBR_EL2. All fields are architecturally UNKNOWN on reset. + * + * VTTBR_EL2.VMID: Set to zero. Even though EL1&0 stage 2 address + * translation is disabled, cache maintenance operations depend on the + * VMID. + * + * VTTBR_EL2.BADDR: Set to zero as EL1&0 stage 2 address translation is + * disabled. + */ + write_vttbr_el2(VTTBR_RESET_VAL & + ~((VTTBR_VMID_MASK << VTTBR_VMID_SHIFT) | + (VTTBR_BADDR_MASK << VTTBR_BADDR_SHIFT))); + + /* + * Initialise MDCR_EL2, setting all fields rather than relying on hw. + * Some fields are architecturally UNKNOWN on reset. + * + * MDCR_EL2.TDRA: Set to zero so that Non-secure EL0 and EL1 System + * register accesses to the Debug ROM registers are not trapped to EL2. + * + * MDCR_EL2.TDOSA: Set to zero so that Non-secure EL1 System register + * accesses to the powerdown debug registers are not trapped to EL2. + * + * MDCR_EL2.TDA: Set to zero so that System register accesses to the + * debug registers do not trap to EL2. + * + * MDCR_EL2.TDE: Set to zero so that debug exceptions are not routed to + * EL2. + */ + mdcr_el2 = MDCR_EL2_RESET_VAL & + ~(MDCR_EL2_TDRA_BIT | MDCR_EL2_TDOSA_BIT | MDCR_EL2_TDA_BIT | + MDCR_EL2_TDE_BIT); + + write_mdcr_el2(mdcr_el2); + + /* + * Initialise HSTR_EL2. All fields are architecturally UNKNOWN on reset. + * + * HSTR_EL2.T<n>: Set all these fields to zero so that Non-secure EL0 or + * EL1 accesses to System registers do not trap to EL2. + */ + write_hstr_el2(HSTR_EL2_RESET_VAL & ~(HSTR_EL2_T_MASK)); + + /* + * Initialise CNTHP_CTL_EL2. All fields are architecturally UNKNOWN on + * reset. + * + * CNTHP_CTL_EL2:ENABLE: Set to zero to disable the EL2 physical timer + * and prevent timer interrupts. + */ + write_cnthp_ctl_el2(CNTHP_CTL_RESET_VAL & ~(CNTHP_CTL_ENABLE_BIT)); + + manage_extensions_nonsecure_el2_unused(); +#endif /* INIT_UNUSED_NS_EL2 */ +} + +/******************************************************************************* + * Prepare the CPU system registers for first entry into realm, secure, or + * normal world. + * + * If execution is requested to EL2 or hyp mode, SCTLR_EL2 is initialized + * If execution is requested to non-secure EL1 or svc mode, and the CPU supports + * EL2 then EL2 is disabled by configuring all necessary EL2 registers. + * For all entries, the EL1 registers are initialized from the cpu_context + ******************************************************************************/ +void cm_prepare_el3_exit(uint32_t security_state) +{ + u_register_t sctlr_elx, scr_el3; + cpu_context_t *ctx = cm_get_context(security_state); + + assert(ctx != NULL); + + if (security_state == NON_SECURE) { + uint64_t el2_implemented = el_implemented(2); + + scr_el3 = read_ctx_reg(get_el3state_ctx(ctx), + CTX_SCR_EL3); + + if (((scr_el3 & SCR_HCE_BIT) != 0U) + || (el2_implemented != EL_IMPL_NONE)) { + /* + * If context is not being used for EL2, initialize + * HCRX_EL2 with its init value here. + */ + if (is_feat_hcx_supported()) { + write_hcrx_el2(HCRX_EL2_INIT_VAL); + } + + /* + * Initialize Fine-grained trap registers introduced + * by FEAT_FGT so all traps are initially disabled when + * switching to EL2 or a lower EL, preventing undesired + * behavior. + */ + if (is_feat_fgt_supported()) { + /* + * Initialize HFG*_EL2 registers with a default + * value so legacy systems unaware of FEAT_FGT + * do not get trapped due to their lack of + * initialization for this feature. + */ + write_hfgitr_el2(HFGITR_EL2_INIT_VAL); + write_hfgrtr_el2(HFGRTR_EL2_INIT_VAL); + write_hfgwtr_el2(HFGWTR_EL2_INIT_VAL); + } + } + + + if ((scr_el3 & SCR_HCE_BIT) != 0U) { + /* Use SCTLR_EL1.EE value to initialise sctlr_el2 */ + sctlr_elx = read_ctx_reg(get_el1_sysregs_ctx(ctx), + CTX_SCTLR_EL1); + sctlr_elx &= SCTLR_EE_BIT; + sctlr_elx |= SCTLR_EL2_RES1; +#if ERRATA_A75_764081 + /* + * If workaround of errata 764081 for Cortex-A75 is used + * then set SCTLR_EL2.IESB to enable Implicit Error + * Synchronization Barrier. + */ + sctlr_elx |= SCTLR_IESB_BIT; +#endif + write_sctlr_el2(sctlr_elx); + } else if (el2_implemented != EL_IMPL_NONE) { + init_nonsecure_el2_unused(ctx); + } + } + + cm_el1_sysregs_context_restore(security_state); + cm_set_next_eret_context(security_state); +} + +#if CTX_INCLUDE_EL2_REGS + +static void el2_sysregs_context_save_fgt(el2_sysregs_t *ctx) +{ + write_ctx_reg(ctx, CTX_HDFGRTR_EL2, read_hdfgrtr_el2()); + if (is_feat_amu_supported()) { + write_ctx_reg(ctx, CTX_HAFGRTR_EL2, read_hafgrtr_el2()); + } + write_ctx_reg(ctx, CTX_HDFGWTR_EL2, read_hdfgwtr_el2()); + write_ctx_reg(ctx, CTX_HFGITR_EL2, read_hfgitr_el2()); + write_ctx_reg(ctx, CTX_HFGRTR_EL2, read_hfgrtr_el2()); + write_ctx_reg(ctx, CTX_HFGWTR_EL2, read_hfgwtr_el2()); +} + +static void el2_sysregs_context_restore_fgt(el2_sysregs_t *ctx) +{ + write_hdfgrtr_el2(read_ctx_reg(ctx, CTX_HDFGRTR_EL2)); + if (is_feat_amu_supported()) { + write_hafgrtr_el2(read_ctx_reg(ctx, CTX_HAFGRTR_EL2)); + } + write_hdfgwtr_el2(read_ctx_reg(ctx, CTX_HDFGWTR_EL2)); + write_hfgitr_el2(read_ctx_reg(ctx, CTX_HFGITR_EL2)); + write_hfgrtr_el2(read_ctx_reg(ctx, CTX_HFGRTR_EL2)); + write_hfgwtr_el2(read_ctx_reg(ctx, CTX_HFGWTR_EL2)); +} + +static void el2_sysregs_context_save_mpam(el2_sysregs_t *ctx) +{ + u_register_t mpam_idr = read_mpamidr_el1(); + + write_ctx_reg(ctx, CTX_MPAM2_EL2, read_mpam2_el2()); + + /* + * The context registers that we intend to save would be part of the + * PE's system register frame only if MPAMIDR_EL1.HAS_HCR == 1. + */ + if ((mpam_idr & MPAMIDR_HAS_HCR_BIT) == 0U) { + return; + } + + /* + * MPAMHCR_EL2, MPAMVPMV_EL2 and MPAMVPM0_EL2 are always present if + * MPAMIDR_HAS_HCR_BIT == 1. + */ + write_ctx_reg(ctx, CTX_MPAMHCR_EL2, read_mpamhcr_el2()); + write_ctx_reg(ctx, CTX_MPAMVPM0_EL2, read_mpamvpm0_el2()); + write_ctx_reg(ctx, CTX_MPAMVPMV_EL2, read_mpamvpmv_el2()); + + /* + * The number of MPAMVPM registers is implementation defined, their + * number is stored in the MPAMIDR_EL1 register. + */ + switch ((mpam_idr >> MPAMIDR_EL1_VPMR_MAX_SHIFT) & MPAMIDR_EL1_VPMR_MAX_MASK) { + case 7: + write_ctx_reg(ctx, CTX_MPAMVPM7_EL2, read_mpamvpm7_el2()); + __fallthrough; + case 6: + write_ctx_reg(ctx, CTX_MPAMVPM6_EL2, read_mpamvpm6_el2()); + __fallthrough; + case 5: + write_ctx_reg(ctx, CTX_MPAMVPM5_EL2, read_mpamvpm5_el2()); + __fallthrough; + case 4: + write_ctx_reg(ctx, CTX_MPAMVPM4_EL2, read_mpamvpm4_el2()); + __fallthrough; + case 3: + write_ctx_reg(ctx, CTX_MPAMVPM3_EL2, read_mpamvpm3_el2()); + __fallthrough; + case 2: + write_ctx_reg(ctx, CTX_MPAMVPM2_EL2, read_mpamvpm2_el2()); + __fallthrough; + case 1: + write_ctx_reg(ctx, CTX_MPAMVPM1_EL2, read_mpamvpm1_el2()); + break; + } +} + +static void el2_sysregs_context_restore_mpam(el2_sysregs_t *ctx) +{ + u_register_t mpam_idr = read_mpamidr_el1(); + + write_mpam2_el2(read_ctx_reg(ctx, CTX_MPAM2_EL2)); + + if ((mpam_idr & MPAMIDR_HAS_HCR_BIT) == 0U) { + return; + } + + write_mpamhcr_el2(read_ctx_reg(ctx, CTX_MPAMHCR_EL2)); + write_mpamvpm0_el2(read_ctx_reg(ctx, CTX_MPAMVPM0_EL2)); + write_mpamvpmv_el2(read_ctx_reg(ctx, CTX_MPAMVPMV_EL2)); + + switch ((mpam_idr >> MPAMIDR_EL1_VPMR_MAX_SHIFT) & MPAMIDR_EL1_VPMR_MAX_MASK) { + case 7: + write_mpamvpm7_el2(read_ctx_reg(ctx, CTX_MPAMVPM7_EL2)); + __fallthrough; + case 6: + write_mpamvpm6_el2(read_ctx_reg(ctx, CTX_MPAMVPM6_EL2)); + __fallthrough; + case 5: + write_mpamvpm5_el2(read_ctx_reg(ctx, CTX_MPAMVPM5_EL2)); + __fallthrough; + case 4: + write_mpamvpm4_el2(read_ctx_reg(ctx, CTX_MPAMVPM4_EL2)); + __fallthrough; + case 3: + write_mpamvpm3_el2(read_ctx_reg(ctx, CTX_MPAMVPM3_EL2)); + __fallthrough; + case 2: + write_mpamvpm2_el2(read_ctx_reg(ctx, CTX_MPAMVPM2_EL2)); + __fallthrough; + case 1: + write_mpamvpm1_el2(read_ctx_reg(ctx, CTX_MPAMVPM1_EL2)); + break; + } +} + +/* ----------------------------------------------------- + * The following registers are not added: + * AMEVCNTVOFF0<n>_EL2 + * AMEVCNTVOFF1<n>_EL2 + * ICH_AP0R<n>_EL2 + * ICH_AP1R<n>_EL2 + * ICH_LR<n>_EL2 + * ----------------------------------------------------- + */ +static void el2_sysregs_context_save_common(el2_sysregs_t *ctx) +{ + write_ctx_reg(ctx, CTX_ACTLR_EL2, read_actlr_el2()); + write_ctx_reg(ctx, CTX_AFSR0_EL2, read_afsr0_el2()); + write_ctx_reg(ctx, CTX_AFSR1_EL2, read_afsr1_el2()); + write_ctx_reg(ctx, CTX_AMAIR_EL2, read_amair_el2()); + write_ctx_reg(ctx, CTX_CNTHCTL_EL2, read_cnthctl_el2()); + write_ctx_reg(ctx, CTX_CNTVOFF_EL2, read_cntvoff_el2()); + write_ctx_reg(ctx, CTX_CPTR_EL2, read_cptr_el2()); + if (CTX_INCLUDE_AARCH32_REGS) { + write_ctx_reg(ctx, CTX_DBGVCR32_EL2, read_dbgvcr32_el2()); + } + write_ctx_reg(ctx, CTX_ELR_EL2, read_elr_el2()); + write_ctx_reg(ctx, CTX_ESR_EL2, read_esr_el2()); + write_ctx_reg(ctx, CTX_FAR_EL2, read_far_el2()); + write_ctx_reg(ctx, CTX_HACR_EL2, read_hacr_el2()); + write_ctx_reg(ctx, CTX_HCR_EL2, read_hcr_el2()); + write_ctx_reg(ctx, CTX_HPFAR_EL2, read_hpfar_el2()); + write_ctx_reg(ctx, CTX_HSTR_EL2, read_hstr_el2()); + + /* + * Set the NS bit to be able to access the ICC_SRE_EL2 register + * TODO: remove with root context + */ + u_register_t scr_el3 = read_scr_el3(); + + write_scr_el3(scr_el3 | SCR_NS_BIT); + isb(); + write_ctx_reg(ctx, CTX_ICC_SRE_EL2, read_icc_sre_el2()); + + write_scr_el3(scr_el3); + isb(); + + write_ctx_reg(ctx, CTX_ICH_HCR_EL2, read_ich_hcr_el2()); + write_ctx_reg(ctx, CTX_ICH_VMCR_EL2, read_ich_vmcr_el2()); + write_ctx_reg(ctx, CTX_MAIR_EL2, read_mair_el2()); + write_ctx_reg(ctx, CTX_MDCR_EL2, read_mdcr_el2()); + write_ctx_reg(ctx, CTX_SCTLR_EL2, read_sctlr_el2()); + write_ctx_reg(ctx, CTX_SPSR_EL2, read_spsr_el2()); + write_ctx_reg(ctx, CTX_SP_EL2, read_sp_el2()); + write_ctx_reg(ctx, CTX_TCR_EL2, read_tcr_el2()); + write_ctx_reg(ctx, CTX_TPIDR_EL2, read_tpidr_el2()); + write_ctx_reg(ctx, CTX_TTBR0_EL2, read_ttbr0_el2()); + write_ctx_reg(ctx, CTX_VBAR_EL2, read_vbar_el2()); + write_ctx_reg(ctx, CTX_VMPIDR_EL2, read_vmpidr_el2()); + write_ctx_reg(ctx, CTX_VPIDR_EL2, read_vpidr_el2()); + write_ctx_reg(ctx, CTX_VTCR_EL2, read_vtcr_el2()); + write_ctx_reg(ctx, CTX_VTTBR_EL2, read_vttbr_el2()); +} + +static void el2_sysregs_context_restore_common(el2_sysregs_t *ctx) +{ + write_actlr_el2(read_ctx_reg(ctx, CTX_ACTLR_EL2)); + write_afsr0_el2(read_ctx_reg(ctx, CTX_AFSR0_EL2)); + write_afsr1_el2(read_ctx_reg(ctx, CTX_AFSR1_EL2)); + write_amair_el2(read_ctx_reg(ctx, CTX_AMAIR_EL2)); + write_cnthctl_el2(read_ctx_reg(ctx, CTX_CNTHCTL_EL2)); + write_cntvoff_el2(read_ctx_reg(ctx, CTX_CNTVOFF_EL2)); + write_cptr_el2(read_ctx_reg(ctx, CTX_CPTR_EL2)); + if (CTX_INCLUDE_AARCH32_REGS) { + write_dbgvcr32_el2(read_ctx_reg(ctx, CTX_DBGVCR32_EL2)); + } + write_elr_el2(read_ctx_reg(ctx, CTX_ELR_EL2)); + write_esr_el2(read_ctx_reg(ctx, CTX_ESR_EL2)); + write_far_el2(read_ctx_reg(ctx, CTX_FAR_EL2)); + write_hacr_el2(read_ctx_reg(ctx, CTX_HACR_EL2)); + write_hcr_el2(read_ctx_reg(ctx, CTX_HCR_EL2)); + write_hpfar_el2(read_ctx_reg(ctx, CTX_HPFAR_EL2)); + write_hstr_el2(read_ctx_reg(ctx, CTX_HSTR_EL2)); + + /* + * Set the NS bit to be able to access the ICC_SRE_EL2 register + * TODO: remove with root context + */ + u_register_t scr_el3 = read_scr_el3(); + + write_scr_el3(scr_el3 | SCR_NS_BIT); + isb(); + write_icc_sre_el2(read_ctx_reg(ctx, CTX_ICC_SRE_EL2)); + + write_scr_el3(scr_el3); + isb(); + + write_ich_hcr_el2(read_ctx_reg(ctx, CTX_ICH_HCR_EL2)); + write_ich_vmcr_el2(read_ctx_reg(ctx, CTX_ICH_VMCR_EL2)); + write_mair_el2(read_ctx_reg(ctx, CTX_MAIR_EL2)); + write_mdcr_el2(read_ctx_reg(ctx, CTX_MDCR_EL2)); + write_sctlr_el2(read_ctx_reg(ctx, CTX_SCTLR_EL2)); + write_spsr_el2(read_ctx_reg(ctx, CTX_SPSR_EL2)); + write_sp_el2(read_ctx_reg(ctx, CTX_SP_EL2)); + write_tcr_el2(read_ctx_reg(ctx, CTX_TCR_EL2)); + write_tpidr_el2(read_ctx_reg(ctx, CTX_TPIDR_EL2)); + write_ttbr0_el2(read_ctx_reg(ctx, CTX_TTBR0_EL2)); + write_vbar_el2(read_ctx_reg(ctx, CTX_VBAR_EL2)); + write_vmpidr_el2(read_ctx_reg(ctx, CTX_VMPIDR_EL2)); + write_vpidr_el2(read_ctx_reg(ctx, CTX_VPIDR_EL2)); + write_vtcr_el2(read_ctx_reg(ctx, CTX_VTCR_EL2)); + write_vttbr_el2(read_ctx_reg(ctx, CTX_VTTBR_EL2)); +} + +/******************************************************************************* + * Save EL2 sysreg context + ******************************************************************************/ +void cm_el2_sysregs_context_save(uint32_t security_state) +{ + cpu_context_t *ctx; + el2_sysregs_t *el2_sysregs_ctx; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + el2_sysregs_ctx = get_el2_sysregs_ctx(ctx); + + el2_sysregs_context_save_common(el2_sysregs_ctx); +#if CTX_INCLUDE_MTE_REGS + write_ctx_reg(el2_sysregs_ctx, CTX_TFSR_EL2, read_tfsr_el2()); +#endif + if (is_feat_mpam_supported()) { + el2_sysregs_context_save_mpam(el2_sysregs_ctx); + } + + if (is_feat_fgt_supported()) { + el2_sysregs_context_save_fgt(el2_sysregs_ctx); + } + + if (is_feat_ecv_v2_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_CNTPOFF_EL2, read_cntpoff_el2()); + } + + if (is_feat_vhe_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_CONTEXTIDR_EL2, read_contextidr_el2()); + write_ctx_reg(el2_sysregs_ctx, CTX_TTBR1_EL2, read_ttbr1_el2()); + } + + if (is_feat_ras_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_VDISR_EL2, read_vdisr_el2()); + write_ctx_reg(el2_sysregs_ctx, CTX_VSESR_EL2, read_vsesr_el2()); + } + + if (is_feat_nv2_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_VNCR_EL2, read_vncr_el2()); + } + + if (is_feat_trf_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_TRFCR_EL2, read_trfcr_el2()); + } + + if (is_feat_csv2_2_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_SCXTNUM_EL2, read_scxtnum_el2()); + } + + if (is_feat_hcx_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_HCRX_EL2, read_hcrx_el2()); + } + if (is_feat_tcr2_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_TCR2_EL2, read_tcr2_el2()); + } + if (is_feat_sxpie_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_PIRE0_EL2, read_pire0_el2()); + write_ctx_reg(el2_sysregs_ctx, CTX_PIR_EL2, read_pir_el2()); + } + if (is_feat_s2pie_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_S2PIR_EL2, read_s2pir_el2()); + } + if (is_feat_sxpoe_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_POR_EL2, read_por_el2()); + } + if (is_feat_gcs_supported()) { + write_ctx_reg(el2_sysregs_ctx, CTX_GCSPR_EL2, read_gcspr_el2()); + write_ctx_reg(el2_sysregs_ctx, CTX_GCSCR_EL2, read_gcscr_el2()); + } +} + +/******************************************************************************* + * Restore EL2 sysreg context + ******************************************************************************/ +void cm_el2_sysregs_context_restore(uint32_t security_state) +{ + cpu_context_t *ctx; + el2_sysregs_t *el2_sysregs_ctx; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + el2_sysregs_ctx = get_el2_sysregs_ctx(ctx); + + el2_sysregs_context_restore_common(el2_sysregs_ctx); +#if CTX_INCLUDE_MTE_REGS + write_tfsr_el2(read_ctx_reg(el2_sysregs_ctx, CTX_TFSR_EL2)); +#endif + if (is_feat_mpam_supported()) { + el2_sysregs_context_restore_mpam(el2_sysregs_ctx); + } + + if (is_feat_fgt_supported()) { + el2_sysregs_context_restore_fgt(el2_sysregs_ctx); + } + + if (is_feat_ecv_v2_supported()) { + write_cntpoff_el2(read_ctx_reg(el2_sysregs_ctx, CTX_CNTPOFF_EL2)); + } + + if (is_feat_vhe_supported()) { + write_contextidr_el2(read_ctx_reg(el2_sysregs_ctx, CTX_CONTEXTIDR_EL2)); + write_ttbr1_el2(read_ctx_reg(el2_sysregs_ctx, CTX_TTBR1_EL2)); + } + + if (is_feat_ras_supported()) { + write_vdisr_el2(read_ctx_reg(el2_sysregs_ctx, CTX_VDISR_EL2)); + write_vsesr_el2(read_ctx_reg(el2_sysregs_ctx, CTX_VSESR_EL2)); + } + + if (is_feat_nv2_supported()) { + write_vncr_el2(read_ctx_reg(el2_sysregs_ctx, CTX_VNCR_EL2)); + } + if (is_feat_trf_supported()) { + write_trfcr_el2(read_ctx_reg(el2_sysregs_ctx, CTX_TRFCR_EL2)); + } + + if (is_feat_csv2_2_supported()) { + write_scxtnum_el2(read_ctx_reg(el2_sysregs_ctx, CTX_SCXTNUM_EL2)); + } + + if (is_feat_hcx_supported()) { + write_hcrx_el2(read_ctx_reg(el2_sysregs_ctx, CTX_HCRX_EL2)); + } + if (is_feat_tcr2_supported()) { + write_tcr2_el2(read_ctx_reg(el2_sysregs_ctx, CTX_TCR2_EL2)); + } + if (is_feat_sxpie_supported()) { + write_pire0_el2(read_ctx_reg(el2_sysregs_ctx, CTX_PIRE0_EL2)); + write_pir_el2(read_ctx_reg(el2_sysregs_ctx, CTX_PIR_EL2)); + } + if (is_feat_s2pie_supported()) { + write_s2pir_el2(read_ctx_reg(el2_sysregs_ctx, CTX_S2PIR_EL2)); + } + if (is_feat_sxpoe_supported()) { + write_por_el2(read_ctx_reg(el2_sysregs_ctx, CTX_POR_EL2)); + } + if (is_feat_gcs_supported()) { + write_gcscr_el2(read_ctx_reg(el2_sysregs_ctx, CTX_GCSCR_EL2)); + write_gcspr_el2(read_ctx_reg(el2_sysregs_ctx, CTX_GCSPR_EL2)); + } +} +#endif /* CTX_INCLUDE_EL2_REGS */ + +/******************************************************************************* + * This function is used to exit to Non-secure world. If CTX_INCLUDE_EL2_REGS + * is enabled, it restores EL1 and EL2 sysreg contexts instead of directly + * updating EL1 and EL2 registers. Otherwise, it calls the generic + * cm_prepare_el3_exit function. + ******************************************************************************/ +void cm_prepare_el3_exit_ns(void) +{ +#if CTX_INCLUDE_EL2_REGS +#if ENABLE_ASSERTIONS + cpu_context_t *ctx = cm_get_context(NON_SECURE); + assert(ctx != NULL); + + /* Assert that EL2 is used. */ + u_register_t scr_el3 = read_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3); + assert(((scr_el3 & SCR_HCE_BIT) != 0UL) && + (el_implemented(2U) != EL_IMPL_NONE)); +#endif /* ENABLE_ASSERTIONS */ + + /* Restore EL2 and EL1 sysreg contexts */ + cm_el2_sysregs_context_restore(NON_SECURE); + cm_el1_sysregs_context_restore(NON_SECURE); + cm_set_next_eret_context(NON_SECURE); +#else + cm_prepare_el3_exit(NON_SECURE); +#endif /* CTX_INCLUDE_EL2_REGS */ +} + +/******************************************************************************* + * The next four functions are used by runtime services to save and restore + * EL1 context on the 'cpu_context' structure for the specified security + * state. + ******************************************************************************/ +void cm_el1_sysregs_context_save(uint32_t security_state) +{ + cpu_context_t *ctx; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + el1_sysregs_context_save(get_el1_sysregs_ctx(ctx)); + +#if IMAGE_BL31 + if (security_state == SECURE) + PUBLISH_EVENT(cm_exited_secure_world); + else + PUBLISH_EVENT(cm_exited_normal_world); +#endif +} + +void cm_el1_sysregs_context_restore(uint32_t security_state) +{ + cpu_context_t *ctx; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + el1_sysregs_context_restore(get_el1_sysregs_ctx(ctx)); + +#if IMAGE_BL31 + if (security_state == SECURE) + PUBLISH_EVENT(cm_entering_secure_world); + else + PUBLISH_EVENT(cm_entering_normal_world); +#endif +} + +/******************************************************************************* + * This function populates ELR_EL3 member of 'cpu_context' pertaining to the + * given security state with the given entrypoint + ******************************************************************************/ +void cm_set_elr_el3(uint32_t security_state, uintptr_t entrypoint) +{ + cpu_context_t *ctx; + el3_state_t *state; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + /* Populate EL3 state so that ERET jumps to the correct entry */ + state = get_el3state_ctx(ctx); + write_ctx_reg(state, CTX_ELR_EL3, entrypoint); +} + +/******************************************************************************* + * This function populates ELR_EL3 and SPSR_EL3 members of 'cpu_context' + * pertaining to the given security state + ******************************************************************************/ +void cm_set_elr_spsr_el3(uint32_t security_state, + uintptr_t entrypoint, uint32_t spsr) +{ + cpu_context_t *ctx; + el3_state_t *state; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + /* Populate EL3 state so that ERET jumps to the correct entry */ + state = get_el3state_ctx(ctx); + write_ctx_reg(state, CTX_ELR_EL3, entrypoint); + write_ctx_reg(state, CTX_SPSR_EL3, spsr); +} + +/******************************************************************************* + * This function updates a single bit in the SCR_EL3 member of the 'cpu_context' + * pertaining to the given security state using the value and bit position + * specified in the parameters. It preserves all other bits. + ******************************************************************************/ +void cm_write_scr_el3_bit(uint32_t security_state, + uint32_t bit_pos, + uint32_t value) +{ + cpu_context_t *ctx; + el3_state_t *state; + u_register_t scr_el3; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + /* Ensure that the bit position is a valid one */ + assert(((1UL << bit_pos) & SCR_VALID_BIT_MASK) != 0U); + + /* Ensure that the 'value' is only a bit wide */ + assert(value <= 1U); + + /* + * Get the SCR_EL3 value from the cpu context, clear the desired bit + * and set it to its new value. + */ + state = get_el3state_ctx(ctx); + scr_el3 = read_ctx_reg(state, CTX_SCR_EL3); + scr_el3 &= ~(1UL << bit_pos); + scr_el3 |= (u_register_t)value << bit_pos; + write_ctx_reg(state, CTX_SCR_EL3, scr_el3); +} + +/******************************************************************************* + * This function retrieves SCR_EL3 member of 'cpu_context' pertaining to the + * given security state. + ******************************************************************************/ +u_register_t cm_get_scr_el3(uint32_t security_state) +{ + cpu_context_t *ctx; + el3_state_t *state; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + /* Populate EL3 state so that ERET jumps to the correct entry */ + state = get_el3state_ctx(ctx); + return read_ctx_reg(state, CTX_SCR_EL3); +} + +/******************************************************************************* + * This function is used to program the context that's used for exception + * return. This initializes the SP_EL3 to a pointer to a 'cpu_context' set for + * the required security state + ******************************************************************************/ +void cm_set_next_eret_context(uint32_t security_state) +{ + cpu_context_t *ctx; + + ctx = cm_get_context(security_state); + assert(ctx != NULL); + + cm_set_next_context(ctx); +} diff --git a/lib/el3_runtime/aarch64/cpu_data.S b/lib/el3_runtime/aarch64/cpu_data.S new file mode 100644 index 0000000..313f882 --- /dev/null +++ b/lib/el3_runtime/aarch64/cpu_data.S @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2014-2020, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <lib/el3_runtime/cpu_data.h> + +.globl init_cpu_data_ptr +.globl _cpu_data_by_index + +/* ----------------------------------------------------------------- + * void init_cpu_data_ptr(void) + * + * Initialise the TPIDR_EL3 register to refer to the cpu_data_t + * for the calling CPU. This must be called before cm_get_cpu_data() + * + * This can be called without a valid stack. It assumes that + * plat_my_core_pos() does not clobber register x10. + * clobbers: x0, x1, x10 + * ----------------------------------------------------------------- + */ +func init_cpu_data_ptr + mov x10, x30 + bl plat_my_core_pos + bl _cpu_data_by_index + msr tpidr_el3, x0 + ret x10 +endfunc init_cpu_data_ptr + +/* ----------------------------------------------------------------- + * cpu_data_t *_cpu_data_by_index(uint32_t cpu_index) + * + * Return the cpu_data structure for the CPU with given linear index + * + * This can be called without a valid stack. + * clobbers: x0, x1 + * ----------------------------------------------------------------- + */ +func _cpu_data_by_index + mov_imm x1, CPU_DATA_SIZE + mul x0, x0, x1 + adrp x1, percpu_data + add x1, x1, :lo12:percpu_data + add x0, x0, x1 + ret +endfunc _cpu_data_by_index diff --git a/lib/el3_runtime/cpu_data_array.c b/lib/el3_runtime/cpu_data_array.c new file mode 100644 index 0000000..2056182 --- /dev/null +++ b/lib/el3_runtime/cpu_data_array.c @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2014-2016, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <platform_def.h> + +#include <lib/cassert.h> +#include <lib/el3_runtime/cpu_data.h> + +/* The per_cpu_ptr_cache_t space allocation */ +cpu_data_t percpu_data[PLATFORM_CORE_COUNT]; diff --git a/lib/extensions/amu/aarch32/amu.c b/lib/extensions/amu/aarch32/amu.c new file mode 100644 index 0000000..351a552 --- /dev/null +++ b/lib/extensions/amu/aarch32/amu.c @@ -0,0 +1,395 @@ +/* + * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <cdefs.h> +#include <stdbool.h> + +#include "../amu_private.h" +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/el3_runtime/pubsub_events.h> +#include <lib/extensions/amu.h> + +#include <plat/common/platform.h> + +struct amu_ctx { + uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; +#endif + + uint16_t group0_enable; +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint16_t group1_enable; +#endif +}; + +static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; + +CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, + amu_ctx_group0_enable_cannot_represent_all_group0_counters); + +#if ENABLE_AMU_AUXILIARY_COUNTERS +CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, + amu_ctx_group1_enable_cannot_represent_all_group1_counters); +#endif + +static inline __unused void write_hcptr_tam(uint32_t value) +{ + write_hcptr((read_hcptr() & ~TAM_BIT) | + ((value << TAM_SHIFT) & TAM_BIT)); +} + +static inline __unused void write_amcr_cg1rz(uint32_t value) +{ + write_amcr((read_amcr() & ~AMCR_CG1RZ_BIT) | + ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); +} + +static inline __unused uint32_t read_amcfgr_ncg(void) +{ + return (read_amcfgr() >> AMCFGR_NCG_SHIFT) & + AMCFGR_NCG_MASK; +} + +static inline __unused uint32_t read_amcgcr_cg0nc(void) +{ + return (read_amcgcr() >> AMCGCR_CG0NC_SHIFT) & + AMCGCR_CG0NC_MASK; +} + +static inline __unused uint32_t read_amcgcr_cg1nc(void) +{ + return (read_amcgcr() >> AMCGCR_CG1NC_SHIFT) & + AMCGCR_CG1NC_MASK; +} + +static inline __unused uint32_t read_amcntenset0_px(void) +{ + return (read_amcntenset0() >> AMCNTENSET0_Pn_SHIFT) & + AMCNTENSET0_Pn_MASK; +} + +static inline __unused uint32_t read_amcntenset1_px(void) +{ + return (read_amcntenset1() >> AMCNTENSET1_Pn_SHIFT) & + AMCNTENSET1_Pn_MASK; +} + +static inline __unused void write_amcntenset0_px(uint32_t px) +{ + uint32_t value = read_amcntenset0(); + + value &= ~AMCNTENSET0_Pn_MASK; + value |= (px << AMCNTENSET0_Pn_SHIFT) & + AMCNTENSET0_Pn_MASK; + + write_amcntenset0(value); +} + +static inline __unused void write_amcntenset1_px(uint32_t px) +{ + uint32_t value = read_amcntenset1(); + + value &= ~AMCNTENSET1_Pn_MASK; + value |= (px << AMCNTENSET1_Pn_SHIFT) & + AMCNTENSET1_Pn_MASK; + + write_amcntenset1(value); +} + +static inline __unused void write_amcntenclr0_px(uint32_t px) +{ + uint32_t value = read_amcntenclr0(); + + value &= ~AMCNTENCLR0_Pn_MASK; + value |= (px << AMCNTENCLR0_Pn_SHIFT) & AMCNTENCLR0_Pn_MASK; + + write_amcntenclr0(value); +} + +static inline __unused void write_amcntenclr1_px(uint32_t px) +{ + uint32_t value = read_amcntenclr1(); + + value &= ~AMCNTENCLR1_Pn_MASK; + value |= (px << AMCNTENCLR1_Pn_SHIFT) & AMCNTENCLR1_Pn_MASK; + + write_amcntenclr1(value); +} + +#if ENABLE_AMU_AUXILIARY_COUNTERS +static __unused bool amu_group1_supported(void) +{ + return read_amcfgr_ncg() > 0U; +} +#endif + +/* + * Enable counters. This function is meant to be invoked by the context + * management library before exiting from EL3. + */ +void amu_enable(bool el2_unused) +{ + uint32_t amcfgr_ncg; /* Number of counter groups */ + uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ + + uint32_t amcntenset0_px = 0x0; /* Group 0 enable mask */ + uint32_t amcntenset1_px = 0x0; /* Group 1 enable mask */ + + if (el2_unused) { + /* + * HCPTR.TAM: Set to zero so any accesses to the Activity + * Monitor registers do not trap to EL2. + */ + write_hcptr_tam(0U); + } + + /* + * Retrieve the number of architected counters. All of these counters + * are enabled by default. + */ + + amcgcr_cg0nc = read_amcgcr_cg0nc(); + amcntenset0_px = (UINT32_C(1) << (amcgcr_cg0nc)) - 1U; + + assert(amcgcr_cg0nc <= AMU_AMCGCR_CG0NC_MAX); + + /* + * The platform may opt to enable specific auxiliary counters. This can + * be done via the common FCONF getter, or via the platform-implemented + * function. + */ + +#if ENABLE_AMU_AUXILIARY_COUNTERS + const struct amu_topology *topology; + +#if ENABLE_AMU_FCONF + topology = FCONF_GET_PROPERTY(amu, config, topology); +#else + topology = plat_amu_topology(); +#endif /* ENABLE_AMU_FCONF */ + + if (topology != NULL) { + unsigned int core_pos = plat_my_core_pos(); + + amcntenset1_el0_px = topology->cores[core_pos].enable; + } else { + ERROR("AMU: failed to generate AMU topology\n"); + } +#endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ + + /* + * Enable the requested counters. + */ + + write_amcntenset0_px(amcntenset0_px); + + amcfgr_ncg = read_amcfgr_ncg(); + if (amcfgr_ncg > 0U) { + write_amcntenset1_px(amcntenset1_px); + +#if !ENABLE_AMU_AUXILIARY_COUNTERS + VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); +#endif + } + + /* Bail out if FEAT_AMUv1p1 features are not present. */ + if (!is_feat_amuv1p1_supported()) { + return; + } + +#if AMU_RESTRICT_COUNTERS + /* + * FEAT_AMUv1p1 adds a register field to restrict access to group 1 + * counters at all but the highest implemented EL. This is controlled + * with the AMU_RESTRICT_COUNTERS compile time flag, when set, system + * register reads at lower ELs return zero. Reads from the memory + * mapped view are unaffected. + */ + VERBOSE("AMU group 1 counter access restricted.\n"); + write_amcr_cg1rz(1U); +#else + write_amcr_cg1rz(0U); +#endif +} + +/* Read the group 0 counter identified by the given `idx`. */ +static uint64_t amu_group0_cnt_read(unsigned int idx) +{ + assert(is_feat_amu_supported()); + assert(idx < read_amcgcr_cg0nc()); + + return amu_group0_cnt_read_internal(idx); +} + +/* Write the group 0 counter identified by the given `idx` with `val` */ +static void amu_group0_cnt_write(unsigned int idx, uint64_t val) +{ + assert(is_feat_amu_supported()); + assert(idx < read_amcgcr_cg0nc()); + + amu_group0_cnt_write_internal(idx, val); + isb(); +} + +#if ENABLE_AMU_AUXILIARY_COUNTERS +/* Read the group 1 counter identified by the given `idx` */ +static uint64_t amu_group1_cnt_read(unsigned int idx) +{ + assert(is_feat_amu_supported()); + assert(amu_group1_supported()); + assert(idx < read_amcgcr_cg1nc()); + + return amu_group1_cnt_read_internal(idx); +} + +/* Write the group 1 counter identified by the given `idx` with `val` */ +static void amu_group1_cnt_write(unsigned int idx, uint64_t val) +{ + assert(is_feat_amu_supported()); + assert(amu_group1_supported()); + assert(idx < read_amcgcr_cg1nc()); + + amu_group1_cnt_write_internal(idx, val); + isb(); +} +#endif + +static void *amu_context_save(const void *arg) +{ + uint32_t i; + + unsigned int core_pos; + struct amu_ctx *ctx; + + uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ + +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint32_t amcfgr_ncg; /* Number of counter groups */ + uint32_t amcgcr_cg1nc; /* Number of group 1 counters */ +#endif + + if (!is_feat_amu_supported()) { + return (void *)0; + } + + core_pos = plat_my_core_pos(); + ctx = &amu_ctxs_[core_pos]; + + amcgcr_cg0nc = read_amcgcr_cg0nc(); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + amcfgr_ncg = read_amcfgr_ncg(); + amcgcr_cg1nc = (amcfgr_ncg > 0U) ? read_amcgcr_cg1nc() : 0U; +#endif + + /* + * Disable all AMU counters. + */ + + ctx->group0_enable = read_amcntenset0_px(); + write_amcntenclr0_px(ctx->group0_enable); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + if (amcfgr_ncg > 0U) { + ctx->group1_enable = read_amcntenset1_px(); + write_amcntenclr1_px(ctx->group1_enable); + } +#endif + + /* + * Save the counters to the local context. + */ + + isb(); /* Ensure counters have been stopped */ + + for (i = 0U; i < amcgcr_cg0nc; i++) { + ctx->group0_cnts[i] = amu_group0_cnt_read(i); + } + +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U; i < amcgcr_cg1nc; i++) { + ctx->group1_cnts[i] = amu_group1_cnt_read(i); + } +#endif + + return (void *)0; +} + +static void *amu_context_restore(const void *arg) +{ + uint32_t i; + + unsigned int core_pos; + struct amu_ctx *ctx; + + uint32_t amcfgr_ncg; /* Number of counter groups */ + uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ + +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint32_t amcgcr_cg1nc; /* Number of group 1 counters */ +#endif + + if (!is_feat_amu_supported()) { + return (void *)0; + } + + core_pos = plat_my_core_pos(); + ctx = &amu_ctxs_[core_pos]; + + amcfgr_ncg = read_amcfgr_ncg(); + amcgcr_cg0nc = read_amcgcr_cg0nc(); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + amcgcr_cg1nc = (amcfgr_ncg > 0U) ? read_amcgcr_cg1nc() : 0U; +#endif + + /* + * Sanity check that all counters were disabled when the context was + * previously saved. + */ + + assert(read_amcntenset0_px() == 0U); + + if (amcfgr_ncg > 0U) { + assert(read_amcntenset1_px() == 0U); + } + + /* + * Restore the counter values from the local context. + */ + + for (i = 0U; i < amcgcr_cg0nc; i++) { + amu_group0_cnt_write(i, ctx->group0_cnts[i]); + } + +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U; i < amcgcr_cg1nc; i++) { + amu_group1_cnt_write(i, ctx->group1_cnts[i]); + } +#endif + + /* + * Re-enable counters that were disabled during context save. + */ + + write_amcntenset0_px(ctx->group0_enable); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + if (amcfgr_ncg > 0U) { + write_amcntenset1_px(ctx->group1_enable); + } +#endif + + return (void *)0; +} + +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); diff --git a/lib/extensions/amu/aarch32/amu_helpers.S b/lib/extensions/amu/aarch32/amu_helpers.S new file mode 100644 index 0000000..7090b2d --- /dev/null +++ b/lib/extensions/amu/aarch32/amu_helpers.S @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <assert_macros.S> +#include <asm_macros.S> + + .globl amu_group0_cnt_read_internal + .globl amu_group0_cnt_write_internal + .globl amu_group1_cnt_read_internal + .globl amu_group1_cnt_write_internal + .globl amu_group1_set_evtype_internal + +/* + * uint64_t amu_group0_cnt_read_internal(int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `r0` and `r1`. + */ +func amu_group0_cnt_read_internal +#if ENABLE_ASSERTIONS + /* `idx` should be between [0, 3] */ + mov r1, r0 + lsr r1, r1, #2 + cmp r1, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of ldcopr16/bx lr instruction pair + * in the table below. + */ + adr r1, 1f + lsl r0, r0, #3 /* each ldcopr16/bx lr sequence is 8 bytes */ + add r1, r1, r0 + bx r1 +1: + ldcopr16 r0, r1, AMEVCNTR00 /* index 0 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR01 /* index 1 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR02 /* index 2 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR03 /* index 3 */ + bx lr +endfunc amu_group0_cnt_read_internal + +/* + * void amu_group0_cnt_write_internal(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + * `idx` is passed in `r0` and `val` is passed in `r2` and `r3`. + * `r1` is used as a scratch register. + */ +func amu_group0_cnt_write_internal +#if ENABLE_ASSERTIONS + /* `idx` should be between [0, 3] */ + mov r1, r0 + lsr r1, r1, #2 + cmp r1, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of stcopr16/bx lr instruction pair + * in the table below. + */ + adr r1, 1f + lsl r0, r0, #3 /* each stcopr16/bx lr sequence is 8 bytes */ + add r1, r1, r0 + bx r1 + +1: + stcopr16 r2, r3, AMEVCNTR00 /* index 0 */ + bx lr + stcopr16 r2, r3, AMEVCNTR01 /* index 1 */ + bx lr + stcopr16 r2, r3, AMEVCNTR02 /* index 2 */ + bx lr + stcopr16 r2, r3, AMEVCNTR03 /* index 3 */ + bx lr +endfunc amu_group0_cnt_write_internal + +#if ENABLE_AMU_AUXILIARY_COUNTERS +/* + * uint64_t amu_group1_cnt_read_internal(int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `r0` and `r1`. + */ +func amu_group1_cnt_read_internal +#if ENABLE_ASSERTIONS + /* `idx` should be between [0, 15] */ + mov r1, r0 + lsr r1, r1, #4 + cmp r1, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of ldcopr16/bx lr instruction pair + * in the table below. + */ + adr r1, 1f + lsl r0, r0, #3 /* each ldcopr16/bx lr sequence is 8 bytes */ + add r1, r1, r0 + bx r1 + +1: + ldcopr16 r0, r1, AMEVCNTR10 /* index 0 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR11 /* index 1 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR12 /* index 2 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR13 /* index 3 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR14 /* index 4 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR15 /* index 5 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR16 /* index 6 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR17 /* index 7 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR18 /* index 8 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR19 /* index 9 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1A /* index 10 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1B /* index 11 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1C /* index 12 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1D /* index 13 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1E /* index 14 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1F /* index 15 */ + bx lr +endfunc amu_group1_cnt_read_internal + +/* + * void amu_group1_cnt_write_internal(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + * `idx` is passed in `r0` and `val` is passed in `r2` and `r3`. + * `r1` is used as a scratch register. + */ +func amu_group1_cnt_write_internal +#if ENABLE_ASSERTIONS + /* `idx` should be between [0, 15] */ + mov r1, r0 + lsr r1, r1, #4 + cmp r1, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of ldcopr16/bx lr instruction pair + * in the table below. + */ + adr r1, 1f + lsl r0, r0, #3 /* each stcopr16/bx lr sequence is 8 bytes */ + add r1, r1, r0 + bx r1 + +1: + stcopr16 r2, r3, AMEVCNTR10 /* index 0 */ + bx lr + stcopr16 r2, r3, AMEVCNTR11 /* index 1 */ + bx lr + stcopr16 r2, r3, AMEVCNTR12 /* index 2 */ + bx lr + stcopr16 r2, r3, AMEVCNTR13 /* index 3 */ + bx lr + stcopr16 r2, r3, AMEVCNTR14 /* index 4 */ + bx lr + stcopr16 r2, r3, AMEVCNTR15 /* index 5 */ + bx lr + stcopr16 r2, r3, AMEVCNTR16 /* index 6 */ + bx lr + stcopr16 r2, r3, AMEVCNTR17 /* index 7 */ + bx lr + stcopr16 r2, r3, AMEVCNTR18 /* index 8 */ + bx lr + stcopr16 r2, r3, AMEVCNTR19 /* index 9 */ + bx lr + stcopr16 r2, r3, AMEVCNTR1A /* index 10 */ + bx lr + stcopr16 r2, r3, AMEVCNTR1B /* index 11 */ + bx lr + stcopr16 r2, r3, AMEVCNTR1C /* index 12 */ + bx lr + stcopr16 r2, r3, AMEVCNTR1D /* index 13 */ + bx lr + stcopr16 r2, r3, AMEVCNTR1E /* index 14 */ + bx lr + stcopr16 r2, r3, AMEVCNTR1F /* index 15 */ + bx lr +endfunc amu_group1_cnt_write_internal + +/* + * void amu_group1_set_evtype_internal(int idx, unsigned int val); + * + * Program the AMU event type register indexed by `idx` + * with the value `val`. + */ +func amu_group1_set_evtype_internal +#if ENABLE_ASSERTIONS + /* `idx` should be between [0, 15] */ + mov r2, r0 + lsr r2, r2, #4 + cmp r2, #0 + ASM_ASSERT(eq) + + /* val should be between [0, 65535] */ + mov r2, r1 + lsr r2, r2, #16 + cmp r2, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of stcopr/bx lr instruction pair + * in the table below. + */ + adr r2, 1f + lsl r0, r0, #3 /* each stcopr/bx lr sequence is 8 bytes */ + add r2, r2, r0 + bx r2 + +1: + stcopr r1, AMEVTYPER10 /* index 0 */ + bx lr + stcopr r1, AMEVTYPER11 /* index 1 */ + bx lr + stcopr r1, AMEVTYPER12 /* index 2 */ + bx lr + stcopr r1, AMEVTYPER13 /* index 3 */ + bx lr + stcopr r1, AMEVTYPER14 /* index 4 */ + bx lr + stcopr r1, AMEVTYPER15 /* index 5 */ + bx lr + stcopr r1, AMEVTYPER16 /* index 6 */ + bx lr + stcopr r1, AMEVTYPER17 /* index 7 */ + bx lr + stcopr r1, AMEVTYPER18 /* index 8 */ + bx lr + stcopr r1, AMEVTYPER19 /* index 9 */ + bx lr + stcopr r1, AMEVTYPER1A /* index 10 */ + bx lr + stcopr r1, AMEVTYPER1B /* index 11 */ + bx lr + stcopr r1, AMEVTYPER1C /* index 12 */ + bx lr + stcopr r1, AMEVTYPER1D /* index 13 */ + bx lr + stcopr r1, AMEVTYPER1E /* index 14 */ + bx lr + stcopr r1, AMEVTYPER1F /* index 15 */ + bx lr +endfunc amu_group1_set_evtype_internal +#endif diff --git a/lib/extensions/amu/aarch64/amu.c b/lib/extensions/amu/aarch64/amu.c new file mode 100644 index 0000000..cb9a0f2 --- /dev/null +++ b/lib/extensions/amu/aarch64/amu.c @@ -0,0 +1,596 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <cdefs.h> +#include <inttypes.h> +#include <stdbool.h> +#include <stdint.h> + +#include "../amu_private.h" +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/el3_runtime/pubsub_events.h> +#include <lib/extensions/amu.h> + +#include <plat/common/platform.h> + +#if ENABLE_AMU_FCONF +# include <lib/fconf/fconf.h> +# include <lib/fconf/fconf_amu_getter.h> +#endif + +#if ENABLE_MPMM +# include <lib/mpmm/mpmm.h> +#endif + +struct amu_ctx { + uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; +#endif + + /* Architected event counter 1 does not have an offset register */ + uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U]; +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS]; +#endif + + uint16_t group0_enable; +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint16_t group1_enable; +#endif +}; + +static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; + +CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, + amu_ctx_group0_enable_cannot_represent_all_group0_counters); + +#if ENABLE_AMU_AUXILIARY_COUNTERS +CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, + amu_ctx_group1_enable_cannot_represent_all_group1_counters); +#endif + +static inline __unused uint64_t read_hcr_el2_amvoffen(void) +{ + return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >> + HCR_AMVOFFEN_SHIFT; +} + +static inline __unused void write_cptr_el2_tam(uint64_t value) +{ + write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) | + ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT)); +} + +static inline __unused void ctx_write_scr_el3_amvoffen(cpu_context_t *ctx, uint64_t amvoffen) +{ + uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3); + + value &= ~SCR_AMVOFFEN_BIT; + value |= (amvoffen << SCR_AMVOFFEN_SHIFT) & SCR_AMVOFFEN_BIT; + + write_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3, value); +} + +static inline __unused void write_hcr_el2_amvoffen(uint64_t value) +{ + write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) | + ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT)); +} + +static inline __unused void write_amcr_el0_cg1rz(uint64_t value) +{ + write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) | + ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); +} + +static inline __unused uint64_t read_amcfgr_el0_ncg(void) +{ + return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) & + AMCFGR_EL0_NCG_MASK; +} + +static inline __unused uint64_t read_amcgcr_el0_cg0nc(void) +{ + return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) & + AMCGCR_EL0_CG0NC_MASK; +} + +static inline __unused uint64_t read_amcg1idr_el0_voff(void) +{ + return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & + AMCG1IDR_VOFF_MASK; +} + +static inline __unused uint64_t read_amcgcr_el0_cg1nc(void) +{ + return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & + AMCGCR_EL0_CG1NC_MASK; +} + +static inline __unused uint64_t read_amcntenset0_el0_px(void) +{ + return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) & + AMCNTENSET0_EL0_Pn_MASK; +} + +static inline __unused uint64_t read_amcntenset1_el0_px(void) +{ + return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) & + AMCNTENSET1_EL0_Pn_MASK; +} + +static inline __unused void write_amcntenset0_el0_px(uint64_t px) +{ + uint64_t value = read_amcntenset0_el0(); + + value &= ~AMCNTENSET0_EL0_Pn_MASK; + value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK; + + write_amcntenset0_el0(value); +} + +static inline __unused void write_amcntenset1_el0_px(uint64_t px) +{ + uint64_t value = read_amcntenset1_el0(); + + value &= ~AMCNTENSET1_EL0_Pn_MASK; + value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK; + + write_amcntenset1_el0(value); +} + +static inline __unused void write_amcntenclr0_el0_px(uint64_t px) +{ + uint64_t value = read_amcntenclr0_el0(); + + value &= ~AMCNTENCLR0_EL0_Pn_MASK; + value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK; + + write_amcntenclr0_el0(value); +} + +static inline __unused void write_amcntenclr1_el0_px(uint64_t px) +{ + uint64_t value = read_amcntenclr1_el0(); + + value &= ~AMCNTENCLR1_EL0_Pn_MASK; + value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK; + + write_amcntenclr1_el0(value); +} + +#if ENABLE_AMU_AUXILIARY_COUNTERS +static __unused bool amu_group1_supported(void) +{ + return read_amcfgr_el0_ncg() > 0U; +} +#endif + +/* + * Enable counters. This function is meant to be invoked by the context + * management library before exiting from EL3. + */ +void amu_enable(cpu_context_t *ctx) +{ + /* Initialize FEAT_AMUv1p1 features if present. */ + if (is_feat_amuv1p1_supported()) { + /* + * Set SCR_EL3.AMVOFFEN to one so that accesses to virtual + * offset registers at EL2 do not trap to EL3 + */ + ctx_write_scr_el3_amvoffen(ctx, 1U); + } +} + +void amu_enable_per_world(per_world_context_t *per_world_ctx) +{ + /* + * Set CPTR_EL3.TAM to zero so that any accesses to the Activity Monitor + * registers do not trap to EL3. + */ + uint64_t cptr_el3 = per_world_ctx->ctx_cptr_el3; + + cptr_el3 &= ~TAM_BIT; + per_world_ctx->ctx_cptr_el3 = cptr_el3; +} + +void amu_init_el3(void) +{ + uint64_t group0_impl_ctr = read_amcgcr_el0_cg0nc(); + uint64_t group0_en_mask = (1 << (group0_impl_ctr)) - 1U; + uint64_t num_ctr_groups = read_amcfgr_el0_ncg(); + + /* Enable all architected counters by default */ + write_amcntenset0_el0_px(group0_en_mask); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + if (num_ctr_groups > 0U) { + uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */ + const struct amu_topology *topology; + + /* + * The platform may opt to enable specific auxiliary counters. + * This can be done via the common FCONF getter, or via the + * platform-implemented function. + */ +#if ENABLE_AMU_FCONF + topology = FCONF_GET_PROPERTY(amu, config, topology); +#else + topology = plat_amu_topology(); +#endif /* ENABLE_AMU_FCONF */ + + if (topology != NULL) { + unsigned int core_pos = plat_my_core_pos(); + + amcntenset1_el0_px = topology->cores[core_pos].enable; + } else { + ERROR("AMU: failed to generate AMU topology\n"); + } + + write_amcntenset1_el0_px(amcntenset1_el0_px); + } +#else /* ENABLE_AMU_AUXILIARY_COUNTERS */ + if (num_ctr_groups > 0U) { + VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); + } +#endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ + + if (is_feat_amuv1p1_supported()) { +#if AMU_RESTRICT_COUNTERS + /* + * FEAT_AMUv1p1 adds a register field to restrict access to + * group 1 counters at all but the highest implemented EL. This + * is controlled with the `AMU_RESTRICT_COUNTERS` compile time + * flag, when set, system register reads at lower ELs return + * zero. Reads from the memory mapped view are unaffected. + */ + VERBOSE("AMU group 1 counter access restricted.\n"); + write_amcr_el0_cg1rz(1U); +#else + write_amcr_el0_cg1rz(0U); +#endif + } + +#if ENABLE_MPMM + mpmm_enable(); +#endif +} + +void amu_init_el2_unused(void) +{ + /* + * CPTR_EL2.TAM: Set to zero so any accesses to the Activity Monitor + * registers do not trap to EL2. + */ + write_cptr_el2_tam(0U); + + /* Initialize FEAT_AMUv1p1 features if present. */ + if (is_feat_amuv1p1_supported()) { + /* Make sure virtual offsets are disabled if EL2 not used. */ + write_hcr_el2_amvoffen(0U); + } +} + +/* Read the group 0 counter identified by the given `idx`. */ +static uint64_t amu_group0_cnt_read(unsigned int idx) +{ + assert(is_feat_amu_supported()); + assert(idx < read_amcgcr_el0_cg0nc()); + + return amu_group0_cnt_read_internal(idx); +} + +/* Write the group 0 counter identified by the given `idx` with `val` */ +static void amu_group0_cnt_write(unsigned int idx, uint64_t val) +{ + assert(is_feat_amu_supported()); + assert(idx < read_amcgcr_el0_cg0nc()); + + amu_group0_cnt_write_internal(idx, val); + isb(); +} + +/* + * Unlike with auxiliary counters, we cannot detect at runtime whether an + * architected counter supports a virtual offset. These are instead fixed + * according to FEAT_AMUv1p1, but this switch will need to be updated if later + * revisions of FEAT_AMU add additional architected counters. + */ +static bool amu_group0_voffset_supported(uint64_t idx) +{ + switch (idx) { + case 0U: + case 2U: + case 3U: + return true; + + case 1U: + return false; + + default: + ERROR("AMU: can't set up virtual offset for unknown " + "architected counter %" PRIu64 "!\n", idx); + + panic(); + } +} + +/* + * Read the group 0 offset register for a given index. Index must be 0, 2, + * or 3, the register for 1 does not exist. + * + * Using this function requires FEAT_AMUv1p1 support. + */ +static uint64_t amu_group0_voffset_read(unsigned int idx) +{ + assert(is_feat_amuv1p1_supported()); + assert(idx < read_amcgcr_el0_cg0nc()); + assert(idx != 1U); + + return amu_group0_voffset_read_internal(idx); +} + +/* + * Write the group 0 offset register for a given index. Index must be 0, 2, or + * 3, the register for 1 does not exist. + * + * Using this function requires FEAT_AMUv1p1 support. + */ +static void amu_group0_voffset_write(unsigned int idx, uint64_t val) +{ + assert(is_feat_amuv1p1_supported()); + assert(idx < read_amcgcr_el0_cg0nc()); + assert(idx != 1U); + + amu_group0_voffset_write_internal(idx, val); + isb(); +} + +#if ENABLE_AMU_AUXILIARY_COUNTERS +/* Read the group 1 counter identified by the given `idx` */ +static uint64_t amu_group1_cnt_read(unsigned int idx) +{ + assert(is_feat_amu_supported()); + assert(amu_group1_supported()); + assert(idx < read_amcgcr_el0_cg1nc()); + + return amu_group1_cnt_read_internal(idx); +} + +/* Write the group 1 counter identified by the given `idx` with `val` */ +static void amu_group1_cnt_write(unsigned int idx, uint64_t val) +{ + assert(is_feat_amu_supported()); + assert(amu_group1_supported()); + assert(idx < read_amcgcr_el0_cg1nc()); + + amu_group1_cnt_write_internal(idx, val); + isb(); +} + +/* + * Read the group 1 offset register for a given index. + * + * Using this function requires FEAT_AMUv1p1 support. + */ +static uint64_t amu_group1_voffset_read(unsigned int idx) +{ + assert(is_feat_amuv1p1_supported()); + assert(amu_group1_supported()); + assert(idx < read_amcgcr_el0_cg1nc()); + assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); + + return amu_group1_voffset_read_internal(idx); +} + +/* + * Write the group 1 offset register for a given index. + * + * Using this function requires FEAT_AMUv1p1 support. + */ +static void amu_group1_voffset_write(unsigned int idx, uint64_t val) +{ + assert(is_feat_amuv1p1_supported()); + assert(amu_group1_supported()); + assert(idx < read_amcgcr_el0_cg1nc()); + assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); + + amu_group1_voffset_write_internal(idx, val); + isb(); +} +#endif + +static void *amu_context_save(const void *arg) +{ + uint64_t i, j; + + unsigned int core_pos; + struct amu_ctx *ctx; + + uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */ + uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ + +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ + uint64_t amcfgr_el0_ncg; /* Number of counter groups */ + uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ +#endif + + if (!is_feat_amu_supported()) { + return (void *)0; + } + + core_pos = plat_my_core_pos(); + ctx = &amu_ctxs_[core_pos]; + + amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); + if (is_feat_amuv1p1_supported()) { + hcr_el2_amvoffen = read_hcr_el2_amvoffen(); + } + +#if ENABLE_AMU_AUXILIARY_COUNTERS + amcfgr_el0_ncg = read_amcfgr_el0_ncg(); + amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; + amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; +#endif + + /* + * Disable all AMU counters. + */ + + ctx->group0_enable = read_amcntenset0_el0_px(); + write_amcntenclr0_el0_px(ctx->group0_enable); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + if (amcfgr_el0_ncg > 0U) { + ctx->group1_enable = read_amcntenset1_el0_px(); + write_amcntenclr1_el0_px(ctx->group1_enable); + } +#endif + + /* + * Save the counters to the local context. + */ + + isb(); /* Ensure counters have been stopped */ + + for (i = 0U; i < amcgcr_el0_cg0nc; i++) { + ctx->group0_cnts[i] = amu_group0_cnt_read(i); + } + +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U; i < amcgcr_el0_cg1nc; i++) { + ctx->group1_cnts[i] = amu_group1_cnt_read(i); + } +#endif + + /* + * Save virtual offsets for counters that offer them. + */ + + if (hcr_el2_amvoffen != 0U) { + for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { + if (!amu_group0_voffset_supported(i)) { + continue; /* No virtual offset */ + } + + ctx->group0_voffsets[j++] = amu_group0_voffset_read(i); + } + +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { + if ((amcg1idr_el0_voff >> i) & 1U) { + continue; /* No virtual offset */ + } + + ctx->group1_voffsets[j++] = amu_group1_voffset_read(i); + } +#endif + } + + return (void *)0; +} + +static void *amu_context_restore(const void *arg) +{ + uint64_t i, j; + + unsigned int core_pos; + struct amu_ctx *ctx; + + uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */ + + uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ + +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t amcfgr_el0_ncg; /* Number of counter groups */ + uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ + uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ +#endif + + if (!is_feat_amu_supported()) { + return (void *)0; + } + + core_pos = plat_my_core_pos(); + ctx = &amu_ctxs_[core_pos]; + + amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); + + if (is_feat_amuv1p1_supported()) { + hcr_el2_amvoffen = read_hcr_el2_amvoffen(); + } + +#if ENABLE_AMU_AUXILIARY_COUNTERS + amcfgr_el0_ncg = read_amcfgr_el0_ncg(); + amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; + amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; +#endif + + /* + * Restore the counter values from the local context. + */ + + for (i = 0U; i < amcgcr_el0_cg0nc; i++) { + amu_group0_cnt_write(i, ctx->group0_cnts[i]); + } + +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U; i < amcgcr_el0_cg1nc; i++) { + amu_group1_cnt_write(i, ctx->group1_cnts[i]); + } +#endif + + /* + * Restore virtual offsets for counters that offer them. + */ + + if (hcr_el2_amvoffen != 0U) { + for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { + if (!amu_group0_voffset_supported(i)) { + continue; /* No virtual offset */ + } + + amu_group0_voffset_write(i, ctx->group0_voffsets[j++]); + } + +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { + if ((amcg1idr_el0_voff >> i) & 1U) { + continue; /* No virtual offset */ + } + + amu_group1_voffset_write(i, ctx->group1_voffsets[j++]); + } +#endif + } + + /* + * Re-enable counters that were disabled during context save. + */ + + write_amcntenset0_el0_px(ctx->group0_enable); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + if (amcfgr_el0_ncg > 0) { + write_amcntenset1_el0_px(ctx->group1_enable); + } +#endif + +#if ENABLE_MPMM + mpmm_enable(); +#endif + + return (void *)0; +} + +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); diff --git a/lib/extensions/amu/aarch64/amu_helpers.S b/lib/extensions/amu/aarch64/amu_helpers.S new file mode 100644 index 0000000..95d4ad6 --- /dev/null +++ b/lib/extensions/amu/aarch64/amu_helpers.S @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <assert_macros.S> +#include <asm_macros.S> + + .globl amu_group0_cnt_read_internal + .globl amu_group0_cnt_write_internal + .globl amu_group1_cnt_read_internal + .globl amu_group1_cnt_write_internal + .globl amu_group1_set_evtype_internal + + /* FEAT_AMUv1p1 virtualisation offset register functions */ + .globl amu_group0_voffset_read_internal + .globl amu_group0_voffset_write_internal + .globl amu_group1_voffset_read_internal + .globl amu_group1_voffset_write_internal + +/* + * uint64_t amu_group0_cnt_read_internal(int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `x0`. + */ +func amu_group0_cnt_read_internal + adr x1, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~3 + ASM_ASSERT(eq) +#endif + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + add x1, x1, x0, lsl #3 /* each mrs/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x1, x1, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x1 + +1: read AMEVCNTR00_EL0 /* index 0 */ + read AMEVCNTR01_EL0 /* index 1 */ + read AMEVCNTR02_EL0 /* index 2 */ + read AMEVCNTR03_EL0 /* index 3 */ +endfunc amu_group0_cnt_read_internal + +/* + * void amu_group0_cnt_write_internal(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + */ +func amu_group0_cnt_write_internal + adr x2, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~3 + ASM_ASSERT(eq) +#endif + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + add x2, x2, x0, lsl #3 /* each msr/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x2, x2, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x2 + +1: write AMEVCNTR00_EL0 /* index 0 */ + write AMEVCNTR01_EL0 /* index 1 */ + write AMEVCNTR02_EL0 /* index 2 */ + write AMEVCNTR03_EL0 /* index 3 */ +endfunc amu_group0_cnt_write_internal + +#if ENABLE_AMU_AUXILIARY_COUNTERS +/* + * uint64_t amu_group1_cnt_read_internal(int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `x0`. + */ +func amu_group1_cnt_read_internal + adr x1, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~0xF + ASM_ASSERT(eq) +#endif + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + add x1, x1, x0, lsl #3 /* each mrs/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x1, x1, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x1 + +1: read AMEVCNTR10_EL0 /* index 0 */ + read AMEVCNTR11_EL0 /* index 1 */ + read AMEVCNTR12_EL0 /* index 2 */ + read AMEVCNTR13_EL0 /* index 3 */ + read AMEVCNTR14_EL0 /* index 4 */ + read AMEVCNTR15_EL0 /* index 5 */ + read AMEVCNTR16_EL0 /* index 6 */ + read AMEVCNTR17_EL0 /* index 7 */ + read AMEVCNTR18_EL0 /* index 8 */ + read AMEVCNTR19_EL0 /* index 9 */ + read AMEVCNTR1A_EL0 /* index 10 */ + read AMEVCNTR1B_EL0 /* index 11 */ + read AMEVCNTR1C_EL0 /* index 12 */ + read AMEVCNTR1D_EL0 /* index 13 */ + read AMEVCNTR1E_EL0 /* index 14 */ + read AMEVCNTR1F_EL0 /* index 15 */ +endfunc amu_group1_cnt_read_internal + +/* + * void amu_group1_cnt_write_internal(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + */ +func amu_group1_cnt_write_internal + adr x2, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~0xF + ASM_ASSERT(eq) +#endif + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + add x2, x2, x0, lsl #3 /* each msr/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x2, x2, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x2 + +1: write AMEVCNTR10_EL0 /* index 0 */ + write AMEVCNTR11_EL0 /* index 1 */ + write AMEVCNTR12_EL0 /* index 2 */ + write AMEVCNTR13_EL0 /* index 3 */ + write AMEVCNTR14_EL0 /* index 4 */ + write AMEVCNTR15_EL0 /* index 5 */ + write AMEVCNTR16_EL0 /* index 6 */ + write AMEVCNTR17_EL0 /* index 7 */ + write AMEVCNTR18_EL0 /* index 8 */ + write AMEVCNTR19_EL0 /* index 9 */ + write AMEVCNTR1A_EL0 /* index 10 */ + write AMEVCNTR1B_EL0 /* index 11 */ + write AMEVCNTR1C_EL0 /* index 12 */ + write AMEVCNTR1D_EL0 /* index 13 */ + write AMEVCNTR1E_EL0 /* index 14 */ + write AMEVCNTR1F_EL0 /* index 15 */ +endfunc amu_group1_cnt_write_internal + +/* + * void amu_group1_set_evtype_internal(int idx, unsigned int val); + * + * Program the AMU event type register indexed by `idx` + * with the value `val`. + */ +func amu_group1_set_evtype_internal + adr x2, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~0xF + ASM_ASSERT(eq) + + /* val should be between [0, 65535] */ + tst x1, #~0xFFFF + ASM_ASSERT(eq) +#endif + /* + * Given `idx` calculate address of msr/ret instruction pair + * in the table below. + */ + add x2, x2, x0, lsl #3 /* each msr/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x2, x2, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x2 + +1: write AMEVTYPER10_EL0 /* index 0 */ + write AMEVTYPER11_EL0 /* index 1 */ + write AMEVTYPER12_EL0 /* index 2 */ + write AMEVTYPER13_EL0 /* index 3 */ + write AMEVTYPER14_EL0 /* index 4 */ + write AMEVTYPER15_EL0 /* index 5 */ + write AMEVTYPER16_EL0 /* index 6 */ + write AMEVTYPER17_EL0 /* index 7 */ + write AMEVTYPER18_EL0 /* index 8 */ + write AMEVTYPER19_EL0 /* index 9 */ + write AMEVTYPER1A_EL0 /* index 10 */ + write AMEVTYPER1B_EL0 /* index 11 */ + write AMEVTYPER1C_EL0 /* index 12 */ + write AMEVTYPER1D_EL0 /* index 13 */ + write AMEVTYPER1E_EL0 /* index 14 */ + write AMEVTYPER1F_EL0 /* index 15 */ +endfunc amu_group1_set_evtype_internal +#endif + +/* + * Accessor functions for virtual offset registers added with FEAT_AMUv1p1 + */ + +/* + * uint64_t amu_group0_voffset_read_internal(int idx); + * + * Given `idx`, read the corresponding AMU virtual offset register + * and return it in `x0`. + */ +func amu_group0_voffset_read_internal + adr x1, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~3 + ASM_ASSERT(eq) + /* Make sure idx != 1 since AMEVCNTVOFF01_EL2 does not exist */ + cmp x0, #1 + ASM_ASSERT(ne) +#endif + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + add x1, x1, x0, lsl #3 /* each mrs/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x1, x1, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x1 + +1: read AMEVCNTVOFF00_EL2 /* index 0 */ + .skip 8 /* AMEVCNTVOFF01_EL2 does not exist */ +#if ENABLE_BTI + .skip 4 +#endif + read AMEVCNTVOFF02_EL2 /* index 2 */ + read AMEVCNTVOFF03_EL2 /* index 3 */ +endfunc amu_group0_voffset_read_internal + +/* + * void amu_group0_voffset_write_internal(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU virtual offset register. + */ +func amu_group0_voffset_write_internal + adr x2, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~3 + ASM_ASSERT(eq) + /* Make sure idx != 1 since AMEVCNTVOFF01_EL2 does not exist */ + cmp x0, #1 + ASM_ASSERT(ne) +#endif + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + add x2, x2, x0, lsl #3 /* each msr/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x2, x2, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x2 + +1: write AMEVCNTVOFF00_EL2 /* index 0 */ + .skip 8 /* AMEVCNTVOFF01_EL2 does not exist */ +#if ENABLE_BTI + .skip 4 +#endif + write AMEVCNTVOFF02_EL2 /* index 2 */ + write AMEVCNTVOFF03_EL2 /* index 3 */ +endfunc amu_group0_voffset_write_internal + +#if ENABLE_AMU_AUXILIARY_COUNTERS +/* + * uint64_t amu_group1_voffset_read_internal(int idx); + * + * Given `idx`, read the corresponding AMU virtual offset register + * and return it in `x0`. + */ +func amu_group1_voffset_read_internal + adr x1, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~0xF + ASM_ASSERT(eq) +#endif + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + add x1, x1, x0, lsl #3 /* each mrs/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x1, x1, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x1 + +1: read AMEVCNTVOFF10_EL2 /* index 0 */ + read AMEVCNTVOFF11_EL2 /* index 1 */ + read AMEVCNTVOFF12_EL2 /* index 2 */ + read AMEVCNTVOFF13_EL2 /* index 3 */ + read AMEVCNTVOFF14_EL2 /* index 4 */ + read AMEVCNTVOFF15_EL2 /* index 5 */ + read AMEVCNTVOFF16_EL2 /* index 6 */ + read AMEVCNTVOFF17_EL2 /* index 7 */ + read AMEVCNTVOFF18_EL2 /* index 8 */ + read AMEVCNTVOFF19_EL2 /* index 9 */ + read AMEVCNTVOFF1A_EL2 /* index 10 */ + read AMEVCNTVOFF1B_EL2 /* index 11 */ + read AMEVCNTVOFF1C_EL2 /* index 12 */ + read AMEVCNTVOFF1D_EL2 /* index 13 */ + read AMEVCNTVOFF1E_EL2 /* index 14 */ + read AMEVCNTVOFF1F_EL2 /* index 15 */ +endfunc amu_group1_voffset_read_internal + +/* + * void amu_group1_voffset_write_internal(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU virtual offset register. + */ +func amu_group1_voffset_write_internal + adr x2, 1f +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + tst x0, #~0xF + ASM_ASSERT(eq) +#endif + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + add x2, x2, x0, lsl #3 /* each msr/ret sequence is 8 bytes */ +#if ENABLE_BTI + add x2, x2, x0, lsl #2 /* + "bti j" instruction */ +#endif + br x2 + +1: write AMEVCNTVOFF10_EL2 /* index 0 */ + write AMEVCNTVOFF11_EL2 /* index 1 */ + write AMEVCNTVOFF12_EL2 /* index 2 */ + write AMEVCNTVOFF13_EL2 /* index 3 */ + write AMEVCNTVOFF14_EL2 /* index 4 */ + write AMEVCNTVOFF15_EL2 /* index 5 */ + write AMEVCNTVOFF16_EL2 /* index 6 */ + write AMEVCNTVOFF17_EL2 /* index 7 */ + write AMEVCNTVOFF18_EL2 /* index 8 */ + write AMEVCNTVOFF19_EL2 /* index 9 */ + write AMEVCNTVOFF1A_EL2 /* index 10 */ + write AMEVCNTVOFF1B_EL2 /* index 11 */ + write AMEVCNTVOFF1C_EL2 /* index 12 */ + write AMEVCNTVOFF1D_EL2 /* index 13 */ + write AMEVCNTVOFF1E_EL2 /* index 14 */ + write AMEVCNTVOFF1F_EL2 /* index 15 */ +endfunc amu_group1_voffset_write_internal +#endif diff --git a/lib/extensions/amu/amu.mk b/lib/extensions/amu/amu.mk new file mode 100644 index 0000000..868ab12 --- /dev/null +++ b/lib/extensions/amu/amu.mk @@ -0,0 +1,24 @@ +# +# Copyright (c) 2021, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +include lib/fconf/fconf.mk + +AMU_SOURCES := lib/extensions/amu/${ARCH}/amu.c \ + lib/extensions/amu/${ARCH}/amu_helpers.S + +ifneq (${ENABLE_AMU_AUXILIARY_COUNTERS},0) + ifeq (${ENABLE_FEAT_AMU},0) + $(error AMU auxiliary counter support (`ENABLE_AMU_AUXILIARY_COUNTERS`) requires AMU support (`ENABLE_FEAT_AMU`)) + endif +endif + +ifneq (${ENABLE_AMU_FCONF},0) + ifeq (${ENABLE_AMU_AUXILIARY_COUNTERS},0) + $(error AMU FCONF support (`ENABLE_AMU_FCONF`) is not necessary when auxiliary counter support (`ENABLE_AMU_AUXILIARY_COUNTERS`) is disabled) + endif + + AMU_SOURCES += ${FCONF_AMU_SOURCES} +endif diff --git a/lib/extensions/amu/amu_private.h b/lib/extensions/amu/amu_private.h new file mode 100644 index 0000000..a3b6845 --- /dev/null +++ b/lib/extensions/amu/amu_private.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef AMU_PRIVATE_H +#define AMU_PRIVATE_H + +#include <stdint.h> + +#include <lib/cassert.h> +#include <lib/extensions/amu.h> +#include <lib/utils_def.h> + +#include <platform_def.h> + +#define AMU_GROUP0_MAX_COUNTERS U(16) +#define AMU_GROUP1_MAX_COUNTERS U(16) + +#define AMU_AMCGCR_CG0NC_MAX U(16) + +uint64_t amu_group0_cnt_read_internal(unsigned int idx); +void amu_group0_cnt_write_internal(unsigned int idx, uint64_t val); + +uint64_t amu_group1_cnt_read_internal(unsigned int idx); +void amu_group1_cnt_write_internal(unsigned int idx, uint64_t val); +void amu_group1_set_evtype_internal(unsigned int idx, unsigned int val); + +#if __aarch64__ +uint64_t amu_group0_voffset_read_internal(unsigned int idx); +void amu_group0_voffset_write_internal(unsigned int idx, uint64_t val); + +uint64_t amu_group1_voffset_read_internal(unsigned int idx); +void amu_group1_voffset_write_internal(unsigned int idx, uint64_t val); +#endif + +#endif /* AMU_PRIVATE_H */ diff --git a/lib/extensions/brbe/brbe.c b/lib/extensions/brbe/brbe.c new file mode 100644 index 0000000..37bd834 --- /dev/null +++ b/lib/extensions/brbe/brbe.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/extensions/brbe.h> + +void brbe_init_el3(void) +{ + uint64_t val; + + /* + * MDCR_EL3.SBRBE = 0b01 + * + * Allows BRBE usage in non-secure world and prohibited in + * secure world. + */ + val = read_mdcr_el3(); + val &= ~(MDCR_SBRBE_MASK << MDCR_SBRBE_SHIFT); + val |= (0x1UL << MDCR_SBRBE_SHIFT); + write_mdcr_el3(val); +} diff --git a/lib/extensions/mpam/mpam.c b/lib/extensions/mpam/mpam.c new file mode 100644 index 0000000..875ad9c --- /dev/null +++ b/lib/extensions/mpam/mpam.c @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2018-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/extensions/mpam.h> + +void mpam_enable(cpu_context_t *context) +{ + u_register_t mpam3_el3; + + mpam3_el3 = read_ctx_reg(get_el3state_ctx(context), CTX_MPAM3_EL3); + + /* + * Enable MPAM, and disable trapping to EL3 when lower ELs access their + * own MPAM registers + */ + mpam3_el3 = (mpam3_el3 | MPAM3_EL3_MPAMEN_BIT) & + ~(MPAM3_EL3_TRAPLOWER_BIT); + write_ctx_reg(get_el3state_ctx(context), CTX_MPAM3_EL3, mpam3_el3); +} + +/* + * If EL2 is implemented but unused, disable trapping to EL2 when lower ELs + * access their own MPAM registers. + */ +void mpam_init_el2_unused(void) +{ + write_mpam2_el2(0ULL); + + if ((read_mpamidr_el1() & MPAMIDR_HAS_HCR_BIT) != 0U) { + write_mpamhcr_el2(0ULL); + } + +} diff --git a/lib/extensions/pauth/pauth_helpers.S b/lib/extensions/pauth/pauth_helpers.S new file mode 100644 index 0000000..fb5fa97 --- /dev/null +++ b/lib/extensions/pauth/pauth_helpers.S @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <lib/el3_runtime/cpu_data.h> + + .global pauth_init_enable_el1 + .global pauth_disable_el1 + .global pauth_init_enable_el3 + .global pauth_disable_el3 + .globl pauth_load_bl31_apiakey + .globl pauth_load_bl1_apiakey_enable + +/* ------------------------------------------------------------- + * Program APIAKey_EL1 and enable pointer authentication in EL1 + * ------------------------------------------------------------- + */ +func pauth_init_enable_el1 + stp x29, x30, [sp, #-16]! + + /* Initialize platform key */ + bl plat_init_apkey + + /* Program instruction key A used by the Trusted Firmware */ + msr APIAKeyLo_EL1, x0 + msr APIAKeyHi_EL1, x1 + + /* Enable pointer authentication */ + mrs x0, sctlr_el1 + orr x0, x0, #SCTLR_EnIA_BIT + +#if ENABLE_BTI + /* Enable PAC branch type compatibility */ + bic x0, x0, #(SCTLR_BT0_BIT | SCTLR_BT1_BIT) +#endif + msr sctlr_el1, x0 + isb + + ldp x29, x30, [sp], #16 + ret +endfunc pauth_init_enable_el1 + +/* ------------------------------------------------------------- + * Disable pointer authentication in EL1 + * ------------------------------------------------------------- + */ +func pauth_disable_el1 + mrs x0, sctlr_el1 + bic x0, x0, #SCTLR_EnIA_BIT + msr sctlr_el1, x0 + isb + ret +endfunc pauth_disable_el1 + +/* ------------------------------------------------------------- + * Program APIAKey_EL1 and enable pointer authentication in EL3 + * ------------------------------------------------------------- + */ +func pauth_init_enable_el3 + stp x29, x30, [sp, #-16]! + + /* Initialize platform key */ + bl plat_init_apkey + + /* Program instruction key A used by the Trusted Firmware */ + msr APIAKeyLo_EL1, x0 + msr APIAKeyHi_EL1, x1 + + /* Enable pointer authentication */ + mrs x0, sctlr_el3 + orr x0, x0, #SCTLR_EnIA_BIT + +#if ENABLE_BTI + /* Enable PAC branch type compatibility */ + bic x0, x0, #SCTLR_BT_BIT +#endif + msr sctlr_el3, x0 + isb + + ldp x29, x30, [sp], #16 + ret +endfunc pauth_init_enable_el3 + +/* ------------------------------------------------------------- + * Disable pointer authentication in EL3 + * ------------------------------------------------------------- + */ +func pauth_disable_el3 + mrs x0, sctlr_el3 + bic x0, x0, #SCTLR_EnIA_BIT + msr sctlr_el3, x0 + isb + ret +endfunc pauth_disable_el3 + +/* ------------------------------------------------------------- + * The following functions strictly follow the AArch64 PCS + * to use x9-x17 (temporary caller-saved registers) to load + * the APIAKey_EL1 and enable pointer authentication. + * ------------------------------------------------------------- + */ +func pauth_load_bl31_apiakey + /* tpidr_el3 contains the address of cpu_data structure */ + mrs x9, tpidr_el3 + + /* Load apiakey from cpu_data */ + ldp x10, x11, [x9, #CPU_DATA_APIAKEY_OFFSET] + + /* Program instruction key A */ + msr APIAKeyLo_EL1, x10 + msr APIAKeyHi_EL1, x11 + isb + ret +endfunc pauth_load_bl31_apiakey + +func pauth_load_bl1_apiakey_enable + /* Load instruction key A used by the Trusted Firmware */ + adrp x9, bl1_apiakey + add x9, x9, :lo12:bl1_apiakey + ldp x10, x11, [x9] + + /* Program instruction key A */ + msr APIAKeyLo_EL1, x10 + msr APIAKeyHi_EL1, x11 + + /* Enable pointer authentication */ + mrs x9, sctlr_el3 + orr x9, x9, #SCTLR_EnIA_BIT + +#if ENABLE_BTI + /* Enable PAC branch type compatibility */ + bic x9, x9, #SCTLR_BT_BIT +#endif + msr sctlr_el3, x9 + isb + ret +endfunc pauth_load_bl1_apiakey_enable diff --git a/lib/extensions/pmuv3/aarch32/pmuv3.c b/lib/extensions/pmuv3/aarch32/pmuv3.c new file mode 100644 index 0000000..effb7e0 --- /dev/null +++ b/lib/extensions/pmuv3/aarch32/pmuv3.c @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/extensions/pmuv3.h> + +static u_register_t mtpmu_disable_el3(u_register_t sdcr) +{ + if (!is_feat_mtpmu_supported()) { + return sdcr; + } + + /* + * SDCR.MTPME = 0 + * FEAT_MTPMU is disabled. The Effective value of PMEVTYPER<n>.MT is + * zero. + */ + sdcr &= ~SDCR_MTPME_BIT; + + return sdcr; +} + +/* + * Applies to all PMU versions. Name is PMUv3 for compatibility with aarch64 and + * to not clash with platforms which reuse the PMU name + */ +void pmuv3_init_el3(void) +{ + u_register_t sdcr = read_sdcr(); + + /* --------------------------------------------------------------------- + * Initialise SDCR, setting all the fields rather than relying on hw. + * + * SDCR.SCCD: Set to one so that cycle counting by PMCCNTR is prohibited + * in Secure state. This bit is RES0 in versions of the architecture + * earlier than ARMv8.5 + * + * SDCR.SPME: Set to zero so that event counting is prohibited in Secure + * state (and explicitly EL3 with later revisions). If ARMv8.2 Debug is + * not implemented this bit does not have any effect on the counters + * unless there is support for the implementation defined + * authentication interface ExternalSecureNoninvasiveDebugEnabled(). + * --------------------------------------------------------------------- + */ + sdcr = (sdcr | SDCR_SCCD_BIT) & ~SDCR_SPME_BIT; + sdcr = mtpmu_disable_el3(sdcr); + write_sdcr(sdcr); + + /* --------------------------------------------------------------------- + * Initialise PMCR, setting all fields rather than relying + * on hw. Some fields are architecturally UNKNOWN on reset. + * + * PMCR.DP: Set to one to prohibit cycle counting whilst in Secure mode. + * + * PMCR.X: Set to zero to disable export of events. + * + * PMCR.C: Set to one to reset PMCCNTR. + * + * PMCR.P: Set to one to reset each event counter PMEVCNTR<n> to zero. + * + * PMCR.E: Set to zero to disable cycle and event counters. + * --------------------------------------------------------------------- + */ + + write_pmcr(read_pmcr() | PMCR_DP_BIT | PMCR_C_BIT | PMCR_P_BIT | + ~(PMCR_X_BIT | PMCR_E_BIT)); +} diff --git a/lib/extensions/pmuv3/aarch64/pmuv3.c b/lib/extensions/pmuv3/aarch64/pmuv3.c new file mode 100644 index 0000000..61fc47d --- /dev/null +++ b/lib/extensions/pmuv3/aarch64/pmuv3.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/extensions/pmuv3.h> + +static u_register_t init_mdcr_el2_hpmn(u_register_t mdcr_el2) +{ + /* + * Initialize MDCR_EL2.HPMN to its hardware reset value so we don't + * throw anyone off who expects this to be sensible. + */ + mdcr_el2 &= ~MDCR_EL2_HPMN_MASK; + mdcr_el2 |= ((read_pmcr_el0() >> PMCR_EL0_N_SHIFT) & PMCR_EL0_N_MASK); + + return mdcr_el2; +} + +void pmuv3_enable(cpu_context_t *ctx) +{ +#if CTX_INCLUDE_EL2_REGS + u_register_t mdcr_el2; + + mdcr_el2 = read_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_MDCR_EL2); + mdcr_el2 = init_mdcr_el2_hpmn(mdcr_el2); + write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_MDCR_EL2, mdcr_el2); +#endif /* CTX_INCLUDE_EL2_REGS */ +} + +static u_register_t mtpmu_disable_el3(u_register_t mdcr_el3) +{ + if (!is_feat_mtpmu_supported()) { + return mdcr_el3; + } + + /* + * MDCR_EL3.MTPME = 0 + * FEAT_MTPMU is disabled. The Effective value of PMEVTYPER<n>_EL0.MT is + * zero. + */ + mdcr_el3 &= ~MDCR_MTPME_BIT; + + return mdcr_el3; +} + +void pmuv3_init_el3(void) +{ + u_register_t mdcr_el3 = read_mdcr_el3(); + + /* --------------------------------------------------------------------- + * Initialise MDCR_EL3, setting all fields rather than relying on hw. + * Some fields are architecturally UNKNOWN on reset. + * + * MDCR_EL3.MPMX: Set to zero to not affect event counters (when + * SPME = 0). + * + * MDCR_EL3.MCCD: Set to one so that cycle counting by PMCCNTR_EL0 is + * prohibited in EL3. This bit is RES0 in versions of the + * architecture with FEAT_PMUv3p7 not implemented. + * + * MDCR_EL3.SCCD: Set to one so that cycle counting by PMCCNTR_EL0 is + * prohibited in Secure state. This bit is RES0 in versions of the + * architecture with FEAT_PMUv3p5 not implemented. + * + * MDCR_EL3.SPME: Set to zero so that event counting is prohibited in + * Secure state (and explicitly EL3 with later revisions). If ARMv8.2 + * Debug is not implemented this bit does not have any effect on the + * counters unless there is support for the implementation defined + * authentication interface ExternalSecureNoninvasiveDebugEnabled(). + * + * The SPME/MPMX combination is a little tricky. Below is a small + * summary if another combination is ever needed: + * SPME | MPMX | secure world | EL3 + * ------------------------------------- + * 0 | 0 | disabled | disabled + * 1 | 0 | enabled | enabled + * 0 | 1 | enabled | disabled + * 1 | 1 | enabled | disabled only for counters 0 to + * MDCR_EL2.HPMN - 1. Enabled for the rest + * + * MDCR_EL3.TPM: Set to zero so that EL0, EL1, and EL2 System register + * accesses to all Performance Monitors registers do not trap to EL3. + */ + mdcr_el3 = (mdcr_el3 | MDCR_SCCD_BIT | MDCR_MCCD_BIT) & + ~(MDCR_MPMX_BIT | MDCR_SPME_BIT | MDCR_TPM_BIT); + mdcr_el3 = mtpmu_disable_el3(mdcr_el3); + write_mdcr_el3(mdcr_el3); + + /* --------------------------------------------------------------------- + * Initialise PMCR_EL0 setting all fields rather than relying + * on hw. Some fields are architecturally UNKNOWN on reset. + * + * PMCR_EL0.DP: Set to one so that the cycle counter, + * PMCCNTR_EL0 does not count when event counting is prohibited. + * Necessary on PMUv3 <= p7 where MDCR_EL3.{SCCD,MCCD} are not + * available + * + * PMCR_EL0.X: Set to zero to disable export of events. + * + * PMCR_EL0.C: Set to one to reset PMCCNTR_EL0 to zero. + * + * PMCR_EL0.P: Set to one to reset each event counter PMEVCNTR<n>_EL0 to + * zero. + * + * PMCR_EL0.E: Set to zero to disable cycle and event counters. + * --------------------------------------------------------------------- + */ + write_pmcr_el0((read_pmcr_el0() | PMCR_EL0_DP_BIT | PMCR_EL0_C_BIT | + PMCR_EL0_P_BIT) & ~(PMCR_EL0_X_BIT | PMCR_EL0_E_BIT)); +} + +static u_register_t mtpmu_disable_el2(u_register_t mdcr_el2) +{ + if (!is_feat_mtpmu_supported()) { + return mdcr_el2; + } + + /* + * MDCR_EL2.MTPME = 0 + * FEAT_MTPMU is disabled. The Effective value of PMEVTYPER<n>_EL0.MT is + * zero. + */ + mdcr_el2 &= ~MDCR_EL2_MTPME; + + return mdcr_el2; +} + +void pmuv3_init_el2_unused(void) +{ + u_register_t mdcr_el2 = read_mdcr_el2(); + + /* + * Initialise MDCR_EL2, setting all fields rather than + * relying on hw. Some fields are architecturally + * UNKNOWN on reset. + * + * MDCR_EL2.HLP: Set to one so that event counter overflow, that is + * recorded in PMOVSCLR_EL0[0-30], occurs on the increment that changes + * PMEVCNTR<n>_EL0[63] from 1 to 0, when ARMv8.5-PMU is implemented. + * This bit is RES0 in versions of the architecture earlier than + * ARMv8.5, setting it to 1 doesn't have any effect on them. + * + * MDCR_EL2.HCCD: Set to one to prohibit cycle counting at EL2. This bit + * is RES0 in versions of the architecture with FEAT_PMUv3p5 not + * implemented. + * + * MDCR_EL2.HPMD: Set to one so that event counting is + * prohibited at EL2 for counter n < MDCR_EL2.HPMN. This bit is RES0 + * in versions of the architecture with FEAT_PMUv3p1 not implemented. + * + * MDCR_EL2.HPME: Set to zero to disable event counters for counters + * n >= MDCR_EL2.HPMN. + * + * MDCR_EL2.TPM: Set to zero so that Non-secure EL0 and + * EL1 accesses to all Performance Monitors registers + * are not trapped to EL2. + * + * MDCR_EL2.TPMCR: Set to zero so that Non-secure EL0 + * and EL1 accesses to the PMCR_EL0 or PMCR are not + * trapped to EL2. + */ + mdcr_el2 = (mdcr_el2 | MDCR_EL2_HLP_BIT | MDCR_EL2_HPMD_BIT | + MDCR_EL2_HCCD_BIT) & + ~(MDCR_EL2_HPME_BIT | MDCR_EL2_TPM_BIT | MDCR_EL2_TPMCR_BIT); + mdcr_el2 = init_mdcr_el2_hpmn(mdcr_el2); + mdcr_el2 = mtpmu_disable_el2(mdcr_el2); + write_mdcr_el2(mdcr_el2); +} diff --git a/lib/extensions/ras/ras_common.c b/lib/extensions/ras/ras_common.c new file mode 100644 index 0000000..50f3727 --- /dev/null +++ b/lib/extensions/ras/ras_common.c @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2018-2021, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <arch_helpers.h> +#include <bl31/ea_handle.h> +#include <bl31/ehf.h> +#include <common/debug.h> +#include <lib/extensions/ras.h> +#include <lib/extensions/ras_arch.h> +#include <plat/common/platform.h> + +#ifndef PLAT_RAS_PRI +# error Platform must define RAS priority value +#endif + +/* + * Function to convert architecturally-defined primary error code SERR, + * bits[7:0] from ERR<n>STATUS to its corresponding error string. + */ +const char *ras_serr_to_str(unsigned int serr) +{ + const char *str[ERROR_STATUS_NUM_SERR] = { + "No error", + "IMPLEMENTATION DEFINED error", + "Data value from (non-associative) internal memory", + "IMPLEMENTATION DEFINED pin", + "Assertion failure", + "Error detected on internal data path", + "Data value from associative memory", + "Address/control value from associative memory", + "Data value from a TLB", + "Address/control value from a TLB", + "Data value from producer", + "Address/control value from producer", + "Data value from (non-associative) external memory", + "Illegal address (software fault)", + "Illegal access (software fault)", + "Illegal state (software fault)", + "Internal data register", + "Internal control register", + "Error response from slave", + "External timeout", + "Internal timeout", + "Deferred error from slave not supported at master" + }; + + /* + * All other values are reserved. Reserved values might be defined + * in a future version of the architecture + */ + if (serr >= ERROR_STATUS_NUM_SERR) + return "unknown SERR"; + + return str[serr]; +} + +/* Handler that receives External Aborts on RAS-capable systems */ +int ras_ea_handler(unsigned int ea_reason, uint64_t syndrome, void *cookie, + void *handle, uint64_t flags) +{ + unsigned int i, n_handled = 0; + int probe_data, ret; + struct err_record_info *info; + + const struct err_handler_data err_data = { + .version = ERR_HANDLER_VERSION, + .ea_reason = ea_reason, + .interrupt = 0, + .syndrome = (uint32_t) syndrome, + .flags = flags, + .cookie = cookie, + .handle = handle + }; + + for_each_err_record_info(i, info) { + assert(info->probe != NULL); + assert(info->handler != NULL); + + /* Continue probing until the record group signals no error */ + while (true) { + if (info->probe(info, &probe_data) == 0) + break; + + /* Handle error */ + ret = info->handler(info, probe_data, &err_data); + if (ret != 0) + return ret; + + n_handled++; + } + } + + return (n_handled != 0U) ? 1 : 0; +} + +#if ENABLE_ASSERTIONS +static void assert_interrupts_sorted(void) +{ + unsigned int i, last; + struct ras_interrupt *start = ras_interrupt_mappings.intrs; + + if (ras_interrupt_mappings.num_intrs == 0UL) + return; + + last = start[0].intr_number; + for (i = 1; i < ras_interrupt_mappings.num_intrs; i++) { + assert(start[i].intr_number > last); + last = start[i].intr_number; + } +} +#endif + +/* + * Given an RAS interrupt number, locate the registered handler and call it. If + * no handler was found for the interrupt number, this function panics. + */ +static int ras_interrupt_handler(uint32_t intr_raw, uint32_t flags, + void *handle, void *cookie) +{ + struct ras_interrupt *ras_inrs = ras_interrupt_mappings.intrs; + struct ras_interrupt *selected = NULL; + int probe_data = 0; + int start, end, mid, ret __unused; + + const struct err_handler_data err_data = { + .version = ERR_HANDLER_VERSION, + .interrupt = intr_raw, + .flags = flags, + .cookie = cookie, + .handle = handle + }; + + assert(ras_interrupt_mappings.num_intrs > 0UL); + + start = 0; + end = (int)ras_interrupt_mappings.num_intrs - 1; + while (start <= end) { + mid = ((end + start) / 2); + if (intr_raw == ras_inrs[mid].intr_number) { + selected = &ras_inrs[mid]; + break; + } else if (intr_raw < ras_inrs[mid].intr_number) { + /* Move left */ + end = mid - 1; + } else { + /* Move right */ + start = mid + 1; + } + } + + if (selected == NULL) { + ERROR("RAS interrupt %u has no handler!\n", intr_raw); + panic(); + } + + if (selected->err_record->probe != NULL) { + ret = selected->err_record->probe(selected->err_record, &probe_data); + assert(ret != 0); + } + + /* Call error handler for the record group */ + assert(selected->err_record->handler != NULL); + (void) selected->err_record->handler(selected->err_record, probe_data, + &err_data); + + return 0; +} + +void __init ras_init(void) +{ +#if ENABLE_ASSERTIONS + /* Check RAS interrupts are sorted */ + assert_interrupts_sorted(); +#endif + + /* Register RAS priority handler */ + ehf_register_priority_handler(PLAT_RAS_PRI, ras_interrupt_handler); +} diff --git a/lib/extensions/ras/std_err_record.c b/lib/extensions/ras/std_err_record.c new file mode 100644 index 0000000..6c1dc9f --- /dev/null +++ b/lib/extensions/ras/std_err_record.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <lib/extensions/ras_arch.h> +#include <lib/utils_def.h> + +/* + * Probe for error in memory-mapped registers containing error records + * implemented Standard Error Record format. Upon detecting an error, set probe + * data to the index of the record in error, and return 1; otherwise, return 0. + */ +int ser_probe_memmap(uintptr_t base, unsigned int size_num_k, int *probe_data) +{ + unsigned int num_records, num_group_regs, i; + uint64_t gsr; + + assert(base != 0UL); + + /* Only 4K supported for now */ + assert(size_num_k == STD_ERR_NODE_SIZE_NUM_K); + + num_records = (unsigned int) + (mmio_read_32(ERR_DEVID(base, size_num_k)) & ERR_DEVID_MASK); + + /* A group register shows error status for 2^6 error records */ + num_group_regs = (num_records >> 6U) + 1U; + + /* Iterate through group registers to find a record in error */ + for (i = 0; i < num_group_regs; i++) { + gsr = mmio_read_64(ERR_GSR(base, size_num_k, i)); + if (gsr == 0ULL) + continue; + + /* Return the index of the record in error */ + if (probe_data != NULL) + *probe_data = (((int) (i << 6U)) + __builtin_ctzll(gsr)); + + return 1; + } + + return 0; +} + +/* + * Probe for error in System Registers where error records are implemented in + * Standard Error Record format. Upon detecting an error, set probe data to the + * index of the record in error, and return 1; otherwise, return 0. + */ +int ser_probe_sysreg(unsigned int idx_start, unsigned int num_idx, int *probe_data) +{ + unsigned int i; + uint64_t status; + unsigned int max_idx __unused = + ((unsigned int) read_erridr_el1()) & ERRIDR_MASK; + + assert(idx_start < max_idx); + assert(check_u32_overflow(idx_start, num_idx) == 0); + assert((idx_start + num_idx - 1U) < max_idx); + + for (i = 0; i < num_idx; i++) { + /* Select the error record */ + ser_sys_select_record(idx_start + i); + + /* Retrieve status register from the error record */ + status = read_erxstatus_el1(); + + /* Check for valid field in status */ + if (ERR_STATUS_GET_FIELD(status, V) != 0U) { + if (probe_data != NULL) + *probe_data = (int) i; + return 1; + } + } + + return 0; +} diff --git a/lib/extensions/sme/sme.c b/lib/extensions/sme/sme.c new file mode 100644 index 0000000..b1409b9 --- /dev/null +++ b/lib/extensions/sme/sme.c @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2021-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/el3_runtime/context_mgmt.h> +#include <lib/extensions/sme.h> +#include <lib/extensions/sve.h> + +void sme_enable(cpu_context_t *context) +{ + u_register_t reg; + el3_state_t *state; + + /* Get the context state. */ + state = get_el3state_ctx(context); + + /* Set the ENTP2 bit in SCR_EL3 to enable access to TPIDR2_EL0. */ + reg = read_ctx_reg(state, CTX_SCR_EL3); + reg |= SCR_ENTP2_BIT; + write_ctx_reg(state, CTX_SCR_EL3, reg); +} + +void sme_enable_per_world(per_world_context_t *per_world_ctx) +{ + u_register_t reg; + + /* Enable SME in CPTR_EL3. */ + reg = per_world_ctx->ctx_cptr_el3; + reg |= ESM_BIT; + per_world_ctx->ctx_cptr_el3 = reg; +} + +void sme_init_el3(void) +{ + u_register_t cptr_el3 = read_cptr_el3(); + u_register_t smcr_el3; + + /* Set CPTR_EL3.ESM bit so we can access SMCR_EL3 without trapping. */ + write_cptr_el3(cptr_el3 | ESM_BIT); + isb(); + + /* + * Set the max LEN value and FA64 bit. This register is set up per_world + * to be the least restrictive, then lower ELs can restrict as needed + * using SMCR_EL2 and SMCR_EL1. + */ + smcr_el3 = SMCR_ELX_LEN_MAX; + if (read_feat_sme_fa64_id_field() != 0U) { + VERBOSE("[SME] FA64 enabled\n"); + smcr_el3 |= SMCR_ELX_FA64_BIT; + } + + /* + * Enable access to ZT0 register. + * Make sure FEAT_SME2 is supported by the hardware before continuing. + * If supported, Set the EZT0 bit in SMCR_EL3 to allow instructions to + * access ZT0 register without trapping. + */ + if (is_feat_sme2_supported()) { + VERBOSE("SME2 enabled\n"); + smcr_el3 |= SMCR_ELX_EZT0_BIT; + } + write_smcr_el3(smcr_el3); + + /* Reset CPTR_EL3 value. */ + write_cptr_el3(cptr_el3); + isb(); +} + +void sme_init_el2_unused(void) +{ + /* + * CPTR_EL2.TCPAC: Set to zero so that Non-secure EL1 accesses to the + * CPACR_EL1 or CPACR from both Execution states do not trap to EL2. + */ + write_cptr_el2(read_cptr_el2() & ~CPTR_EL2_TCPAC_BIT); +} + +void sme_disable(cpu_context_t *context) +{ + u_register_t reg; + el3_state_t *state; + + /* Get the context state. */ + state = get_el3state_ctx(context); + + /* Disable access to TPIDR2_EL0. */ + reg = read_ctx_reg(state, CTX_SCR_EL3); + reg &= ~SCR_ENTP2_BIT; + write_ctx_reg(state, CTX_SCR_EL3, reg); +} + +void sme_disable_per_world(per_world_context_t *per_world_ctx) +{ + u_register_t reg; + + /* Disable SME, SVE, and FPU since they all share registers. */ + reg = per_world_ctx->ctx_cptr_el3; + reg &= ~ESM_BIT; /* Trap SME */ + reg &= ~CPTR_EZ_BIT; /* Trap SVE */ + reg |= TFP_BIT; /* Trap FPU/SIMD */ + per_world_ctx->ctx_cptr_el3 = reg; +} diff --git a/lib/extensions/spe/spe.c b/lib/extensions/spe/spe.c new file mode 100644 index 0000000..2c25a9d --- /dev/null +++ b/lib/extensions/spe/spe.c @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/el3_runtime/pubsub.h> +#include <lib/extensions/spe.h> + +static inline void psb_csync(void) +{ + /* + * The assembler does not yet understand the psb csync mnemonic + * so use the equivalent hint instruction. + */ + __asm__ volatile("hint #17"); +} + +void spe_init_el3(void) +{ + uint64_t v; + + /* + * MDCR_EL3.NSPB (ARM v8.2): SPE enabled in Non-secure state + * and disabled in secure state. Accesses to SPE registers at + * S-EL1 generate trap exceptions to EL3. + * + * MDCR_EL3.NSPBE: Profiling Buffer uses Non-secure Virtual Addresses. + * When FEAT_RME is not implemented, this field is RES0. + * + * MDCR_EL3.EnPMSN (ARM v8.7): Do not trap access to PMSNEVFR_EL1 + * register at NS-EL1 or NS-EL2 to EL3 if FEAT_SPEv1p2 is implemented. + * Setting this bit to 1 doesn't have any effect on it when + * FEAT_SPEv1p2 not implemented. + */ + v = read_mdcr_el3(); + v |= MDCR_NSPB(MDCR_NSPB_EL1) | MDCR_EnPMSN_BIT; + v &= ~(MDCR_NSPBE_BIT); + write_mdcr_el3(v); +} + +void spe_init_el2_unused(void) +{ + uint64_t v; + + /* + * MDCR_EL2.TPMS (ARM v8.2): Do not trap statistical + * profiling controls to EL2. + * + * MDCR_EL2.E2PB (ARM v8.2): SPE enabled in Non-secure + * state. Accesses to profiling buffer controls at + * Non-secure EL1 are not trapped to EL2. + */ + v = read_mdcr_el2(); + v &= ~MDCR_EL2_TPMS; + v |= MDCR_EL2_E2PB(MDCR_EL2_E2PB_EL1); + write_mdcr_el2(v); +} + +void spe_disable(void) +{ + uint64_t v; + + /* Drain buffered data */ + psb_csync(); + dsbnsh(); + + /* Disable profiling buffer */ + v = read_pmblimitr_el1(); + v &= ~(1ULL << 0); + write_pmblimitr_el1(v); + isb(); +} + +static void *spe_drain_buffers_hook(const void *arg) +{ + if (!is_feat_spe_supported()) + return (void *)-1; + + /* Drain buffered data */ + psb_csync(); + dsbnsh(); + + return (void *)0; +} + +SUBSCRIBE_TO_EVENT(cm_entering_secure_world, spe_drain_buffers_hook); diff --git a/lib/extensions/sve/sve.c b/lib/extensions/sve/sve.c new file mode 100644 index 0000000..143717e --- /dev/null +++ b/lib/extensions/sve/sve.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <lib/cassert.h> +#include <lib/el3_runtime/pubsub.h> +#include <lib/extensions/sve.h> + +CASSERT(SVE_VECTOR_LEN <= 2048, assert_sve_vl_too_long); +CASSERT(SVE_VECTOR_LEN >= 128, assert_sve_vl_too_short); +CASSERT((SVE_VECTOR_LEN % 128) == 0, assert_sve_vl_granule); + +/* + * Converts SVE vector size restriction in bytes to LEN according to ZCR_EL3 documentation. + * VECTOR_SIZE = (LEN+1) * 128 + */ +#define CONVERT_SVE_LENGTH(x) (((x / 128) - 1)) + +void sve_enable_per_world(per_world_context_t *per_world_ctx) +{ + u_register_t cptr_el3; + + /* Enable access to SVE functionality for all ELs. */ + cptr_el3 = per_world_ctx->ctx_cptr_el3; + cptr_el3 = (cptr_el3 | CPTR_EZ_BIT) & ~(TFP_BIT); + per_world_ctx->ctx_cptr_el3 = cptr_el3; + + /* Restrict maximum SVE vector length (SVE_VECTOR_LEN+1) * 128. */ + per_world_ctx->ctx_zcr_el3 = (ZCR_EL3_LEN_MASK & CONVERT_SVE_LENGTH(SVE_VECTOR_LEN)); +} + +void sve_init_el2_unused(void) +{ + /* + * CPTR_EL2.TFP: Set to zero so that Non-secure accesses to Advanced + * SIMD and floating-point functionality from both Execution states do + * not trap to EL2. + */ + write_cptr_el2(read_cptr_el2() & ~CPTR_EL2_TFP_BIT); +} + +void sve_disable_per_world(per_world_context_t *per_world_ctx) +{ + u_register_t reg; + + /* Disable SVE and FPU since they share registers. */ + reg = per_world_ctx->ctx_cptr_el3; + reg &= ~CPTR_EZ_BIT; /* Trap SVE */ + reg |= TFP_BIT; /* Trap FPU/SIMD */ + per_world_ctx->ctx_cptr_el3 = reg; +} diff --git a/lib/extensions/sys_reg_trace/aarch32/sys_reg_trace.c b/lib/extensions/sys_reg_trace/aarch32/sys_reg_trace.c new file mode 100644 index 0000000..6da504e --- /dev/null +++ b/lib/extensions/sys_reg_trace/aarch32/sys_reg_trace.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <lib/extensions/sys_reg_trace.h> + +void sys_reg_trace_init_el3(void) +{ + uint32_t val; + + /* + * NSACR.NSTRCDIS = b0 + * enable NS system register access to implemented trace + * registers. + */ + val = read_nsacr(); + val &= ~NSTRCDIS_BIT; + write_nsacr(val); +} diff --git a/lib/extensions/sys_reg_trace/aarch64/sys_reg_trace.c b/lib/extensions/sys_reg_trace/aarch64/sys_reg_trace.c new file mode 100644 index 0000000..2170763 --- /dev/null +++ b/lib/extensions/sys_reg_trace/aarch64/sys_reg_trace.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <lib/extensions/sys_reg_trace.h> + +void sys_reg_trace_enable_per_world(per_world_context_t *per_world_ctx) +{ + /* + * CPTR_EL3.TTA: Set to zero so that System register accesses to the + * trace registers do not trap to EL3. + */ + uint64_t val = per_world_ctx->ctx_cptr_el3; + val &= ~(TTA_BIT); + per_world_ctx->ctx_cptr_el3 = val; +} + +void sys_reg_trace_disable_per_world(per_world_context_t *per_world_ctx) +{ + /* + * CPTR_EL3.TTA: Set to one so that System register accesses to the + * trace registers trap to EL3, unless it is trapped by CPACR.TRCDIS, + * CPACR_EL1.TTA, or CPTR_EL2.TTA + */ + uint64_t val = per_world_ctx->ctx_cptr_el3; + val |= TTA_BIT; + per_world_ctx->ctx_cptr_el3 = val; +} + +void sys_reg_trace_init_el2_unused(void) +{ + /* + * CPTR_EL2.TTA: Set to zero so that Non-secure System register accesses + * to the trace registers from both Execution states do not trap to + * EL2. If PE trace unit System registers are not implemented then this + * bit is reserved, and must be set to zero. + */ + write_cptr_el2(read_cptr_el2() & ~CPTR_EL2_TTA_BIT); +} diff --git a/lib/extensions/trbe/trbe.c b/lib/extensions/trbe/trbe.c new file mode 100644 index 0000000..d4fbdfb --- /dev/null +++ b/lib/extensions/trbe/trbe.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/el3_runtime/pubsub.h> +#include <lib/extensions/trbe.h> + +static void tsb_csync(void) +{ + /* + * The assembler does not yet understand the tsb csync mnemonic + * so use the equivalent hint instruction. + */ + __asm__ volatile("hint #18"); +} + +void trbe_init_el3(void) +{ + u_register_t val; + + /* + * MDCR_EL3.NSTBE = 0b0 + * Trace Buffer owning Security state is Non-secure state. If FEAT_RME + * is not implemented, this field is RES0. + * + * MDCR_EL3.NSTB = 0b11 + * Allow access of trace buffer control registers from NS-EL1 and + * NS-EL2, tracing is prohibited in Secure and Realm state (if + * implemented). + */ + val = read_mdcr_el3(); + val |= MDCR_NSTB(MDCR_NSTB_EL1); + val &= ~(MDCR_NSTBE_BIT); + write_mdcr_el3(val); +} + +void trbe_init_el2_unused(void) +{ + /* + * MDCR_EL2.E2TB: Set to zero so that the trace Buffer + * owning exception level is NS-EL1 and, tracing is + * prohibited at NS-EL2. These bits are RES0 when + * FEAT_TRBE is not implemented. + */ + write_mdcr_el2(read_mdcr_el2() & ~MDCR_EL2_E2TB(MDCR_EL2_E2TB_EL1)); +} + +static void *trbe_drain_trace_buffers_hook(const void *arg __unused) +{ + if (is_feat_trbe_supported()) { + /* + * Before switching from normal world to secure world + * the trace buffers need to be drained out to memory. This is + * required to avoid an invalid memory access when TTBR is switched + * for entry to S-EL1. + */ + tsb_csync(); + dsbnsh(); + } + + return (void *)0; +} + +SUBSCRIBE_TO_EVENT(cm_entering_secure_world, trbe_drain_trace_buffers_hook); diff --git a/lib/extensions/trf/aarch32/trf.c b/lib/extensions/trf/aarch32/trf.c new file mode 100644 index 0000000..321c089 --- /dev/null +++ b/lib/extensions/trf/aarch32/trf.c @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <lib/extensions/trf.h> + +void trf_init_el3(void) +{ + uint32_t val; + + /* + * Allow access of trace filter control registers from + * non-monitor mode. + */ + val = read_sdcr(); + val &= ~(SDCR_TTRF_BIT); + write_sdcr(val); +} diff --git a/lib/extensions/trf/aarch64/trf.c b/lib/extensions/trf/aarch64/trf.c new file mode 100644 index 0000000..83fbf85 --- /dev/null +++ b/lib/extensions/trf/aarch64/trf.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/extensions/trf.h> + +void trf_init_el3(void) +{ + u_register_t val; + + /* + * MDCR_EL3.STE = b0 + * Trace prohibited in Secure state unless overridden by the + * IMPLEMENTATION DEFINED authentication interface. + * + * MDCR_EL3.TTRF = b0 + * Allow access of trace filter control registers from NS-EL2 + * and NS-EL1 when NS-EL2 is implemented but not used + */ + val = read_mdcr_el3(); + val &= ~(MDCR_STE_BIT | MDCR_TTRF_BIT); + write_mdcr_el3(val); +} + +void trf_init_el2_unused(void) +{ + /* + * MDCR_EL2.TTRF: Set to zero so that access to Trace + * Filter Control register TRFCR_EL1 at EL1 is not + * trapped to EL2. This bit is RES0 in versions of + * the architecture earlier than ARMv8.4. + * + */ + write_mdcr_el2(read_mdcr_el2() & ~MDCR_EL2_TTRF); +} diff --git a/lib/fconf/fconf.c b/lib/fconf/fconf.c new file mode 100644 index 0000000..136e056 --- /dev/null +++ b/lib/fconf/fconf.c @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019-2022, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> + +#include <common/debug.h> +#include <common/fdt_wrappers.h> +#include <lib/fconf/fconf.h> +#include <lib/fconf/fconf_dyn_cfg_getter.h> +#include <libfdt.h> +#include <plat/common/platform.h> +#include <platform_def.h> + +int fconf_load_config(unsigned int image_id) +{ + int err; + const struct dyn_cfg_dtb_info_t *config_info; + + assert((image_id == FW_CONFIG_ID) || (image_id == TB_FW_CONFIG_ID)); + + image_info_t config_image_info = { + .h.type = (uint8_t)PARAM_IMAGE_BINARY, + .h.version = (uint8_t)VERSION_2, + .h.size = (uint16_t)sizeof(image_info_t), + .h.attr = 0 + }; + + config_info = FCONF_GET_PROPERTY(dyn_cfg, dtb, image_id); + assert(config_info != NULL); + + config_image_info.image_base = config_info->config_addr; + config_image_info.image_max_size = config_info->config_max_size; + + VERBOSE("FCONF: Loading config with image ID: %u\n", image_id); + err = load_auth_image(image_id, &config_image_info); + if (err != 0) { + VERBOSE("Failed to load config %u\n", image_id); + return err; + } + + INFO("FCONF: Config file with image ID:%u loaded at address = 0x%lx\n", + image_id, config_image_info.image_base); + + return 0; +} + +void fconf_populate(const char *config_type, uintptr_t config) +{ + assert(config != 0UL); + + /* Check if the pointer to DTB is correct */ + if (fdt_check_header((void *)config) != 0) { + ERROR("FCONF: Invalid DTB file passed for %s\n", config_type); + panic(); + } + + INFO("FCONF: Reading %s firmware configuration file from: 0x%lx\n", config_type, config); + + /* Go through all registered populate functions */ + IMPORT_SYM(struct fconf_populator *, __FCONF_POPULATOR_START__, start); + IMPORT_SYM(struct fconf_populator *, __FCONF_POPULATOR_END__, end); + const struct fconf_populator *populator; + + for (populator = start; populator != end; populator++) { + assert((populator->info != NULL) && (populator->populate != NULL)); + + if (strcmp(populator->config_type, config_type) == 0) { + INFO("FCONF: Reading firmware configuration information for: %s\n", populator->info); + if (populator->populate(config) != 0) { + /* TODO: handle property miss */ + panic(); + } + } + } +} diff --git a/lib/fconf/fconf.mk b/lib/fconf/fconf.mk new file mode 100644 index 0000000..ff781aa --- /dev/null +++ b/lib/fconf/fconf.mk @@ -0,0 +1,19 @@ +# +# Copyright (c) 2019-2021, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +include common/fdt_wrappers.mk + +FCONF_SOURCES := lib/fconf/fconf.c +FCONF_SOURCES += ${FDT_WRAPPERS_SOURCES} + +FCONF_DYN_SOURCES := lib/fconf/fconf_dyn_cfg_getter.c +FCONF_DYN_SOURCES += ${FDT_WRAPPERS_SOURCES} + +FCONF_AMU_SOURCES := lib/fconf/fconf_amu_getter.c +FCONF_AMU_SOURCES += ${FDT_WRAPPERS_SOURCES} + +FCONF_MPMM_SOURCES := lib/fconf/fconf_mpmm_getter.c +FCONF_MPMM_SOURCES += ${FDT_WRAPPERS_SOURCES} diff --git a/lib/fconf/fconf_amu_getter.c b/lib/fconf/fconf_amu_getter.c new file mode 100644 index 0000000..eff309c --- /dev/null +++ b/lib/fconf/fconf_amu_getter.c @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2021, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stddef.h> +#include <stdint.h> + +#include <common/debug.h> +#include <common/fdt_wrappers.h> +#include <lib/fconf/fconf.h> +#include <lib/fconf/fconf_amu_getter.h> +#include <libfdt.h> + +#include <plat/common/platform.h> + +struct fconf_amu_config fconf_amu_config; +static struct amu_topology fconf_amu_topology_; + +/* + * Populate the core-specific AMU structure with information retrieved from a + * device tree. + * + * Returns `0` on success, or a negative integer representing an error code. + */ +static int fconf_populate_amu_cpu_amu(const void *fdt, int parent, + struct amu_core *amu) +{ + int ret = 0; + int node = 0; + + fdt_for_each_subnode(node, fdt, parent) { + const char *name; + const char *value; + int len; + + uintptr_t idx = 0U; + + name = fdt_get_name(fdt, node, &len); + if (strncmp(name, "counter@", 8) != 0) { + continue; + } + + ret = fdt_get_reg_props_by_index(fdt, node, 0, &idx, NULL); + if (ret < 0) { + break; + } + + value = fdt_getprop(fdt, node, "enable-at-el3", &len); + if ((value == NULL) && (len != -FDT_ERR_NOTFOUND)) { + break; + } + + if (len != -FDT_ERR_NOTFOUND) { + amu->enable |= (1 << idx); + } + } + + if ((node < 0) && (node != -FDT_ERR_NOTFOUND)) { + return node; + } + + return ret; +} + +/* + * Within a `cpu` node, attempt to dereference the `amu` property, and populate + * the AMU information for the core. + * + * Returns `0` on success, or a negative integer representing an error code. + */ +static int fconf_populate_amu_cpu(const void *fdt, int node, uintptr_t mpidr) +{ + int ret; + int idx; + + uint32_t amu_phandle; + struct amu_core *amu; + + ret = fdt_read_uint32(fdt, node, "amu", &amu_phandle); + if (ret < 0) { + if (ret == -FDT_ERR_NOTFOUND) { + ret = 0; + } + + return ret; + } + + node = fdt_node_offset_by_phandle(fdt, amu_phandle); + if (node < 0) { + return node; + } + + idx = plat_core_pos_by_mpidr(mpidr); + if (idx < 0) { + return -FDT_ERR_BADVALUE; + } + + amu = &fconf_amu_topology_.cores[idx]; + + return fconf_populate_amu_cpu_amu(fdt, node, amu); +} + +/* + * Populates the global `amu_topology` structure based on what's described by + * the hardware configuration device tree blob. + * + * The device tree is expected to provide an `amu` property for each `cpu` node, + * like so: + * + * cpu@0 { + * amu = <&cpu0_amu>; + * }; + * + * amus { + * cpu0_amu: amu-0 { + * counters { + * #address-cells = <2>; + * #size-cells = <0>; + * + * counter@x,y { + * reg = <x y>; // Group x, counter y + * }; + * }; + * }; + * }; + */ +static int fconf_populate_amu(uintptr_t config) +{ + int ret = fdtw_for_each_cpu( + (const void *)config, fconf_populate_amu_cpu); + if (ret == 0) { + fconf_amu_config.topology = &fconf_amu_topology_; + } else { + ERROR("FCONF: failed to parse AMU information: %d\n", ret); + } + + return ret; +} + +FCONF_REGISTER_POPULATOR(HW_CONFIG, amu, fconf_populate_amu); diff --git a/lib/fconf/fconf_cot_getter.c b/lib/fconf/fconf_cot_getter.c new file mode 100644 index 0000000..1033018 --- /dev/null +++ b/lib/fconf/fconf_cot_getter.c @@ -0,0 +1,499 @@ +/* + * Copyright (c) 2020-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stddef.h> + +#include <mbedtls/version.h> + +#include <common/fdt_wrappers.h> +#include <common/tbbr/cot_def.h> +#include <drivers/auth/auth_mod.h> +#include <lib/fconf/fconf.h> +#include <lib/object_pool.h> +#include <libfdt.h> + +#include <tools_share/tbbr_oid.h> + +/* static structures used during authentication process */ +static auth_param_type_desc_t sig = AUTH_PARAM_TYPE_DESC( + AUTH_PARAM_SIG, 0); +static auth_param_type_desc_t sig_alg = AUTH_PARAM_TYPE_DESC( + AUTH_PARAM_SIG_ALG, 0); +static auth_param_type_desc_t raw_data = AUTH_PARAM_TYPE_DESC( + AUTH_PARAM_RAW_DATA, 0); + +/* pointers to an array of CoT descriptors */ +static const auth_img_desc_t *cot_desc[MAX_NUMBER_IDS]; +/* array of CoT descriptors */ +static auth_img_desc_t auth_img_descs[MAX_NUMBER_IDS]; + +/* array of authentication methods structures */ +static auth_method_desc_t auth_methods[MAX_NUMBER_IDS * AUTH_METHOD_NUM]; +static OBJECT_POOL_ARRAY(auth_methods_pool, auth_methods); + +/* array of authentication params structures */ +static auth_param_desc_t auth_params[MAX_NUMBER_IDS * COT_MAX_VERIFIED_PARAMS]; +static OBJECT_POOL_ARRAY(auth_params_pool, auth_params); + +/* array of authentication param type structures */ +static auth_param_type_desc_t auth_param_type_descs[MAX_NUMBER_IDS]; +static OBJECT_POOL_ARRAY(auth_param_type_descs_pool, auth_param_type_descs); + +/* + * array of OIDs + * Object IDs are used to search hash, pk, counter values in certificate. + * As per binding we have below 2 combinations: + * 1. Certificates are validated using nv-cntr and pk + * 2. Raw images are authenticated using hash + * Hence in worst case, there are maximum 2 OIDs per image/certificate + */ +static unsigned char oids[(MAX_NUMBER_IDS * 2)][MAX_OID_NAME_LEN]; +static OBJECT_POOL_ARRAY(oid_pool, oids); + +/* An array of auth buffer which holds hashes and pk + * ToDo: Size decided with the current number of images and + * certificates which are available in CoT. Size of these buffers bound to + * increase in the future on the addition of images/certificates. + */ +static unsigned char hash_auth_bufs[20][HASH_DER_LEN]; +static OBJECT_POOL_ARRAY(hash_auth_buf_pool, hash_auth_bufs); +static unsigned char pk_auth_bufs[12][PK_DER_LEN]; +static OBJECT_POOL_ARRAY(pk_auth_buf_pool, pk_auth_bufs); + +/******************************************************************************* + * update_parent_auth_data() - Update authentication data structure + * @auth_desc[in]: Pointer to the auth image descriptor + * @type_desc[in]: Pointer to authentication parameter + * @auth_buf_size[in]: Buffer size to hold pk or hash + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int update_parent_auth_data(const auth_img_desc_t *auth_desc, + auth_param_type_desc_t *type_desc, + unsigned int auth_buf_size) +{ + unsigned int i; + auth_param_desc_t *auth_data = &auth_desc->authenticated_data[0]; + unsigned char *auth_buf; + + for (i = 0U; i < COT_MAX_VERIFIED_PARAMS; i++) { + if (auth_data[i].type_desc == type_desc) { + return 0; + } + if (auth_data[i].type_desc == NULL) { + break; + } + } + + if (auth_buf_size == HASH_DER_LEN) { + auth_buf = pool_alloc(&hash_auth_buf_pool); + } else if (auth_buf_size == PK_DER_LEN) { + auth_buf = pool_alloc(&pk_auth_buf_pool); + } else { + return -1; + } + + if (i < COT_MAX_VERIFIED_PARAMS) { + auth_data[i].type_desc = type_desc; + auth_data[i].data.ptr = auth_buf; + auth_data[i].data.len = auth_buf_size; + } else { + ERROR("Out of authentication data array\n"); + return -1; + } + + return 0; +} + +/******************************************************************************* + * get_auth_param_type_desc() - Get pointer of authentication parameter + * @img_id[in]: Image Id + * @type_desc[out]: Pointer to authentication parameter + * @buf_size[out]: Buffer size which hold hash/pk + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int get_auth_param_type_desc(unsigned int img_id, + auth_param_type_desc_t **type_desc, + unsigned int *buf_size) +{ + auth_method_desc_t *img_auth_method = NULL; + img_type_t type = auth_img_descs[img_id].img_type; + + if (type == IMG_CERT) { + img_auth_method = + &auth_img_descs[img_id].img_auth_methods[AUTH_METHOD_SIG]; + *type_desc = img_auth_method->param.sig.pk; + *buf_size = PK_DER_LEN; + } else if (type == IMG_RAW) { + img_auth_method = + &auth_img_descs[img_id].img_auth_methods[AUTH_METHOD_HASH]; + *type_desc = img_auth_method->param.hash.hash; + *buf_size = HASH_DER_LEN; + } else { + return -1; + } + + return 0; +} + +/******************************************************************************* + * set_auth_method() - Update global auth image descriptors with authentication + * method data + * @auth_method_type[in]: Type of authentication method + * @oid[in]: Object Idetifier for pk/hash search + * @auth_method[in]: Pointer to authentication method to set + ******************************************************************************/ +static void set_auth_method(auth_method_type_t auth_method_type, char *oid, + auth_method_desc_t *auth_method) +{ + auth_param_type_t auth_param_type = AUTH_PARAM_NONE; + auth_param_type_desc_t *auth_param_type_desc; + + assert(auth_method != NULL); + + auth_param_type_desc = pool_alloc(&auth_param_type_descs_pool); + auth_method->type = auth_method_type; + + if (auth_method_type == AUTH_METHOD_SIG) { + auth_param_type = AUTH_PARAM_PUB_KEY; + auth_method->param.sig.sig = &sig; + auth_method->param.sig.alg = &sig_alg; + auth_method->param.sig.data = &raw_data; + auth_method->param.sig.pk = auth_param_type_desc; + } else if (auth_method_type == AUTH_METHOD_HASH) { + auth_param_type = AUTH_PARAM_HASH; + auth_method->param.hash.data = &raw_data; + auth_method->param.hash.hash = auth_param_type_desc; + } else if (auth_method_type == AUTH_METHOD_NV_CTR) { + auth_param_type = AUTH_PARAM_NV_CTR; + auth_method->param.nv_ctr.cert_nv_ctr = auth_param_type_desc; + auth_method->param.nv_ctr.plat_nv_ctr = auth_param_type_desc; + } + + auth_param_type_desc->type = auth_param_type; + auth_param_type_desc->cookie = (void *)oid; +} + +/******************************************************************************* + * get_oid() - get object identifier from device tree + * @dtb[in]: Pointer to the device tree blob in memory + * @node[in]: Offset of the node + * @prop[in]: Property to read from the given node + * @oid[out]: Object Indentifier of key/hash/nv-counter in certificate + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int get_oid(const void *dtb, int node, const char *prop, char **oid) +{ + uint32_t phandle; + int rc; + + rc = fdt_read_uint32(dtb, node, prop, &phandle); + if (rc < 0) { + return rc; + } + + node = fdt_node_offset_by_phandle(dtb, phandle); + if (node < 0) { + return node; + } + + *oid = pool_alloc(&oid_pool); + rc = fdtw_read_string(dtb, node, "oid", *oid, MAX_OID_NAME_LEN); + + return rc; +} + +/******************************************************************************* + * populate_and_set_auth_methods() - Populate auth method parameters from + * device tree and set authentication method + * structure. + * @dtb[in]: Pointer to the device tree blob in memory + * @node[in]: Offset of the node + * @img_id[in]: Image identifier + * @type[in]: Type of image + * @root_certificate[in]:Root certificate (authenticated by ROTPK) + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int populate_and_set_auth_methods(const void *dtb, int node, + unsigned int img_id, img_type_t type, + bool root_certificate) +{ + auth_method_type_t auth_method_type = AUTH_METHOD_NONE; + int rc; + char *oid = NULL; + + auth_method_desc_t *auth_method = pool_alloc_n(&auth_methods_pool, + AUTH_METHOD_NUM); + + /* + * This is as per binding document where certificates are + * verified by signature and images are verified by hash. + */ + if (type == IMG_CERT) { + if (root_certificate) { + oid = NULL; + } else { + rc = get_oid(dtb, node, "signing-key", &oid); + if (rc < 0) { + ERROR("FCONF: Can't read %s property\n", + "signing-key"); + return rc; + } + } + auth_method_type = AUTH_METHOD_SIG; + } else if (type == IMG_RAW) { + rc = get_oid(dtb, node, "hash", &oid); + if (rc < 0) { + ERROR("FCONF: Can't read %s property\n", + "hash"); + return rc; + } + auth_method_type = AUTH_METHOD_HASH; + } else { + return -1; + } + + set_auth_method(auth_method_type, oid, + &auth_method[auth_method_type]); + + /* Retrieve the optional property */ + rc = get_oid(dtb, node, "antirollback-counter", &oid); + if (rc == 0) { + auth_method_type = AUTH_METHOD_NV_CTR; + set_auth_method(auth_method_type, oid, + &auth_method[auth_method_type]); + } + + auth_img_descs[img_id].img_auth_methods = &auth_method[0]; + + return 0; +} + +/******************************************************************************* + * get_parent_img_id() - Get parent image id for given child node + * @dtb[in]: Pointer to the device tree blob in memory + * @node[in]: Offset of the child node + * @parent_img_id[out]: Image id of parent + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int get_parent_img_id(const void *dtb, int node, + unsigned int *parent_img_id) +{ + uint32_t phandle; + int err; + + err = fdt_read_uint32(dtb, node, "parent", &phandle); + if (err < 0) { + ERROR("FCONF: Could not read %s property in node\n", + "parent"); + return err; + } + + node = fdt_node_offset_by_phandle(dtb, phandle); + if (node < 0) { + ERROR("FCONF: Failed to locate node using its phandle\n"); + return node; + } + + err = fdt_read_uint32(dtb, node, "image-id", parent_img_id); + if (err < 0) { + ERROR("FCONF: Could not read %s property in node\n", + "image-id"); + } + + return err; +} + +/******************************************************************************* + * set_desc_data() - Update data in descriptor's structure + * @dtb[in]: Pointer to the device tree blob in memory + * @node[in]: Offset of the node + * @type[in]: Type of image (RAW/CERT) + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int set_desc_data(const void *dtb, int node, img_type_t type) +{ + int rc; + bool root_certificate = false; + unsigned int img_id, parent_img_id; + + rc = fdt_read_uint32(dtb, node, "image-id", &img_id); + if (rc < 0) { + ERROR("FCONF: Can't find property %s in node\n", + "image-id"); + return rc; + } + + if (fdt_getprop(dtb, node, "root-certificate", + NULL) != NULL) { + root_certificate = true; + } + + if (!root_certificate) { + rc = get_parent_img_id(dtb, node, &parent_img_id); + if (rc < 0) { + return rc; + } + auth_img_descs[img_id].parent = &auth_img_descs[parent_img_id]; + } + + auth_img_descs[img_id].img_id = img_id; + auth_img_descs[img_id].img_type = type; + + rc = populate_and_set_auth_methods(dtb, node, img_id, type, + root_certificate); + if (rc < 0) { + return rc; + } + + if (type == IMG_CERT) { + auth_param_desc_t *auth_param = + pool_alloc_n(&auth_params_pool, + COT_MAX_VERIFIED_PARAMS); + auth_img_descs[img_id].authenticated_data = &auth_param[0]; + } + + cot_desc[img_id] = &auth_img_descs[img_id]; + + return rc; +} + +/******************************************************************************* + * populate_manifest_descs() - Populate CoT descriptors and update global + * certificate structures + * @dtb[in]: Pointer to the device tree blob in memory + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int populate_manifest_descs(const void *dtb) +{ + int node, child; + int rc; + + /* + * Assert the node offset points to "arm, cert-descs" + * compatible property + */ + const char *compatible_str = "arm, cert-descs"; + + node = fdt_node_offset_by_compatible(dtb, -1, compatible_str); + if (node < 0) { + ERROR("FCONF: Can't find %s compatible in node\n", + compatible_str); + return node; + } + + fdt_for_each_subnode(child, dtb, node) { + rc = set_desc_data(dtb, child, IMG_CERT); + if (rc < 0) { + return rc; + } + } + + return 0; +} + +/******************************************************************************* + * populate_image_descs() - Populate CoT descriptors and update global + * image descriptor structures. + * @dtb[in]: Pointer to the device tree blob in memory + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int populate_image_descs(const void *dtb) +{ + int node, child; + int rc; + + /* + * Assert the node offset points to "arm, img-descs" + * compatible property + */ + const char *compatible_str = "arm, img-descs"; + + node = fdt_node_offset_by_compatible(dtb, -1, compatible_str); + if (node < 0) { + ERROR("FCONF: Can't find %s compatible in node\n", + compatible_str); + return node; + } + + fdt_for_each_subnode(child, dtb, node) { + rc = set_desc_data(dtb, child, IMG_RAW); + if (rc < 0) { + return rc; + } + } + + return 0; +} + +/******************************************************************************* + * fconf_populate_cot_descs() - Populate CoT descriptors and update global + * structures + * @config[in]: Pointer to the device tree blob in memory + * + * Return 0 on success or an error value otherwise. + ******************************************************************************/ +static int fconf_populate_cot_descs(uintptr_t config) +{ + auth_param_type_desc_t *type_desc = NULL; + unsigned int auth_buf_size = 0U; + int rc; + + /* As libfdt uses void *, we can't avoid this cast */ + const void *dtb = (void *)config; + + /* populate manifest descs information */ + rc = populate_manifest_descs(dtb); + if (rc < 0) { + ERROR("FCONF: population of %s descs failed %d\n", + "manifest", rc); + return rc; + } + + /* populate image descs information */ + rc = populate_image_descs(dtb); + if (rc < 0) { + ERROR("FCONF: population of %s descs failed %d\n", + "images", rc); + return rc; + } + + /* update parent's authentication data */ + for (unsigned int i = 0U; i < MAX_NUMBER_IDS; i++) { + if (auth_img_descs[i].parent != NULL) { + rc = get_auth_param_type_desc(i, + &type_desc, + &auth_buf_size); + if (rc < 0) { + ERROR("FCONF: failed to get auth data %d\n", + rc); + return rc; + } + + rc = update_parent_auth_data(auth_img_descs[i].parent, + type_desc, + auth_buf_size); + if (rc < 0) { + ERROR("FCONF: auth data update failed %d\n", + rc); + return rc; + } + } + } + + return rc; +} + +FCONF_REGISTER_POPULATOR(TB_FW, cot_desc, fconf_populate_cot_descs); +REGISTER_COT(cot_desc); diff --git a/lib/fconf/fconf_dyn_cfg_getter.c b/lib/fconf/fconf_dyn_cfg_getter.c new file mode 100644 index 0000000..58956d6 --- /dev/null +++ b/lib/fconf/fconf_dyn_cfg_getter.c @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2019-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> + +#include <common/debug.h> +#include <common/fdt_wrappers.h> +#include <lib/fconf/fconf_dyn_cfg_getter.h> +#include <lib/object_pool.h> +#include <libfdt.h> + +#include <platform_def.h> + +/* We currently use FW, TB_FW, SOC_FW, TOS_FW, NT_FW and HW configs */ +#define MAX_DTB_INFO U(6) +/* + * Compile time assert if FW_CONFIG_ID is 0 which is more + * unlikely as 0 is a valid image ID for FIP as per the current + * code but still to avoid code breakage in case of unlikely + * event when image IDs get changed. + */ +CASSERT(FW_CONFIG_ID != U(0), assert_invalid_fw_config_id); + +static struct dyn_cfg_dtb_info_t dtb_infos[MAX_DTB_INFO]; +static OBJECT_POOL_ARRAY(dtb_info_pool, dtb_infos); + +/* + * This function is used to alloc memory for config information from + * global pool and set the configuration information. + */ +void set_config_info(uintptr_t config_addr, uintptr_t secondary_config_addr, + uint32_t config_max_size, + unsigned int config_id) +{ + struct dyn_cfg_dtb_info_t *dtb_info; + + dtb_info = pool_alloc(&dtb_info_pool); + dtb_info->config_addr = config_addr; + dtb_info->secondary_config_addr = secondary_config_addr; + dtb_info->config_max_size = config_max_size; + dtb_info->config_id = config_id; +} + +/* Get index of the config_id image */ +unsigned int dyn_cfg_dtb_info_get_index(unsigned int config_id) +{ + unsigned int index; + + /* Positions index to the proper config-id */ + for (index = 0U; index < MAX_DTB_INFO; index++) { + if (dtb_infos[index].config_id == config_id) { + return index; + } + } + + return FCONF_INVALID_IDX; +} + +struct dyn_cfg_dtb_info_t *dyn_cfg_dtb_info_getter(unsigned int config_id) +{ + /* Positions index to the proper config-id */ + unsigned int index = dyn_cfg_dtb_info_get_index(config_id); + + if (index < MAX_DTB_INFO) { + return &dtb_infos[index]; + } + + WARN("FCONF: Invalid config id %u\n", config_id); + + return NULL; +} + +int fconf_populate_dtb_registry(uintptr_t config) +{ + int rc; + int node, child; + + /* As libfdt use void *, we can't avoid this cast */ + const void *dtb = (void *)config; + + /* + * In case of BL1, fw_config dtb information is already + * populated in global dtb_infos array by 'set_config_info' + * function, Below check is present to avoid re-population of + * fw_config information. + * + * Other BLs, satisfy below check and populate fw_config information + * in global dtb_infos array. + */ + if (dtb_infos[0].config_id == 0U) { + uint32_t config_max_size = fdt_totalsize(dtb); + set_config_info(config, ~0UL, config_max_size, FW_CONFIG_ID); + } + + /* Find the node offset point to "fconf,dyn_cfg-dtb_registry" compatible property */ + const char *compatible_str = "fconf,dyn_cfg-dtb_registry"; + node = fdt_node_offset_by_compatible(dtb, -1, compatible_str); + if (node < 0) { + ERROR("FCONF: Can't find %s compatible in dtb\n", compatible_str); + return node; + } + + fdt_for_each_subnode(child, dtb, node) { + uint32_t config_max_size, config_id; + uintptr_t config_addr; + uintptr_t secondary_config_addr = ~0UL; + uint64_t val64; + + /* Read configuration dtb information */ + rc = fdt_read_uint64(dtb, child, "load-address", &val64); + if (rc < 0) { + ERROR("FCONF: Incomplete configuration property in dtb-registry.\n"); + return rc; + } + config_addr = (uintptr_t)val64; + + rc = fdt_read_uint32(dtb, child, "max-size", &config_max_size); + if (rc < 0) { + ERROR("FCONF: Incomplete configuration property in dtb-registry.\n"); + return rc; + } + + rc = fdt_read_uint32(dtb, child, "id", &config_id); + if (rc < 0) { + ERROR("FCONF: Incomplete configuration property in dtb-registry.\n"); + return rc; + } + + VERBOSE("FCONF: dyn_cfg.dtb_registry cell found with:\n"); + VERBOSE("\tload-address = %lx\n", config_addr); + VERBOSE("\tmax-size = 0x%x\n", config_max_size); + VERBOSE("\tconfig-id = %u\n", config_id); + + rc = fdt_read_uint64(dtb, child, "secondary-load-address", + &val64); + if (rc == 0) { + secondary_config_addr = (uintptr_t)val64; + VERBOSE("\tsecondary-load-address = %lx\n", + secondary_config_addr); + } + + set_config_info(config_addr, secondary_config_addr, + config_max_size, config_id); + } + + if ((child < 0) && (child != -FDT_ERR_NOTFOUND)) { + ERROR("%d: fdt_for_each_subnode(): %d\n", __LINE__, node); + return child; + } + + return 0; +} + +FCONF_REGISTER_POPULATOR(FW_CONFIG, dyn_cfg, fconf_populate_dtb_registry); diff --git a/lib/fconf/fconf_mpmm_getter.c b/lib/fconf/fconf_mpmm_getter.c new file mode 100644 index 0000000..02a566d --- /dev/null +++ b/lib/fconf/fconf_mpmm_getter.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stddef.h> +#include <stdint.h> + +#include <common/debug.h> +#include <common/fdt_wrappers.h> +#include <lib/fconf/fconf.h> +#include <lib/fconf/fconf_mpmm_getter.h> +#include <libfdt.h> + +#include <plat/common/platform.h> + +struct fconf_mpmm_config fconf_mpmm_config; +static struct mpmm_topology fconf_mpmm_topology; + +/* + * Within a `cpu` node, determine support for MPMM via the `supports-mpmm` + * property. + * + * Returns `0` on success, or a negative integer representing an error code. + */ +static int fconf_populate_mpmm_cpu(const void *fdt, int off, uintptr_t mpidr) +{ + int ret, len; + + int core_pos; + struct mpmm_core *core; + + core_pos = plat_core_pos_by_mpidr(mpidr); + if (core_pos < 0) { + return -FDT_ERR_BADVALUE; + } + + core = &fconf_mpmm_topology.cores[core_pos]; + + fdt_getprop(fdt, off, "supports-mpmm", &len); + if (len >= 0) { + core->supported = true; + ret = 0; + } else { + core->supported = false; + ret = len; + } + + return ret; +} + +/* + * Populates the global `fconf_mpmm_config` structure based on what's described + * by the hardware configuration device tree blob. + * + * The device tree is expected to provide a `supports-mpmm` property for each + * `cpu` node, like so: + * + * cpu@0 { + * supports-mpmm; + * }; + * + * This property indicates whether the core implements MPMM, as we cannot detect + * support for it dynamically. + */ +static int fconf_populate_mpmm(uintptr_t config) +{ + int ret = fdtw_for_each_cpu( + (const void *)config, fconf_populate_mpmm_cpu); + if (ret == 0) { + fconf_mpmm_config.topology = &fconf_mpmm_topology; + } else { + ERROR("FCONF: failed to configure MPMM: %d\n", ret); + } + + return ret; +} + +FCONF_REGISTER_POPULATOR(HW_CONFIG, mpmm, fconf_populate_mpmm); diff --git a/lib/fconf/fconf_tbbr_getter.c b/lib/fconf/fconf_tbbr_getter.c new file mode 100644 index 0000000..c3b4b7e --- /dev/null +++ b/lib/fconf/fconf_tbbr_getter.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2019-2022, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <assert.h> + +#include <common/bl_common.h> +#include <common/debug.h> +#include <common/fdt_wrappers.h> +#include <lib/fconf/fconf_tbbr_getter.h> +#include <libfdt.h> + +struct tbbr_dyn_config_t tbbr_dyn_config; + +int fconf_populate_tbbr_dyn_config(uintptr_t config) +{ + int err; + int node; + uint64_t val64; + uint32_t val32; + + /* As libfdt use void *, we can't avoid this cast */ + const void *dtb = (void *)config; + + /* Assert the node offset point to "arm,tb_fw" compatible property */ + const char *compatible_str = "arm,tb_fw"; + node = fdt_node_offset_by_compatible(dtb, -1, compatible_str); + if (node < 0) { + ERROR("FCONF: Can't find `%s` compatible in dtb\n", + compatible_str); + return node; + } + + /* Locate the disable_auth cell and read the value */ + err = fdt_read_uint32(dtb, node, "disable_auth", + &tbbr_dyn_config.disable_auth); + if (err < 0) { + WARN("FCONF: Read %s failed for `%s`\n", + "cell", "disable_auth"); + return err; + } + + /* Check if the value is boolean */ + if ((tbbr_dyn_config.disable_auth != 0U) && + (tbbr_dyn_config.disable_auth != 1U)) { + WARN("Invalid value for `%s` cell %u\n", + "disable_auth", tbbr_dyn_config.disable_auth); + return -1; + } + +#if defined(DYN_DISABLE_AUTH) + if (tbbr_dyn_config.disable_auth == 1) + dyn_disable_auth(); +#endif + + /* Retrieve the Mbed TLS heap details from the DTB */ + err = fdt_read_uint64(dtb, node, "mbedtls_heap_addr", &val64); + if (err < 0) { + ERROR("FCONF: Read %s failed for `%s`\n", + "cell", "mbedtls_heap_addr"); + return err; + } + tbbr_dyn_config.mbedtls_heap_addr = (void *)(uintptr_t)val64; + + err = fdt_read_uint32(dtb, node, "mbedtls_heap_size", &val32); + if (err < 0) { + ERROR("FCONF: Read %s failed for `%s`\n", + "cell", "mbedtls_heap_size"); + return err; + } + tbbr_dyn_config.mbedtls_heap_size = val32; + + VERBOSE("%s%s%s %u\n", "FCONF: `tbbr.", "disable_auth", + "` cell found with value =", tbbr_dyn_config.disable_auth); + VERBOSE("%s%s%s %p\n", "FCONF: `tbbr.", "mbedtls_heap_addr", + "` cell found with value =", tbbr_dyn_config.mbedtls_heap_addr); + VERBOSE("%s%s%s %zu\n", "FCONF: `tbbr.", "mbedtls_heap_size", + "` cell found with value =", tbbr_dyn_config.mbedtls_heap_size); + + return 0; +} + +FCONF_REGISTER_POPULATOR(TB_FW, tbbr, fconf_populate_tbbr_dyn_config); diff --git a/lib/gpt_rme/gpt_rme.c b/lib/gpt_rme/gpt_rme.c new file mode 100644 index 0000000..f5353cb --- /dev/null +++ b/lib/gpt_rme/gpt_rme.c @@ -0,0 +1,1260 @@ +/* + * Copyright (c) 2022, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <errno.h> +#include <inttypes.h> +#include <limits.h> +#include <stdint.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include "gpt_rme_private.h" +#include <lib/gpt_rme/gpt_rme.h> +#include <lib/smccc.h> +#include <lib/spinlock.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +#if !ENABLE_RME +#error "ENABLE_RME must be enabled to use the GPT library." +#endif + +/* + * Lookup T from PPS + * + * PPS Size T + * 0b000 4GB 32 + * 0b001 64GB 36 + * 0b010 1TB 40 + * 0b011 4TB 42 + * 0b100 16TB 44 + * 0b101 256TB 48 + * 0b110 4PB 52 + * + * See section 15.1.27 of the RME specification. + */ +static const gpt_t_val_e gpt_t_lookup[] = {PPS_4GB_T, PPS_64GB_T, + PPS_1TB_T, PPS_4TB_T, + PPS_16TB_T, PPS_256TB_T, + PPS_4PB_T}; + +/* + * Lookup P from PGS + * + * PGS Size P + * 0b00 4KB 12 + * 0b10 16KB 14 + * 0b01 64KB 16 + * + * Note that pgs=0b10 is 16KB and pgs=0b01 is 64KB, this is not a typo. + * + * See section 15.1.27 of the RME specification. + */ +static const gpt_p_val_e gpt_p_lookup[] = {PGS_4KB_P, PGS_64KB_P, PGS_16KB_P}; + +/* + * This structure contains GPT configuration data. + */ +typedef struct { + uintptr_t plat_gpt_l0_base; + gpccr_pps_e pps; + gpt_t_val_e t; + gpccr_pgs_e pgs; + gpt_p_val_e p; +} gpt_config_t; + +static gpt_config_t gpt_config; + +/* These variables are used during initialization of the L1 tables. */ +static unsigned int gpt_next_l1_tbl_idx; +static uintptr_t gpt_l1_tbl; + +/* + * This function checks to see if a GPI value is valid. + * + * These are valid GPI values. + * GPT_GPI_NO_ACCESS U(0x0) + * GPT_GPI_SECURE U(0x8) + * GPT_GPI_NS U(0x9) + * GPT_GPI_ROOT U(0xA) + * GPT_GPI_REALM U(0xB) + * GPT_GPI_ANY U(0xF) + * + * Parameters + * gpi GPI to check for validity. + * + * Return + * true for a valid GPI, false for an invalid one. + */ +static bool gpt_is_gpi_valid(unsigned int gpi) +{ + if ((gpi == GPT_GPI_NO_ACCESS) || (gpi == GPT_GPI_ANY) || + ((gpi >= GPT_GPI_SECURE) && (gpi <= GPT_GPI_REALM))) { + return true; + } + return false; +} + +/* + * This function checks to see if two PAS regions overlap. + * + * Parameters + * base_1: base address of first PAS + * size_1: size of first PAS + * base_2: base address of second PAS + * size_2: size of second PAS + * + * Return + * True if PAS regions overlap, false if they do not. + */ +static bool gpt_check_pas_overlap(uintptr_t base_1, size_t size_1, + uintptr_t base_2, size_t size_2) +{ + if (((base_1 + size_1) > base_2) && ((base_2 + size_2) > base_1)) { + return true; + } + return false; +} + +/* + * This helper function checks to see if a PAS region from index 0 to + * (pas_idx - 1) occupies the L0 region at index l0_idx in the L0 table. + * + * Parameters + * l0_idx: Index of the L0 entry to check + * pas_regions: PAS region array + * pas_idx: Upper bound of the PAS array index. + * + * Return + * True if a PAS region occupies the L0 region in question, false if not. + */ +static bool gpt_does_previous_pas_exist_here(unsigned int l0_idx, + pas_region_t *pas_regions, + unsigned int pas_idx) +{ + /* Iterate over PAS regions up to pas_idx. */ + for (unsigned int i = 0U; i < pas_idx; i++) { + if (gpt_check_pas_overlap((GPT_L0GPTSZ_ACTUAL_SIZE * l0_idx), + GPT_L0GPTSZ_ACTUAL_SIZE, + pas_regions[i].base_pa, pas_regions[i].size)) { + return true; + } + } + return false; +} + +/* + * This function iterates over all of the PAS regions and checks them to ensure + * proper alignment of base and size, that the GPI is valid, and that no regions + * overlap. As a part of the overlap checks, this function checks existing L0 + * mappings against the new PAS regions in the event that gpt_init_pas_l1_tables + * is called multiple times to place L1 tables in different areas of memory. It + * also counts the number of L1 tables needed and returns it on success. + * + * Parameters + * *pas_regions Pointer to array of PAS region structures. + * pas_region_cnt Total number of PAS regions in the array. + * + * Return + * Negative Linux error code in the event of a failure, number of L1 regions + * required when successful. + */ +static int gpt_validate_pas_mappings(pas_region_t *pas_regions, + unsigned int pas_region_cnt) +{ + unsigned int idx; + unsigned int l1_cnt = 0U; + unsigned int pas_l1_cnt; + uint64_t *l0_desc = (uint64_t *)gpt_config.plat_gpt_l0_base; + + assert(pas_regions != NULL); + assert(pas_region_cnt != 0U); + + for (idx = 0U; idx < pas_region_cnt; idx++) { + /* Check for arithmetic overflow in region. */ + if ((ULONG_MAX - pas_regions[idx].base_pa) < + pas_regions[idx].size) { + ERROR("[GPT] Address overflow in PAS[%u]!\n", idx); + return -EOVERFLOW; + } + + /* Initial checks for PAS validity. */ + if (((pas_regions[idx].base_pa + pas_regions[idx].size) > + GPT_PPS_ACTUAL_SIZE(gpt_config.t)) || + !gpt_is_gpi_valid(GPT_PAS_ATTR_GPI(pas_regions[idx].attrs))) { + ERROR("[GPT] PAS[%u] is invalid!\n", idx); + return -EFAULT; + } + + /* + * Make sure this PAS does not overlap with another one. We + * start from idx + 1 instead of 0 since prior PAS mappings will + * have already checked themselves against this one. + */ + for (unsigned int i = idx + 1; i < pas_region_cnt; i++) { + if (gpt_check_pas_overlap(pas_regions[idx].base_pa, + pas_regions[idx].size, + pas_regions[i].base_pa, + pas_regions[i].size)) { + ERROR("[GPT] PAS[%u] overlaps with PAS[%u]\n", + i, idx); + return -EFAULT; + } + } + + /* + * Since this function can be called multiple times with + * separate L1 tables we need to check the existing L0 mapping + * to see if this PAS would fall into one that has already been + * initialized. + */ + for (unsigned int i = GPT_L0_IDX(pas_regions[idx].base_pa); + i <= GPT_L0_IDX(pas_regions[idx].base_pa + pas_regions[idx].size - 1); + i++) { + if ((GPT_L0_TYPE(l0_desc[i]) == GPT_L0_TYPE_BLK_DESC) && + (GPT_L0_BLKD_GPI(l0_desc[i]) == GPT_GPI_ANY)) { + /* This descriptor is unused so continue. */ + continue; + } + + /* + * This descriptor has been initialized in a previous + * call to this function so cannot be initialized again. + */ + ERROR("[GPT] PAS[%u] overlaps with previous L0[%d]!\n", + idx, i); + return -EFAULT; + } + + /* Check for block mapping (L0) type. */ + if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == + GPT_PAS_ATTR_MAP_TYPE_BLOCK) { + /* Make sure base and size are block-aligned. */ + if (!GPT_IS_L0_ALIGNED(pas_regions[idx].base_pa) || + !GPT_IS_L0_ALIGNED(pas_regions[idx].size)) { + ERROR("[GPT] PAS[%u] is not block-aligned!\n", + idx); + return -EFAULT; + } + + continue; + } + + /* Check for granule mapping (L1) type. */ + if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == + GPT_PAS_ATTR_MAP_TYPE_GRANULE) { + /* Make sure base and size are granule-aligned. */ + if (!GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].base_pa) || + !GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].size)) { + ERROR("[GPT] PAS[%u] is not granule-aligned!\n", + idx); + return -EFAULT; + } + + /* Find how many L1 tables this PAS occupies. */ + pas_l1_cnt = (GPT_L0_IDX(pas_regions[idx].base_pa + + pas_regions[idx].size - 1) - + GPT_L0_IDX(pas_regions[idx].base_pa) + 1); + + /* + * This creates a situation where, if multiple PAS + * regions occupy the same table descriptor, we can get + * an artificially high total L1 table count. The way we + * handle this is by checking each PAS against those + * before it in the array, and if they both occupy the + * same PAS we subtract from pas_l1_cnt and only the + * first PAS in the array gets to count it. + */ + + /* + * If L1 count is greater than 1 we know the start and + * end PAs are in different L0 regions so we must check + * both for overlap against other PAS. + */ + if (pas_l1_cnt > 1) { + if (gpt_does_previous_pas_exist_here( + GPT_L0_IDX(pas_regions[idx].base_pa + + pas_regions[idx].size - 1), + pas_regions, idx)) { + pas_l1_cnt = pas_l1_cnt - 1; + } + } + + if (gpt_does_previous_pas_exist_here( + GPT_L0_IDX(pas_regions[idx].base_pa), + pas_regions, idx)) { + pas_l1_cnt = pas_l1_cnt - 1; + } + + l1_cnt += pas_l1_cnt; + continue; + } + + /* If execution reaches this point, mapping type is invalid. */ + ERROR("[GPT] PAS[%u] has invalid mapping type 0x%x.\n", idx, + GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs)); + return -EINVAL; + } + + return l1_cnt; +} + +/* + * This function validates L0 initialization parameters. + * + * Parameters + * l0_mem_base Base address of memory used for L0 tables. + * l1_mem_size Size of memory available for L0 tables. + * + * Return + * Negative Linux error code in the event of a failure, 0 for success. + */ +static int gpt_validate_l0_params(gpccr_pps_e pps, uintptr_t l0_mem_base, + size_t l0_mem_size) +{ + size_t l0_alignment; + + /* + * Make sure PPS is valid and then store it since macros need this value + * to work. + */ + if (pps > GPT_PPS_MAX) { + ERROR("[GPT] Invalid PPS: 0x%x\n", pps); + return -EINVAL; + } + gpt_config.pps = pps; + gpt_config.t = gpt_t_lookup[pps]; + + /* Alignment must be the greater of 4k or l0 table size. */ + l0_alignment = PAGE_SIZE_4KB; + if (l0_alignment < GPT_L0_TABLE_SIZE(gpt_config.t)) { + l0_alignment = GPT_L0_TABLE_SIZE(gpt_config.t); + } + + /* Check base address. */ + if ((l0_mem_base == 0U) || ((l0_mem_base & (l0_alignment - 1)) != 0U)) { + ERROR("[GPT] Invalid L0 base address: 0x%lx\n", l0_mem_base); + return -EFAULT; + } + + /* Check size. */ + if (l0_mem_size < GPT_L0_TABLE_SIZE(gpt_config.t)) { + ERROR("[GPT] Inadequate L0 memory: need 0x%lx, have 0x%lx)\n", + GPT_L0_TABLE_SIZE(gpt_config.t), + l0_mem_size); + return -ENOMEM; + } + + return 0; +} + +/* + * In the event that L1 tables are needed, this function validates + * the L1 table generation parameters. + * + * Parameters + * l1_mem_base Base address of memory used for L1 table allocation. + * l1_mem_size Total size of memory available for L1 tables. + * l1_gpt_cnt Number of L1 tables needed. + * + * Return + * Negative Linux error code in the event of a failure, 0 for success. + */ +static int gpt_validate_l1_params(uintptr_t l1_mem_base, size_t l1_mem_size, + unsigned int l1_gpt_cnt) +{ + size_t l1_gpt_mem_sz; + + /* Check if the granularity is supported */ + if (!xlat_arch_is_granule_size_supported( + GPT_PGS_ACTUAL_SIZE(gpt_config.p))) { + return -EPERM; + } + + /* Make sure L1 tables are aligned to their size. */ + if ((l1_mem_base & (GPT_L1_TABLE_SIZE(gpt_config.p) - 1)) != 0U) { + ERROR("[GPT] Unaligned L1 GPT base address: 0x%lx\n", + l1_mem_base); + return -EFAULT; + } + + /* Get total memory needed for L1 tables. */ + l1_gpt_mem_sz = l1_gpt_cnt * GPT_L1_TABLE_SIZE(gpt_config.p); + + /* Check for overflow. */ + if ((l1_gpt_mem_sz / GPT_L1_TABLE_SIZE(gpt_config.p)) != l1_gpt_cnt) { + ERROR("[GPT] Overflow calculating L1 memory size.\n"); + return -ENOMEM; + } + + /* Make sure enough space was supplied. */ + if (l1_mem_size < l1_gpt_mem_sz) { + ERROR("[GPT] Inadequate memory for L1 GPTs. "); + ERROR(" Expected 0x%lx bytes. Got 0x%lx bytes\n", + l1_gpt_mem_sz, l1_mem_size); + return -ENOMEM; + } + + VERBOSE("[GPT] Requested 0x%lx bytes for L1 GPTs.\n", l1_gpt_mem_sz); + return 0; +} + +/* + * This function initializes L0 block descriptors (regions that cannot be + * transitioned at the granule level) according to the provided PAS. + * + * Parameters + * *pas Pointer to the structure defining the PAS region to + * initialize. + */ +static void gpt_generate_l0_blk_desc(pas_region_t *pas) +{ + uint64_t gpt_desc; + unsigned int end_idx; + unsigned int idx; + uint64_t *l0_gpt_arr; + + assert(gpt_config.plat_gpt_l0_base != 0U); + assert(pas != NULL); + + /* + * Checking of PAS parameters has already been done in + * gpt_validate_pas_mappings so no need to check the same things again. + */ + + l0_gpt_arr = (uint64_t *)gpt_config.plat_gpt_l0_base; + + /* Create the GPT Block descriptor for this PAS region */ + gpt_desc = GPT_L0_BLK_DESC(GPT_PAS_ATTR_GPI(pas->attrs)); + + /* Start index of this region in L0 GPTs */ + idx = GPT_L0_IDX(pas->base_pa); + + /* + * Determine number of L0 GPT descriptors covered by + * this PAS region and use the count to populate these + * descriptors. + */ + end_idx = GPT_L0_IDX(pas->base_pa + pas->size); + + /* Generate the needed block descriptors. */ + for (; idx < end_idx; idx++) { + l0_gpt_arr[idx] = gpt_desc; + VERBOSE("[GPT] L0 entry (BLOCK) index %u [%p]: GPI = 0x%" PRIx64 " (0x%" PRIx64 ")\n", + idx, &l0_gpt_arr[idx], + (gpt_desc >> GPT_L0_BLK_DESC_GPI_SHIFT) & + GPT_L0_BLK_DESC_GPI_MASK, l0_gpt_arr[idx]); + } +} + +/* + * Helper function to determine if the end physical address lies in the same L0 + * region as the current physical address. If true, the end physical address is + * returned else, the start address of the next region is returned. + * + * Parameters + * cur_pa Physical address of the current PA in the loop through + * the range. + * end_pa Physical address of the end PA in a PAS range. + * + * Return + * The PA of the end of the current range. + */ +static uintptr_t gpt_get_l1_end_pa(uintptr_t cur_pa, uintptr_t end_pa) +{ + uintptr_t cur_idx; + uintptr_t end_idx; + + cur_idx = GPT_L0_IDX(cur_pa); + end_idx = GPT_L0_IDX(end_pa); + + assert(cur_idx <= end_idx); + + if (cur_idx == end_idx) { + return end_pa; + } + + return (cur_idx + 1U) << GPT_L0_IDX_SHIFT; +} + +/* + * Helper function to fill out GPI entries in a single L1 table. This function + * fills out entire L1 descriptors at a time to save memory writes. + * + * Parameters + * gpi GPI to set this range to + * l1 Pointer to L1 table to fill out + * first Address of first granule in range. + * last Address of last granule in range (inclusive). + */ +static void gpt_fill_l1_tbl(uint64_t gpi, uint64_t *l1, uintptr_t first, + uintptr_t last) +{ + uint64_t gpi_field = GPT_BUILD_L1_DESC(gpi); + uint64_t gpi_mask = 0xFFFFFFFFFFFFFFFF; + + assert(first <= last); + assert((first & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) == 0U); + assert((last & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) == 0U); + assert(GPT_L0_IDX(first) == GPT_L0_IDX(last)); + assert(l1 != NULL); + + /* Shift the mask if we're starting in the middle of an L1 entry. */ + gpi_mask = gpi_mask << (GPT_L1_GPI_IDX(gpt_config.p, first) << 2); + + /* Fill out each L1 entry for this region. */ + for (unsigned int i = GPT_L1_IDX(gpt_config.p, first); + i <= GPT_L1_IDX(gpt_config.p, last); i++) { + /* Account for stopping in the middle of an L1 entry. */ + if (i == GPT_L1_IDX(gpt_config.p, last)) { + gpi_mask &= (gpi_mask >> ((15 - + GPT_L1_GPI_IDX(gpt_config.p, last)) << 2)); + } + + /* Write GPI values. */ + assert((l1[i] & gpi_mask) == + (GPT_BUILD_L1_DESC(GPT_GPI_ANY) & gpi_mask)); + l1[i] = (l1[i] & ~gpi_mask) | (gpi_mask & gpi_field); + + /* Reset mask. */ + gpi_mask = 0xFFFFFFFFFFFFFFFF; + } +} + +/* + * This function finds the next available unused L1 table and initializes all + * granules descriptor entries to GPI_ANY. This ensures that there are no chunks + * of GPI_NO_ACCESS (0b0000) memory floating around in the system in the + * event that a PAS region stops midway through an L1 table, thus guaranteeing + * that all memory not explicitly assigned is GPI_ANY. This function does not + * check for overflow conditions, that should be done by the caller. + * + * Return + * Pointer to the next available L1 table. + */ +static uint64_t *gpt_get_new_l1_tbl(void) +{ + /* Retrieve the next L1 table. */ + uint64_t *l1 = (uint64_t *)((uint64_t)(gpt_l1_tbl) + + (GPT_L1_TABLE_SIZE(gpt_config.p) * + gpt_next_l1_tbl_idx)); + + /* Increment L1 counter. */ + gpt_next_l1_tbl_idx++; + + /* Initialize all GPIs to GPT_GPI_ANY */ + for (unsigned int i = 0U; i < GPT_L1_ENTRY_COUNT(gpt_config.p); i++) { + l1[i] = GPT_BUILD_L1_DESC(GPT_GPI_ANY); + } + + return l1; +} + +/* + * When L1 tables are needed, this function creates the necessary L0 table + * descriptors and fills out the L1 table entries according to the supplied + * PAS range. + * + * Parameters + * *pas Pointer to the structure defining the PAS region. + */ +static void gpt_generate_l0_tbl_desc(pas_region_t *pas) +{ + uintptr_t end_pa; + uintptr_t cur_pa; + uintptr_t last_gran_pa; + uint64_t *l0_gpt_base; + uint64_t *l1_gpt_arr; + unsigned int l0_idx; + + assert(gpt_config.plat_gpt_l0_base != 0U); + assert(pas != NULL); + + /* + * Checking of PAS parameters has already been done in + * gpt_validate_pas_mappings so no need to check the same things again. + */ + + end_pa = pas->base_pa + pas->size; + l0_gpt_base = (uint64_t *)gpt_config.plat_gpt_l0_base; + + /* We start working from the granule at base PA */ + cur_pa = pas->base_pa; + + /* Iterate over each L0 region in this memory range. */ + for (l0_idx = GPT_L0_IDX(pas->base_pa); + l0_idx <= GPT_L0_IDX(end_pa - 1U); + l0_idx++) { + + /* + * See if the L0 entry is already a table descriptor or if we + * need to create one. + */ + if (GPT_L0_TYPE(l0_gpt_base[l0_idx]) == GPT_L0_TYPE_TBL_DESC) { + /* Get the L1 array from the L0 entry. */ + l1_gpt_arr = GPT_L0_TBLD_ADDR(l0_gpt_base[l0_idx]); + } else { + /* Get a new L1 table from the L1 memory space. */ + l1_gpt_arr = gpt_get_new_l1_tbl(); + + /* Fill out the L0 descriptor and flush it. */ + l0_gpt_base[l0_idx] = GPT_L0_TBL_DESC(l1_gpt_arr); + } + + VERBOSE("[GPT] L0 entry (TABLE) index %u [%p] ==> L1 Addr 0x%llx (0x%" PRIx64 ")\n", + l0_idx, &l0_gpt_base[l0_idx], + (unsigned long long)(l1_gpt_arr), + l0_gpt_base[l0_idx]); + + /* + * Determine the PA of the last granule in this L0 descriptor. + */ + last_gran_pa = gpt_get_l1_end_pa(cur_pa, end_pa) - + GPT_PGS_ACTUAL_SIZE(gpt_config.p); + + /* + * Fill up L1 GPT entries between these two addresses. This + * function needs the addresses of the first granule and last + * granule in the range. + */ + gpt_fill_l1_tbl(GPT_PAS_ATTR_GPI(pas->attrs), l1_gpt_arr, + cur_pa, last_gran_pa); + + /* Advance cur_pa to first granule in next L0 region. */ + cur_pa = gpt_get_l1_end_pa(cur_pa, end_pa); + } +} + +/* + * This function flushes a range of L0 descriptors used by a given PAS region + * array. There is a chance that some unmodified L0 descriptors would be flushed + * in the case that there are "holes" in an array of PAS regions but overall + * this should be faster than individually flushing each modified L0 descriptor + * as they are created. + * + * Parameters + * *pas Pointer to an array of PAS regions. + * pas_count Number of entries in the PAS array. + */ +static void flush_l0_for_pas_array(pas_region_t *pas, unsigned int pas_count) +{ + unsigned int idx; + unsigned int start_idx; + unsigned int end_idx; + uint64_t *l0 = (uint64_t *)gpt_config.plat_gpt_l0_base; + + assert(pas != NULL); + assert(pas_count > 0); + + /* Initial start and end values. */ + start_idx = GPT_L0_IDX(pas[0].base_pa); + end_idx = GPT_L0_IDX(pas[0].base_pa + pas[0].size - 1); + + /* Find lowest and highest L0 indices used in this PAS array. */ + for (idx = 1; idx < pas_count; idx++) { + if (GPT_L0_IDX(pas[idx].base_pa) < start_idx) { + start_idx = GPT_L0_IDX(pas[idx].base_pa); + } + if (GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1) > end_idx) { + end_idx = GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1); + } + } + + /* + * Flush all covered L0 descriptors, add 1 because we need to include + * the end index value. + */ + flush_dcache_range((uintptr_t)&l0[start_idx], + ((end_idx + 1) - start_idx) * sizeof(uint64_t)); +} + +/* + * Public API to enable granule protection checks once the tables have all been + * initialized. This function is called at first initialization and then again + * later during warm boots of CPU cores. + * + * Return + * Negative Linux error code in the event of a failure, 0 for success. + */ +int gpt_enable(void) +{ + u_register_t gpccr_el3; + + /* + * Granule tables must be initialised before enabling + * granule protection. + */ + if (gpt_config.plat_gpt_l0_base == 0U) { + ERROR("[GPT] Tables have not been initialized!\n"); + return -EPERM; + } + + /* Write the base address of the L0 tables into GPTBR */ + write_gptbr_el3(((gpt_config.plat_gpt_l0_base >> GPTBR_BADDR_VAL_SHIFT) + >> GPTBR_BADDR_SHIFT) & GPTBR_BADDR_MASK); + + /* GPCCR_EL3.PPS */ + gpccr_el3 = SET_GPCCR_PPS(gpt_config.pps); + + /* GPCCR_EL3.PGS */ + gpccr_el3 |= SET_GPCCR_PGS(gpt_config.pgs); + + /* + * Since EL3 maps the L1 region as Inner shareable, use the same + * shareability attribute for GPC as well so that + * GPC fetches are visible to PEs + */ + gpccr_el3 |= SET_GPCCR_SH(GPCCR_SH_IS); + + /* Outer and Inner cacheability set to Normal memory, WB, RA, WA. */ + gpccr_el3 |= SET_GPCCR_ORGN(GPCCR_ORGN_WB_RA_WA); + gpccr_el3 |= SET_GPCCR_IRGN(GPCCR_IRGN_WB_RA_WA); + + /* Prepopulate GPCCR_EL3 but don't enable GPC yet */ + write_gpccr_el3(gpccr_el3); + isb(); + + /* Invalidate any stale TLB entries and any cached register fields */ + tlbipaallos(); + dsb(); + isb(); + + /* Enable GPT */ + gpccr_el3 |= GPCCR_GPC_BIT; + + /* TODO: Configure GPCCR_EL3_GPCP for Fault control. */ + write_gpccr_el3(gpccr_el3); + isb(); + tlbipaallos(); + dsb(); + isb(); + + return 0; +} + +/* + * Public API to disable granule protection checks. + */ +void gpt_disable(void) +{ + u_register_t gpccr_el3 = read_gpccr_el3(); + + write_gpccr_el3(gpccr_el3 & ~GPCCR_GPC_BIT); + dsbsy(); + isb(); +} + +/* + * Public API that initializes the entire protected space to GPT_GPI_ANY using + * the L0 tables (block descriptors). Ideally, this function is invoked prior + * to DDR discovery and initialization. The MMU must be initialized before + * calling this function. + * + * Parameters + * pps PPS value to use for table generation + * l0_mem_base Base address of L0 tables in memory. + * l0_mem_size Total size of memory available for L0 tables. + * + * Return + * Negative Linux error code in the event of a failure, 0 for success. + */ +int gpt_init_l0_tables(gpccr_pps_e pps, uintptr_t l0_mem_base, + size_t l0_mem_size) +{ + int ret; + uint64_t gpt_desc; + + /* Ensure that MMU and Data caches are enabled. */ + assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U); + + /* Validate other parameters. */ + ret = gpt_validate_l0_params(pps, l0_mem_base, l0_mem_size); + if (ret != 0) { + return ret; + } + + /* Create the descriptor to initialize L0 entries with. */ + gpt_desc = GPT_L0_BLK_DESC(GPT_GPI_ANY); + + /* Iterate through all L0 entries */ + for (unsigned int i = 0U; i < GPT_L0_REGION_COUNT(gpt_config.t); i++) { + ((uint64_t *)l0_mem_base)[i] = gpt_desc; + } + + /* Flush updated L0 tables to memory. */ + flush_dcache_range((uintptr_t)l0_mem_base, + (size_t)GPT_L0_TABLE_SIZE(gpt_config.t)); + + /* Stash the L0 base address once initial setup is complete. */ + gpt_config.plat_gpt_l0_base = l0_mem_base; + + return 0; +} + +/* + * Public API that carves out PAS regions from the L0 tables and builds any L1 + * tables that are needed. This function ideally is run after DDR discovery and + * initialization. The L0 tables must have already been initialized to GPI_ANY + * when this function is called. + * + * This function can be called multiple times with different L1 memory ranges + * and PAS regions if it is desirable to place L1 tables in different locations + * in memory. (ex: you have multiple DDR banks and want to place the L1 tables + * in the DDR bank that they control) + * + * Parameters + * pgs PGS value to use for table generation. + * l1_mem_base Base address of memory used for L1 tables. + * l1_mem_size Total size of memory available for L1 tables. + * *pas_regions Pointer to PAS regions structure array. + * pas_count Total number of PAS regions. + * + * Return + * Negative Linux error code in the event of a failure, 0 for success. + */ +int gpt_init_pas_l1_tables(gpccr_pgs_e pgs, uintptr_t l1_mem_base, + size_t l1_mem_size, pas_region_t *pas_regions, + unsigned int pas_count) +{ + int ret; + int l1_gpt_cnt; + + /* Ensure that MMU and Data caches are enabled. */ + assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U); + + /* PGS is needed for gpt_validate_pas_mappings so check it now. */ + if (pgs > GPT_PGS_MAX) { + ERROR("[GPT] Invalid PGS: 0x%x\n", pgs); + return -EINVAL; + } + gpt_config.pgs = pgs; + gpt_config.p = gpt_p_lookup[pgs]; + + /* Make sure L0 tables have been initialized. */ + if (gpt_config.plat_gpt_l0_base == 0U) { + ERROR("[GPT] L0 tables must be initialized first!\n"); + return -EPERM; + } + + /* Check if L1 GPTs are required and how many. */ + l1_gpt_cnt = gpt_validate_pas_mappings(pas_regions, pas_count); + if (l1_gpt_cnt < 0) { + return l1_gpt_cnt; + } + + VERBOSE("[GPT] %u L1 GPTs requested.\n", l1_gpt_cnt); + + /* If L1 tables are needed then validate the L1 parameters. */ + if (l1_gpt_cnt > 0) { + ret = gpt_validate_l1_params(l1_mem_base, l1_mem_size, + l1_gpt_cnt); + if (ret != 0) { + return ret; + } + + /* Set up parameters for L1 table generation. */ + gpt_l1_tbl = l1_mem_base; + gpt_next_l1_tbl_idx = 0U; + } + + INFO("[GPT] Boot Configuration\n"); + INFO(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t); + INFO(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p); + INFO(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL); + INFO(" PAS count: 0x%x\n", pas_count); + INFO(" L0 base: 0x%lx\n", gpt_config.plat_gpt_l0_base); + + /* Generate the tables in memory. */ + for (unsigned int idx = 0U; idx < pas_count; idx++) { + INFO("[GPT] PAS[%u]: base 0x%lx, size 0x%lx, GPI 0x%x, type 0x%x\n", + idx, pas_regions[idx].base_pa, pas_regions[idx].size, + GPT_PAS_ATTR_GPI(pas_regions[idx].attrs), + GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs)); + + /* Check if a block or table descriptor is required */ + if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == + GPT_PAS_ATTR_MAP_TYPE_BLOCK) { + gpt_generate_l0_blk_desc(&pas_regions[idx]); + + } else { + gpt_generate_l0_tbl_desc(&pas_regions[idx]); + } + } + + /* Flush modified L0 tables. */ + flush_l0_for_pas_array(pas_regions, pas_count); + + /* Flush L1 tables if needed. */ + if (l1_gpt_cnt > 0) { + flush_dcache_range(l1_mem_base, + GPT_L1_TABLE_SIZE(gpt_config.p) * + l1_gpt_cnt); + } + + /* Make sure that all the entries are written to the memory. */ + dsbishst(); + tlbipaallos(); + dsb(); + isb(); + + return 0; +} + +/* + * Public API to initialize the runtime gpt_config structure based on the values + * present in the GPTBR_EL3 and GPCCR_EL3 registers. GPT initialization + * typically happens in a bootloader stage prior to setting up the EL3 runtime + * environment for the granule transition service so this function detects the + * initialization from a previous stage. Granule protection checks must be + * enabled already or this function will return an error. + * + * Return + * Negative Linux error code in the event of a failure, 0 for success. + */ +int gpt_runtime_init(void) +{ + u_register_t reg; + + /* Ensure that MMU and Data caches are enabled. */ + assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U); + + /* Ensure GPC are already enabled. */ + if ((read_gpccr_el3() & GPCCR_GPC_BIT) == 0U) { + ERROR("[GPT] Granule protection checks are not enabled!\n"); + return -EPERM; + } + + /* + * Read the L0 table address from GPTBR, we don't need the L1 base + * address since those are included in the L0 tables as needed. + */ + reg = read_gptbr_el3(); + gpt_config.plat_gpt_l0_base = ((reg >> GPTBR_BADDR_SHIFT) & + GPTBR_BADDR_MASK) << + GPTBR_BADDR_VAL_SHIFT; + + /* Read GPCCR to get PGS and PPS values. */ + reg = read_gpccr_el3(); + gpt_config.pps = (reg >> GPCCR_PPS_SHIFT) & GPCCR_PPS_MASK; + gpt_config.t = gpt_t_lookup[gpt_config.pps]; + gpt_config.pgs = (reg >> GPCCR_PGS_SHIFT) & GPCCR_PGS_MASK; + gpt_config.p = gpt_p_lookup[gpt_config.pgs]; + + VERBOSE("[GPT] Runtime Configuration\n"); + VERBOSE(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t); + VERBOSE(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p); + VERBOSE(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL); + VERBOSE(" L0 base: 0x%lx\n", gpt_config.plat_gpt_l0_base); + + return 0; +} + +/* + * The L1 descriptors are protected by a spinlock to ensure that multiple + * CPUs do not attempt to change the descriptors at once. In the future it + * would be better to have separate spinlocks for each L1 descriptor. + */ +static spinlock_t gpt_lock; + +/* + * A helper to write the value (target_pas << gpi_shift) to the index of + * the gpt_l1_addr + */ +static inline void write_gpt(uint64_t *gpt_l1_desc, uint64_t *gpt_l1_addr, + unsigned int gpi_shift, unsigned int idx, + unsigned int target_pas) +{ + *gpt_l1_desc &= ~(GPT_L1_GRAN_DESC_GPI_MASK << gpi_shift); + *gpt_l1_desc |= ((uint64_t)target_pas << gpi_shift); + gpt_l1_addr[idx] = *gpt_l1_desc; +} + +/* + * Helper to retrieve the gpt_l1_* information from the base address + * returned in gpi_info + */ +static int get_gpi_params(uint64_t base, gpi_info_t *gpi_info) +{ + uint64_t gpt_l0_desc, *gpt_l0_base; + + gpt_l0_base = (uint64_t *)gpt_config.plat_gpt_l0_base; + gpt_l0_desc = gpt_l0_base[GPT_L0_IDX(base)]; + if (GPT_L0_TYPE(gpt_l0_desc) != GPT_L0_TYPE_TBL_DESC) { + VERBOSE("[GPT] Granule is not covered by a table descriptor!\n"); + VERBOSE(" Base=0x%" PRIx64 "\n", base); + return -EINVAL; + } + + /* Get the table index and GPI shift from PA. */ + gpi_info->gpt_l1_addr = GPT_L0_TBLD_ADDR(gpt_l0_desc); + gpi_info->idx = GPT_L1_IDX(gpt_config.p, base); + gpi_info->gpi_shift = GPT_L1_GPI_IDX(gpt_config.p, base) << 2; + + gpi_info->gpt_l1_desc = (gpi_info->gpt_l1_addr)[gpi_info->idx]; + gpi_info->gpi = (gpi_info->gpt_l1_desc >> gpi_info->gpi_shift) & + GPT_L1_GRAN_DESC_GPI_MASK; + return 0; +} + +/* + * This function is the granule transition delegate service. When a granule + * transition request occurs it is routed to this function to have the request, + * if valid, fulfilled following A1.1.1 Delegate of RME supplement + * + * TODO: implement support for transitioning multiple granules at once. + * + * Parameters + * base Base address of the region to transition, must be + * aligned to granule size. + * size Size of region to transition, must be aligned to granule + * size. + * src_sec_state Security state of the caller. + * + * Return + * Negative Linux error code in the event of a failure, 0 for success. + */ +int gpt_delegate_pas(uint64_t base, size_t size, unsigned int src_sec_state) +{ + gpi_info_t gpi_info; + uint64_t nse; + int res; + unsigned int target_pas; + + /* Ensure that the tables have been set up before taking requests. */ + assert(gpt_config.plat_gpt_l0_base != 0UL); + + /* Ensure that caches are enabled. */ + assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); + + /* Delegate request can only come from REALM or SECURE */ + assert(src_sec_state == SMC_FROM_REALM || + src_sec_state == SMC_FROM_SECURE); + + /* See if this is a single or a range of granule transition. */ + if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) { + return -EINVAL; + } + + /* Check that base and size are valid */ + if ((ULONG_MAX - base) < size) { + VERBOSE("[GPT] Transition request address overflow!\n"); + VERBOSE(" Base=0x%" PRIx64 "\n", base); + VERBOSE(" Size=0x%lx\n", size); + return -EINVAL; + } + + /* Make sure base and size are valid. */ + if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) != 0UL) || + ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) != 0UL) || + (size == 0UL) || + ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) { + VERBOSE("[GPT] Invalid granule transition address range!\n"); + VERBOSE(" Base=0x%" PRIx64 "\n", base); + VERBOSE(" Size=0x%lx\n", size); + return -EINVAL; + } + + target_pas = GPT_GPI_REALM; + if (src_sec_state == SMC_FROM_SECURE) { + target_pas = GPT_GPI_SECURE; + } + + /* + * Access to L1 tables is controlled by a global lock to ensure + * that no more than one CPU is allowed to make changes at any + * given time. + */ + spin_lock(&gpt_lock); + res = get_gpi_params(base, &gpi_info); + if (res != 0) { + spin_unlock(&gpt_lock); + return res; + } + + /* Check that the current address is in NS state */ + if (gpi_info.gpi != GPT_GPI_NS) { + VERBOSE("[GPT] Only Granule in NS state can be delegated.\n"); + VERBOSE(" Caller: %u, Current GPI: %u\n", src_sec_state, + gpi_info.gpi); + spin_unlock(&gpt_lock); + return -EPERM; + } + + if (src_sec_state == SMC_FROM_SECURE) { + nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT; + } else { + nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT; + } + + /* + * In order to maintain mutual distrust between Realm and Secure + * states, remove any data speculatively fetched into the target + * physical address space. Issue DC CIPAPA over address range + */ + flush_dcache_to_popa_range(nse | base, + GPT_PGS_ACTUAL_SIZE(gpt_config.p)); + + write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, + gpi_info.gpi_shift, gpi_info.idx, target_pas); + dsboshst(); + + gpt_tlbi_by_pa_ll(base, GPT_PGS_ACTUAL_SIZE(gpt_config.p)); + dsbosh(); + + nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT; + + flush_dcache_to_popa_range(nse | base, + GPT_PGS_ACTUAL_SIZE(gpt_config.p)); + + /* Unlock access to the L1 tables. */ + spin_unlock(&gpt_lock); + + /* + * The isb() will be done as part of context + * synchronization when returning to lower EL + */ + VERBOSE("[GPT] Granule 0x%" PRIx64 ", GPI 0x%x->0x%x\n", + base, gpi_info.gpi, target_pas); + + return 0; +} + +/* + * This function is the granule transition undelegate service. When a granule + * transition request occurs it is routed to this function where the request is + * validated then fulfilled if possible. + * + * TODO: implement support for transitioning multiple granules at once. + * + * Parameters + * base Base address of the region to transition, must be + * aligned to granule size. + * size Size of region to transition, must be aligned to granule + * size. + * src_sec_state Security state of the caller. + * + * Return + * Negative Linux error code in the event of a failure, 0 for success. + */ +int gpt_undelegate_pas(uint64_t base, size_t size, unsigned int src_sec_state) +{ + gpi_info_t gpi_info; + uint64_t nse; + int res; + + /* Ensure that the tables have been set up before taking requests. */ + assert(gpt_config.plat_gpt_l0_base != 0UL); + + /* Ensure that MMU and caches are enabled. */ + assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); + + /* Delegate request can only come from REALM or SECURE */ + assert(src_sec_state == SMC_FROM_REALM || + src_sec_state == SMC_FROM_SECURE); + + /* See if this is a single or a range of granule transition. */ + if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) { + return -EINVAL; + } + + /* Check that base and size are valid */ + if ((ULONG_MAX - base) < size) { + VERBOSE("[GPT] Transition request address overflow!\n"); + VERBOSE(" Base=0x%" PRIx64 "\n", base); + VERBOSE(" Size=0x%lx\n", size); + return -EINVAL; + } + + /* Make sure base and size are valid. */ + if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) != 0UL) || + ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) != 0UL) || + (size == 0UL) || + ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) { + VERBOSE("[GPT] Invalid granule transition address range!\n"); + VERBOSE(" Base=0x%" PRIx64 "\n", base); + VERBOSE(" Size=0x%lx\n", size); + return -EINVAL; + } + + /* + * Access to L1 tables is controlled by a global lock to ensure + * that no more than one CPU is allowed to make changes at any + * given time. + */ + spin_lock(&gpt_lock); + + res = get_gpi_params(base, &gpi_info); + if (res != 0) { + spin_unlock(&gpt_lock); + return res; + } + + /* Check that the current address is in the delegated state */ + if ((src_sec_state == SMC_FROM_REALM && + gpi_info.gpi != GPT_GPI_REALM) || + (src_sec_state == SMC_FROM_SECURE && + gpi_info.gpi != GPT_GPI_SECURE)) { + VERBOSE("[GPT] Only Granule in REALM or SECURE state can be undelegated.\n"); + VERBOSE(" Caller: %u, Current GPI: %u\n", src_sec_state, + gpi_info.gpi); + spin_unlock(&gpt_lock); + return -EPERM; + } + + + /* In order to maintain mutual distrust between Realm and Secure + * states, remove access now, in order to guarantee that writes + * to the currently-accessible physical address space will not + * later become observable. + */ + write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, + gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NO_ACCESS); + dsboshst(); + + gpt_tlbi_by_pa_ll(base, GPT_PGS_ACTUAL_SIZE(gpt_config.p)); + dsbosh(); + + if (src_sec_state == SMC_FROM_SECURE) { + nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT; + } else { + nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT; + } + + /* Ensure that the scrubbed data has made it past the PoPA */ + flush_dcache_to_popa_range(nse | base, + GPT_PGS_ACTUAL_SIZE(gpt_config.p)); + + /* + * Remove any data loaded speculatively + * in NS space from before the scrubbing + */ + nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT; + + flush_dcache_to_popa_range(nse | base, + GPT_PGS_ACTUAL_SIZE(gpt_config.p)); + + /* Clear existing GPI encoding and transition granule. */ + write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, + gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NS); + dsboshst(); + + /* Ensure that all agents observe the new NS configuration */ + gpt_tlbi_by_pa_ll(base, GPT_PGS_ACTUAL_SIZE(gpt_config.p)); + dsbosh(); + + /* Unlock access to the L1 tables. */ + spin_unlock(&gpt_lock); + + /* + * The isb() will be done as part of context + * synchronization when returning to lower EL + */ + VERBOSE("[GPT] Granule 0x%" PRIx64 ", GPI 0x%x->0x%x\n", + base, gpi_info.gpi, GPT_GPI_NS); + + return 0; +} diff --git a/lib/gpt_rme/gpt_rme.mk b/lib/gpt_rme/gpt_rme.mk new file mode 100644 index 0000000..60176f4 --- /dev/null +++ b/lib/gpt_rme/gpt_rme.mk @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +GPT_LIB_SRCS := $(addprefix lib/gpt_rme/, \ + gpt_rme.c) diff --git a/lib/gpt_rme/gpt_rme_private.h b/lib/gpt_rme/gpt_rme_private.h new file mode 100644 index 0000000..3c817f3 --- /dev/null +++ b/lib/gpt_rme/gpt_rme_private.h @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2022, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef GPT_RME_PRIVATE_H +#define GPT_RME_PRIVATE_H + +#include <arch.h> +#include <lib/gpt_rme/gpt_rme.h> +#include <lib/utils_def.h> + +/******************************************************************************/ +/* GPT descriptor definitions */ +/******************************************************************************/ + +/* GPT level 0 descriptor bit definitions. */ +#define GPT_L0_TYPE_MASK UL(0xF) +#define GPT_L0_TYPE_SHIFT U(0) + +/* For now, we don't support contiguous descriptors, only table and block. */ +#define GPT_L0_TYPE_TBL_DESC UL(0x3) +#define GPT_L0_TYPE_BLK_DESC UL(0x1) + +#define GPT_L0_TBL_DESC_L1ADDR_MASK UL(0xFFFFFFFFFF) +#define GPT_L0_TBL_DESC_L1ADDR_SHIFT U(12) + +#define GPT_L0_BLK_DESC_GPI_MASK UL(0xF) +#define GPT_L0_BLK_DESC_GPI_SHIFT U(4) + +/* GPT level 1 descriptor bit definitions */ +#define GPT_L1_GRAN_DESC_GPI_MASK UL(0xF) + +/* + * This macro fills out every GPI entry in a granules descriptor to the same + * value. + */ +#define GPT_BUILD_L1_DESC(_gpi) (((uint64_t)(_gpi) << 4*0) | \ + ((uint64_t)(_gpi) << 4*1) | \ + ((uint64_t)(_gpi) << 4*2) | \ + ((uint64_t)(_gpi) << 4*3) | \ + ((uint64_t)(_gpi) << 4*4) | \ + ((uint64_t)(_gpi) << 4*5) | \ + ((uint64_t)(_gpi) << 4*6) | \ + ((uint64_t)(_gpi) << 4*7) | \ + ((uint64_t)(_gpi) << 4*8) | \ + ((uint64_t)(_gpi) << 4*9) | \ + ((uint64_t)(_gpi) << 4*10) | \ + ((uint64_t)(_gpi) << 4*11) | \ + ((uint64_t)(_gpi) << 4*12) | \ + ((uint64_t)(_gpi) << 4*13) | \ + ((uint64_t)(_gpi) << 4*14) | \ + ((uint64_t)(_gpi) << 4*15)) + +/******************************************************************************/ +/* GPT platform configuration */ +/******************************************************************************/ + +/* This value comes from GPCCR_EL3 so no externally supplied definition. */ +#define GPT_L0GPTSZ ((unsigned int)((read_gpccr_el3() >> \ + GPCCR_L0GPTSZ_SHIFT) & GPCCR_L0GPTSZ_MASK)) + +/* The "S" value is directly related to L0GPTSZ */ +#define GPT_S_VAL (GPT_L0GPTSZ + 30U) + +/* + * Map PPS values to T values. + * + * PPS Size T + * 0b000 4GB 32 + * 0b001 64GB 36 + * 0b010 1TB 40 + * 0b011 4TB 42 + * 0b100 16TB 44 + * 0b101 256TB 48 + * 0b110 4PB 52 + * + * See section 15.1.27 of the RME specification. + */ +typedef enum { + PPS_4GB_T = 32U, + PPS_64GB_T = 36U, + PPS_1TB_T = 40U, + PPS_4TB_T = 42U, + PPS_16TB_T = 44U, + PPS_256TB_T = 48U, + PPS_4PB_T = 52U +} gpt_t_val_e; + +/* + * Map PGS values to P values. + * + * PGS Size P + * 0b00 4KB 12 + * 0b10 16KB 14 + * 0b01 64KB 16 + * + * Note that pgs=0b10 is 16KB and pgs=0b01 is 64KB, this is not a typo. + * + * See section 15.1.27 of the RME specification. + */ +typedef enum { + PGS_4KB_P = 12U, + PGS_16KB_P = 14U, + PGS_64KB_P = 16U +} gpt_p_val_e; + +/* + * Internal structure to retrieve the values from get_gpi_info(); + */ +typedef struct gpi_info { + uint64_t gpt_l1_desc; + uint64_t *gpt_l1_addr; + unsigned int idx; + unsigned int gpi_shift; + unsigned int gpi; +} gpi_info_t; + +/* Max valid value for PGS. */ +#define GPT_PGS_MAX (2U) + +/* Max valid value for PPS. */ +#define GPT_PPS_MAX (6U) + +/******************************************************************************/ +/* L0 address attribute macros */ +/******************************************************************************/ + +/* + * Width of the L0 index field. + * + * If S is greater than or equal to T then there is a single L0 region covering + * the entire protected space so there is no L0 index, so the width (and the + * derivative mask value) are both zero. If we don't specifically handle this + * special case we'll get a negative width value which does not make sense and + * would cause problems. + */ +#define GPT_L0_IDX_WIDTH(_t) (((_t) > GPT_S_VAL) ? \ + ((_t) - GPT_S_VAL) : (0U)) + +/* Bit shift for the L0 index field in a PA. */ +#define GPT_L0_IDX_SHIFT (GPT_S_VAL) + +/* + * Mask for the L0 index field, must be shifted. + * + * The value 0x3FFFFF is 22 bits wide which is the maximum possible width of the + * L0 index within a physical address. This is calculated by + * ((t_max - 1) - s_min + 1) where t_max is 52 for 4PB, the largest PPS, and + * s_min is 30 for 1GB, the smallest L0GPTSZ. + */ +#define GPT_L0_IDX_MASK(_t) (0x3FFFFFUL >> (22U - \ + (GPT_L0_IDX_WIDTH(_t)))) + +/* Total number of L0 regions. */ +#define GPT_L0_REGION_COUNT(_t) ((GPT_L0_IDX_MASK(_t)) + 1U) + +/* Total size of each GPT L0 region in bytes. */ +#define GPT_L0_REGION_SIZE (1UL << (GPT_L0_IDX_SHIFT)) + +/* Total size in bytes of the whole L0 table. */ +#define GPT_L0_TABLE_SIZE(_t) ((GPT_L0_REGION_COUNT(_t)) << 3U) + +/******************************************************************************/ +/* L1 address attribute macros */ +/******************************************************************************/ + +/* + * Width of the L1 index field. + * + * This field does not have a special case to handle widths less than zero like + * the L0 index field above since all valid combinations of PGS (p) and L0GPTSZ + * (s) will result in a positive width value. + */ +#define GPT_L1_IDX_WIDTH(_p) ((GPT_S_VAL - 1U) - ((_p) + 3U)) + +/* Bit shift for the L1 index field. */ +#define GPT_L1_IDX_SHIFT(_p) ((_p) + 4U) + +/* + * Mask for the L1 index field, must be shifted. + * + * The value 0x7FFFFF is 23 bits wide and is the maximum possible width of the + * L1 index within a physical address. It is calculated by + * ((s_max - 1) - (p_min + 4) + 1) where s_max is 39 for 512gb, the largest + * L0GPTSZ, and p_min is 12 for 4KB granules, the smallest PGS. + */ +#define GPT_L1_IDX_MASK(_p) (0x7FFFFFUL >> (23U - \ + (GPT_L1_IDX_WIDTH(_p)))) + +/* Bit shift for the index of the L1 GPI in a PA. */ +#define GPT_L1_GPI_IDX_SHIFT(_p) (_p) + +/* Mask for the index of the L1 GPI in a PA. */ +#define GPT_L1_GPI_IDX_MASK (0xF) + +/* Total number of entries in each L1 table. */ +#define GPT_L1_ENTRY_COUNT(_p) ((GPT_L1_IDX_MASK(_p)) + 1U) + +/* Total size in bytes of each L1 table. */ +#define GPT_L1_TABLE_SIZE(_p) ((GPT_L1_ENTRY_COUNT(_p)) << 3U) + +/******************************************************************************/ +/* General helper macros */ +/******************************************************************************/ + +/* Protected space actual size in bytes. */ +#define GPT_PPS_ACTUAL_SIZE(_t) (1UL << (_t)) + +/* Granule actual size in bytes. */ +#define GPT_PGS_ACTUAL_SIZE(_p) (1UL << (_p)) + +/* L0 GPT region size in bytes. */ +#define GPT_L0GPTSZ_ACTUAL_SIZE (1UL << GPT_S_VAL) + +/* Get the index of the L0 entry from a physical address. */ +#define GPT_L0_IDX(_pa) ((_pa) >> GPT_L0_IDX_SHIFT) + +/* + * This definition is used to determine if a physical address lies on an L0 + * region boundary. + */ +#define GPT_IS_L0_ALIGNED(_pa) (((_pa) & (GPT_L0_REGION_SIZE - U(1))) == U(0)) + +/* Get the type field from an L0 descriptor. */ +#define GPT_L0_TYPE(_desc) (((_desc) >> GPT_L0_TYPE_SHIFT) & \ + GPT_L0_TYPE_MASK) + +/* Create an L0 block descriptor. */ +#define GPT_L0_BLK_DESC(_gpi) (GPT_L0_TYPE_BLK_DESC | \ + (((_gpi) & GPT_L0_BLK_DESC_GPI_MASK) << \ + GPT_L0_BLK_DESC_GPI_SHIFT)) + +/* Create an L0 table descriptor with an L1 table address. */ +#define GPT_L0_TBL_DESC(_pa) (GPT_L0_TYPE_TBL_DESC | ((uint64_t)(_pa) & \ + (GPT_L0_TBL_DESC_L1ADDR_MASK << \ + GPT_L0_TBL_DESC_L1ADDR_SHIFT))) + +/* Get the GPI from an L0 block descriptor. */ +#define GPT_L0_BLKD_GPI(_desc) (((_desc) >> GPT_L0_BLK_DESC_GPI_SHIFT) & \ + GPT_L0_BLK_DESC_GPI_MASK) + +/* Get the L1 address from an L0 table descriptor. */ +#define GPT_L0_TBLD_ADDR(_desc) ((uint64_t *)(((_desc) & \ + (GPT_L0_TBL_DESC_L1ADDR_MASK << \ + GPT_L0_TBL_DESC_L1ADDR_SHIFT)))) + +/* Get the index into the L1 table from a physical address. */ +#define GPT_L1_IDX(_p, _pa) (((_pa) >> GPT_L1_IDX_SHIFT(_p)) & \ + GPT_L1_IDX_MASK(_p)) + +/* Get the index of the GPI within an L1 table entry from a physical address. */ +#define GPT_L1_GPI_IDX(_p, _pa) (((_pa) >> GPT_L1_GPI_IDX_SHIFT(_p)) & \ + GPT_L1_GPI_IDX_MASK) + +/* Determine if an address is granule-aligned. */ +#define GPT_IS_L1_ALIGNED(_p, _pa) (((_pa) & (GPT_PGS_ACTUAL_SIZE(_p) - U(1))) \ + == U(0)) + +#endif /* GPT_RME_PRIVATE_H */ diff --git a/lib/libc/aarch32/memset.S b/lib/libc/aarch32/memset.S new file mode 100644 index 0000000..880ba83 --- /dev/null +++ b/lib/libc/aarch32/memset.S @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + + .syntax unified + .global memset + +/* ----------------------------------------------------------------------- + * void *memset(void *dst, int val, size_t count) + * + * Copy the value of 'val' (converted to an unsigned char) into + * each of the first 'count' characters of the object pointed to by 'dst'. + * + * Returns the value of 'dst'. + * ----------------------------------------------------------------------- + */ +func memset + mov r12, r0 /* keep r0 */ + tst r0, #3 + beq aligned /* 4-bytes aligned */ + + /* Unaligned 'dst' */ +unaligned: + subs r2, r2, #1 + strbhs r1, [r12], #1 + bxls lr /* return if 0 */ + tst r12, #3 + bne unaligned /* continue while unaligned */ + + /* 4-bytes aligned */ +aligned:bfi r1, r1, #8, #8 /* propagate 'val' */ + bfi r1, r1, #16, #16 + + mov r3, r1 + + cmp r2, #16 + blo less_16 /* < 16 */ + + push {r4, lr} + mov r4, r1 + mov lr, r1 + +write_32: + subs r2, r2, #32 + stmiahs r12!, {r1, r3, r4, lr} + stmiahs r12!, {r1, r3, r4, lr} + bhi write_32 /* write 32 bytes in a loop */ + popeq {r4, pc} /* return if 0 */ + lsls r2, r2, #28 /* C = r2[4]; N = r2[3]; Z = r2[3:0] */ + stmiacs r12!, {r1, r3, r4, lr} /* write 16 bytes */ + popeq {r4, pc} /* return if 16 */ + stmiami r12!, {r1, r3} /* write 8 bytes */ + lsls r2, r2, #2 /* C = r2[2]; N = r2[1]; Z = r2[1:0] */ + strcs r1, [r12], #4 /* write 4 bytes */ + popeq {r4, pc} /* return if 8 or 4 */ + strhmi r1, [r12], #2 /* write 2 bytes */ + lsls r2, r2, #1 /* N = Z = r2[0] */ + strbmi r1, [r12] /* write 1 byte */ + pop {r4, pc} + +less_16:lsls r2, r2, #29 /* C = r2[3]; N = r2[2]; Z = r2[2:0] */ + stmiacs r12!, {r1, r3} /* write 8 bytes */ + bxeq lr /* return if 8 */ + strmi r1, [r12], #4 /* write 4 bytes */ + lsls r2, r2, #2 /* C = r2[1]; N = Z = r2[0] */ + strhcs r1, [r12], #2 /* write 2 bytes */ + strbmi r1, [r12] /* write 1 byte */ + bx lr + +endfunc memset diff --git a/lib/libc/aarch64/memset.S b/lib/libc/aarch64/memset.S new file mode 100644 index 0000000..0543704 --- /dev/null +++ b/lib/libc/aarch64/memset.S @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + + .global memset + +/* ----------------------------------------------------------------------- + * void *memset(void *dst, int val, size_t count) + * + * Copy the value of 'val' (converted to an unsigned char) into + * each of the first 'count' characters of the object pointed to by 'dst'. + * + * Returns the value of 'dst'. + * ----------------------------------------------------------------------- + */ +func memset + cbz x2, exit /* exit if 'count' = 0 */ + mov x3, x0 /* keep x0 */ + tst x0, #7 + b.eq aligned /* 8-bytes aligned */ + + /* Unaligned 'dst' */ +unaligned: + strb w1, [x3], #1 + subs x2, x2, #1 + b.eq exit /* exit if 0 */ + tst x3, #7 + b.ne unaligned /* continue while unaligned */ + + /* 8-bytes aligned */ +aligned:cbz x1, x1_zero + bfi w1, w1, #8, #8 /* propagate 'val' */ + bfi w1, w1, #16, #16 + bfi x1, x1, #32, #32 + +x1_zero:ands x4, x2, #~0x3f + b.eq less_64 + +write_64: + .rept 4 + stp x1, x1, [x3], #16 /* write 64 bytes in a loop */ + .endr + subs x4, x4, #64 + b.ne write_64 +less_64:tbz w2, #5, less_32 /* < 32 bytes */ + stp x1, x1, [x3], #16 /* write 32 bytes */ + stp x1, x1, [x3], #16 +less_32:tbz w2, #4, less_16 /* < 16 bytes */ + stp x1, x1, [x3], #16 /* write 16 bytes */ +less_16:tbz w2, #3, less_8 /* < 8 bytes */ + str x1, [x3], #8 /* write 8 bytes */ +less_8: tbz w2, #2, less_4 /* < 4 bytes */ + str w1, [x3], #4 /* write 4 bytes */ +less_4: tbz w2, #1, less_2 /* < 2 bytes */ + strh w1, [x3], #2 /* write 2 bytes */ +less_2: tbz w2, #0, exit + strb w1, [x3] /* write 1 byte */ +exit: ret + +endfunc memset diff --git a/lib/libc/aarch64/setjmp.S b/lib/libc/aarch64/setjmp.S new file mode 100644 index 0000000..9cb31ea --- /dev/null +++ b/lib/libc/aarch64/setjmp.S @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <assert_macros.S> +#include <setjmp.h> + + .globl setjmp + .globl longjmp + +/* + * int setjmp(jmp_buf env); + */ +func setjmp + mov x7, sp + + stp x19, x20, [x0, #JMP_CTX_X19] + stp x21, x22, [x0, #JMP_CTX_X21] + stp x23, x24, [x0, #JMP_CTX_X23] + stp x25, x26, [x0, #JMP_CTX_X25] + stp x27, x28, [x0, #JMP_CTX_X27] + stp x29, x30, [x0, #JMP_CTX_X29] + stp x7, xzr, [x0, #JMP_CTX_SP] + + mov x0, #0 + ret +endfunc setjmp + + +/* + * void longjmp(jmp_buf env, int val); + */ +func longjmp + ldp x7, xzr, [x0, #JMP_CTX_SP] + +#if ENABLE_ASSERTIONS + /* + * Since we're unwinding the stack, assert that the stack being reset to + * is shallower. + */ + mov x19, sp + cmp x7, x19 + ASM_ASSERT(ge) +#endif + + ldp x19, x20, [x0, #JMP_CTX_X19] + ldp x21, x22, [x0, #JMP_CTX_X21] + ldp x23, x24, [x0, #JMP_CTX_X23] + ldp x25, x26, [x0, #JMP_CTX_X25] + ldp x27, x28, [x0, #JMP_CTX_X27] + ldp x29, x30, [x0, #JMP_CTX_X29] + + mov sp, x7 + + ands x0, x1, x1 /* Move val to x0 and set flags */ + cinc x0, x0, eq /* If val is 0, return 1 */ + ret +endfunc longjmp diff --git a/lib/libc/abort.c b/lib/libc/abort.c new file mode 100644 index 0000000..ac27f62 --- /dev/null +++ b/lib/libc/abort.c @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2013-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdlib.h> + +#include <common/debug.h> + +void abort(void) +{ + ERROR("ABORT\n"); + panic(); +} diff --git a/lib/libc/assert.c b/lib/libc/assert.c new file mode 100644 index 0000000..301d142 --- /dev/null +++ b/lib/libc/assert.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <cdefs.h> +#include <stdio.h> + +#include <common/debug.h> +#include <drivers/console.h> +#include <plat/common/platform.h> + +/* + * Only print the output if PLAT_LOG_LEVEL_ASSERT is higher or equal to + * LOG_LEVEL_INFO, which is the default value for builds with DEBUG=1. + */ + +#if PLAT_LOG_LEVEL_ASSERT >= LOG_LEVEL_INFO +void __dead2 __assert(const char *file, unsigned int line) +{ + printf("ASSERT: %s:%u\n", file, line); + backtrace("assert"); + console_flush(); + plat_panic_handler(); +} +#else +void __dead2 __assert(void) +{ + backtrace("assert"); + console_flush(); + plat_panic_handler(); +} +#endif diff --git a/lib/libc/exit.c b/lib/libc/exit.c new file mode 100644 index 0000000..6de2e93 --- /dev/null +++ b/lib/libc/exit.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2015-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdlib.h> + +static void (*exitfun)(void); + +void exit(int status) +{ + if (exitfun != NULL) + (*exitfun)(); + for (;;) + ; +} + +int atexit(void (*fun)(void)) +{ + if (exitfun != NULL) + return -1; + exitfun = fun; + + return 0; +} diff --git a/lib/libc/libc.mk b/lib/libc/libc.mk new file mode 100644 index 0000000..95da68c --- /dev/null +++ b/lib/libc/libc.mk @@ -0,0 +1,42 @@ +# +# Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +LIBC_SRCS := $(addprefix lib/libc/, \ + abort.c \ + assert.c \ + exit.c \ + memchr.c \ + memcmp.c \ + memcpy.c \ + memcpy_s.c \ + memmove.c \ + memrchr.c \ + memset.c \ + printf.c \ + putchar.c \ + puts.c \ + snprintf.c \ + strchr.c \ + strcmp.c \ + strlcat.c \ + strlcpy.c \ + strlen.c \ + strncmp.c \ + strnlen.c \ + strrchr.c \ + strtok.c \ + strtoul.c \ + strtoll.c \ + strtoull.c \ + strtol.c) + +ifeq (${ARCH},aarch64) +LIBC_SRCS += $(addprefix lib/libc/aarch64/, \ + setjmp.S) +endif + +INCLUDES += -Iinclude/lib/libc \ + -Iinclude/lib/libc/$(ARCH) \ diff --git a/lib/libc/libc_asm.mk b/lib/libc/libc_asm.mk new file mode 100644 index 0000000..2f27265 --- /dev/null +++ b/lib/libc/libc_asm.mk @@ -0,0 +1,44 @@ +# +# Copyright (c) 2020-2021, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +LIBC_SRCS := $(addprefix lib/libc/, \ + abort.c \ + assert.c \ + exit.c \ + memchr.c \ + memcmp.c \ + memcpy.c \ + memmove.c \ + memrchr.c \ + printf.c \ + putchar.c \ + puts.c \ + snprintf.c \ + strchr.c \ + strcmp.c \ + strlcat.c \ + strlcpy.c \ + strlen.c \ + strncmp.c \ + strnlen.c \ + strrchr.c \ + strtok.c \ + strtoul.c \ + strtoll.c \ + strtoull.c \ + strtol.c) + +ifeq (${ARCH},aarch64) +LIBC_SRCS += $(addprefix lib/libc/aarch64/, \ + memset.S \ + setjmp.S) +else +LIBC_SRCS += $(addprefix lib/libc/aarch32/, \ + memset.S) +endif + +INCLUDES += -Iinclude/lib/libc \ + -Iinclude/lib/libc/$(ARCH) \ diff --git a/lib/libc/memchr.c b/lib/libc/memchr.c new file mode 100644 index 0000000..66d7ba1 --- /dev/null +++ b/lib/libc/memchr.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2013-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stddef.h> +#include <string.h> + +void *memchr(const void *src, int c, size_t len) +{ + const unsigned char *s = src; + + while (len--) { + if (*s == (unsigned char)c) + return (void *) s; + s++; + } + + return NULL; +} diff --git a/lib/libc/memcmp.c b/lib/libc/memcmp.c new file mode 100644 index 0000000..db2701b --- /dev/null +++ b/lib/libc/memcmp.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2013-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stddef.h> +#include <string.h> + +int memcmp(const void *s1, const void *s2, size_t len) +{ + const unsigned char *s = s1; + const unsigned char *d = s2; + unsigned char sc; + unsigned char dc; + + while (len--) { + sc = *s++; + dc = *d++; + if (sc - dc) + return (sc - dc); + } + + return 0; +} diff --git a/lib/libc/memcpy.c b/lib/libc/memcpy.c new file mode 100644 index 0000000..af9ed45 --- /dev/null +++ b/lib/libc/memcpy.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2013-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stddef.h> +#include <string.h> + +void *memcpy(void *dst, const void *src, size_t len) +{ + const char *s = src; + char *d = dst; + + while (len--) + *d++ = *s++; + + return dst; +} diff --git a/lib/libc/memcpy_s.c b/lib/libc/memcpy_s.c new file mode 100644 index 0000000..26953bf --- /dev/null +++ b/lib/libc/memcpy_s.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2023, Intel Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <errno.h> +#include <stddef.h> +#include <string.h> + +int memcpy_s(void *dst, size_t dsize, void *src, size_t ssize) +{ + unsigned int *s = (unsigned int *)src; + unsigned int *d = (unsigned int *)dst; + + /* + * Check source and destination size is NULL + */ + if ((dst == NULL) || (src == NULL)) { + return -ENOMEM; + } + + /* + * Check source and destination size validity + */ + if ((dsize == 0) || (ssize == 0)) { + return -ERANGE; + } + + /* + * Check both source and destination size range + */ + if ((ssize > dsize) || (dsize > ssize)) { + return -EINVAL; + } + + /* + * Check both source and destination address overlapping + * When (s > d < s + ssize) + * Or (d > s < d + dsize) + */ + + if (d > s) { + if ((d) < (s + ssize)) { + return -EOPNOTSUPP; + } + } + + if (s > d) { + if ((s) < (d + dsize)) { + return -EOPNOTSUPP; + } + } + + /* + * Start copy process when there is no error + */ + while (ssize--) { + d[ssize] = s[ssize]; + } + + return 0; +} diff --git a/lib/libc/memmove.c b/lib/libc/memmove.c new file mode 100644 index 0000000..5c2b661 --- /dev/null +++ b/lib/libc/memmove.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2013-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <string.h> + +void *memmove(void *dst, const void *src, size_t len) +{ + /* + * The following test makes use of unsigned arithmetic overflow to + * more efficiently test the condition !(src <= dst && dst < str+len). + * It also avoids the situation where the more explicit test would give + * incorrect results were the calculation str+len to overflow (though + * that issue is probably moot as such usage is probably undefined + * behaviour and a bug anyway. + */ + if ((size_t)dst - (size_t)src >= len) { + /* destination not in source data, so can safely use memcpy */ + return memcpy(dst, src, len); + } else { + /* copy backwards... */ + const char *end = dst; + const char *s = (const char *)src + len; + char *d = (char *)dst + len; + while (d != end) + *--d = *--s; + } + return dst; +} diff --git a/lib/libc/memrchr.c b/lib/libc/memrchr.c new file mode 100644 index 0000000..01caef3 --- /dev/null +++ b/lib/libc/memrchr.c @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <string.h> + +#undef memrchr + +void *memrchr(const void *src, int c, size_t len) +{ + const unsigned char *s = src + (len - 1); + + while (len--) { + if (*s == (unsigned char)c) { + return (void*) s; + } + + s--; + } + + return NULL; +} diff --git a/lib/libc/memset.c b/lib/libc/memset.c new file mode 100644 index 0000000..c5bac8d --- /dev/null +++ b/lib/libc/memset.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2013-2020, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stddef.h> +#include <string.h> +#include <stdint.h> + +void *memset(void *dst, int val, size_t count) +{ + uint8_t *ptr = dst; + uint64_t *ptr64; + uint64_t fill = (unsigned char)val; + + /* Simplify code below by making sure we write at least one byte. */ + if (count == 0U) { + return dst; + } + + /* Handle the first part, until the pointer becomes 64-bit aligned. */ + while (((uintptr_t)ptr & 7U) != 0U) { + *ptr = (uint8_t)val; + ptr++; + if (--count == 0U) { + return dst; + } + } + + /* Duplicate the fill byte to the rest of the 64-bit word. */ + fill |= fill << 8; + fill |= fill << 16; + fill |= fill << 32; + + /* Use 64-bit writes for as long as possible. */ + ptr64 = (uint64_t *)ptr; + for (; count >= 8U; count -= 8) { + *ptr64 = fill; + ptr64++; + } + + /* Handle the remaining part byte-per-byte. */ + ptr = (uint8_t *)ptr64; + while (count-- > 0U) { + *ptr = (uint8_t)val; + ptr++; + } + + return dst; +} diff --git a/lib/libc/printf.c b/lib/libc/printf.c new file mode 100644 index 0000000..6931a7e --- /dev/null +++ b/lib/libc/printf.c @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2014-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> + +#define get_num_va_args(_args, _lcount) \ + (((_lcount) > 1) ? va_arg(_args, long long int) : \ + (((_lcount) == 1) ? va_arg(_args, long int) : \ + va_arg(_args, int))) + +#define get_unum_va_args(_args, _lcount) \ + (((_lcount) > 1) ? va_arg(_args, unsigned long long int) : \ + (((_lcount) == 1) ? va_arg(_args, unsigned long int) : \ + va_arg(_args, unsigned int))) + +static int string_print(const char *str) +{ + int count = 0; + + assert(str != NULL); + + for ( ; *str != '\0'; str++) { + (void)putchar(*str); + count++; + } + + return count; +} + +static int unsigned_num_print(unsigned long long int unum, unsigned int radix, + char padc, int padn, bool uppercase) +{ + /* Just need enough space to store 64 bit decimal integer */ + char num_buf[20]; + int i = 0, count = 0; + unsigned int rem; + + /* num_buf is only large enough for radix >= 10 */ + if (radix < 10) { + assert(0); + return 0; + } + + do { + rem = unum % radix; + if (rem < 0xa) { + num_buf[i] = '0' + rem; + } else if (uppercase) { + num_buf[i] = 'A' + (rem - 0xa); + } else { + num_buf[i] = 'a' + (rem - 0xa); + } + i++; + unum /= radix; + } while (unum > 0U); + + if (padn > 0) { + while (i < padn) { + (void)putchar(padc); + count++; + padn--; + } + } + + while (--i >= 0) { + (void)putchar(num_buf[i]); + count++; + } + + return count; +} + +/******************************************************************* + * Reduced format print for Trusted firmware. + * The following type specifiers are supported by this print + * %x - hexadecimal format + * %s - string format + * %d or %i - signed decimal format + * %c - character format + * %u - unsigned decimal format + * %p - pointer format + * + * The following length specifiers are supported by this print + * %l - long int (64-bit on AArch64) + * %ll - long long int (64-bit on AArch64) + * %z - size_t sized integer formats (64 bit on AArch64) + * + * The following padding specifiers are supported by this print + * %0NN - Left-pad the number with 0s (NN is a decimal number) + * + * The print exits on all other formats specifiers other than valid + * combinations of the above specifiers. + *******************************************************************/ +int vprintf(const char *fmt, va_list args) +{ + int l_count; + long long int num; + unsigned long long int unum; + char *str; + char padc = '\0'; /* Padding character */ + int padn; /* Number of characters to pad */ + int count = 0; /* Number of printed characters */ + bool uppercase; /* Print characters in uppercase */ + + while (*fmt != '\0') { + uppercase = false; + l_count = 0; + padn = 0; + + if (*fmt == '%') { + fmt++; + /* Check the format specifier */ +loop: + switch (*fmt) { + case '%': + (void)putchar('%'); + break; + case 'i': /* Fall through to next one */ + case 'd': + num = get_num_va_args(args, l_count); + if (num < 0) { + (void)putchar('-'); + unum = (unsigned long long int)-num; + padn--; + } else + unum = (unsigned long long int)num; + + count += unsigned_num_print(unum, 10, + padc, padn, uppercase); + break; + case 'c': + (void)putchar(va_arg(args, int)); + count++; + break; + case 's': + str = va_arg(args, char *); + count += string_print(str); + break; + case 'p': + unum = (uintptr_t)va_arg(args, void *); + if (unum > 0U) { + count += string_print("0x"); + padn -= 2; + } + + count += unsigned_num_print(unum, 16, + padc, padn, uppercase); + break; + case 'X': + uppercase = true; + // fall through + case 'x': + unum = get_unum_va_args(args, l_count); + count += unsigned_num_print(unum, 16, + padc, padn, uppercase); + break; + case 'z': + if (sizeof(size_t) == 8U) + l_count = 2; + + fmt++; + goto loop; + case 'l': + l_count++; + fmt++; + goto loop; + case 'u': + unum = get_unum_va_args(args, l_count); + count += unsigned_num_print(unum, 10, + padc, padn, uppercase); + break; + case '0': + padc = '0'; + padn = 0; + fmt++; + + for (;;) { + char ch = *fmt; + if ((ch < '0') || (ch > '9')) { + goto loop; + } + padn = (padn * 10) + (ch - '0'); + fmt++; + } + assert(0); /* Unreachable */ + default: + /* Exit on any other format specifier */ + return -1; + } + fmt++; + continue; + } + (void)putchar(*fmt); + fmt++; + count++; + } + + return count; +} + +int printf(const char *fmt, ...) +{ + int count; + va_list va; + + va_start(va, fmt); + count = vprintf(fmt, va); + va_end(va); + + return count; +} diff --git a/lib/libc/putchar.c b/lib/libc/putchar.c new file mode 100644 index 0000000..1f919d1 --- /dev/null +++ b/lib/libc/putchar.c @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> + +#pragma weak putchar +int putchar(int c) +{ + return c; +} diff --git a/lib/libc/puts.c b/lib/libc/puts.c new file mode 100644 index 0000000..f57fc98 --- /dev/null +++ b/lib/libc/puts.c @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2013-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> + +int puts(const char *s) +{ + int count = 0; + + while (*s != '\0') { + if (putchar(*s) == EOF) + return EOF; + s++; + count++; + } + + if (putchar('\n') == EOF) + return EOF; + + return count + 1; +} diff --git a/lib/libc/snprintf.c b/lib/libc/snprintf.c new file mode 100644 index 0000000..21d3416 --- /dev/null +++ b/lib/libc/snprintf.c @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +#define get_num_va_args(_args, _lcount) \ + (((_lcount) > 1) ? va_arg(_args, long long int) : \ + (((_lcount) == 1) ? va_arg(_args, long int) : \ + va_arg(_args, int))) + +#define get_unum_va_args(_args, _lcount) \ + (((_lcount) > 1) ? va_arg(_args, unsigned long long int) : \ + (((_lcount) == 1) ? va_arg(_args, unsigned long int) : \ + va_arg(_args, unsigned int))) + +#define CHECK_AND_PUT_CHAR(buf, size, chars_printed, ch) \ + do { \ + if ((chars_printed) < (size)) { \ + *(buf) = (ch); \ + (buf)++; \ + } \ + (chars_printed)++; \ + } while (false) + +static void string_print(char **s, size_t n, size_t *chars_printed, + const char *str) +{ + while (*str != '\0') { + CHECK_AND_PUT_CHAR(*s, n, *chars_printed, *str); + str++; + } +} + +static void unsigned_num_print(char **s, size_t n, size_t *chars_printed, + unsigned long long int unum, + unsigned int radix, char padc, int padn, + bool capitalise) +{ + /* Just need enough space to store 64 bit decimal integer */ + char num_buf[20]; + int i = 0; + int width; + unsigned int rem; + char ascii_a = capitalise ? 'A' : 'a'; + + /* num_buf is only large enough for radix >= 10 */ + if (radix < 10) { + assert(0); + return; + } + + do { + rem = unum % radix; + if (rem < 10U) { + num_buf[i] = '0' + rem; + } else { + num_buf[i] = ascii_a + (rem - 10U); + } + i++; + unum /= radix; + } while (unum > 0U); + + width = i; + for (i = padn - width; i > 0; i--) { + CHECK_AND_PUT_CHAR(*s, n, *chars_printed, padc); + } + for (i = width; i > 0; i--) { + CHECK_AND_PUT_CHAR(*s, n, *chars_printed, num_buf[i - 1]); + } + for (i = width + padn; i < 0; i++) { + CHECK_AND_PUT_CHAR(*s, n, *chars_printed, padc); + } +} + +/******************************************************************* + * Reduced vsnprintf to be used for Trusted firmware. + * The following type specifiers are supported: + * + * %x (or %X) - hexadecimal format + * %d or %i - signed decimal format + * %c - character format + * %s - string format + * %u - unsigned decimal format + * %p - pointer format + * + * The following length specifiers are supported by this print + * %l - long int + * %ll - long long int + * %z - size_t sized integer formats + * + * The following padding specifiers are supported by this print + * %0NN - Left-pad the number with 0s (NN is a decimal number) + * %NN - Left-pad the number or string with spaces (NN is a decimal number) + * %-NN - Right-pad the number or string with spaces (NN is a decimal number) + * + * The function panics on all other formats specifiers. + * + * It returns the number of characters that would be written if the + * buffer was big enough. If it returns a value lower than n, the + * whole string has been written. + *******************************************************************/ +int vsnprintf(char *s, size_t n, const char *fmt, va_list args) +{ + int num; + unsigned long long int unum; + char *str; + char padc; /* Padding character */ + int padn; /* Number of characters to pad */ + bool left; + bool capitalise; + size_t chars_printed = 0U; + unsigned int l_count; + + if (n == 0U) { + /* There isn't space for anything. */ + } else if (n == 1U) { + /* Buffer is too small to actually write anything else. */ + *s = '\0'; + n = 0U; + } else { + /* Reserve space for the terminator character. */ + n--; + } + + while (*fmt != '\0') { + left = false; + padc ='\0'; + padn = 0; + capitalise = false; + l_count = 0; + + if (*fmt == '%') { + fmt++; + /* Check the format specifier. */ +loop: + switch (*fmt) { + case '%': + CHECK_AND_PUT_CHAR(s, n, chars_printed, '%'); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + padc = (*fmt == '0') ? '0' : ' '; + for (padn = 0; *fmt >= '0' && *fmt <= '9'; fmt++) { + padn = (padn * 10) + (*fmt - '0'); + } + if (left) { + padn = -padn; + } + goto loop; + case '-': + left = true; + fmt++; + goto loop; + + case 'i': + case 'd': + num = get_num_va_args(args, l_count); + + if (num < 0) { + CHECK_AND_PUT_CHAR(s, n, chars_printed, + '-'); + unum = (unsigned int)-num; + } else { + unum = (unsigned int)num; + } + + unsigned_num_print(&s, n, &chars_printed, + unum, 10, padc, padn, false); + break; + case 'c': + CHECK_AND_PUT_CHAR(s, n, chars_printed, va_arg(args, int)); + break; + case 's': + str = va_arg(args, char *); + string_print(&s, n, &chars_printed, str); + break; + case 'u': + unum = get_unum_va_args(args, l_count); + unsigned_num_print(&s, n, &chars_printed, + unum, 10, padc, padn, false); + break; + case 'z': + l_count = 1; + fmt++; + goto loop; + case 'l': + l_count++; + fmt++; + goto loop; + case 'p': + unum = (uintptr_t)va_arg(args, void *); + if (unum > 0U) { + string_print(&s, n, &chars_printed, "0x"); + padn -= 2; + } + unsigned_num_print(&s, n, &chars_printed, + unum, 16, padc, padn, false); + break; + case 'X': + capitalise = true; + /* fallthrough */ + case 'x': + unum = get_unum_va_args(args, l_count); + unsigned_num_print(&s, n, &chars_printed, + unum, 16, padc, padn, + capitalise); + break; + + default: + CHECK_AND_PUT_CHAR(s, n, chars_printed, '%'); + CHECK_AND_PUT_CHAR(s, n, chars_printed, *fmt); + } + fmt++; + continue; + } + + CHECK_AND_PUT_CHAR(s, n, chars_printed, *fmt); + + fmt++; + } + + if (n > 0U) { + *s = '\0'; + } + + return (int)chars_printed; +} + +/******************************************************************* + * Reduced snprintf to be used for Trusted firmware. + * The following type specifiers are supported: + * + * %x (or %X) - hexadecimal format + * %d or %i - signed decimal format + * %s - string format + * %u - unsigned decimal format + * %p - pointer format + * + * The following padding specifiers are supported by this print + * %0NN - Left-pad the number with 0s (NN is a decimal number) + * %NN - Left-pad the number or string with spaces (NN is a decimal number) + * %-NN - Right-pad the number or string with spaces (NN is a decimal number) + * + * The function panics on all other formats specifiers. + * + * It returns the number of characters that would be written if the + * buffer was big enough. If it returns a value lower than n, the + * whole string has been written. + *******************************************************************/ +int snprintf(char *s, size_t n, const char *fmt, ...) +{ + int count; + va_list all_args; + + va_start(all_args, fmt); + count = vsnprintf(s, n, fmt, all_args); + va_end(all_args); + + return count; +} diff --git a/lib/libc/strchr.c b/lib/libc/strchr.c new file mode 100644 index 0000000..1cd03ca --- /dev/null +++ b/lib/libc/strchr.c @@ -0,0 +1,53 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Portions copyright (c) 2018, Arm Limited and Contributors. + * All rights reserved. + */ + +#include <stddef.h> +#include <string.h> + +char * +strchr(const char *p, int ch) +{ + char c; + + c = ch; + for (;; ++p) { + if (*p == c) + return ((char *)p); + if (*p == '\0') + return (NULL); + } + /* NOTREACHED */ +} diff --git a/lib/libc/strcmp.c b/lib/libc/strcmp.c new file mode 100644 index 0000000..290db4c --- /dev/null +++ b/lib/libc/strcmp.c @@ -0,0 +1,52 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Chris Torek. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Portions copyright (c) 2018, Arm Limited and Contributors. + * All rights reserved. + */ + +#include <string.h> + +/* + * Compare strings. + */ +int +strcmp(const char *s1, const char *s2) +{ + while (*s1 == *s2++) + if (*s1++ == '\0') + return (0); + return (*(const unsigned char *)s1 - *(const unsigned char *)(s2 - 1)); +} diff --git a/lib/libc/strlcat.c b/lib/libc/strlcat.c new file mode 100644 index 0000000..e60c863 --- /dev/null +++ b/lib/libc/strlcat.c @@ -0,0 +1,56 @@ +/* $OpenBSD: strlcat.c,v 1.15 2015/03/02 21:41:08 millert Exp $ */ + +/* + * SPDX-License-Identifier: ISC + * + * Copyright (c) 1998, 2015 Todd C. Miller <Todd.Miller@courtesan.com> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <string.h> + +/* + * Appends src to string dst of size dsize (unlike strncat, dsize is the + * full size of dst, not space left). At most dsize-1 characters + * will be copied. Always NUL terminates (unless dsize <= strlen(dst)). + * Returns strlen(src) + MIN(dsize, strlen(initial dst)). + * If retval >= dsize, truncation occurred. + */ +size_t +strlcat(char * dst, const char * src, size_t dsize) +{ + const char *odst = dst; + const char *osrc = src; + size_t n = dsize; + size_t dlen; + + /* Find the end of dst and adjust bytes left but don't go past end. */ + while (n-- != 0 && *dst != '\0') + dst++; + dlen = dst - odst; + n = dsize - dlen; + + if (n-- == 0) + return(dlen + strlen(src)); + while (*src != '\0') { + if (n != 0) { + *dst++ = *src; + n--; + } + src++; + } + *dst = '\0'; + + return(dlen + (src - osrc)); /* count does not include NUL */ +} diff --git a/lib/libc/strlcpy.c b/lib/libc/strlcpy.c new file mode 100644 index 0000000..c4f39bb --- /dev/null +++ b/lib/libc/strlcpy.c @@ -0,0 +1,52 @@ +/* $OpenBSD: strlcpy.c,v 1.12 2015/01/15 03:54:12 millert Exp $ */ + +/* + * SPDX-License-Identifier: ISC + * + * Copyright (c) 1998, 2015 Todd C. Miller <Todd.Miller@courtesan.com> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <stdint.h> +#include <string.h> + +/* + * Copy string src to buffer dst of size dsize. At most dsize-1 + * chars will be copied. Always NUL terminates (unless dsize == 0). + * Returns strlen(src); if retval >= dsize, truncation occurred. + */ +size_t +strlcpy(char * dst, const char * src, size_t dsize) +{ + const char *osrc = src; + size_t nleft = dsize; + + /* Copy as many bytes as will fit. */ + if (nleft != 0) { + while (--nleft != 0) { + if ((*dst++ = *src++) == '\0') + break; + } + } + + /* Not enough room in dst, add NUL and traverse rest of src. */ + if (nleft == 0) { + if (dsize != 0) + *dst = '\0'; /* NUL-terminate dst */ + while (*src++) + ; + } + + return(src - osrc - 1); /* count does not include NUL */ +} diff --git a/lib/libc/strlen.c b/lib/libc/strlen.c new file mode 100644 index 0000000..e4b79d9 --- /dev/null +++ b/lib/libc/strlen.c @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <string.h> + +size_t strlen(const char *s) +{ + const char *cursor = s; + + while (*cursor) + cursor++; + + return cursor - s; +} diff --git a/lib/libc/strncmp.c b/lib/libc/strncmp.c new file mode 100644 index 0000000..f0bbadc --- /dev/null +++ b/lib/libc/strncmp.c @@ -0,0 +1,53 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Portions copyright (c) 2018, Arm Limited and Contributors. + * All rights reserved. + */ + +#include <string.h> + +int +strncmp(const char *s1, const char *s2, size_t n) +{ + + if (n == 0) + return (0); + do { + if (*s1 != *s2++) + return (*(const unsigned char *)s1 - + *(const unsigned char *)(s2 - 1)); + if (*s1++ == '\0') + break; + } while (--n != 0); + return (0); +} diff --git a/lib/libc/strnlen.c b/lib/libc/strnlen.c new file mode 100644 index 0000000..49e9f7a --- /dev/null +++ b/lib/libc/strnlen.c @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2009 David Schultz <das@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Portions copyright (c) 2018, Arm Limited and Contributors. + * All rights reserved. + */ + +#include <string.h> + +size_t +strnlen(const char *s, size_t maxlen) +{ + size_t len; + + for (len = 0; len < maxlen; len++, s++) { + if (!*s) + break; + } + return (len); +} diff --git a/lib/libc/strrchr.c b/lib/libc/strrchr.c new file mode 100644 index 0000000..cd435ff --- /dev/null +++ b/lib/libc/strrchr.c @@ -0,0 +1,49 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1988, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <stddef.h> +#include <string.h> + +char * +strrchr(const char *p, int ch) +{ + char *save; + char c; + + c = ch; + for (save = NULL;; ++p) { + if (*p == c) + save = (char *)p; + if (*p == '\0') + return (save); + } + /* NOTREACHED */ +} diff --git a/lib/libc/strtok.c b/lib/libc/strtok.c new file mode 100644 index 0000000..7e1a4d2 --- /dev/null +++ b/lib/libc/strtok.c @@ -0,0 +1,83 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1998 Softweyr LLC. All rights reserved. + * + * strtok_r, from Berkeley strtok + * Oct 13, 1998 by Wes Peters <wes@softweyr.com> + * + * Copyright (c) 1988, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notices, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notices, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY SOFTWEYR LLC, THE REGENTS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SOFTWEYR LLC, THE + * REGENTS, OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> + +char * +strtok_r(char *s, const char *delim, char **last) +{ + char *spanp, *tok; + int c, sc; + + if (s == NULL && (s = *last) == NULL) + return (NULL); + + /* + * Skip (span) leading delimiters (s += strspn(s, delim), sort of). + */ +cont: + c = *s++; + for (spanp = (char *)delim; (sc = *spanp++) != 0;) { + if (c == sc) + goto cont; + } + + if (c == 0) { /* no non-delimiter characters */ + *last = NULL; + return (NULL); + } + tok = s - 1; + + /* + * Scan token (scan for delimiters: s += strcspn(s, delim), sort of). + * Note that delim must have one NUL; we stop if we see that, too. + */ + for (;;) { + c = *s++; + spanp = (char *)delim; + do { + if ((sc = *spanp++) == c) { + if (c == 0) + s = NULL; + else + s[-1] = '\0'; + *last = s; + return (tok); + } + } while (sc != 0); + } + /* NOTREACHED */ +} diff --git a/lib/libc/strtol.c b/lib/libc/strtol.c new file mode 100644 index 0000000..deb862c --- /dev/null +++ b/lib/libc/strtol.c @@ -0,0 +1,133 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Copyright (c) 2011 The FreeBSD Foundation + * All rights reserved. + * Portions of this software were developed by David Chisnall + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <errno.h> +#include <limits.h> +#include <stddef.h> +#include <stdlib.h> + +/* + * Convert a string to a long integer. + * + * Assumes that the upper and lower case + * alphabets and digits are each contiguous. + */ +long strtol(const char *nptr, char **endptr, int base) +{ + const char *s; + unsigned long acc; + char c; + unsigned long cutoff; + int neg, any, cutlim; + + /* + * Skip white space and pick up leading +/- sign if any. + * If base is 0, allow 0x for hex and 0 for octal, else + * assume decimal; if base is already 16, allow 0x. + */ + s = nptr; + do { + c = *s++; + } while (isspace((unsigned char)c)); + if (c == '-') { + neg = 1; + c = *s++; + } else { + neg = 0; + if (c == '+') + c = *s++; + } + if ((base == 0 || base == 16) && + c == '0' && (*s == 'x' || *s == 'X') && + ((s[1] >= '0' && s[1] <= '9') || + (s[1] >= 'A' && s[1] <= 'F') || + (s[1] >= 'a' && s[1] <= 'f'))) { + c = s[1]; + s += 2; + base = 16; + } + if (base == 0) + base = c == '0' ? 8 : 10; + acc = any = 0; + + /* + * Compute the cutoff value between legal numbers and illegal + * numbers. That is the largest legal value, divided by the + * base. An input number that is greater than this value, if + * followed by a legal input character, is too big. One that + * is equal to this value may be valid or not; the limit + * between valid and invalid numbers is then based on the last + * digit. For instance, if the range for longs is + * [-2147483648..2147483647] and the input base is 10, + * cutoff will be set to 214748364 and cutlim to either + * 7 (neg==0) or 8 (neg==1), meaning that if we have accumulated + * a value > 214748364, or equal but the next digit is > 7 (or 8), + * the number is too big, and we will return a range error. + * + * Set 'any' if any `digits' consumed; make it negative to indicate + * overflow. + */ + cutoff = neg ? (unsigned long)-(LONG_MIN + LONG_MAX) + LONG_MAX + : LONG_MAX; + cutlim = cutoff % base; + cutoff /= base; + for ( ; ; c = *s++) { + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'A' && c <= 'Z') + c -= 'A' - 10; + else if (c >= 'a' && c <= 'z') + c -= 'a' - 10; + else + break; + if (c >= base) + break; + if (any < 0 || acc > cutoff || (acc == cutoff && c > cutlim)) + any = -1; + else { + any = 1; + acc *= base; + acc += c; + } + } + if (any < 0) { + acc = neg ? LONG_MIN : LONG_MAX; + } else if (neg) + acc = -acc; + if (endptr != NULL) + *endptr = (char *)(any ? s - 1 : nptr); + return (acc); +} diff --git a/lib/libc/strtoll.c b/lib/libc/strtoll.c new file mode 100644 index 0000000..4e101e8 --- /dev/null +++ b/lib/libc/strtoll.c @@ -0,0 +1,134 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Copyright (c) 2011 The FreeBSD Foundation + * All rights reserved. + * Portions of this software were developed by David Chisnall + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <errno.h> +#include <limits.h> +#include <stddef.h> +#include <stdlib.h> + +/* + * Convert a string to a long long integer. + * + * Assumes that the upper and lower case + * alphabets and digits are each contiguous. + */ +long long strtoll(const char *nptr, char **endptr, int base) +{ + const char *s; + unsigned long long acc; + char c; + unsigned long long cutoff; + int neg, any, cutlim; + + /* + * Skip white space and pick up leading +/- sign if any. + * If base is 0, allow 0x for hex and 0 for octal, else + * assume decimal; if base is already 16, allow 0x. + */ + s = nptr; + do { + c = *s++; + } while (isspace((unsigned char)c)); + if (c == '-') { + neg = 1; + c = *s++; + } else { + neg = 0; + if (c == '+') + c = *s++; + } + if ((base == 0 || base == 16) && + c == '0' && (*s == 'x' || *s == 'X') && + ((s[1] >= '0' && s[1] <= '9') || + (s[1] >= 'A' && s[1] <= 'F') || + (s[1] >= 'a' && s[1] <= 'f'))) { + c = s[1]; + s += 2; + base = 16; + } + if (base == 0) + base = c == '0' ? 8 : 10; + acc = any = 0; + + /* + * Compute the cutoff value between legal numbers and illegal + * numbers. That is the largest legal value, divided by the + * base. An input number that is greater than this value, if + * followed by a legal input character, is too big. One that + * is equal to this value may be valid or not; the limit + * between valid and invalid numbers is then based on the last + * digit. For instance, if the range for quads is + * [-9223372036854775808..9223372036854775807] and the input base + * is 10, cutoff will be set to 922337203685477580 and cutlim to + * either 7 (neg==0) or 8 (neg==1), meaning that if we have + * accumulated a value > 922337203685477580, or equal but the + * next digit is > 7 (or 8), the number is too big, and we will + * return a range error. + * + * Set 'any' if any `digits' consumed; make it negative to indicate + * overflow. + */ + cutoff = neg ? (unsigned long long)-(LLONG_MIN + LLONG_MAX) + LLONG_MAX + : LLONG_MAX; + cutlim = cutoff % base; + cutoff /= base; + for ( ; ; c = *s++) { + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'A' && c <= 'Z') + c -= 'A' - 10; + else if (c >= 'a' && c <= 'z') + c -= 'a' - 10; + else + break; + if (c >= base) + break; + if (any < 0 || acc > cutoff || (acc == cutoff && c > cutlim)) + any = -1; + else { + any = 1; + acc *= base; + acc += c; + } + } + if (any < 0) { + acc = neg ? LLONG_MIN : LLONG_MAX; + } else if (neg) + acc = -acc; + if (endptr != NULL) + *endptr = (char *)(any ? s - 1 : nptr); + return (acc); +} diff --git a/lib/libc/strtoul.c b/lib/libc/strtoul.c new file mode 100644 index 0000000..b42fb14 --- /dev/null +++ b/lib/libc/strtoul.c @@ -0,0 +1,112 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Copyright (c) 2011 The FreeBSD Foundation + * All rights reserved. + * Portions of this software were developed by David Chisnall + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <errno.h> +#include <limits.h> +#include <stddef.h> +#include <stdlib.h> + +/* + * Convert a string to an unsigned long integer. + * + * Assumes that the upper and lower case + * alphabets and digits are each contiguous. + */ +unsigned long strtoul(const char *nptr, char **endptr, int base) +{ + const char *s; + unsigned long acc; + char c; + unsigned long cutoff; + int neg, any, cutlim; + + /* + * See strtol for comments as to the logic used. + */ + s = nptr; + do { + c = *s++; + } while (isspace((unsigned char)c)); + if (c == '-') { + neg = 1; + c = *s++; + } else { + neg = 0; + if (c == '+') + c = *s++; + } + if ((base == 0 || base == 16) && + c == '0' && (*s == 'x' || *s == 'X') && + ((s[1] >= '0' && s[1] <= '9') || + (s[1] >= 'A' && s[1] <= 'F') || + (s[1] >= 'a' && s[1] <= 'f'))) { + c = s[1]; + s += 2; + base = 16; + } + if (base == 0) + base = c == '0' ? 8 : 10; + acc = any = 0; + + cutoff = ULONG_MAX / base; + cutlim = ULONG_MAX % base; + for ( ; ; c = *s++) { + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'A' && c <= 'Z') + c -= 'A' - 10; + else if (c >= 'a' && c <= 'z') + c -= 'a' - 10; + else + break; + if (c >= base) + break; + if (any < 0 || acc > cutoff || (acc == cutoff && c > cutlim)) + any = -1; + else { + any = 1; + acc *= base; + acc += c; + } + } + if (any < 0) { + acc = ULONG_MAX; + } else if (neg) + acc = -acc; + if (endptr != NULL) + *endptr = (char *)(any ? s - 1 : nptr); + return (acc); +} diff --git a/lib/libc/strtoull.c b/lib/libc/strtoull.c new file mode 100644 index 0000000..2e65a43 --- /dev/null +++ b/lib/libc/strtoull.c @@ -0,0 +1,112 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Copyright (c) 2011 The FreeBSD Foundation + * All rights reserved. + * Portions of this software were developed by David Chisnall + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <errno.h> +#include <limits.h> +#include <stddef.h> +#include <stdlib.h> + +/* + * Convert a string to an unsigned long long integer. + * + * Assumes that the upper and lower case + * alphabets and digits are each contiguous. + */ +unsigned long long strtoull(const char *nptr, char **endptr, int base) +{ + const char *s; + unsigned long long acc; + char c; + unsigned long long cutoff; + int neg, any, cutlim; + + /* + * See strtoq for comments as to the logic used. + */ + s = nptr; + do { + c = *s++; + } while (isspace((unsigned char)c)); + if (c == '-') { + neg = 1; + c = *s++; + } else { + neg = 0; + if (c == '+') + c = *s++; + } + if ((base == 0 || base == 16) && + c == '0' && (*s == 'x' || *s == 'X') && + ((s[1] >= '0' && s[1] <= '9') || + (s[1] >= 'A' && s[1] <= 'F') || + (s[1] >= 'a' && s[1] <= 'f'))) { + c = s[1]; + s += 2; + base = 16; + } + if (base == 0) + base = c == '0' ? 8 : 10; + acc = any = 0; + + cutoff = ULLONG_MAX / base; + cutlim = ULLONG_MAX % base; + for ( ; ; c = *s++) { + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'A' && c <= 'Z') + c -= 'A' - 10; + else if (c >= 'a' && c <= 'z') + c -= 'a' - 10; + else + break; + if (c >= base) + break; + if (any < 0 || acc > cutoff || (acc == cutoff && c > cutlim)) + any = -1; + else { + any = 1; + acc *= base; + acc += c; + } + } + if (any < 0) { + acc = ULLONG_MAX; + } else if (neg) + acc = -acc; + if (endptr != NULL) + *endptr = (char *)(any ? s - 1 : nptr); + return (acc); +} diff --git a/lib/libfdt/fdt.c b/lib/libfdt/fdt.c new file mode 100644 index 0000000..20c6415 --- /dev/null +++ b/lib/libfdt/fdt.c @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +/* + * Minimal sanity check for a read-only tree. fdt_ro_probe_() checks + * that the given buffer contains what appears to be a flattened + * device tree with sane information in its header. + */ +int32_t fdt_ro_probe_(const void *fdt) +{ + uint32_t totalsize = fdt_totalsize(fdt); + + if (can_assume(VALID_DTB)) + return totalsize; + + /* The device tree must be at an 8-byte aligned address */ + if ((uintptr_t)fdt & 7) + return -FDT_ERR_ALIGNMENT; + + if (fdt_magic(fdt) == FDT_MAGIC) { + /* Complete tree */ + if (!can_assume(LATEST)) { + if (fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION) + return -FDT_ERR_BADVERSION; + if (fdt_last_comp_version(fdt) > + FDT_LAST_SUPPORTED_VERSION) + return -FDT_ERR_BADVERSION; + } + } else if (fdt_magic(fdt) == FDT_SW_MAGIC) { + /* Unfinished sequential-write blob */ + if (!can_assume(VALID_INPUT) && fdt_size_dt_struct(fdt) == 0) + return -FDT_ERR_BADSTATE; + } else { + return -FDT_ERR_BADMAGIC; + } + + if (totalsize < INT32_MAX) + return totalsize; + else + return -FDT_ERR_TRUNCATED; +} + +static int check_off_(uint32_t hdrsize, uint32_t totalsize, uint32_t off) +{ + return (off >= hdrsize) && (off <= totalsize); +} + +static int check_block_(uint32_t hdrsize, uint32_t totalsize, + uint32_t base, uint32_t size) +{ + if (!check_off_(hdrsize, totalsize, base)) + return 0; /* block start out of bounds */ + if ((base + size) < base) + return 0; /* overflow */ + if (!check_off_(hdrsize, totalsize, base + size)) + return 0; /* block end out of bounds */ + return 1; +} + +size_t fdt_header_size_(uint32_t version) +{ + if (version <= 1) + return FDT_V1_SIZE; + else if (version <= 2) + return FDT_V2_SIZE; + else if (version <= 3) + return FDT_V3_SIZE; + else if (version <= 16) + return FDT_V16_SIZE; + else + return FDT_V17_SIZE; +} + +size_t fdt_header_size(const void *fdt) +{ + return can_assume(LATEST) ? FDT_V17_SIZE : + fdt_header_size_(fdt_version(fdt)); +} + +int fdt_check_header(const void *fdt) +{ + size_t hdrsize; + + /* The device tree must be at an 8-byte aligned address */ + if ((uintptr_t)fdt & 7) + return -FDT_ERR_ALIGNMENT; + + if (fdt_magic(fdt) != FDT_MAGIC) + return -FDT_ERR_BADMAGIC; + if (!can_assume(LATEST)) { + if ((fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION) + || (fdt_last_comp_version(fdt) > + FDT_LAST_SUPPORTED_VERSION)) + return -FDT_ERR_BADVERSION; + if (fdt_version(fdt) < fdt_last_comp_version(fdt)) + return -FDT_ERR_BADVERSION; + } + hdrsize = fdt_header_size(fdt); + if (!can_assume(VALID_DTB)) { + if ((fdt_totalsize(fdt) < hdrsize) + || (fdt_totalsize(fdt) > INT_MAX)) + return -FDT_ERR_TRUNCATED; + + /* Bounds check memrsv block */ + if (!check_off_(hdrsize, fdt_totalsize(fdt), + fdt_off_mem_rsvmap(fdt))) + return -FDT_ERR_TRUNCATED; + + /* Bounds check structure block */ + if (!can_assume(LATEST) && fdt_version(fdt) < 17) { + if (!check_off_(hdrsize, fdt_totalsize(fdt), + fdt_off_dt_struct(fdt))) + return -FDT_ERR_TRUNCATED; + } else { + if (!check_block_(hdrsize, fdt_totalsize(fdt), + fdt_off_dt_struct(fdt), + fdt_size_dt_struct(fdt))) + return -FDT_ERR_TRUNCATED; + } + + /* Bounds check strings block */ + if (!check_block_(hdrsize, fdt_totalsize(fdt), + fdt_off_dt_strings(fdt), + fdt_size_dt_strings(fdt))) + return -FDT_ERR_TRUNCATED; + } + + return 0; +} + +const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int len) +{ + unsigned int uoffset = offset; + unsigned int absoffset = offset + fdt_off_dt_struct(fdt); + + if (offset < 0) + return NULL; + + if (!can_assume(VALID_INPUT)) + if ((absoffset < uoffset) + || ((absoffset + len) < absoffset) + || (absoffset + len) > fdt_totalsize(fdt)) + return NULL; + + if (can_assume(LATEST) || fdt_version(fdt) >= 0x11) + if (((uoffset + len) < uoffset) + || ((offset + len) > fdt_size_dt_struct(fdt))) + return NULL; + + return fdt_offset_ptr_(fdt, offset); +} + +uint32_t fdt_next_tag(const void *fdt, int startoffset, int *nextoffset) +{ + const fdt32_t *tagp, *lenp; + uint32_t tag, len, sum; + int offset = startoffset; + const char *p; + + *nextoffset = -FDT_ERR_TRUNCATED; + tagp = fdt_offset_ptr(fdt, offset, FDT_TAGSIZE); + if (!can_assume(VALID_DTB) && !tagp) + return FDT_END; /* premature end */ + tag = fdt32_to_cpu(*tagp); + offset += FDT_TAGSIZE; + + *nextoffset = -FDT_ERR_BADSTRUCTURE; + switch (tag) { + case FDT_BEGIN_NODE: + /* skip name */ + do { + p = fdt_offset_ptr(fdt, offset++, 1); + } while (p && (*p != '\0')); + if (!can_assume(VALID_DTB) && !p) + return FDT_END; /* premature end */ + break; + + case FDT_PROP: + lenp = fdt_offset_ptr(fdt, offset, sizeof(*lenp)); + if (!can_assume(VALID_DTB) && !lenp) + return FDT_END; /* premature end */ + + len = fdt32_to_cpu(*lenp); + sum = len + offset; + if (!can_assume(VALID_DTB) && + (INT_MAX <= sum || sum < (uint32_t) offset)) + return FDT_END; /* premature end */ + + /* skip-name offset, length and value */ + offset += sizeof(struct fdt_property) - FDT_TAGSIZE + len; + + if (!can_assume(LATEST) && + fdt_version(fdt) < 0x10 && len >= 8 && + ((offset - len) % 8) != 0) + offset += 4; + break; + + case FDT_END: + case FDT_END_NODE: + case FDT_NOP: + break; + + default: + return FDT_END; + } + + if (!fdt_offset_ptr(fdt, startoffset, offset - startoffset)) + return FDT_END; /* premature end */ + + *nextoffset = FDT_TAGALIGN(offset); + return tag; +} + +int fdt_check_node_offset_(const void *fdt, int offset) +{ + if (!can_assume(VALID_INPUT) + && ((offset < 0) || (offset % FDT_TAGSIZE))) + return -FDT_ERR_BADOFFSET; + + if (fdt_next_tag(fdt, offset, &offset) != FDT_BEGIN_NODE) + return -FDT_ERR_BADOFFSET; + + return offset; +} + +int fdt_check_prop_offset_(const void *fdt, int offset) +{ + if (!can_assume(VALID_INPUT) + && ((offset < 0) || (offset % FDT_TAGSIZE))) + return -FDT_ERR_BADOFFSET; + + if (fdt_next_tag(fdt, offset, &offset) != FDT_PROP) + return -FDT_ERR_BADOFFSET; + + return offset; +} + +int fdt_next_node(const void *fdt, int offset, int *depth) +{ + int nextoffset = 0; + uint32_t tag; + + if (offset >= 0) + if ((nextoffset = fdt_check_node_offset_(fdt, offset)) < 0) + return nextoffset; + + do { + offset = nextoffset; + tag = fdt_next_tag(fdt, offset, &nextoffset); + + switch (tag) { + case FDT_PROP: + case FDT_NOP: + break; + + case FDT_BEGIN_NODE: + if (depth) + (*depth)++; + break; + + case FDT_END_NODE: + if (depth && ((--(*depth)) < 0)) + return nextoffset; + break; + + case FDT_END: + if ((nextoffset >= 0) + || ((nextoffset == -FDT_ERR_TRUNCATED) && !depth)) + return -FDT_ERR_NOTFOUND; + else + return nextoffset; + } + } while (tag != FDT_BEGIN_NODE); + + return offset; +} + +int fdt_first_subnode(const void *fdt, int offset) +{ + int depth = 0; + + offset = fdt_next_node(fdt, offset, &depth); + if (offset < 0 || depth != 1) + return -FDT_ERR_NOTFOUND; + + return offset; +} + +int fdt_next_subnode(const void *fdt, int offset) +{ + int depth = 1; + + /* + * With respect to the parent, the depth of the next subnode will be + * the same as the last. + */ + do { + offset = fdt_next_node(fdt, offset, &depth); + if (offset < 0 || depth < 1) + return -FDT_ERR_NOTFOUND; + } while (depth > 1); + + return offset; +} + +const char *fdt_find_string_(const char *strtab, int tabsize, const char *s) +{ + int len = strlen(s) + 1; + const char *last = strtab + tabsize - len; + const char *p; + + for (p = strtab; p <= last; p++) + if (memcmp(p, s, len) == 0) + return p; + return NULL; +} + +int fdt_move(const void *fdt, void *buf, int bufsize) +{ + if (!can_assume(VALID_INPUT) && bufsize < 0) + return -FDT_ERR_NOSPACE; + + FDT_RO_PROBE(fdt); + + if (fdt_totalsize(fdt) > (unsigned int)bufsize) + return -FDT_ERR_NOSPACE; + + memmove(buf, fdt, fdt_totalsize(fdt)); + return 0; +} diff --git a/lib/libfdt/fdt_addresses.c b/lib/libfdt/fdt_addresses.c new file mode 100644 index 0000000..c40ba09 --- /dev/null +++ b/lib/libfdt/fdt_addresses.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2014 David Gibson <david@gibson.dropbear.id.au> + * Copyright (C) 2018 embedded brains GmbH + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +static int fdt_cells(const void *fdt, int nodeoffset, const char *name) +{ + const fdt32_t *c; + uint32_t val; + int len; + + c = fdt_getprop(fdt, nodeoffset, name, &len); + if (!c) + return len; + + if (len != sizeof(*c)) + return -FDT_ERR_BADNCELLS; + + val = fdt32_to_cpu(*c); + if (val > FDT_MAX_NCELLS) + return -FDT_ERR_BADNCELLS; + + return (int)val; +} + +int fdt_address_cells(const void *fdt, int nodeoffset) +{ + int val; + + val = fdt_cells(fdt, nodeoffset, "#address-cells"); + if (val == 0) + return -FDT_ERR_BADNCELLS; + if (val == -FDT_ERR_NOTFOUND) + return 2; + return val; +} + +int fdt_size_cells(const void *fdt, int nodeoffset) +{ + int val; + + val = fdt_cells(fdt, nodeoffset, "#size-cells"); + if (val == -FDT_ERR_NOTFOUND) + return 1; + return val; +} + +/* This function assumes that [address|size]_cells is 1 or 2 */ +int fdt_appendprop_addrrange(void *fdt, int parent, int nodeoffset, + const char *name, uint64_t addr, uint64_t size) +{ + int addr_cells, size_cells, ret; + uint8_t data[sizeof(fdt64_t) * 2], *prop; + + ret = fdt_address_cells(fdt, parent); + if (ret < 0) + return ret; + addr_cells = ret; + + ret = fdt_size_cells(fdt, parent); + if (ret < 0) + return ret; + size_cells = ret; + + /* check validity of address */ + prop = data; + if (addr_cells == 1) { + if ((addr > UINT32_MAX) || (((uint64_t) UINT32_MAX + 1 - addr) < size)) + return -FDT_ERR_BADVALUE; + + fdt32_st(prop, (uint32_t)addr); + } else if (addr_cells == 2) { + fdt64_st(prop, addr); + } else { + return -FDT_ERR_BADNCELLS; + } + + /* check validity of size */ + prop += addr_cells * sizeof(fdt32_t); + if (size_cells == 1) { + if (size > UINT32_MAX) + return -FDT_ERR_BADVALUE; + + fdt32_st(prop, (uint32_t)size); + } else if (size_cells == 2) { + fdt64_st(prop, size); + } else { + return -FDT_ERR_BADNCELLS; + } + + return fdt_appendprop(fdt, nodeoffset, name, data, + (addr_cells + size_cells) * sizeof(fdt32_t)); +} diff --git a/lib/libfdt/fdt_empty_tree.c b/lib/libfdt/fdt_empty_tree.c new file mode 100644 index 0000000..49d54d4 --- /dev/null +++ b/lib/libfdt/fdt_empty_tree.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2012 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +int fdt_create_empty_tree(void *buf, int bufsize) +{ + int err; + + err = fdt_create(buf, bufsize); + if (err) + return err; + + err = fdt_finish_reservemap(buf); + if (err) + return err; + + err = fdt_begin_node(buf, ""); + if (err) + return err; + + err = fdt_end_node(buf); + if (err) + return err; + + err = fdt_finish(buf); + if (err) + return err; + + return fdt_open_into(buf, buf, bufsize); +} diff --git a/lib/libfdt/fdt_overlay.c b/lib/libfdt/fdt_overlay.c new file mode 100644 index 0000000..5c0c398 --- /dev/null +++ b/lib/libfdt/fdt_overlay.c @@ -0,0 +1,867 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2016 Free Electrons + * Copyright (C) 2016 NextThing Co. + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +/** + * overlay_get_target_phandle - retrieves the target phandle of a fragment + * @fdto: pointer to the device tree overlay blob + * @fragment: node offset of the fragment in the overlay + * + * overlay_get_target_phandle() retrieves the target phandle of an + * overlay fragment when that fragment uses a phandle (target + * property) instead of a path (target-path property). + * + * returns: + * the phandle pointed by the target property + * 0, if the phandle was not found + * -1, if the phandle was malformed + */ +static uint32_t overlay_get_target_phandle(const void *fdto, int fragment) +{ + const fdt32_t *val; + int len; + + val = fdt_getprop(fdto, fragment, "target", &len); + if (!val) + return 0; + + if ((len != sizeof(*val)) || (fdt32_to_cpu(*val) == (uint32_t)-1)) + return (uint32_t)-1; + + return fdt32_to_cpu(*val); +} + +int fdt_overlay_target_offset(const void *fdt, const void *fdto, + int fragment_offset, char const **pathp) +{ + uint32_t phandle; + const char *path = NULL; + int path_len = 0, ret; + + /* Try first to do a phandle based lookup */ + phandle = overlay_get_target_phandle(fdto, fragment_offset); + if (phandle == (uint32_t)-1) + return -FDT_ERR_BADPHANDLE; + + /* no phandle, try path */ + if (!phandle) { + /* And then a path based lookup */ + path = fdt_getprop(fdto, fragment_offset, "target-path", &path_len); + if (path) + ret = fdt_path_offset(fdt, path); + else + ret = path_len; + } else + ret = fdt_node_offset_by_phandle(fdt, phandle); + + /* + * If we haven't found either a target or a + * target-path property in a node that contains a + * __overlay__ subnode (we wouldn't be called + * otherwise), consider it a improperly written + * overlay + */ + if (ret < 0 && path_len == -FDT_ERR_NOTFOUND) + ret = -FDT_ERR_BADOVERLAY; + + /* return on error */ + if (ret < 0) + return ret; + + /* return pointer to path (if available) */ + if (pathp) + *pathp = path ? path : NULL; + + return ret; +} + +/** + * overlay_phandle_add_offset - Increases a phandle by an offset + * @fdt: Base device tree blob + * @node: Device tree overlay blob + * @name: Name of the property to modify (phandle or linux,phandle) + * @delta: offset to apply + * + * overlay_phandle_add_offset() increments a node phandle by a given + * offset. + * + * returns: + * 0 on success. + * Negative error code on error + */ +static int overlay_phandle_add_offset(void *fdt, int node, + const char *name, uint32_t delta) +{ + const fdt32_t *val; + uint32_t adj_val; + int len; + + val = fdt_getprop(fdt, node, name, &len); + if (!val) + return len; + + if (len != sizeof(*val)) + return -FDT_ERR_BADPHANDLE; + + adj_val = fdt32_to_cpu(*val); + if ((adj_val + delta) < adj_val) + return -FDT_ERR_NOPHANDLES; + + adj_val += delta; + if (adj_val == (uint32_t)-1) + return -FDT_ERR_NOPHANDLES; + + return fdt_setprop_inplace_u32(fdt, node, name, adj_val); +} + +/** + * overlay_adjust_node_phandles - Offsets the phandles of a node + * @fdto: Device tree overlay blob + * @node: Offset of the node we want to adjust + * @delta: Offset to shift the phandles of + * + * overlay_adjust_node_phandles() adds a constant to all the phandles + * of a given node. This is mainly use as part of the overlay + * application process, when we want to update all the overlay + * phandles to not conflict with the overlays of the base device tree. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_adjust_node_phandles(void *fdto, int node, + uint32_t delta) +{ + int child; + int ret; + + ret = overlay_phandle_add_offset(fdto, node, "phandle", delta); + if (ret && ret != -FDT_ERR_NOTFOUND) + return ret; + + ret = overlay_phandle_add_offset(fdto, node, "linux,phandle", delta); + if (ret && ret != -FDT_ERR_NOTFOUND) + return ret; + + fdt_for_each_subnode(child, fdto, node) { + ret = overlay_adjust_node_phandles(fdto, child, delta); + if (ret) + return ret; + } + + return 0; +} + +/** + * overlay_adjust_local_phandles - Adjust the phandles of a whole overlay + * @fdto: Device tree overlay blob + * @delta: Offset to shift the phandles of + * + * overlay_adjust_local_phandles() adds a constant to all the + * phandles of an overlay. This is mainly use as part of the overlay + * application process, when we want to update all the overlay + * phandles to not conflict with the overlays of the base device tree. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_adjust_local_phandles(void *fdto, uint32_t delta) +{ + /* + * Start adjusting the phandles from the overlay root + */ + return overlay_adjust_node_phandles(fdto, 0, delta); +} + +/** + * overlay_update_local_node_references - Adjust the overlay references + * @fdto: Device tree overlay blob + * @tree_node: Node offset of the node to operate on + * @fixup_node: Node offset of the matching local fixups node + * @delta: Offset to shift the phandles of + * + * overlay_update_local_nodes_references() update the phandles + * pointing to a node within the device tree overlay by adding a + * constant delta. + * + * This is mainly used as part of a device tree application process, + * where you want the device tree overlays phandles to not conflict + * with the ones from the base device tree before merging them. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_update_local_node_references(void *fdto, + int tree_node, + int fixup_node, + uint32_t delta) +{ + int fixup_prop; + int fixup_child; + int ret; + + fdt_for_each_property_offset(fixup_prop, fdto, fixup_node) { + const fdt32_t *fixup_val; + const char *tree_val; + const char *name; + int fixup_len; + int tree_len; + int i; + + fixup_val = fdt_getprop_by_offset(fdto, fixup_prop, + &name, &fixup_len); + if (!fixup_val) + return fixup_len; + + if (fixup_len % sizeof(uint32_t)) + return -FDT_ERR_BADOVERLAY; + fixup_len /= sizeof(uint32_t); + + tree_val = fdt_getprop(fdto, tree_node, name, &tree_len); + if (!tree_val) { + if (tree_len == -FDT_ERR_NOTFOUND) + return -FDT_ERR_BADOVERLAY; + + return tree_len; + } + + for (i = 0; i < fixup_len; i++) { + fdt32_t adj_val; + uint32_t poffset; + + poffset = fdt32_to_cpu(fixup_val[i]); + + /* + * phandles to fixup can be unaligned. + * + * Use a memcpy for the architectures that do + * not support unaligned accesses. + */ + memcpy(&adj_val, tree_val + poffset, sizeof(adj_val)); + + adj_val = cpu_to_fdt32(fdt32_to_cpu(adj_val) + delta); + + ret = fdt_setprop_inplace_namelen_partial(fdto, + tree_node, + name, + strlen(name), + poffset, + &adj_val, + sizeof(adj_val)); + if (ret == -FDT_ERR_NOSPACE) + return -FDT_ERR_BADOVERLAY; + + if (ret) + return ret; + } + } + + fdt_for_each_subnode(fixup_child, fdto, fixup_node) { + const char *fixup_child_name = fdt_get_name(fdto, fixup_child, + NULL); + int tree_child; + + tree_child = fdt_subnode_offset(fdto, tree_node, + fixup_child_name); + if (tree_child == -FDT_ERR_NOTFOUND) + return -FDT_ERR_BADOVERLAY; + if (tree_child < 0) + return tree_child; + + ret = overlay_update_local_node_references(fdto, + tree_child, + fixup_child, + delta); + if (ret) + return ret; + } + + return 0; +} + +/** + * overlay_update_local_references - Adjust the overlay references + * @fdto: Device tree overlay blob + * @delta: Offset to shift the phandles of + * + * overlay_update_local_references() update all the phandles pointing + * to a node within the device tree overlay by adding a constant + * delta to not conflict with the base overlay. + * + * This is mainly used as part of a device tree application process, + * where you want the device tree overlays phandles to not conflict + * with the ones from the base device tree before merging them. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_update_local_references(void *fdto, uint32_t delta) +{ + int fixups; + + fixups = fdt_path_offset(fdto, "/__local_fixups__"); + if (fixups < 0) { + /* There's no local phandles to adjust, bail out */ + if (fixups == -FDT_ERR_NOTFOUND) + return 0; + + return fixups; + } + + /* + * Update our local references from the root of the tree + */ + return overlay_update_local_node_references(fdto, 0, fixups, + delta); +} + +/** + * overlay_fixup_one_phandle - Set an overlay phandle to the base one + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * @symbols_off: Node offset of the symbols node in the base device tree + * @path: Path to a node holding a phandle in the overlay + * @path_len: number of path characters to consider + * @name: Name of the property holding the phandle reference in the overlay + * @name_len: number of name characters to consider + * @poffset: Offset within the overlay property where the phandle is stored + * @label: Label of the node referenced by the phandle + * + * overlay_fixup_one_phandle() resolves an overlay phandle pointing to + * a node in the base device tree. + * + * This is part of the device tree overlay application process, when + * you want all the phandles in the overlay to point to the actual + * base dt nodes. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_fixup_one_phandle(void *fdt, void *fdto, + int symbols_off, + const char *path, uint32_t path_len, + const char *name, uint32_t name_len, + int poffset, const char *label) +{ + const char *symbol_path; + uint32_t phandle; + fdt32_t phandle_prop; + int symbol_off, fixup_off; + int prop_len; + + if (symbols_off < 0) + return symbols_off; + + symbol_path = fdt_getprop(fdt, symbols_off, label, + &prop_len); + if (!symbol_path) + return prop_len; + + symbol_off = fdt_path_offset(fdt, symbol_path); + if (symbol_off < 0) + return symbol_off; + + phandle = fdt_get_phandle(fdt, symbol_off); + if (!phandle) + return -FDT_ERR_NOTFOUND; + + fixup_off = fdt_path_offset_namelen(fdto, path, path_len); + if (fixup_off == -FDT_ERR_NOTFOUND) + return -FDT_ERR_BADOVERLAY; + if (fixup_off < 0) + return fixup_off; + + phandle_prop = cpu_to_fdt32(phandle); + return fdt_setprop_inplace_namelen_partial(fdto, fixup_off, + name, name_len, poffset, + &phandle_prop, + sizeof(phandle_prop)); +}; + +/** + * overlay_fixup_phandle - Set an overlay phandle to the base one + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * @symbols_off: Node offset of the symbols node in the base device tree + * @property: Property offset in the overlay holding the list of fixups + * + * overlay_fixup_phandle() resolves all the overlay phandles pointed + * to in a __fixups__ property, and updates them to match the phandles + * in use in the base device tree. + * + * This is part of the device tree overlay application process, when + * you want all the phandles in the overlay to point to the actual + * base dt nodes. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_fixup_phandle(void *fdt, void *fdto, int symbols_off, + int property) +{ + const char *value; + const char *label; + int len; + + value = fdt_getprop_by_offset(fdto, property, + &label, &len); + if (!value) { + if (len == -FDT_ERR_NOTFOUND) + return -FDT_ERR_INTERNAL; + + return len; + } + + do { + const char *path, *name, *fixup_end; + const char *fixup_str = value; + uint32_t path_len, name_len; + uint32_t fixup_len; + char *sep, *endptr; + int poffset, ret; + + fixup_end = memchr(value, '\0', len); + if (!fixup_end) + return -FDT_ERR_BADOVERLAY; + fixup_len = fixup_end - fixup_str; + + len -= fixup_len + 1; + value += fixup_len + 1; + + path = fixup_str; + sep = memchr(fixup_str, ':', fixup_len); + if (!sep || *sep != ':') + return -FDT_ERR_BADOVERLAY; + + path_len = sep - path; + if (path_len == (fixup_len - 1)) + return -FDT_ERR_BADOVERLAY; + + fixup_len -= path_len + 1; + name = sep + 1; + sep = memchr(name, ':', fixup_len); + if (!sep || *sep != ':') + return -FDT_ERR_BADOVERLAY; + + name_len = sep - name; + if (!name_len) + return -FDT_ERR_BADOVERLAY; + + poffset = strtoul(sep + 1, &endptr, 10); + if ((*endptr != '\0') || (endptr <= (sep + 1))) + return -FDT_ERR_BADOVERLAY; + + ret = overlay_fixup_one_phandle(fdt, fdto, symbols_off, + path, path_len, name, name_len, + poffset, label); + if (ret) + return ret; + } while (len > 0); + + return 0; +} + +/** + * overlay_fixup_phandles - Resolve the overlay phandles to the base + * device tree + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * + * overlay_fixup_phandles() resolves all the overlay phandles pointing + * to nodes in the base device tree. + * + * This is one of the steps of the device tree overlay application + * process, when you want all the phandles in the overlay to point to + * the actual base dt nodes. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_fixup_phandles(void *fdt, void *fdto) +{ + int fixups_off, symbols_off; + int property; + + /* We can have overlays without any fixups */ + fixups_off = fdt_path_offset(fdto, "/__fixups__"); + if (fixups_off == -FDT_ERR_NOTFOUND) + return 0; /* nothing to do */ + if (fixups_off < 0) + return fixups_off; + + /* And base DTs without symbols */ + symbols_off = fdt_path_offset(fdt, "/__symbols__"); + if ((symbols_off < 0 && (symbols_off != -FDT_ERR_NOTFOUND))) + return symbols_off; + + fdt_for_each_property_offset(property, fdto, fixups_off) { + int ret; + + ret = overlay_fixup_phandle(fdt, fdto, symbols_off, property); + if (ret) + return ret; + } + + return 0; +} + +/** + * overlay_apply_node - Merges a node into the base device tree + * @fdt: Base Device Tree blob + * @target: Node offset in the base device tree to apply the fragment to + * @fdto: Device tree overlay blob + * @node: Node offset in the overlay holding the changes to merge + * + * overlay_apply_node() merges a node into a target base device tree + * node pointed. + * + * This is part of the final step in the device tree overlay + * application process, when all the phandles have been adjusted and + * resolved and you just have to merge overlay into the base device + * tree. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_apply_node(void *fdt, int target, + void *fdto, int node) +{ + int property; + int subnode; + + fdt_for_each_property_offset(property, fdto, node) { + const char *name; + const void *prop; + int prop_len; + int ret; + + prop = fdt_getprop_by_offset(fdto, property, &name, + &prop_len); + if (prop_len == -FDT_ERR_NOTFOUND) + return -FDT_ERR_INTERNAL; + if (prop_len < 0) + return prop_len; + + ret = fdt_setprop(fdt, target, name, prop, prop_len); + if (ret) + return ret; + } + + fdt_for_each_subnode(subnode, fdto, node) { + const char *name = fdt_get_name(fdto, subnode, NULL); + int nnode; + int ret; + + nnode = fdt_add_subnode(fdt, target, name); + if (nnode == -FDT_ERR_EXISTS) { + nnode = fdt_subnode_offset(fdt, target, name); + if (nnode == -FDT_ERR_NOTFOUND) + return -FDT_ERR_INTERNAL; + } + + if (nnode < 0) + return nnode; + + ret = overlay_apply_node(fdt, nnode, fdto, subnode); + if (ret) + return ret; + } + + return 0; +} + +/** + * overlay_merge - Merge an overlay into its base device tree + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * + * overlay_merge() merges an overlay into its base device tree. + * + * This is the next to last step in the device tree overlay application + * process, when all the phandles have been adjusted and resolved and + * you just have to merge overlay into the base device tree. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_merge(void *fdt, void *fdto) +{ + int fragment; + + fdt_for_each_subnode(fragment, fdto, 0) { + int overlay; + int target; + int ret; + + /* + * Each fragments will have an __overlay__ node. If + * they don't, it's not supposed to be merged + */ + overlay = fdt_subnode_offset(fdto, fragment, "__overlay__"); + if (overlay == -FDT_ERR_NOTFOUND) + continue; + + if (overlay < 0) + return overlay; + + target = fdt_overlay_target_offset(fdt, fdto, fragment, NULL); + if (target < 0) + return target; + + ret = overlay_apply_node(fdt, target, fdto, overlay); + if (ret) + return ret; + } + + return 0; +} + +static int get_path_len(const void *fdt, int nodeoffset) +{ + int len = 0, namelen; + const char *name; + + FDT_RO_PROBE(fdt); + + for (;;) { + name = fdt_get_name(fdt, nodeoffset, &namelen); + if (!name) + return namelen; + + /* root? we're done */ + if (namelen == 0) + break; + + nodeoffset = fdt_parent_offset(fdt, nodeoffset); + if (nodeoffset < 0) + return nodeoffset; + len += namelen + 1; + } + + /* in case of root pretend it's "/" */ + if (len == 0) + len++; + return len; +} + +/** + * overlay_symbol_update - Update the symbols of base tree after a merge + * @fdt: Base Device Tree blob + * @fdto: Device tree overlay blob + * + * overlay_symbol_update() updates the symbols of the base tree with the + * symbols of the applied overlay + * + * This is the last step in the device tree overlay application + * process, allowing the reference of overlay symbols by subsequent + * overlay operations. + * + * returns: + * 0 on success + * Negative error code on failure + */ +static int overlay_symbol_update(void *fdt, void *fdto) +{ + int root_sym, ov_sym, prop, path_len, fragment, target; + int len, frag_name_len, ret, rel_path_len; + const char *s, *e; + const char *path; + const char *name; + const char *frag_name; + const char *rel_path; + const char *target_path; + char *buf; + void *p; + + ov_sym = fdt_subnode_offset(fdto, 0, "__symbols__"); + + /* if no overlay symbols exist no problem */ + if (ov_sym < 0) + return 0; + + root_sym = fdt_subnode_offset(fdt, 0, "__symbols__"); + + /* it no root symbols exist we should create them */ + if (root_sym == -FDT_ERR_NOTFOUND) + root_sym = fdt_add_subnode(fdt, 0, "__symbols__"); + + /* any error is fatal now */ + if (root_sym < 0) + return root_sym; + + /* iterate over each overlay symbol */ + fdt_for_each_property_offset(prop, fdto, ov_sym) { + path = fdt_getprop_by_offset(fdto, prop, &name, &path_len); + if (!path) + return path_len; + + /* verify it's a string property (terminated by a single \0) */ + if (path_len < 1 || memchr(path, '\0', path_len) != &path[path_len - 1]) + return -FDT_ERR_BADVALUE; + + /* keep end marker to avoid strlen() */ + e = path + path_len; + + if (*path != '/') + return -FDT_ERR_BADVALUE; + + /* get fragment name first */ + s = strchr(path + 1, '/'); + if (!s) { + /* Symbol refers to something that won't end + * up in the target tree */ + continue; + } + + frag_name = path + 1; + frag_name_len = s - path - 1; + + /* verify format; safe since "s" lies in \0 terminated prop */ + len = sizeof("/__overlay__/") - 1; + if ((e - s) > len && (memcmp(s, "/__overlay__/", len) == 0)) { + /* /<fragment-name>/__overlay__/<relative-subnode-path> */ + rel_path = s + len; + rel_path_len = e - rel_path - 1; + } else if ((e - s) == len + && (memcmp(s, "/__overlay__", len - 1) == 0)) { + /* /<fragment-name>/__overlay__ */ + rel_path = ""; + rel_path_len = 0; + } else { + /* Symbol refers to something that won't end + * up in the target tree */ + continue; + } + + /* find the fragment index in which the symbol lies */ + ret = fdt_subnode_offset_namelen(fdto, 0, frag_name, + frag_name_len); + /* not found? */ + if (ret < 0) + return -FDT_ERR_BADOVERLAY; + fragment = ret; + + /* an __overlay__ subnode must exist */ + ret = fdt_subnode_offset(fdto, fragment, "__overlay__"); + if (ret < 0) + return -FDT_ERR_BADOVERLAY; + + /* get the target of the fragment */ + ret = fdt_overlay_target_offset(fdt, fdto, fragment, &target_path); + if (ret < 0) + return ret; + target = ret; + + /* if we have a target path use */ + if (!target_path) { + ret = get_path_len(fdt, target); + if (ret < 0) + return ret; + len = ret; + } else { + len = strlen(target_path); + } + + ret = fdt_setprop_placeholder(fdt, root_sym, name, + len + (len > 1) + rel_path_len + 1, &p); + if (ret < 0) + return ret; + + if (!target_path) { + /* again in case setprop_placeholder changed it */ + ret = fdt_overlay_target_offset(fdt, fdto, fragment, &target_path); + if (ret < 0) + return ret; + target = ret; + } + + buf = p; + if (len > 1) { /* target is not root */ + if (!target_path) { + ret = fdt_get_path(fdt, target, buf, len + 1); + if (ret < 0) + return ret; + } else + memcpy(buf, target_path, len + 1); + + } else + len--; + + buf[len] = '/'; + memcpy(buf + len + 1, rel_path, rel_path_len); + buf[len + 1 + rel_path_len] = '\0'; + } + + return 0; +} + +int fdt_overlay_apply(void *fdt, void *fdto) +{ + uint32_t delta; + int ret; + + FDT_RO_PROBE(fdt); + FDT_RO_PROBE(fdto); + + ret = fdt_find_max_phandle(fdt, &delta); + if (ret) + goto err; + + ret = overlay_adjust_local_phandles(fdto, delta); + if (ret) + goto err; + + ret = overlay_update_local_references(fdto, delta); + if (ret) + goto err; + + ret = overlay_fixup_phandles(fdt, fdto); + if (ret) + goto err; + + ret = overlay_merge(fdt, fdto); + if (ret) + goto err; + + ret = overlay_symbol_update(fdt, fdto); + if (ret) + goto err; + + /* + * The overlay has been damaged, erase its magic. + */ + fdt_set_magic(fdto, ~0); + + return 0; + +err: + /* + * The overlay might have been damaged, erase its magic. + */ + fdt_set_magic(fdto, ~0); + + /* + * The base device tree might have been damaged, erase its + * magic. + */ + fdt_set_magic(fdt, ~0); + + return ret; +} diff --git a/lib/libfdt/fdt_ro.c b/lib/libfdt/fdt_ro.c new file mode 100644 index 0000000..9f6c551 --- /dev/null +++ b/lib/libfdt/fdt_ro.c @@ -0,0 +1,859 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +static int fdt_nodename_eq_(const void *fdt, int offset, + const char *s, int len) +{ + int olen; + const char *p = fdt_get_name(fdt, offset, &olen); + + if (!p || olen < len) + /* short match */ + return 0; + + if (memcmp(p, s, len) != 0) + return 0; + + if (p[len] == '\0') + return 1; + else if (!memchr(s, '@', len) && (p[len] == '@')) + return 1; + else + return 0; +} + +const char *fdt_get_string(const void *fdt, int stroffset, int *lenp) +{ + int32_t totalsize; + uint32_t absoffset; + size_t len; + int err; + const char *s, *n; + + if (can_assume(VALID_INPUT)) { + s = (const char *)fdt + fdt_off_dt_strings(fdt) + stroffset; + + if (lenp) + *lenp = strlen(s); + return s; + } + totalsize = fdt_ro_probe_(fdt); + err = totalsize; + if (totalsize < 0) + goto fail; + + err = -FDT_ERR_BADOFFSET; + absoffset = stroffset + fdt_off_dt_strings(fdt); + if (absoffset >= (unsigned)totalsize) + goto fail; + len = totalsize - absoffset; + + if (fdt_magic(fdt) == FDT_MAGIC) { + if (stroffset < 0) + goto fail; + if (can_assume(LATEST) || fdt_version(fdt) >= 17) { + if ((unsigned)stroffset >= fdt_size_dt_strings(fdt)) + goto fail; + if ((fdt_size_dt_strings(fdt) - stroffset) < len) + len = fdt_size_dt_strings(fdt) - stroffset; + } + } else if (fdt_magic(fdt) == FDT_SW_MAGIC) { + unsigned int sw_stroffset = -stroffset; + + if ((stroffset >= 0) || + (sw_stroffset > fdt_size_dt_strings(fdt))) + goto fail; + if (sw_stroffset < len) + len = sw_stroffset; + } else { + err = -FDT_ERR_INTERNAL; + goto fail; + } + + s = (const char *)fdt + absoffset; + n = memchr(s, '\0', len); + if (!n) { + /* missing terminating NULL */ + err = -FDT_ERR_TRUNCATED; + goto fail; + } + + if (lenp) + *lenp = n - s; + return s; + +fail: + if (lenp) + *lenp = err; + return NULL; +} + +const char *fdt_string(const void *fdt, int stroffset) +{ + return fdt_get_string(fdt, stroffset, NULL); +} + +static int fdt_string_eq_(const void *fdt, int stroffset, + const char *s, int len) +{ + int slen; + const char *p = fdt_get_string(fdt, stroffset, &slen); + + return p && (slen == len) && (memcmp(p, s, len) == 0); +} + +int fdt_find_max_phandle(const void *fdt, uint32_t *phandle) +{ + uint32_t max = 0; + int offset = -1; + + while (true) { + uint32_t value; + + offset = fdt_next_node(fdt, offset, NULL); + if (offset < 0) { + if (offset == -FDT_ERR_NOTFOUND) + break; + + return offset; + } + + value = fdt_get_phandle(fdt, offset); + + if (value > max) + max = value; + } + + if (phandle) + *phandle = max; + + return 0; +} + +int fdt_generate_phandle(const void *fdt, uint32_t *phandle) +{ + uint32_t max; + int err; + + err = fdt_find_max_phandle(fdt, &max); + if (err < 0) + return err; + + if (max == FDT_MAX_PHANDLE) + return -FDT_ERR_NOPHANDLES; + + if (phandle) + *phandle = max + 1; + + return 0; +} + +static const struct fdt_reserve_entry *fdt_mem_rsv(const void *fdt, int n) +{ + unsigned int offset = n * sizeof(struct fdt_reserve_entry); + unsigned int absoffset = fdt_off_mem_rsvmap(fdt) + offset; + + if (!can_assume(VALID_INPUT)) { + if (absoffset < fdt_off_mem_rsvmap(fdt)) + return NULL; + if (absoffset > fdt_totalsize(fdt) - + sizeof(struct fdt_reserve_entry)) + return NULL; + } + return fdt_mem_rsv_(fdt, n); +} + +int fdt_get_mem_rsv(const void *fdt, int n, uint64_t *address, uint64_t *size) +{ + const struct fdt_reserve_entry *re; + + FDT_RO_PROBE(fdt); + re = fdt_mem_rsv(fdt, n); + if (!can_assume(VALID_INPUT) && !re) + return -FDT_ERR_BADOFFSET; + + *address = fdt64_ld_(&re->address); + *size = fdt64_ld_(&re->size); + return 0; +} + +int fdt_num_mem_rsv(const void *fdt) +{ + int i; + const struct fdt_reserve_entry *re; + + for (i = 0; (re = fdt_mem_rsv(fdt, i)) != NULL; i++) { + if (fdt64_ld_(&re->size) == 0) + return i; + } + return -FDT_ERR_TRUNCATED; +} + +static int nextprop_(const void *fdt, int offset) +{ + uint32_t tag; + int nextoffset; + + do { + tag = fdt_next_tag(fdt, offset, &nextoffset); + + switch (tag) { + case FDT_END: + if (nextoffset >= 0) + return -FDT_ERR_BADSTRUCTURE; + else + return nextoffset; + + case FDT_PROP: + return offset; + } + offset = nextoffset; + } while (tag == FDT_NOP); + + return -FDT_ERR_NOTFOUND; +} + +int fdt_subnode_offset_namelen(const void *fdt, int offset, + const char *name, int namelen) +{ + int depth; + + FDT_RO_PROBE(fdt); + + for (depth = 0; + (offset >= 0) && (depth >= 0); + offset = fdt_next_node(fdt, offset, &depth)) + if ((depth == 1) + && fdt_nodename_eq_(fdt, offset, name, namelen)) + return offset; + + if (depth < 0) + return -FDT_ERR_NOTFOUND; + return offset; /* error */ +} + +int fdt_subnode_offset(const void *fdt, int parentoffset, + const char *name) +{ + return fdt_subnode_offset_namelen(fdt, parentoffset, name, strlen(name)); +} + +int fdt_path_offset_namelen(const void *fdt, const char *path, int namelen) +{ + const char *end = path + namelen; + const char *p = path; + int offset = 0; + + FDT_RO_PROBE(fdt); + + /* see if we have an alias */ + if (*path != '/') { + const char *q = memchr(path, '/', end - p); + + if (!q) + q = end; + + p = fdt_get_alias_namelen(fdt, p, q - p); + if (!p) + return -FDT_ERR_BADPATH; + offset = fdt_path_offset(fdt, p); + + p = q; + } + + while (p < end) { + const char *q; + + while (*p == '/') { + p++; + if (p == end) + return offset; + } + q = memchr(p, '/', end - p); + if (! q) + q = end; + + offset = fdt_subnode_offset_namelen(fdt, offset, p, q-p); + if (offset < 0) + return offset; + + p = q; + } + + return offset; +} + +int fdt_path_offset(const void *fdt, const char *path) +{ + return fdt_path_offset_namelen(fdt, path, strlen(path)); +} + +const char *fdt_get_name(const void *fdt, int nodeoffset, int *len) +{ + const struct fdt_node_header *nh = fdt_offset_ptr_(fdt, nodeoffset); + const char *nameptr; + int err; + + if (((err = fdt_ro_probe_(fdt)) < 0) + || ((err = fdt_check_node_offset_(fdt, nodeoffset)) < 0)) + goto fail; + + nameptr = nh->name; + + if (!can_assume(LATEST) && fdt_version(fdt) < 0x10) { + /* + * For old FDT versions, match the naming conventions of V16: + * give only the leaf name (after all /). The actual tree + * contents are loosely checked. + */ + const char *leaf; + leaf = strrchr(nameptr, '/'); + if (leaf == NULL) { + err = -FDT_ERR_BADSTRUCTURE; + goto fail; + } + nameptr = leaf+1; + } + + if (len) + *len = strlen(nameptr); + + return nameptr; + + fail: + if (len) + *len = err; + return NULL; +} + +int fdt_first_property_offset(const void *fdt, int nodeoffset) +{ + int offset; + + if ((offset = fdt_check_node_offset_(fdt, nodeoffset)) < 0) + return offset; + + return nextprop_(fdt, offset); +} + +int fdt_next_property_offset(const void *fdt, int offset) +{ + if ((offset = fdt_check_prop_offset_(fdt, offset)) < 0) + return offset; + + return nextprop_(fdt, offset); +} + +static const struct fdt_property *fdt_get_property_by_offset_(const void *fdt, + int offset, + int *lenp) +{ + int err; + const struct fdt_property *prop; + + if (!can_assume(VALID_INPUT) && + (err = fdt_check_prop_offset_(fdt, offset)) < 0) { + if (lenp) + *lenp = err; + return NULL; + } + + prop = fdt_offset_ptr_(fdt, offset); + + if (lenp) + *lenp = fdt32_ld_(&prop->len); + + return prop; +} + +const struct fdt_property *fdt_get_property_by_offset(const void *fdt, + int offset, + int *lenp) +{ + /* Prior to version 16, properties may need realignment + * and this API does not work. fdt_getprop_*() will, however. */ + + if (!can_assume(LATEST) && fdt_version(fdt) < 0x10) { + if (lenp) + *lenp = -FDT_ERR_BADVERSION; + return NULL; + } + + return fdt_get_property_by_offset_(fdt, offset, lenp); +} + +static const struct fdt_property *fdt_get_property_namelen_(const void *fdt, + int offset, + const char *name, + int namelen, + int *lenp, + int *poffset) +{ + for (offset = fdt_first_property_offset(fdt, offset); + (offset >= 0); + (offset = fdt_next_property_offset(fdt, offset))) { + const struct fdt_property *prop; + + prop = fdt_get_property_by_offset_(fdt, offset, lenp); + if (!can_assume(LIBFDT_FLAWLESS) && !prop) { + offset = -FDT_ERR_INTERNAL; + break; + } + if (fdt_string_eq_(fdt, fdt32_ld_(&prop->nameoff), + name, namelen)) { + if (poffset) + *poffset = offset; + return prop; + } + } + + if (lenp) + *lenp = offset; + return NULL; +} + + +const struct fdt_property *fdt_get_property_namelen(const void *fdt, + int offset, + const char *name, + int namelen, int *lenp) +{ + /* Prior to version 16, properties may need realignment + * and this API does not work. fdt_getprop_*() will, however. */ + if (!can_assume(LATEST) && fdt_version(fdt) < 0x10) { + if (lenp) + *lenp = -FDT_ERR_BADVERSION; + return NULL; + } + + return fdt_get_property_namelen_(fdt, offset, name, namelen, lenp, + NULL); +} + + +const struct fdt_property *fdt_get_property(const void *fdt, + int nodeoffset, + const char *name, int *lenp) +{ + return fdt_get_property_namelen(fdt, nodeoffset, name, + strlen(name), lenp); +} + +const void *fdt_getprop_namelen(const void *fdt, int nodeoffset, + const char *name, int namelen, int *lenp) +{ + int poffset; + const struct fdt_property *prop; + + prop = fdt_get_property_namelen_(fdt, nodeoffset, name, namelen, lenp, + &poffset); + if (!prop) + return NULL; + + /* Handle realignment */ + if (!can_assume(LATEST) && fdt_version(fdt) < 0x10 && + (poffset + sizeof(*prop)) % 8 && fdt32_ld_(&prop->len) >= 8) + return prop->data + 4; + return prop->data; +} + +const void *fdt_getprop_by_offset(const void *fdt, int offset, + const char **namep, int *lenp) +{ + const struct fdt_property *prop; + + prop = fdt_get_property_by_offset_(fdt, offset, lenp); + if (!prop) + return NULL; + if (namep) { + const char *name; + int namelen; + + if (!can_assume(VALID_INPUT)) { + name = fdt_get_string(fdt, fdt32_ld_(&prop->nameoff), + &namelen); + *namep = name; + if (!name) { + if (lenp) + *lenp = namelen; + return NULL; + } + } else { + *namep = fdt_string(fdt, fdt32_ld_(&prop->nameoff)); + } + } + + /* Handle realignment */ + if (!can_assume(LATEST) && fdt_version(fdt) < 0x10 && + (offset + sizeof(*prop)) % 8 && fdt32_ld_(&prop->len) >= 8) + return prop->data + 4; + return prop->data; +} + +const void *fdt_getprop(const void *fdt, int nodeoffset, + const char *name, int *lenp) +{ + return fdt_getprop_namelen(fdt, nodeoffset, name, strlen(name), lenp); +} + +uint32_t fdt_get_phandle(const void *fdt, int nodeoffset) +{ + const fdt32_t *php; + int len; + + /* FIXME: This is a bit sub-optimal, since we potentially scan + * over all the properties twice. */ + php = fdt_getprop(fdt, nodeoffset, "phandle", &len); + if (!php || (len != sizeof(*php))) { + php = fdt_getprop(fdt, nodeoffset, "linux,phandle", &len); + if (!php || (len != sizeof(*php))) + return 0; + } + + return fdt32_ld_(php); +} + +const char *fdt_get_alias_namelen(const void *fdt, + const char *name, int namelen) +{ + int aliasoffset; + + aliasoffset = fdt_path_offset(fdt, "/aliases"); + if (aliasoffset < 0) + return NULL; + + return fdt_getprop_namelen(fdt, aliasoffset, name, namelen, NULL); +} + +const char *fdt_get_alias(const void *fdt, const char *name) +{ + return fdt_get_alias_namelen(fdt, name, strlen(name)); +} + +int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen) +{ + int pdepth = 0, p = 0; + int offset, depth, namelen; + const char *name; + + FDT_RO_PROBE(fdt); + + if (buflen < 2) + return -FDT_ERR_NOSPACE; + + for (offset = 0, depth = 0; + (offset >= 0) && (offset <= nodeoffset); + offset = fdt_next_node(fdt, offset, &depth)) { + while (pdepth > depth) { + do { + p--; + } while (buf[p-1] != '/'); + pdepth--; + } + + if (pdepth >= depth) { + name = fdt_get_name(fdt, offset, &namelen); + if (!name) + return namelen; + if ((p + namelen + 1) <= buflen) { + memcpy(buf + p, name, namelen); + p += namelen; + buf[p++] = '/'; + pdepth++; + } + } + + if (offset == nodeoffset) { + if (pdepth < (depth + 1)) + return -FDT_ERR_NOSPACE; + + if (p > 1) /* special case so that root path is "/", not "" */ + p--; + buf[p] = '\0'; + return 0; + } + } + + if ((offset == -FDT_ERR_NOTFOUND) || (offset >= 0)) + return -FDT_ERR_BADOFFSET; + else if (offset == -FDT_ERR_BADOFFSET) + return -FDT_ERR_BADSTRUCTURE; + + return offset; /* error from fdt_next_node() */ +} + +int fdt_supernode_atdepth_offset(const void *fdt, int nodeoffset, + int supernodedepth, int *nodedepth) +{ + int offset, depth; + int supernodeoffset = -FDT_ERR_INTERNAL; + + FDT_RO_PROBE(fdt); + + if (supernodedepth < 0) + return -FDT_ERR_NOTFOUND; + + for (offset = 0, depth = 0; + (offset >= 0) && (offset <= nodeoffset); + offset = fdt_next_node(fdt, offset, &depth)) { + if (depth == supernodedepth) + supernodeoffset = offset; + + if (offset == nodeoffset) { + if (nodedepth) + *nodedepth = depth; + + if (supernodedepth > depth) + return -FDT_ERR_NOTFOUND; + else + return supernodeoffset; + } + } + + if (!can_assume(VALID_INPUT)) { + if ((offset == -FDT_ERR_NOTFOUND) || (offset >= 0)) + return -FDT_ERR_BADOFFSET; + else if (offset == -FDT_ERR_BADOFFSET) + return -FDT_ERR_BADSTRUCTURE; + } + + return offset; /* error from fdt_next_node() */ +} + +int fdt_node_depth(const void *fdt, int nodeoffset) +{ + int nodedepth; + int err; + + err = fdt_supernode_atdepth_offset(fdt, nodeoffset, 0, &nodedepth); + if (err) + return (can_assume(LIBFDT_FLAWLESS) || err < 0) ? err : + -FDT_ERR_INTERNAL; + return nodedepth; +} + +int fdt_parent_offset(const void *fdt, int nodeoffset) +{ + int nodedepth = fdt_node_depth(fdt, nodeoffset); + + if (nodedepth < 0) + return nodedepth; + return fdt_supernode_atdepth_offset(fdt, nodeoffset, + nodedepth - 1, NULL); +} + +int fdt_node_offset_by_prop_value(const void *fdt, int startoffset, + const char *propname, + const void *propval, int proplen) +{ + int offset; + const void *val; + int len; + + FDT_RO_PROBE(fdt); + + /* FIXME: The algorithm here is pretty horrible: we scan each + * property of a node in fdt_getprop(), then if that didn't + * find what we want, we scan over them again making our way + * to the next node. Still it's the easiest to implement + * approach; performance can come later. */ + for (offset = fdt_next_node(fdt, startoffset, NULL); + offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + val = fdt_getprop(fdt, offset, propname, &len); + if (val && (len == proplen) + && (memcmp(val, propval, len) == 0)) + return offset; + } + + return offset; /* error from fdt_next_node() */ +} + +int fdt_node_offset_by_phandle(const void *fdt, uint32_t phandle) +{ + int offset; + + if ((phandle == 0) || (phandle == ~0U)) + return -FDT_ERR_BADPHANDLE; + + FDT_RO_PROBE(fdt); + + /* FIXME: The algorithm here is pretty horrible: we + * potentially scan each property of a node in + * fdt_get_phandle(), then if that didn't find what + * we want, we scan over them again making our way to the next + * node. Still it's the easiest to implement approach; + * performance can come later. */ + for (offset = fdt_next_node(fdt, -1, NULL); + offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + if (fdt_get_phandle(fdt, offset) == phandle) + return offset; + } + + return offset; /* error from fdt_next_node() */ +} + +int fdt_stringlist_contains(const char *strlist, int listlen, const char *str) +{ + int len = strlen(str); + const char *p; + + while (listlen >= len) { + if (memcmp(str, strlist, len+1) == 0) + return 1; + p = memchr(strlist, '\0', listlen); + if (!p) + return 0; /* malformed strlist.. */ + listlen -= (p-strlist) + 1; + strlist = p + 1; + } + return 0; +} + +int fdt_stringlist_count(const void *fdt, int nodeoffset, const char *property) +{ + const char *list, *end; + int length, count = 0; + + list = fdt_getprop(fdt, nodeoffset, property, &length); + if (!list) + return length; + + end = list + length; + + while (list < end) { + length = strnlen(list, end - list) + 1; + + /* Abort if the last string isn't properly NUL-terminated. */ + if (list + length > end) + return -FDT_ERR_BADVALUE; + + list += length; + count++; + } + + return count; +} + +int fdt_stringlist_search(const void *fdt, int nodeoffset, const char *property, + const char *string) +{ + int length, len, idx = 0; + const char *list, *end; + + list = fdt_getprop(fdt, nodeoffset, property, &length); + if (!list) + return length; + + len = strlen(string) + 1; + end = list + length; + + while (list < end) { + length = strnlen(list, end - list) + 1; + + /* Abort if the last string isn't properly NUL-terminated. */ + if (list + length > end) + return -FDT_ERR_BADVALUE; + + if (length == len && memcmp(list, string, length) == 0) + return idx; + + list += length; + idx++; + } + + return -FDT_ERR_NOTFOUND; +} + +const char *fdt_stringlist_get(const void *fdt, int nodeoffset, + const char *property, int idx, + int *lenp) +{ + const char *list, *end; + int length; + + list = fdt_getprop(fdt, nodeoffset, property, &length); + if (!list) { + if (lenp) + *lenp = length; + + return NULL; + } + + end = list + length; + + while (list < end) { + length = strnlen(list, end - list) + 1; + + /* Abort if the last string isn't properly NUL-terminated. */ + if (list + length > end) { + if (lenp) + *lenp = -FDT_ERR_BADVALUE; + + return NULL; + } + + if (idx == 0) { + if (lenp) + *lenp = length - 1; + + return list; + } + + list += length; + idx--; + } + + if (lenp) + *lenp = -FDT_ERR_NOTFOUND; + + return NULL; +} + +int fdt_node_check_compatible(const void *fdt, int nodeoffset, + const char *compatible) +{ + const void *prop; + int len; + + prop = fdt_getprop(fdt, nodeoffset, "compatible", &len); + if (!prop) + return len; + + return !fdt_stringlist_contains(prop, len, compatible); +} + +int fdt_node_offset_by_compatible(const void *fdt, int startoffset, + const char *compatible) +{ + int offset, err; + + FDT_RO_PROBE(fdt); + + /* FIXME: The algorithm here is pretty horrible: we scan each + * property of a node in fdt_node_check_compatible(), then if + * that didn't find what we want, we scan over them again + * making our way to the next node. Still it's the easiest to + * implement approach; performance can come later. */ + for (offset = fdt_next_node(fdt, startoffset, NULL); + offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + err = fdt_node_check_compatible(fdt, offset, compatible); + if ((err < 0) && (err != -FDT_ERR_NOTFOUND)) + return err; + else if (err == 0) + return offset; + } + + return offset; /* error from fdt_next_node() */ +} diff --git a/lib/libfdt/fdt_rw.c b/lib/libfdt/fdt_rw.c new file mode 100644 index 0000000..3621d36 --- /dev/null +++ b/lib/libfdt/fdt_rw.c @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +static int fdt_blocks_misordered_(const void *fdt, + int mem_rsv_size, int struct_size) +{ + return (fdt_off_mem_rsvmap(fdt) < FDT_ALIGN(sizeof(struct fdt_header), 8)) + || (fdt_off_dt_struct(fdt) < + (fdt_off_mem_rsvmap(fdt) + mem_rsv_size)) + || (fdt_off_dt_strings(fdt) < + (fdt_off_dt_struct(fdt) + struct_size)) + || (fdt_totalsize(fdt) < + (fdt_off_dt_strings(fdt) + fdt_size_dt_strings(fdt))); +} + +static int fdt_rw_probe_(void *fdt) +{ + if (can_assume(VALID_DTB)) + return 0; + FDT_RO_PROBE(fdt); + + if (!can_assume(LATEST) && fdt_version(fdt) < 17) + return -FDT_ERR_BADVERSION; + if (fdt_blocks_misordered_(fdt, sizeof(struct fdt_reserve_entry), + fdt_size_dt_struct(fdt))) + return -FDT_ERR_BADLAYOUT; + if (!can_assume(LATEST) && fdt_version(fdt) > 17) + fdt_set_version(fdt, 17); + + return 0; +} + +#define FDT_RW_PROBE(fdt) \ + { \ + int err_; \ + if ((err_ = fdt_rw_probe_(fdt)) != 0) \ + return err_; \ + } + +static inline unsigned int fdt_data_size_(void *fdt) +{ + return fdt_off_dt_strings(fdt) + fdt_size_dt_strings(fdt); +} + +static int fdt_splice_(void *fdt, void *splicepoint, int oldlen, int newlen) +{ + char *p = splicepoint; + unsigned int dsize = fdt_data_size_(fdt); + size_t soff = p - (char *)fdt; + + if ((oldlen < 0) || (soff + oldlen < soff) || (soff + oldlen > dsize)) + return -FDT_ERR_BADOFFSET; + if ((p < (char *)fdt) || (dsize + newlen < (unsigned)oldlen)) + return -FDT_ERR_BADOFFSET; + if (dsize - oldlen + newlen > fdt_totalsize(fdt)) + return -FDT_ERR_NOSPACE; + memmove(p + newlen, p + oldlen, ((char *)fdt + dsize) - (p + oldlen)); + return 0; +} + +static int fdt_splice_mem_rsv_(void *fdt, struct fdt_reserve_entry *p, + int oldn, int newn) +{ + int delta = (newn - oldn) * sizeof(*p); + int err; + err = fdt_splice_(fdt, p, oldn * sizeof(*p), newn * sizeof(*p)); + if (err) + return err; + fdt_set_off_dt_struct(fdt, fdt_off_dt_struct(fdt) + delta); + fdt_set_off_dt_strings(fdt, fdt_off_dt_strings(fdt) + delta); + return 0; +} + +static int fdt_splice_struct_(void *fdt, void *p, + int oldlen, int newlen) +{ + int delta = newlen - oldlen; + int err; + + if ((err = fdt_splice_(fdt, p, oldlen, newlen))) + return err; + + fdt_set_size_dt_struct(fdt, fdt_size_dt_struct(fdt) + delta); + fdt_set_off_dt_strings(fdt, fdt_off_dt_strings(fdt) + delta); + return 0; +} + +/* Must only be used to roll back in case of error */ +static void fdt_del_last_string_(void *fdt, const char *s) +{ + int newlen = strlen(s) + 1; + + fdt_set_size_dt_strings(fdt, fdt_size_dt_strings(fdt) - newlen); +} + +static int fdt_splice_string_(void *fdt, int newlen) +{ + void *p = (char *)fdt + + fdt_off_dt_strings(fdt) + fdt_size_dt_strings(fdt); + int err; + + if ((err = fdt_splice_(fdt, p, 0, newlen))) + return err; + + fdt_set_size_dt_strings(fdt, fdt_size_dt_strings(fdt) + newlen); + return 0; +} + +/** + * fdt_find_add_string_() - Find or allocate a string + * + * @fdt: pointer to the device tree to check/adjust + * @s: string to find/add + * @allocated: Set to 0 if the string was found, 1 if not found and so + * allocated. Ignored if can_assume(NO_ROLLBACK) + * @return offset of string in the string table (whether found or added) + */ +static int fdt_find_add_string_(void *fdt, const char *s, int *allocated) +{ + char *strtab = (char *)fdt + fdt_off_dt_strings(fdt); + const char *p; + char *new; + int len = strlen(s) + 1; + int err; + + if (!can_assume(NO_ROLLBACK)) + *allocated = 0; + + p = fdt_find_string_(strtab, fdt_size_dt_strings(fdt), s); + if (p) + /* found it */ + return (p - strtab); + + new = strtab + fdt_size_dt_strings(fdt); + err = fdt_splice_string_(fdt, len); + if (err) + return err; + + if (!can_assume(NO_ROLLBACK)) + *allocated = 1; + + memcpy(new, s, len); + return (new - strtab); +} + +int fdt_add_mem_rsv(void *fdt, uint64_t address, uint64_t size) +{ + struct fdt_reserve_entry *re; + int err; + + FDT_RW_PROBE(fdt); + + re = fdt_mem_rsv_w_(fdt, fdt_num_mem_rsv(fdt)); + err = fdt_splice_mem_rsv_(fdt, re, 0, 1); + if (err) + return err; + + re->address = cpu_to_fdt64(address); + re->size = cpu_to_fdt64(size); + return 0; +} + +int fdt_del_mem_rsv(void *fdt, int n) +{ + struct fdt_reserve_entry *re = fdt_mem_rsv_w_(fdt, n); + + FDT_RW_PROBE(fdt); + + if (n >= fdt_num_mem_rsv(fdt)) + return -FDT_ERR_NOTFOUND; + + return fdt_splice_mem_rsv_(fdt, re, 1, 0); +} + +static int fdt_resize_property_(void *fdt, int nodeoffset, const char *name, + int len, struct fdt_property **prop) +{ + int oldlen; + int err; + + *prop = fdt_get_property_w(fdt, nodeoffset, name, &oldlen); + if (!*prop) + return oldlen; + + if ((err = fdt_splice_struct_(fdt, (*prop)->data, FDT_TAGALIGN(oldlen), + FDT_TAGALIGN(len)))) + return err; + + (*prop)->len = cpu_to_fdt32(len); + return 0; +} + +static int fdt_add_property_(void *fdt, int nodeoffset, const char *name, + int len, struct fdt_property **prop) +{ + int proplen; + int nextoffset; + int namestroff; + int err; + int allocated; + + if ((nextoffset = fdt_check_node_offset_(fdt, nodeoffset)) < 0) + return nextoffset; + + namestroff = fdt_find_add_string_(fdt, name, &allocated); + if (namestroff < 0) + return namestroff; + + *prop = fdt_offset_ptr_w_(fdt, nextoffset); + proplen = sizeof(**prop) + FDT_TAGALIGN(len); + + err = fdt_splice_struct_(fdt, *prop, 0, proplen); + if (err) { + /* Delete the string if we failed to add it */ + if (!can_assume(NO_ROLLBACK) && allocated) + fdt_del_last_string_(fdt, name); + return err; + } + + (*prop)->tag = cpu_to_fdt32(FDT_PROP); + (*prop)->nameoff = cpu_to_fdt32(namestroff); + (*prop)->len = cpu_to_fdt32(len); + return 0; +} + +int fdt_set_name(void *fdt, int nodeoffset, const char *name) +{ + char *namep; + int oldlen, newlen; + int err; + + FDT_RW_PROBE(fdt); + + namep = (char *)(uintptr_t)fdt_get_name(fdt, nodeoffset, &oldlen); + if (!namep) + return oldlen; + + newlen = strlen(name); + + err = fdt_splice_struct_(fdt, namep, FDT_TAGALIGN(oldlen+1), + FDT_TAGALIGN(newlen+1)); + if (err) + return err; + + memcpy(namep, name, newlen+1); + return 0; +} + +int fdt_setprop_placeholder(void *fdt, int nodeoffset, const char *name, + int len, void **prop_data) +{ + struct fdt_property *prop; + int err; + + FDT_RW_PROBE(fdt); + + err = fdt_resize_property_(fdt, nodeoffset, name, len, &prop); + if (err == -FDT_ERR_NOTFOUND) + err = fdt_add_property_(fdt, nodeoffset, name, len, &prop); + if (err) + return err; + + *prop_data = prop->data; + return 0; +} + +int fdt_setprop(void *fdt, int nodeoffset, const char *name, + const void *val, int len) +{ + void *prop_data; + int err; + + err = fdt_setprop_placeholder(fdt, nodeoffset, name, len, &prop_data); + if (err) + return err; + + if (len) + memcpy(prop_data, val, len); + return 0; +} + +int fdt_appendprop(void *fdt, int nodeoffset, const char *name, + const void *val, int len) +{ + struct fdt_property *prop; + int err, oldlen, newlen; + + FDT_RW_PROBE(fdt); + + prop = fdt_get_property_w(fdt, nodeoffset, name, &oldlen); + if (prop) { + newlen = len + oldlen; + err = fdt_splice_struct_(fdt, prop->data, + FDT_TAGALIGN(oldlen), + FDT_TAGALIGN(newlen)); + if (err) + return err; + prop->len = cpu_to_fdt32(newlen); + memcpy(prop->data + oldlen, val, len); + } else { + err = fdt_add_property_(fdt, nodeoffset, name, len, &prop); + if (err) + return err; + memcpy(prop->data, val, len); + } + return 0; +} + +int fdt_delprop(void *fdt, int nodeoffset, const char *name) +{ + struct fdt_property *prop; + int len, proplen; + + FDT_RW_PROBE(fdt); + + prop = fdt_get_property_w(fdt, nodeoffset, name, &len); + if (!prop) + return len; + + proplen = sizeof(*prop) + FDT_TAGALIGN(len); + return fdt_splice_struct_(fdt, prop, proplen, 0); +} + +int fdt_add_subnode_namelen(void *fdt, int parentoffset, + const char *name, int namelen) +{ + struct fdt_node_header *nh; + int offset, nextoffset; + int nodelen; + int err; + uint32_t tag; + fdt32_t *endtag; + + FDT_RW_PROBE(fdt); + + offset = fdt_subnode_offset_namelen(fdt, parentoffset, name, namelen); + if (offset >= 0) + return -FDT_ERR_EXISTS; + else if (offset != -FDT_ERR_NOTFOUND) + return offset; + + /* Try to place the new node after the parent's properties */ + tag = fdt_next_tag(fdt, parentoffset, &nextoffset); + /* the fdt_subnode_offset_namelen() should ensure this never hits */ + if (!can_assume(LIBFDT_FLAWLESS) && (tag != FDT_BEGIN_NODE)) + return -FDT_ERR_INTERNAL; + do { + offset = nextoffset; + tag = fdt_next_tag(fdt, offset, &nextoffset); + } while ((tag == FDT_PROP) || (tag == FDT_NOP)); + + nh = fdt_offset_ptr_w_(fdt, offset); + nodelen = sizeof(*nh) + FDT_TAGALIGN(namelen+1) + FDT_TAGSIZE; + + err = fdt_splice_struct_(fdt, nh, 0, nodelen); + if (err) + return err; + + nh->tag = cpu_to_fdt32(FDT_BEGIN_NODE); + memset(nh->name, 0, FDT_TAGALIGN(namelen+1)); + memcpy(nh->name, name, namelen); + endtag = (fdt32_t *)((char *)nh + nodelen - FDT_TAGSIZE); + *endtag = cpu_to_fdt32(FDT_END_NODE); + + return offset; +} + +int fdt_add_subnode(void *fdt, int parentoffset, const char *name) +{ + return fdt_add_subnode_namelen(fdt, parentoffset, name, strlen(name)); +} + +int fdt_del_node(void *fdt, int nodeoffset) +{ + int endoffset; + + FDT_RW_PROBE(fdt); + + endoffset = fdt_node_end_offset_(fdt, nodeoffset); + if (endoffset < 0) + return endoffset; + + return fdt_splice_struct_(fdt, fdt_offset_ptr_w_(fdt, nodeoffset), + endoffset - nodeoffset, 0); +} + +static void fdt_packblocks_(const char *old, char *new, + int mem_rsv_size, + int struct_size, + int strings_size) +{ + int mem_rsv_off, struct_off, strings_off; + + mem_rsv_off = FDT_ALIGN(sizeof(struct fdt_header), 8); + struct_off = mem_rsv_off + mem_rsv_size; + strings_off = struct_off + struct_size; + + memmove(new + mem_rsv_off, old + fdt_off_mem_rsvmap(old), mem_rsv_size); + fdt_set_off_mem_rsvmap(new, mem_rsv_off); + + memmove(new + struct_off, old + fdt_off_dt_struct(old), struct_size); + fdt_set_off_dt_struct(new, struct_off); + fdt_set_size_dt_struct(new, struct_size); + + memmove(new + strings_off, old + fdt_off_dt_strings(old), strings_size); + fdt_set_off_dt_strings(new, strings_off); + fdt_set_size_dt_strings(new, fdt_size_dt_strings(old)); +} + +int fdt_open_into(const void *fdt, void *buf, int bufsize) +{ + int err; + int mem_rsv_size, struct_size; + int newsize; + const char *fdtstart = fdt; + const char *fdtend = fdtstart + fdt_totalsize(fdt); + char *tmp; + + FDT_RO_PROBE(fdt); + + mem_rsv_size = (fdt_num_mem_rsv(fdt)+1) + * sizeof(struct fdt_reserve_entry); + + if (can_assume(LATEST) || fdt_version(fdt) >= 17) { + struct_size = fdt_size_dt_struct(fdt); + } else if (fdt_version(fdt) == 16) { + struct_size = 0; + while (fdt_next_tag(fdt, struct_size, &struct_size) != FDT_END) + ; + if (struct_size < 0) + return struct_size; + } else { + return -FDT_ERR_BADVERSION; + } + + if (can_assume(LIBFDT_ORDER) || + !fdt_blocks_misordered_(fdt, mem_rsv_size, struct_size)) { + /* no further work necessary */ + err = fdt_move(fdt, buf, bufsize); + if (err) + return err; + fdt_set_version(buf, 17); + fdt_set_size_dt_struct(buf, struct_size); + fdt_set_totalsize(buf, bufsize); + return 0; + } + + /* Need to reorder */ + newsize = FDT_ALIGN(sizeof(struct fdt_header), 8) + mem_rsv_size + + struct_size + fdt_size_dt_strings(fdt); + + if (bufsize < newsize) + return -FDT_ERR_NOSPACE; + + /* First attempt to build converted tree at beginning of buffer */ + tmp = buf; + /* But if that overlaps with the old tree... */ + if (((tmp + newsize) > fdtstart) && (tmp < fdtend)) { + /* Try right after the old tree instead */ + tmp = (char *)(uintptr_t)fdtend; + if ((tmp + newsize) > ((char *)buf + bufsize)) + return -FDT_ERR_NOSPACE; + } + + fdt_packblocks_(fdt, tmp, mem_rsv_size, struct_size, + fdt_size_dt_strings(fdt)); + memmove(buf, tmp, newsize); + + fdt_set_magic(buf, FDT_MAGIC); + fdt_set_totalsize(buf, bufsize); + fdt_set_version(buf, 17); + fdt_set_last_comp_version(buf, 16); + fdt_set_boot_cpuid_phys(buf, fdt_boot_cpuid_phys(fdt)); + + return 0; +} + +int fdt_pack(void *fdt) +{ + int mem_rsv_size; + + FDT_RW_PROBE(fdt); + + mem_rsv_size = (fdt_num_mem_rsv(fdt)+1) + * sizeof(struct fdt_reserve_entry); + fdt_packblocks_(fdt, fdt, mem_rsv_size, fdt_size_dt_struct(fdt), + fdt_size_dt_strings(fdt)); + fdt_set_totalsize(fdt, fdt_data_size_(fdt)); + + return 0; +} diff --git a/lib/libfdt/fdt_strerror.c b/lib/libfdt/fdt_strerror.c new file mode 100644 index 0000000..d852b77 --- /dev/null +++ b/lib/libfdt/fdt_strerror.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +struct fdt_errtabent { + const char *str; +}; + +#define FDT_ERRTABENT(val) \ + [(val)] = { .str = #val, } + +static struct fdt_errtabent fdt_errtable[] = { + FDT_ERRTABENT(FDT_ERR_NOTFOUND), + FDT_ERRTABENT(FDT_ERR_EXISTS), + FDT_ERRTABENT(FDT_ERR_NOSPACE), + + FDT_ERRTABENT(FDT_ERR_BADOFFSET), + FDT_ERRTABENT(FDT_ERR_BADPATH), + FDT_ERRTABENT(FDT_ERR_BADPHANDLE), + FDT_ERRTABENT(FDT_ERR_BADSTATE), + + FDT_ERRTABENT(FDT_ERR_TRUNCATED), + FDT_ERRTABENT(FDT_ERR_BADMAGIC), + FDT_ERRTABENT(FDT_ERR_BADVERSION), + FDT_ERRTABENT(FDT_ERR_BADSTRUCTURE), + FDT_ERRTABENT(FDT_ERR_BADLAYOUT), + FDT_ERRTABENT(FDT_ERR_INTERNAL), + FDT_ERRTABENT(FDT_ERR_BADNCELLS), + FDT_ERRTABENT(FDT_ERR_BADVALUE), + FDT_ERRTABENT(FDT_ERR_BADOVERLAY), + FDT_ERRTABENT(FDT_ERR_NOPHANDLES), + FDT_ERRTABENT(FDT_ERR_BADFLAGS), + FDT_ERRTABENT(FDT_ERR_ALIGNMENT), +}; +#define FDT_ERRTABSIZE ((int)(sizeof(fdt_errtable) / sizeof(fdt_errtable[0]))) + +const char *fdt_strerror(int errval) +{ + if (errval > 0) + return "<valid offset/length>"; + else if (errval == 0) + return "<no error>"; + else if (-errval < FDT_ERRTABSIZE) { + const char *s = fdt_errtable[-errval].str; + + if (s) + return s; + } + + return "<unknown error>"; +} diff --git a/lib/libfdt/fdt_sw.c b/lib/libfdt/fdt_sw.c new file mode 100644 index 0000000..4c569ee --- /dev/null +++ b/lib/libfdt/fdt_sw.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +static int fdt_sw_probe_(void *fdt) +{ + if (!can_assume(VALID_INPUT)) { + if (fdt_magic(fdt) == FDT_MAGIC) + return -FDT_ERR_BADSTATE; + else if (fdt_magic(fdt) != FDT_SW_MAGIC) + return -FDT_ERR_BADMAGIC; + } + + return 0; +} + +#define FDT_SW_PROBE(fdt) \ + { \ + int err; \ + if ((err = fdt_sw_probe_(fdt)) != 0) \ + return err; \ + } + +/* 'memrsv' state: Initial state after fdt_create() + * + * Allowed functions: + * fdt_add_reservemap_entry() + * fdt_finish_reservemap() [moves to 'struct' state] + */ +static int fdt_sw_probe_memrsv_(void *fdt) +{ + int err = fdt_sw_probe_(fdt); + if (err) + return err; + + if (!can_assume(VALID_INPUT) && fdt_off_dt_strings(fdt) != 0) + return -FDT_ERR_BADSTATE; + return 0; +} + +#define FDT_SW_PROBE_MEMRSV(fdt) \ + { \ + int err; \ + if ((err = fdt_sw_probe_memrsv_(fdt)) != 0) \ + return err; \ + } + +/* 'struct' state: Enter this state after fdt_finish_reservemap() + * + * Allowed functions: + * fdt_begin_node() + * fdt_end_node() + * fdt_property*() + * fdt_finish() [moves to 'complete' state] + */ +static int fdt_sw_probe_struct_(void *fdt) +{ + int err = fdt_sw_probe_(fdt); + if (err) + return err; + + if (!can_assume(VALID_INPUT) && + fdt_off_dt_strings(fdt) != fdt_totalsize(fdt)) + return -FDT_ERR_BADSTATE; + return 0; +} + +#define FDT_SW_PROBE_STRUCT(fdt) \ + { \ + int err; \ + if ((err = fdt_sw_probe_struct_(fdt)) != 0) \ + return err; \ + } + +static inline uint32_t sw_flags(void *fdt) +{ + /* assert: (fdt_magic(fdt) == FDT_SW_MAGIC) */ + return fdt_last_comp_version(fdt); +} + +/* 'complete' state: Enter this state after fdt_finish() + * + * Allowed functions: none + */ + +static void *fdt_grab_space_(void *fdt, size_t len) +{ + unsigned int offset = fdt_size_dt_struct(fdt); + unsigned int spaceleft; + + spaceleft = fdt_totalsize(fdt) - fdt_off_dt_struct(fdt) + - fdt_size_dt_strings(fdt); + + if ((offset + len < offset) || (offset + len > spaceleft)) + return NULL; + + fdt_set_size_dt_struct(fdt, offset + len); + return fdt_offset_ptr_w_(fdt, offset); +} + +int fdt_create_with_flags(void *buf, int bufsize, uint32_t flags) +{ + const int hdrsize = FDT_ALIGN(sizeof(struct fdt_header), + sizeof(struct fdt_reserve_entry)); + void *fdt = buf; + + if (bufsize < hdrsize) + return -FDT_ERR_NOSPACE; + + if (flags & ~FDT_CREATE_FLAGS_ALL) + return -FDT_ERR_BADFLAGS; + + memset(buf, 0, bufsize); + + /* + * magic and last_comp_version keep intermediate state during the fdt + * creation process, which is replaced with the proper FDT format by + * fdt_finish(). + * + * flags should be accessed with sw_flags(). + */ + fdt_set_magic(fdt, FDT_SW_MAGIC); + fdt_set_version(fdt, FDT_LAST_SUPPORTED_VERSION); + fdt_set_last_comp_version(fdt, flags); + + fdt_set_totalsize(fdt, bufsize); + + fdt_set_off_mem_rsvmap(fdt, hdrsize); + fdt_set_off_dt_struct(fdt, fdt_off_mem_rsvmap(fdt)); + fdt_set_off_dt_strings(fdt, 0); + + return 0; +} + +int fdt_create(void *buf, int bufsize) +{ + return fdt_create_with_flags(buf, bufsize, 0); +} + +int fdt_resize(void *fdt, void *buf, int bufsize) +{ + size_t headsize, tailsize; + char *oldtail, *newtail; + + FDT_SW_PROBE(fdt); + + if (bufsize < 0) + return -FDT_ERR_NOSPACE; + + headsize = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt); + tailsize = fdt_size_dt_strings(fdt); + + if (!can_assume(VALID_DTB) && + headsize + tailsize > fdt_totalsize(fdt)) + return -FDT_ERR_INTERNAL; + + if ((headsize + tailsize) > (unsigned)bufsize) + return -FDT_ERR_NOSPACE; + + oldtail = (char *)fdt + fdt_totalsize(fdt) - tailsize; + newtail = (char *)buf + bufsize - tailsize; + + /* Two cases to avoid clobbering data if the old and new + * buffers partially overlap */ + if (buf <= fdt) { + memmove(buf, fdt, headsize); + memmove(newtail, oldtail, tailsize); + } else { + memmove(newtail, oldtail, tailsize); + memmove(buf, fdt, headsize); + } + + fdt_set_totalsize(buf, bufsize); + if (fdt_off_dt_strings(buf)) + fdt_set_off_dt_strings(buf, bufsize); + + return 0; +} + +int fdt_add_reservemap_entry(void *fdt, uint64_t addr, uint64_t size) +{ + struct fdt_reserve_entry *re; + int offset; + + FDT_SW_PROBE_MEMRSV(fdt); + + offset = fdt_off_dt_struct(fdt); + if ((offset + sizeof(*re)) > fdt_totalsize(fdt)) + return -FDT_ERR_NOSPACE; + + re = (struct fdt_reserve_entry *)((char *)fdt + offset); + re->address = cpu_to_fdt64(addr); + re->size = cpu_to_fdt64(size); + + fdt_set_off_dt_struct(fdt, offset + sizeof(*re)); + + return 0; +} + +int fdt_finish_reservemap(void *fdt) +{ + int err = fdt_add_reservemap_entry(fdt, 0, 0); + + if (err) + return err; + + fdt_set_off_dt_strings(fdt, fdt_totalsize(fdt)); + return 0; +} + +int fdt_begin_node(void *fdt, const char *name) +{ + struct fdt_node_header *nh; + int namelen; + + FDT_SW_PROBE_STRUCT(fdt); + + namelen = strlen(name) + 1; + nh = fdt_grab_space_(fdt, sizeof(*nh) + FDT_TAGALIGN(namelen)); + if (! nh) + return -FDT_ERR_NOSPACE; + + nh->tag = cpu_to_fdt32(FDT_BEGIN_NODE); + memcpy(nh->name, name, namelen); + return 0; +} + +int fdt_end_node(void *fdt) +{ + fdt32_t *en; + + FDT_SW_PROBE_STRUCT(fdt); + + en = fdt_grab_space_(fdt, FDT_TAGSIZE); + if (! en) + return -FDT_ERR_NOSPACE; + + *en = cpu_to_fdt32(FDT_END_NODE); + return 0; +} + +static int fdt_add_string_(void *fdt, const char *s) +{ + char *strtab = (char *)fdt + fdt_totalsize(fdt); + unsigned int strtabsize = fdt_size_dt_strings(fdt); + unsigned int len = strlen(s) + 1; + unsigned int struct_top, offset; + + offset = strtabsize + len; + struct_top = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt); + if (fdt_totalsize(fdt) - offset < struct_top) + return 0; /* no more room :( */ + + memcpy(strtab - offset, s, len); + fdt_set_size_dt_strings(fdt, strtabsize + len); + return -offset; +} + +/* Must only be used to roll back in case of error */ +static void fdt_del_last_string_(void *fdt, const char *s) +{ + int strtabsize = fdt_size_dt_strings(fdt); + int len = strlen(s) + 1; + + fdt_set_size_dt_strings(fdt, strtabsize - len); +} + +static int fdt_find_add_string_(void *fdt, const char *s, int *allocated) +{ + char *strtab = (char *)fdt + fdt_totalsize(fdt); + int strtabsize = fdt_size_dt_strings(fdt); + const char *p; + + *allocated = 0; + + p = fdt_find_string_(strtab - strtabsize, strtabsize, s); + if (p) + return p - strtab; + + *allocated = 1; + + return fdt_add_string_(fdt, s); +} + +int fdt_property_placeholder(void *fdt, const char *name, int len, void **valp) +{ + struct fdt_property *prop; + int nameoff; + int allocated; + + FDT_SW_PROBE_STRUCT(fdt); + + /* String de-duplication can be slow, _NO_NAME_DEDUP skips it */ + if (sw_flags(fdt) & FDT_CREATE_FLAG_NO_NAME_DEDUP) { + allocated = 1; + nameoff = fdt_add_string_(fdt, name); + } else { + nameoff = fdt_find_add_string_(fdt, name, &allocated); + } + if (nameoff == 0) + return -FDT_ERR_NOSPACE; + + prop = fdt_grab_space_(fdt, sizeof(*prop) + FDT_TAGALIGN(len)); + if (! prop) { + if (allocated) + fdt_del_last_string_(fdt, name); + return -FDT_ERR_NOSPACE; + } + + prop->tag = cpu_to_fdt32(FDT_PROP); + prop->nameoff = cpu_to_fdt32(nameoff); + prop->len = cpu_to_fdt32(len); + *valp = prop->data; + return 0; +} + +int fdt_property(void *fdt, const char *name, const void *val, int len) +{ + void *ptr; + int ret; + + ret = fdt_property_placeholder(fdt, name, len, &ptr); + if (ret) + return ret; + memcpy(ptr, val, len); + return 0; +} + +int fdt_finish(void *fdt) +{ + char *p = (char *)fdt; + fdt32_t *end; + int oldstroffset, newstroffset; + uint32_t tag; + int offset, nextoffset; + + FDT_SW_PROBE_STRUCT(fdt); + + /* Add terminator */ + end = fdt_grab_space_(fdt, sizeof(*end)); + if (! end) + return -FDT_ERR_NOSPACE; + *end = cpu_to_fdt32(FDT_END); + + /* Relocate the string table */ + oldstroffset = fdt_totalsize(fdt) - fdt_size_dt_strings(fdt); + newstroffset = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt); + memmove(p + newstroffset, p + oldstroffset, fdt_size_dt_strings(fdt)); + fdt_set_off_dt_strings(fdt, newstroffset); + + /* Walk the structure, correcting string offsets */ + offset = 0; + while ((tag = fdt_next_tag(fdt, offset, &nextoffset)) != FDT_END) { + if (tag == FDT_PROP) { + struct fdt_property *prop = + fdt_offset_ptr_w_(fdt, offset); + int nameoff; + + nameoff = fdt32_to_cpu(prop->nameoff); + nameoff += fdt_size_dt_strings(fdt); + prop->nameoff = cpu_to_fdt32(nameoff); + } + offset = nextoffset; + } + if (nextoffset < 0) + return nextoffset; + + /* Finally, adjust the header */ + fdt_set_totalsize(fdt, newstroffset + fdt_size_dt_strings(fdt)); + + /* And fix up fields that were keeping intermediate state. */ + fdt_set_last_comp_version(fdt, FDT_LAST_COMPATIBLE_VERSION); + fdt_set_magic(fdt, FDT_MAGIC); + + return 0; +} diff --git a/lib/libfdt/fdt_wip.c b/lib/libfdt/fdt_wip.c new file mode 100644 index 0000000..c2d7566 --- /dev/null +++ b/lib/libfdt/fdt_wip.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include "libfdt_env.h" + +#include <fdt.h> +#include <libfdt.h> + +#include "libfdt_internal.h" + +int fdt_setprop_inplace_namelen_partial(void *fdt, int nodeoffset, + const char *name, int namelen, + uint32_t idx, const void *val, + int len) +{ + void *propval; + int proplen; + + propval = fdt_getprop_namelen_w(fdt, nodeoffset, name, namelen, + &proplen); + if (!propval) + return proplen; + + if ((unsigned)proplen < (len + idx)) + return -FDT_ERR_NOSPACE; + + memcpy((char *)propval + idx, val, len); + return 0; +} + +int fdt_setprop_inplace(void *fdt, int nodeoffset, const char *name, + const void *val, int len) +{ + const void *propval; + int proplen; + + propval = fdt_getprop(fdt, nodeoffset, name, &proplen); + if (!propval) + return proplen; + + if (proplen != len) + return -FDT_ERR_NOSPACE; + + return fdt_setprop_inplace_namelen_partial(fdt, nodeoffset, name, + strlen(name), 0, + val, len); +} + +static void fdt_nop_region_(void *start, int len) +{ + fdt32_t *p; + + for (p = start; (char *)p < ((char *)start + len); p++) + *p = cpu_to_fdt32(FDT_NOP); +} + +int fdt_nop_property(void *fdt, int nodeoffset, const char *name) +{ + struct fdt_property *prop; + int len; + + prop = fdt_get_property_w(fdt, nodeoffset, name, &len); + if (!prop) + return len; + + fdt_nop_region_(prop, len + sizeof(*prop)); + + return 0; +} + +int fdt_node_end_offset_(void *fdt, int offset) +{ + int depth = 0; + + while ((offset >= 0) && (depth >= 0)) + offset = fdt_next_node(fdt, offset, &depth); + + return offset; +} + +int fdt_nop_node(void *fdt, int nodeoffset) +{ + int endoffset; + + endoffset = fdt_node_end_offset_(fdt, nodeoffset); + if (endoffset < 0) + return endoffset; + + fdt_nop_region_(fdt_offset_ptr_w(fdt, nodeoffset, 0), + endoffset - nodeoffset); + return 0; +} diff --git a/lib/libfdt/libfdt.mk b/lib/libfdt/libfdt.mk new file mode 100644 index 0000000..812057d --- /dev/null +++ b/lib/libfdt/libfdt.mk @@ -0,0 +1,19 @@ +# +# Copyright (c) 2016, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +LIBFDT_SRCS := $(addprefix lib/libfdt/, \ + fdt.c \ + fdt_addresses.c \ + fdt_empty_tree.c \ + fdt_ro.c \ + fdt_rw.c \ + fdt_strerror.c \ + fdt_sw.c \ + fdt_wip.c) \ + +INCLUDES += -Iinclude/lib/libfdt + +$(eval $(call MAKE_LIB,fdt)) diff --git a/lib/libfdt/libfdt_internal.h b/lib/libfdt/libfdt_internal.h new file mode 100644 index 0000000..16bda19 --- /dev/null +++ b/lib/libfdt/libfdt_internal.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +#ifndef LIBFDT_INTERNAL_H +#define LIBFDT_INTERNAL_H +/* + * libfdt - Flat Device Tree manipulation + * Copyright (C) 2006 David Gibson, IBM Corporation. + */ +#include <fdt.h> + +#define FDT_ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) +#define FDT_TAGALIGN(x) (FDT_ALIGN((x), FDT_TAGSIZE)) + +int32_t fdt_ro_probe_(const void *fdt); +#define FDT_RO_PROBE(fdt) \ + { \ + int32_t totalsize_; \ + if ((totalsize_ = fdt_ro_probe_(fdt)) < 0) \ + return totalsize_; \ + } + +int fdt_check_node_offset_(const void *fdt, int offset); +int fdt_check_prop_offset_(const void *fdt, int offset); +const char *fdt_find_string_(const char *strtab, int tabsize, const char *s); +int fdt_node_end_offset_(void *fdt, int nodeoffset); + +static inline const void *fdt_offset_ptr_(const void *fdt, int offset) +{ + return (const char *)fdt + fdt_off_dt_struct(fdt) + offset; +} + +static inline void *fdt_offset_ptr_w_(void *fdt, int offset) +{ + return (void *)(uintptr_t)fdt_offset_ptr_(fdt, offset); +} + +static inline const struct fdt_reserve_entry *fdt_mem_rsv_(const void *fdt, int n) +{ + const struct fdt_reserve_entry *rsv_table = + (const struct fdt_reserve_entry *) + ((const char *)fdt + fdt_off_mem_rsvmap(fdt)); + + return rsv_table + n; +} +static inline struct fdt_reserve_entry *fdt_mem_rsv_w_(void *fdt, int n) +{ + return (void *)(uintptr_t)fdt_mem_rsv_(fdt, n); +} + +/* + * Internal helpers to access tructural elements of the device tree + * blob (rather than for exaple reading integers from within property + * values). We assume that we are either given a naturally aligned + * address for the platform or if we are not, we are on a platform + * where unaligned memory reads will be handled in a graceful manner. + * If not the external helpers fdtXX_ld() from libfdt.h can be used + * instead. + */ +static inline uint32_t fdt32_ld_(const fdt32_t *p) +{ + return fdt32_to_cpu(*p); +} + +static inline uint64_t fdt64_ld_(const fdt64_t *p) +{ + return fdt64_to_cpu(*p); +} + +#define FDT_SW_MAGIC (~FDT_MAGIC) + +/**********************************************************************/ +/* Checking controls */ +/**********************************************************************/ + +#ifndef FDT_ASSUME_MASK +#define FDT_ASSUME_MASK 0 +#endif + +/* + * Defines assumptions which can be enabled. Each of these can be enabled + * individually. For maximum safety, don't enable any assumptions! + * + * For minimal code size and no safety, use ASSUME_PERFECT at your own risk. + * You should have another method of validating the device tree, such as a + * signature or hash check before using libfdt. + * + * For situations where security is not a concern it may be safe to enable + * ASSUME_SANE. + */ +enum { + /* + * This does essentially no checks. Only the latest device-tree + * version is correctly handled. Inconsistencies or errors in the device + * tree may cause undefined behaviour or crashes. Invalid parameters + * passed to libfdt may do the same. + * + * If an error occurs when modifying the tree it may leave the tree in + * an intermediate (but valid) state. As an example, adding a property + * where there is insufficient space may result in the property name + * being added to the string table even though the property itself is + * not added to the struct section. + * + * Only use this if you have a fully validated device tree with + * the latest supported version and wish to minimise code size. + */ + ASSUME_PERFECT = 0xff, + + /* + * This assumes that the device tree is sane. i.e. header metadata + * and basic hierarchy are correct. + * + * With this assumption enabled, normal device trees produced by libfdt + * and the compiler should be handled safely. Malicious device trees and + * complete garbage may cause libfdt to behave badly or crash. Truncated + * device trees (e.g. those only partially loaded) can also cause + * problems. + * + * Note: Only checks that relate exclusively to the device tree itself + * (not the parameters passed to libfdt) are disabled by this + * assumption. This includes checking headers, tags and the like. + */ + ASSUME_VALID_DTB = 1 << 0, + + /* + * This builds on ASSUME_VALID_DTB and further assumes that libfdt + * functions are called with valid parameters, i.e. not trigger + * FDT_ERR_BADOFFSET or offsets that are out of bounds. It disables any + * extensive checking of parameters and the device tree, making various + * assumptions about correctness. + * + * It doesn't make sense to enable this assumption unless + * ASSUME_VALID_DTB is also enabled. + */ + ASSUME_VALID_INPUT = 1 << 1, + + /* + * This disables checks for device-tree version and removes all code + * which handles older versions. + * + * Only enable this if you know you have a device tree with the latest + * version. + */ + ASSUME_LATEST = 1 << 2, + + /* + * This assumes that it is OK for a failed addition to the device tree, + * due to lack of space or some other problem, to skip any rollback + * steps (such as dropping the property name from the string table). + * This is safe to enable in most circumstances, even though it may + * leave the tree in a sub-optimal state. + */ + ASSUME_NO_ROLLBACK = 1 << 3, + + /* + * This assumes that the device tree components appear in a 'convenient' + * order, i.e. the memory reservation block first, then the structure + * block and finally the string block. + * + * This order is not specified by the device-tree specification, + * but is expected by libfdt. The device-tree compiler always created + * device trees with this order. + * + * This assumption disables a check in fdt_open_into() and removes the + * ability to fix the problem there. This is safe if you know that the + * device tree is correctly ordered. See fdt_blocks_misordered_(). + */ + ASSUME_LIBFDT_ORDER = 1 << 4, + + /* + * This assumes that libfdt itself does not have any internal bugs. It + * drops certain checks that should never be needed unless libfdt has an + * undiscovered bug. + * + * This can generally be considered safe to enable. + */ + ASSUME_LIBFDT_FLAWLESS = 1 << 5, +}; + +/** + * can_assume_() - check if a particular assumption is enabled + * + * @mask: Mask to check (ASSUME_...) + * @return true if that assumption is enabled, else false + */ +static inline bool can_assume_(int mask) +{ + return FDT_ASSUME_MASK & mask; +} + +/** helper macros for checking assumptions */ +#define can_assume(_assume) can_assume_(ASSUME_ ## _assume) + +#endif /* LIBFDT_INTERNAL_H */ diff --git a/lib/locks/bakery/bakery_lock_coherent.c b/lib/locks/bakery/bakery_lock_coherent.c new file mode 100644 index 0000000..bcd89ef --- /dev/null +++ b/lib/locks/bakery/bakery_lock_coherent.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2013-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <string.h> + +#include <arch_helpers.h> +#include <lib/bakery_lock.h> +#include <lib/el3_runtime/cpu_data.h> +#include <plat/common/platform.h> + +/* + * Functions in this file implement Bakery Algorithm for mutual exclusion with the + * bakery lock data structures in coherent memory. + * + * ARM architecture offers a family of exclusive access instructions to + * efficiently implement mutual exclusion with hardware support. However, as + * well as depending on external hardware, the these instructions have defined + * behavior only on certain memory types (cacheable and Normal memory in + * particular; see ARMv8 Architecture Reference Manual section B2.10). Use cases + * in trusted firmware are such that mutual exclusion implementation cannot + * expect that accesses to the lock have the specific type required by the + * architecture for these primitives to function (for example, not all + * contenders may have address translation enabled). + * + * This implementation does not use mutual exclusion primitives. It expects + * memory regions where the locks reside to be fully ordered and coherent + * (either by disabling address translation, or by assigning proper attributes + * when translation is enabled). + * + * Note that the ARM architecture guarantees single-copy atomicity for aligned + * accesses regardless of status of address translation. + */ + +#define assert_bakery_entry_valid(_entry, _bakery) do { \ + assert((_bakery) != NULL); \ + assert((_entry) < BAKERY_LOCK_MAX_CPUS); \ +} while (false) + +/* Obtain a ticket for a given CPU */ +static unsigned int bakery_get_ticket(bakery_lock_t *bakery, unsigned int me) +{ + unsigned int my_ticket, their_ticket; + unsigned int they; + + /* Prevent recursive acquisition */ + assert(bakery_ticket_number(bakery->lock_data[me]) == 0U); + + /* + * Flag that we're busy getting our ticket. All CPUs are iterated in the + * order of their ordinal position to decide the maximum ticket value + * observed so far. Our priority is set to be greater than the maximum + * observed priority + * + * Note that it's possible that more than one contender gets the same + * ticket value. That's OK as the lock is acquired based on the priority + * value, not the ticket value alone. + */ + my_ticket = 0U; + bakery->lock_data[me] = make_bakery_data(CHOOSING_TICKET, my_ticket); + for (they = 0U; they < BAKERY_LOCK_MAX_CPUS; they++) { + their_ticket = bakery_ticket_number(bakery->lock_data[they]); + if (their_ticket > my_ticket) + my_ticket = their_ticket; + } + + /* + * Compute ticket; then signal to other contenders waiting for us to + * finish calculating our ticket value that we're done + */ + ++my_ticket; + bakery->lock_data[me] = make_bakery_data(CHOSEN_TICKET, my_ticket); + + return my_ticket; +} + + +/* + * Acquire bakery lock + * + * Contending CPUs need first obtain a non-zero ticket and then calculate + * priority value. A contending CPU iterate over all other CPUs in the platform, + * which may be contending for the same lock, in the order of their ordinal + * position (CPU0, CPU1 and so on). A non-contending CPU will have its ticket + * (and priority) value as 0. The contending CPU compares its priority with that + * of others'. The CPU with the highest priority (lowest numerical value) + * acquires the lock + */ +void bakery_lock_get(bakery_lock_t *bakery) +{ + unsigned int they, me; + unsigned int my_ticket, my_prio, their_ticket; + unsigned int their_bakery_data; + + me = plat_my_core_pos(); + + assert_bakery_entry_valid(me, bakery); + + /* Get a ticket */ + my_ticket = bakery_get_ticket(bakery, me); + + /* + * Now that we got our ticket, compute our priority value, then compare + * with that of others, and proceed to acquire the lock + */ + my_prio = bakery_get_priority(my_ticket, me); + for (they = 0U; they < BAKERY_LOCK_MAX_CPUS; they++) { + if (me == they) + continue; + + /* Wait for the contender to get their ticket */ + do { + their_bakery_data = bakery->lock_data[they]; + } while (bakery_is_choosing(their_bakery_data)); + + /* + * If the other party is a contender, they'll have non-zero + * (valid) ticket value. If they do, compare priorities + */ + their_ticket = bakery_ticket_number(their_bakery_data); + if ((their_ticket != 0U) && + (bakery_get_priority(their_ticket, they) < my_prio)) { + /* + * They have higher priority (lower value). Wait for + * their ticket value to change (either release the lock + * to have it dropped to 0; or drop and probably content + * again for the same lock to have an even higher value) + */ + do { + wfe(); + } while (their_ticket == + bakery_ticket_number(bakery->lock_data[they])); + } + } + + /* + * Lock acquired. Ensure that any reads and writes from a shared + * resource in the critical section read/write values after the lock is + * acquired. + */ + dmbish(); +} + + +/* Release the lock and signal contenders */ +void bakery_lock_release(bakery_lock_t *bakery) +{ + unsigned int me = plat_my_core_pos(); + + assert_bakery_entry_valid(me, bakery); + assert(bakery_ticket_number(bakery->lock_data[me]) != 0U); + + /* + * Ensure that other observers see any stores in the critical section + * before releasing the lock. Also ensure all loads in the critical + * section are complete before releasing the lock. Release the lock by + * resetting ticket. Then signal other waiting contenders. + */ + dmbish(); + bakery->lock_data[me] = 0U; + + /* Required to ensure ordering of the following sev */ + dsb(); + sev(); +} diff --git a/lib/locks/bakery/bakery_lock_normal.c b/lib/locks/bakery/bakery_lock_normal.c new file mode 100644 index 0000000..9f27322 --- /dev/null +++ b/lib/locks/bakery/bakery_lock_normal.c @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2015-2020, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <string.h> + +#include <arch_helpers.h> +#include <lib/bakery_lock.h> +#include <lib/el3_runtime/cpu_data.h> +#include <lib/utils_def.h> +#include <plat/common/platform.h> + +/* + * Functions in this file implement Bakery Algorithm for mutual exclusion with the + * bakery lock data structures in cacheable and Normal memory. + * + * ARM architecture offers a family of exclusive access instructions to + * efficiently implement mutual exclusion with hardware support. However, as + * well as depending on external hardware, these instructions have defined + * behavior only on certain memory types (cacheable and Normal memory in + * particular; see ARMv8 Architecture Reference Manual section B2.10). Use cases + * in trusted firmware are such that mutual exclusion implementation cannot + * expect that accesses to the lock have the specific type required by the + * architecture for these primitives to function (for example, not all + * contenders may have address translation enabled). + * + * This implementation does not use mutual exclusion primitives. It expects + * memory regions where the locks reside to be cacheable and Normal. + * + * Note that the ARM architecture guarantees single-copy atomicity for aligned + * accesses regardless of status of address translation. + */ + +#ifdef PLAT_PERCPU_BAKERY_LOCK_SIZE +/* + * Verify that the platform defined value for the per-cpu space for bakery locks is + * a multiple of the cache line size, to prevent multiple CPUs writing to the same + * bakery lock cache line + * + * Using this value, if provided, rather than the linker generated value results in + * more efficient code + */ +CASSERT((PLAT_PERCPU_BAKERY_LOCK_SIZE & (CACHE_WRITEBACK_GRANULE - 1)) == 0, + PLAT_PERCPU_BAKERY_LOCK_SIZE_not_cacheline_multiple); +#define PERCPU_BAKERY_LOCK_SIZE (PLAT_PERCPU_BAKERY_LOCK_SIZE) +#else +/* + * Use the linker defined symbol which has evaluated the size reqiurement. + * This is not as efficient as using a platform defined constant + */ +IMPORT_SYM(uintptr_t, __PERCPU_BAKERY_LOCK_START__, BAKERY_LOCK_START); +IMPORT_SYM(uintptr_t, __PERCPU_BAKERY_LOCK_END__, BAKERY_LOCK_END); +#define PERCPU_BAKERY_LOCK_SIZE (BAKERY_LOCK_END - BAKERY_LOCK_START) +#endif + +static inline bakery_lock_t *get_bakery_info(unsigned int cpu_ix, + bakery_lock_t *lock) +{ + return (bakery_info_t *)((uintptr_t)lock + + cpu_ix * PERCPU_BAKERY_LOCK_SIZE); +} + +static inline void write_cache_op(uintptr_t addr, bool cached) +{ + if (cached) + dccvac(addr); + else + dcivac(addr); + + dsbish(); +} + +static inline void read_cache_op(uintptr_t addr, bool cached) +{ + if (cached) + dccivac(addr); + + dmbish(); +} + +/* Helper function to check if the lock is acquired */ +static inline __unused bool is_lock_acquired(const bakery_info_t *my_bakery_info, + bool is_cached) +{ + /* + * Even though lock data is updated only by the owning cpu and + * appropriate cache maintenance operations are performed, + * if the previous update was done when the cpu was not participating + * in coherency, then there is a chance that cache maintenance + * operations were not propagated to all the caches in the system. + * Hence do a `read_cache_op()` prior to read. + */ + read_cache_op((uintptr_t)my_bakery_info, is_cached); + return bakery_ticket_number(my_bakery_info->lock_data) != 0U; +} + +static unsigned int bakery_get_ticket(bakery_lock_t *lock, + unsigned int me, bool is_cached) +{ + unsigned int my_ticket, their_ticket; + unsigned int they; + bakery_info_t *my_bakery_info, *their_bakery_info; + + /* + * Obtain a reference to the bakery information for this cpu and ensure + * it is not NULL. + */ + my_bakery_info = get_bakery_info(me, lock); + assert(my_bakery_info != NULL); + + /* Prevent recursive acquisition.*/ + assert(!is_lock_acquired(my_bakery_info, is_cached)); + + /* + * Tell other contenders that we are through the bakery doorway i.e. + * going to allocate a ticket for this cpu. + */ + my_ticket = 0U; + my_bakery_info->lock_data = make_bakery_data(CHOOSING_TICKET, my_ticket); + + write_cache_op((uintptr_t)my_bakery_info, is_cached); + + /* + * Iterate through the bakery information of each contender to allocate + * the highest ticket number for this cpu. + */ + for (they = 0U; they < BAKERY_LOCK_MAX_CPUS; they++) { + if (me == they) + continue; + + /* + * Get a reference to the other contender's bakery info and + * ensure that a stale copy is not read. + */ + their_bakery_info = get_bakery_info(they, lock); + assert(their_bakery_info != NULL); + + read_cache_op((uintptr_t)their_bakery_info, is_cached); + + /* + * Update this cpu's ticket number if a higher ticket number is + * seen + */ + their_ticket = bakery_ticket_number(their_bakery_info->lock_data); + if (their_ticket > my_ticket) + my_ticket = their_ticket; + } + + /* + * Compute ticket; then signal to other contenders waiting for us to + * finish calculating our ticket value that we're done + */ + ++my_ticket; + my_bakery_info->lock_data = make_bakery_data(CHOSEN_TICKET, my_ticket); + + write_cache_op((uintptr_t)my_bakery_info, is_cached); + + return my_ticket; +} + +void bakery_lock_get(bakery_lock_t *lock) +{ + unsigned int they, me; + unsigned int my_ticket, my_prio, their_ticket; + bakery_info_t *their_bakery_info; + unsigned int their_bakery_data; + bool is_cached; + + me = plat_my_core_pos(); + is_cached = is_dcache_enabled(); + + /* Get a ticket */ + my_ticket = bakery_get_ticket(lock, me, is_cached); + + /* + * Now that we got our ticket, compute our priority value, then compare + * with that of others, and proceed to acquire the lock + */ + my_prio = bakery_get_priority(my_ticket, me); + for (they = 0U; they < BAKERY_LOCK_MAX_CPUS; they++) { + if (me == they) + continue; + + /* + * Get a reference to the other contender's bakery info and + * ensure that a stale copy is not read. + */ + their_bakery_info = get_bakery_info(they, lock); + assert(their_bakery_info != NULL); + + /* Wait for the contender to get their ticket */ + do { + read_cache_op((uintptr_t)their_bakery_info, is_cached); + their_bakery_data = their_bakery_info->lock_data; + } while (bakery_is_choosing(their_bakery_data)); + + /* + * If the other party is a contender, they'll have non-zero + * (valid) ticket value. If they do, compare priorities + */ + their_ticket = bakery_ticket_number(their_bakery_data); + if (their_ticket && (bakery_get_priority(their_ticket, they) < my_prio)) { + /* + * They have higher priority (lower value). Wait for + * their ticket value to change (either release the lock + * to have it dropped to 0; or drop and probably content + * again for the same lock to have an even higher value) + */ + do { + wfe(); + read_cache_op((uintptr_t)their_bakery_info, is_cached); + } while (their_ticket + == bakery_ticket_number(their_bakery_info->lock_data)); + } + } + + /* + * Lock acquired. Ensure that any reads and writes from a shared + * resource in the critical section read/write values after the lock is + * acquired. + */ + dmbish(); +} + +void bakery_lock_release(bakery_lock_t *lock) +{ + bakery_info_t *my_bakery_info; + bool is_cached = is_dcache_enabled(); + + my_bakery_info = get_bakery_info(plat_my_core_pos(), lock); + + assert(is_lock_acquired(my_bakery_info, is_cached)); + + /* + * Ensure that other observers see any stores in the critical section + * before releasing the lock. Also ensure all loads in the critical + * section are complete before releasing the lock. Release the lock by + * resetting ticket. Then signal other waiting contenders. + */ + dmbish(); + my_bakery_info->lock_data = 0U; + write_cache_op((uintptr_t)my_bakery_info, is_cached); + + /* This sev is ordered by the dsbish in write_cahce_op */ + sev(); +} diff --git a/lib/locks/exclusive/aarch32/spinlock.S b/lib/locks/exclusive/aarch32/spinlock.S new file mode 100644 index 0000000..853d096 --- /dev/null +++ b/lib/locks/exclusive/aarch32/spinlock.S @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + + .globl spin_lock + .globl spin_unlock + +#if ARM_ARCH_AT_LEAST(8, 0) +/* + * According to the ARMv8-A Architecture Reference Manual, "when the global + * monitor for a PE changes from Exclusive Access state to Open Access state, + * an event is generated.". This applies to both AArch32 and AArch64 modes of + * ARMv8-A. As a result, no explicit SEV with unlock is required. + */ +#define COND_SEV() +#else +#define COND_SEV() sev +#endif + +func spin_lock + mov r2, #1 +1: + ldrex r1, [r0] + cmp r1, #0 + wfene + strexeq r1, r2, [r0] + cmpeq r1, #0 + bne 1b + dmb + bx lr +endfunc spin_lock + + +func spin_unlock + mov r1, #0 + stl r1, [r0] + COND_SEV() + bx lr +endfunc spin_unlock diff --git a/lib/locks/exclusive/aarch64/spinlock.S b/lib/locks/exclusive/aarch64/spinlock.S new file mode 100644 index 0000000..5144bf7 --- /dev/null +++ b/lib/locks/exclusive/aarch64/spinlock.S @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2013-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + + .globl spin_lock + .globl spin_unlock + +#if USE_SPINLOCK_CAS +#if !ARM_ARCH_AT_LEAST(8, 1) +#error USE_SPINLOCK_CAS option requires at least an ARMv8.1 platform +#endif + +/* + * When compiled for ARMv8.1 or later, choose spin locks based on Compare and + * Swap instruction. + */ + +/* + * Acquire lock using Compare and Swap instruction. + * + * Compare for 0 with acquire semantics, and swap 1. If failed to acquire, use + * load exclusive semantics to monitor the address and enter WFE. + * + * void spin_lock(spinlock_t *lock); + */ +func spin_lock + mov w2, #1 +1: mov w1, wzr +2: casa w1, w2, [x0] + cbz w1, 3f + ldxr w1, [x0] + cbz w1, 2b + wfe + b 1b +3: + ret +endfunc spin_lock + +#else /* !USE_SPINLOCK_CAS */ + +/* + * Acquire lock using load-/store-exclusive instruction pair. + * + * void spin_lock(spinlock_t *lock); + */ +func spin_lock + mov w2, #1 + sevl +l1: wfe +l2: ldaxr w1, [x0] + cbnz w1, l1 + stxr w1, w2, [x0] + cbnz w1, l2 + ret +endfunc spin_lock + +#endif /* USE_SPINLOCK_CAS */ + +/* + * Release lock previously acquired by spin_lock. + * + * Use store-release to unconditionally clear the spinlock variable. + * Store operation generates an event to all cores waiting in WFE + * when address is monitored by the global monitor. + * + * void spin_unlock(spinlock_t *lock); + */ +func spin_unlock + stlr wzr, [x0] + ret +endfunc spin_unlock diff --git a/lib/mpmm/mpmm.c b/lib/mpmm/mpmm.c new file mode 100644 index 0000000..dc61cf6 --- /dev/null +++ b/lib/mpmm/mpmm.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021-2022, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdbool.h> + +#include <common/debug.h> +#include <lib/mpmm/mpmm.h> + +#include <plat/common/platform.h> + +#if ENABLE_MPMM_FCONF +# include <lib/fconf/fconf.h> +# include <lib/fconf/fconf_mpmm_getter.h> +#endif + +static uint64_t read_cpuppmcr_el3_mpmmpinctl(void) +{ + return (read_cpuppmcr_el3() >> CPUPPMCR_EL3_MPMMPINCTL_SHIFT) & + CPUPPMCR_EL3_MPMMPINCTL_MASK; +} + +static void write_cpumpmmcr_el3_mpmm_en(uint64_t mpmm_en) +{ + uint64_t value = read_cpumpmmcr_el3(); + + value &= ~(CPUMPMMCR_EL3_MPMM_EN_MASK << CPUMPMMCR_EL3_MPMM_EN_SHIFT); + value |= (mpmm_en & CPUMPMMCR_EL3_MPMM_EN_MASK) << + CPUMPMMCR_EL3_MPMM_EN_SHIFT; + + write_cpumpmmcr_el3(value); +} + +static bool mpmm_supported(void) +{ + bool supported = false; + const struct mpmm_topology *topology; + +#if ENABLE_MPMM_FCONF + topology = FCONF_GET_PROPERTY(mpmm, config, topology); +#else + topology = plat_mpmm_topology(); +#endif /* ENABLE_MPMM_FCONF */ + + /* + * For the current core firstly try to find out if the platform + * configuration has claimed support for MPMM, then make sure that MPMM + * is controllable through the system registers. + */ + + if (topology != NULL) { + unsigned int core_pos = plat_my_core_pos(); + + supported = topology->cores[core_pos].supported && + (read_cpuppmcr_el3_mpmmpinctl() == 0U); + } else { + ERROR("MPMM: failed to generate MPMM topology\n"); + } + + return supported; +} + +/* Defaults to false */ +static bool mpmm_disable_for_errata; + +void mpmm_enable(void) +{ + if (mpmm_supported()) { + if (mpmm_disable_for_errata) { + WARN("MPMM: disabled by errata workaround\n"); + return; + } + write_cpumpmmcr_el3_mpmm_en(1U); + } +} + +/* + * This function is called from assembly code very early in BL31 so it must be + * small and simple. + */ +void mpmm_errata_disable(void) +{ + mpmm_disable_for_errata = true; +} diff --git a/lib/mpmm/mpmm.mk b/lib/mpmm/mpmm.mk new file mode 100644 index 0000000..826f925 --- /dev/null +++ b/lib/mpmm/mpmm.mk @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +include lib/extensions/amu/amu.mk +include lib/fconf/fconf.mk + +ifneq (${ENABLE_MPMM},0) + ifneq ($(ARCH),aarch64) + $(error MPMM support (`ENABLE_MPMM`) can only be enabled in AArch64 images (`ARCH`)) + endif + + ifeq (${ENABLE_AMU_AUXILIARY_COUNTERS},0) # For MPMM gear AMU counters + $(error MPMM support (`ENABLE_MPM`) requires auxiliary AMU counter support (`ENABLE_AMU_AUXILIARY_COUNTERS`)) + endif +endif + +MPMM_SOURCES := lib/mpmm/mpmm.c +MPMM_SOURCES += ${AMU_SOURCES} + +ifneq (${ENABLE_MPMM_FCONF},0) + ifeq (${ENABLE_MPMM},0) + $(error MPMM FCONF support (`ENABLE_MPMM_FCONF`) requires MPMM support (`ENABLE_MPMM`)) + endif + + MPMM_SOURCES += ${FCONF_MPMM_SOURCES} +endif diff --git a/lib/optee/optee_utils.c b/lib/optee/optee_utils.c new file mode 100644 index 0000000..14f5533 --- /dev/null +++ b/lib/optee/optee_utils.c @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> + +#include <common/debug.h> +#include <lib/optee_utils.h> + +#include <platform_def.h> + +/******************************************************************************* + * Check if it is a valid tee header + * Return true if valid + * Return false if invalid + ******************************************************************************/ +static bool tee_validate_header(optee_header_t *header) +{ + if ((header->magic == TEE_MAGIC_NUM_OPTEE) && + (header->version == 2u) && + (header->nb_images > 0u) && + (header->nb_images <= OPTEE_MAX_NUM_IMAGES)) { + return true; + } + + return false; +} + +bool optee_header_is_valid(uintptr_t header_base) +{ + return tee_validate_header((optee_header_t *)header_base); +} + +/******************************************************************************* + * Parse the OPTEE image + * Return 0 on success or a negative error code otherwise. + ******************************************************************************/ +static int parse_optee_image(image_info_t *image_info, + optee_image_t *image) +{ + uintptr_t init_load_addr, free_end, requested_end; + size_t init_size; + + init_load_addr = ((uint64_t)image->load_addr_hi << 32) | + image->load_addr_lo; + init_size = image->size; + + /* + * image->load_addr_hi & image->load_addr_lo set to UINT32_MAX indicate + * loader decided address; take our pre-mapped area for current image + * since arm-tf could not allocate memory dynamically + */ + if ((image->load_addr_hi == UINT32_MAX) && + (image->load_addr_lo == UINT32_MAX)) { + init_load_addr = image_info->image_base; + } + + /* Check that the default end address doesn't overflow */ + if (check_uptr_overflow(image_info->image_base, + image_info->image_max_size - 1)) + return -1; + free_end = image_info->image_base + (image_info->image_max_size - 1); + + /* Check that the image end address doesn't overflow */ + if (check_uptr_overflow(init_load_addr, init_size - 1)) + return -1; + requested_end = init_load_addr + (init_size - 1); + /* + * Check that the requested RAM location is within reserved + * space for OPTEE. + */ + if (!((init_load_addr >= image_info->image_base) && + (requested_end <= free_end))) { + WARN("The load address in optee header %p - %p is not in reserved area: %p - %p.\n", + (void *)init_load_addr, + (void *)(init_load_addr + init_size), + (void *)image_info->image_base, + (void *)(image_info->image_base + + image_info->image_max_size)); + return -1; + } + + /* + * Remove the skip attr from image_info, the image will be loaded. + * The default attr in image_info is "IMAGE_ATTRIB_SKIP_LOADING", which + * mean the image will not be loaded. Here, we parse the header image to + * know that the extra image need to be loaded, so remove the skip attr. + */ + image_info->h.attr &= ~IMAGE_ATTRIB_SKIP_LOADING; + + /* Update image base and size of image_info */ + image_info->image_base = init_load_addr; + image_info->image_size = init_size; + + return 0; +} + +/******************************************************************************* + * Parse the OPTEE header + * Return 0 on success or a negative error code otherwise. + ******************************************************************************/ +int parse_optee_header(entry_point_info_t *header_ep, + image_info_t *pager_image_info, + image_info_t *paged_image_info) + +{ + optee_header_t *header; + uint32_t num; + int ret; + + assert(header_ep); + header = (optee_header_t *)header_ep->pc; + assert(header); + + /* Print the OPTEE header information */ + INFO("OPTEE ep=0x%x\n", (unsigned int)header_ep->pc); + INFO("OPTEE header info:\n"); + INFO(" magic=0x%x\n", header->magic); + INFO(" version=0x%x\n", header->version); + INFO(" arch=0x%x\n", header->arch); + INFO(" flags=0x%x\n", header->flags); + INFO(" nb_images=0x%x\n", header->nb_images); + + /* + * OPTEE image has 3 types: + * + * 1. Plain OPTEE bin without header. + * Original bin without header, return directly, + * BL32_EXTRA1_IMAGE_ID and BL32_EXTRA2_IMAGE_ID will be skipped. + * + * 2. OPTEE bin with header bin, but no paging. + * Header available and nb_images = 1, remove skip attr for + * BL32_EXTRA1_IMAGE_ID. BL32_EXTRA1_IMAGE_ID will be loaded, + * and BL32_EXTRA2_IMAGE_ID be skipped. + * + * 3. OPTEE image with paging support. + * Header available and nb_images = 2, there are 3 bins: header, + * pager and pageable. Remove skip attr for BL32_EXTRA1_IMAGE_ID + * and BL32_EXTRA2_IMAGE_ID to load pager and paged bin. + */ + if (!tee_validate_header(header)) { + INFO("Invalid OPTEE header, set legacy mode.\n"); +#ifdef __aarch64__ + header_ep->args.arg0 = MODE_RW_64; +#else + header_ep->args.arg0 = MODE_RW_32; +#endif + return 0; + } + + /* Parse OPTEE image */ + for (num = 0U; num < header->nb_images; num++) { + if (header->optee_image_list[num].image_id == + OPTEE_PAGER_IMAGE_ID) { + ret = parse_optee_image(pager_image_info, + &header->optee_image_list[num]); + } else if (header->optee_image_list[num].image_id == + OPTEE_PAGED_IMAGE_ID) { + if (paged_image_info == NULL) { + if (header->optee_image_list[num].size != 0U) { + ERROR("Paged image is not supported\n"); + return -1; + } + + continue; + } else { + ret = parse_optee_image(paged_image_info, + &header->optee_image_list[num]); + } + } else { + ERROR("Parse optee image failed.\n"); + return -1; + } + + if (ret != 0) + return -1; + } + + /* + * Update "pc" value which should comes from pager image. After the + * header image is parsed, it will be useless, and the actual + * execution image after BL31 is pager image. + */ + header_ep->pc = pager_image_info->image_base; + + /* + * The paged load address and size are populated in + * header image arguments so that can be read by the + * BL32 SPD. + */ + if (paged_image_info != NULL) { + header_ep->args.arg1 = paged_image_info->image_base; + header_ep->args.arg2 = paged_image_info->image_size; + } + + /* Set OPTEE runtime arch - aarch32/aarch64 */ + if (header->arch == 0) { + header_ep->args.arg0 = MODE_RW_32; + } else { +#ifdef __aarch64__ + header_ep->args.arg0 = MODE_RW_64; +#else + ERROR("Cannot boot an AArch64 OP-TEE\n"); + return -1; +#endif + } + + return 0; +} diff --git a/lib/pmf/pmf_main.c b/lib/pmf/pmf_main.c new file mode 100644 index 0000000..b33f49c --- /dev/null +++ b/lib/pmf/pmf_main.c @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2016-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/pmf/pmf.h> +#include <lib/utils_def.h> +#include <plat/common/platform.h> + +/******************************************************************************* + * The 'pmf_svc_descs' array holds the PMF service descriptors exported by + * services by placing them in the '.pmf_svc_descs' linker section. + * The 'pmf_svc_descs_indices' array holds the index of a descriptor in the + * 'pmf_svc_descs' array. The TIF[15:10] bits in the time-stamp id are used + * to get an index into the 'pmf_svc_descs_indices' array. This gives the + * index of the descriptor in the 'pmf_svc_descs' array which contains the + * service function pointers. + ******************************************************************************/ + +IMPORT_SYM(uintptr_t, __PMF_SVC_DESCS_START__, PMF_SVC_DESCS_START); +IMPORT_SYM(uintptr_t, __PMF_SVC_DESCS_END__, PMF_SVC_DESCS_END); +IMPORT_SYM(uintptr_t, __PMF_PERCPU_TIMESTAMP_END__, PMF_PERCPU_TIMESTAMP_END); +IMPORT_SYM(uintptr_t, __PMF_TIMESTAMP_START__, PMF_TIMESTAMP_ARRAY_START); + +#define PMF_PERCPU_TIMESTAMP_SIZE (PMF_PERCPU_TIMESTAMP_END - PMF_TIMESTAMP_ARRAY_START) + +#define PMF_SVC_DESCS_MAX 10 + +/* + * This is used to traverse through registered PMF services. + */ +static pmf_svc_desc_t *pmf_svc_descs; + +/* + * This array is used to store registered PMF services in sorted order. + */ +static int pmf_svc_descs_indices[PMF_SVC_DESCS_MAX]; + +/* + * This is used to track total number of successfully registered PMF services. + */ +static int pmf_num_services; + +/* + * This is the main PMF function that initialize registered + * PMF services and also sort them in ascending order. + */ +int pmf_setup(void) +{ + int rc, ii, jj = 0; + int pmf_svc_descs_num, temp_val; + + /* If no PMF services are registered then simply bail out */ + pmf_svc_descs_num = (PMF_SVC_DESCS_END - PMF_SVC_DESCS_START)/ + sizeof(pmf_svc_desc_t); + if (pmf_svc_descs_num == 0) + return 0; + + assert(pmf_svc_descs_num < PMF_SVC_DESCS_MAX); + + pmf_svc_descs = (pmf_svc_desc_t *) PMF_SVC_DESCS_START; + for (ii = 0; ii < pmf_svc_descs_num; ii++) { + + assert(pmf_svc_descs[ii].get_ts != NULL); + + /* + * Call the initialization routine for this + * PMF service, if it is defined. + */ + if (pmf_svc_descs[ii].init != NULL) { + rc = pmf_svc_descs[ii].init(); + if (rc != 0) { + WARN("Could not initialize PMF" + "service %s - skipping \n", + pmf_svc_descs[ii].name); + continue; + } + } + + /* Update the pmf_svc_descs_indices array */ + pmf_svc_descs_indices[jj++] = ii; + } + + pmf_num_services = jj; + + /* + * Sort the successfully registered PMF services + * according to service ID + */ + for (ii = 1; ii < pmf_num_services; ii++) { + for (jj = 0; jj < (pmf_num_services - ii); jj++) { + if ((pmf_svc_descs[jj].svc_config & PMF_SVC_ID_MASK) > + (pmf_svc_descs[jj + 1].svc_config & + PMF_SVC_ID_MASK)) { + temp_val = pmf_svc_descs_indices[jj]; + pmf_svc_descs_indices[jj] = + pmf_svc_descs_indices[jj+1]; + pmf_svc_descs_indices[jj+1] = temp_val; + } + } + } + + return 0; +} + +/* + * This function implements binary search to find registered + * PMF service based on Service ID provided in `tid` argument. + */ +static pmf_svc_desc_t *get_service(unsigned int tid) +{ + int low = 0; + int mid; + int high = pmf_num_services; + unsigned int svc_id = tid & PMF_SVC_ID_MASK; + int index; + unsigned int desc_svc_id; + + if (pmf_num_services == 0) + return NULL; + + assert(pmf_svc_descs != NULL); + + do { + mid = (low + high) / 2; + index = pmf_svc_descs_indices[mid]; + + desc_svc_id = pmf_svc_descs[index].svc_config & PMF_SVC_ID_MASK; + if (svc_id < desc_svc_id) + high = mid - 1; + if (svc_id > desc_svc_id) + low = mid + 1; + } while ((svc_id != desc_svc_id) && (low <= high)); + + /* + * Make sure the Service found supports the tid range. + */ + if ((svc_id == desc_svc_id) && ((tid & PMF_TID_MASK) < + (pmf_svc_descs[index].svc_config & PMF_TID_MASK))) + return (pmf_svc_desc_t *)&pmf_svc_descs[index]; + + return NULL; +} + +/* + * This function gets the time-stamp value for the PMF services + * registered for SMC interface based on `tid` and `mpidr`. + */ +int pmf_get_timestamp_smc(unsigned int tid, + u_register_t mpidr, + unsigned int flags, + unsigned long long *ts_value) +{ + pmf_svc_desc_t *svc_desc; + assert(ts_value != NULL); + + /* Search for registered service. */ + svc_desc = get_service(tid); + + if (svc_desc == NULL) { + *ts_value = 0; + return -EINVAL; + } else { + /* Call the service time-stamp handler. */ + *ts_value = svc_desc->get_ts(tid, mpidr, flags); + return 0; + } +} + +/* + * This function can be used to dump `ts` value for given `tid`. + * Assumption is that the console is already initialized. + */ +void __pmf_dump_timestamp(unsigned int tid, unsigned long long ts) +{ + printf("PMF:cpu %u tid %u ts %llu\n", + plat_my_core_pos(), tid, ts); +} + +/* + * This function calculate the address identified by + * `base_addr`, `tid` and `cpuid`. + */ +static inline uintptr_t calc_ts_addr(uintptr_t base_addr, + unsigned int tid, + unsigned int cpuid) +{ + assert(cpuid < PLATFORM_CORE_COUNT); + assert(base_addr >= PMF_TIMESTAMP_ARRAY_START); + assert(base_addr < ((PMF_TIMESTAMP_ARRAY_START + + PMF_PERCPU_TIMESTAMP_SIZE) - ((tid & PMF_TID_MASK) * + sizeof(unsigned long long)))); + + base_addr += ((cpuid * PMF_PERCPU_TIMESTAMP_SIZE) + + ((tid & PMF_TID_MASK) * sizeof(unsigned long long))); + + return base_addr; +} + +/* + * This function stores the `ts` value to the storage identified by + * `base_addr`, `tid` and current cpu id. + * Note: The timestamp addresses are cache line aligned per cpu + * and only the owning CPU would ever write into it. + */ +void __pmf_store_timestamp(uintptr_t base_addr, + unsigned int tid, + unsigned long long ts) +{ + unsigned long long *ts_addr = (unsigned long long *)calc_ts_addr(base_addr, + tid, plat_my_core_pos()); + *ts_addr = ts; +} + +/* + * This is the cached version of `pmf_store_my_timestamp` + * Note: The timestamp addresses are cache line aligned per cpu + * and only the owning CPU would ever write into it. + */ +void __pmf_store_timestamp_with_cache_maint(uintptr_t base_addr, + unsigned int tid, + unsigned long long ts) +{ + unsigned long long *ts_addr = (unsigned long long *)calc_ts_addr(base_addr, + tid, plat_my_core_pos()); + *ts_addr = ts; + flush_dcache_range((uintptr_t)ts_addr, sizeof(unsigned long long)); +} + +/* + * This function retrieves the `ts` value from the storage identified by + * `base_addr`, `tid` and `cpuid`. + * Note: The timestamp addresses are cache line aligned per cpu. + */ +unsigned long long __pmf_get_timestamp(uintptr_t base_addr, + unsigned int tid, + unsigned int cpuid, + unsigned int flags) +{ + assert(cpuid < PLATFORM_CORE_COUNT); + unsigned long long *ts_addr = (unsigned long long *)calc_ts_addr(base_addr, + tid, cpuid); + + if ((flags & PMF_CACHE_MAINT) != 0U) + inv_dcache_range((uintptr_t)ts_addr, sizeof(unsigned long long)); + + return *ts_addr; +} diff --git a/lib/pmf/pmf_smc.c b/lib/pmf/pmf_smc.c new file mode 100644 index 0000000..f3dd112 --- /dev/null +++ b/lib/pmf/pmf_smc.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> + +#include <common/debug.h> +#include <lib/pmf/pmf.h> +#include <plat/common/platform.h> +#include <smccc_helpers.h> + +/* + * This function is responsible for handling all PMF SMC calls. + */ +uintptr_t pmf_smc_handler(unsigned int smc_fid, + u_register_t x1, + u_register_t x2, + u_register_t x3, + u_register_t x4, + void *cookie, + void *handle, + u_register_t flags) +{ + int rc; + unsigned long long ts_value; + + /* Determine if the cpu exists of not */ + if (!is_valid_mpidr(x2)) + return PSCI_E_INVALID_PARAMS; + + if (((smc_fid >> FUNCID_CC_SHIFT) & FUNCID_CC_MASK) == SMC_32) { + + x1 = (uint32_t)x1; + x2 = (uint32_t)x2; + x3 = (uint32_t)x3; + + if (smc_fid == PMF_SMC_GET_TIMESTAMP_32) { + /* + * Return error code and the captured + * time-stamp to the caller. + * x0 --> error code. + * x1 - x2 --> time-stamp value. + */ + rc = pmf_get_timestamp_smc((unsigned int)x1, x2, + (unsigned int)x3, &ts_value); + SMC_RET3(handle, rc, (uint32_t)ts_value, + (uint32_t)(ts_value >> 32)); + } + } else { + if (smc_fid == PMF_SMC_GET_TIMESTAMP_64) { + /* + * Return error code and the captured + * time-stamp to the caller. + * x0 --> error code. + * x1 --> time-stamp value. + */ + rc = pmf_get_timestamp_smc((unsigned int)x1, x2, + (unsigned int)x3, &ts_value); + SMC_RET2(handle, rc, ts_value); + } + } + + WARN("Unimplemented PMF Call: 0x%x \n", smc_fid); + SMC_RET1(handle, SMC_UNK); +} diff --git a/lib/psa/delegated_attestation.c b/lib/psa/delegated_attestation.c new file mode 100644 index 0000000..81e2621 --- /dev/null +++ b/lib/psa/delegated_attestation.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022-2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + * + */ + +#include <delegated_attestation.h> +#include <psa/client.h> +#include <psa_manifest/sid.h> + +psa_status_t +rss_delegated_attest_get_delegated_key(uint8_t ecc_curve, + uint32_t key_bits, + uint8_t *key_buf, + size_t key_buf_size, + size_t *key_size, + uint32_t hash_algo) +{ + psa_status_t status; + psa_invec in_vec[] = { + {&ecc_curve, sizeof(ecc_curve)}, + {&key_bits, sizeof(key_bits)}, + {&hash_algo, sizeof(hash_algo)} + }; + psa_outvec out_vec[] = { + {key_buf, key_buf_size} + }; + + if (key_size == NULL) { + return PSA_ERROR_INVALID_ARGUMENT; + } + + status = psa_call(RSS_DELEGATED_SERVICE_HANDLE, + RSS_DELEGATED_ATTEST_GET_DELEGATED_KEY, + in_vec, IOVEC_LEN(in_vec), + out_vec, IOVEC_LEN(out_vec)); + if (status == PSA_SUCCESS) { + *key_size = out_vec[0].len; + } + + return status; +} + +psa_status_t +rss_delegated_attest_get_token(const uint8_t *dak_pub_hash, + size_t dak_pub_hash_size, + uint8_t *token_buf, + size_t token_buf_size, + size_t *token_size) +{ + psa_status_t status; + psa_invec in_vec[] = { + {dak_pub_hash, dak_pub_hash_size} + }; + psa_outvec out_vec[] = { + {token_buf, token_buf_size} + }; + + if (token_size == NULL) { + return PSA_ERROR_INVALID_ARGUMENT; + } + + status = psa_call(RSS_DELEGATED_SERVICE_HANDLE, + RSS_DELEGATED_ATTEST_GET_PLATFORM_TOKEN, + in_vec, IOVEC_LEN(in_vec), + out_vec, IOVEC_LEN(out_vec)); + if (status == PSA_SUCCESS) { + *token_size = out_vec[0].len; + } + + return status; +} diff --git a/lib/psa/measured_boot.c b/lib/psa/measured_boot.c new file mode 100644 index 0000000..c359e9f --- /dev/null +++ b/lib/psa/measured_boot.c @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2022, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + * + */ + +#include <string.h> + +#include <common/debug.h> +#include <measured_boot.h> +#include <psa/client.h> +#include <psa_manifest/sid.h> + +#include "measured_boot_private.h" + +static void print_byte_array(const uint8_t *array __unused, size_t len __unused) +{ +#if LOG_LEVEL >= LOG_LEVEL_INFO + size_t i; + + if (array == NULL || len == 0U) { + (void)printf("\n"); + } else { + for (i = 0U; i < len; ++i) { + (void)printf(" %02x", array[i]); + if ((i & U(0xF)) == U(0xF)) { + (void)printf("\n"); + if (i < (len - 1U)) { + INFO("\t\t:"); + } + } + } + } +#endif +} + +static void log_measurement(uint8_t index, + const uint8_t *signer_id, + size_t signer_id_size, + const uint8_t *version, /* string */ + size_t version_size, + const uint8_t *sw_type, /* string */ + size_t sw_type_size, + uint32_t measurement_algo, + const uint8_t *measurement_value, + size_t measurement_value_size, + bool lock_measurement) +{ + INFO("Measured boot extend measurement:\n"); + INFO(" - slot : %u\n", index); + INFO(" - signer_id :"); + print_byte_array(signer_id, signer_id_size); + INFO(" - version : %s\n", version); + INFO(" - version_size: %zu\n", version_size); + INFO(" - sw_type : %s\n", sw_type); + INFO(" - sw_type_size: %zu\n", sw_type_size); + INFO(" - algorithm : %x\n", measurement_algo); + INFO(" - measurement :"); + print_byte_array(measurement_value, measurement_value_size); + INFO(" - locking : %s\n", lock_measurement ? "true" : "false"); +} + +#if !PLAT_RSS_NOT_SUPPORTED +psa_status_t +rss_measured_boot_extend_measurement(uint8_t index, + const uint8_t *signer_id, + size_t signer_id_size, + const uint8_t *version, + size_t version_size, + uint32_t measurement_algo, + const uint8_t *sw_type, + size_t sw_type_size, + const uint8_t *measurement_value, + size_t measurement_value_size, + bool lock_measurement) +{ + struct measured_boot_extend_iovec_t extend_iov = { + .index = index, + .lock_measurement = lock_measurement, + .measurement_algo = measurement_algo, + .sw_type = {0}, + .sw_type_size = sw_type_size, + }; + + if (version_size > VERSION_MAX_SIZE) { + return PSA_ERROR_INVALID_ARGUMENT; + } + + + if (version_size > 0 && version[version_size - 1] == '\0') { + version_size--; + } + + psa_invec in_vec[] = { + {.base = &extend_iov, + .len = sizeof(struct measured_boot_extend_iovec_t)}, + {.base = signer_id, .len = signer_id_size}, + {.base = version, .len = version_size }, + {.base = measurement_value, .len = measurement_value_size} + }; + + if (sw_type != NULL) { + if (extend_iov.sw_type_size > SW_TYPE_MAX_SIZE) { + return PSA_ERROR_INVALID_ARGUMENT; + } + if (sw_type_size > 0 && sw_type[sw_type_size - 1] == '\0') { + extend_iov.sw_type_size--; + } + memcpy(extend_iov.sw_type, sw_type, extend_iov.sw_type_size); + } + + log_measurement(index, signer_id, signer_id_size, + version, version_size, sw_type, sw_type_size, + measurement_algo, measurement_value, + measurement_value_size, lock_measurement); + + return psa_call(RSS_MEASURED_BOOT_HANDLE, + RSS_MEASURED_BOOT_EXTEND, + in_vec, IOVEC_LEN(in_vec), + NULL, 0); +} + +psa_status_t rss_measured_boot_read_measurement(uint8_t index, + uint8_t *signer_id, + size_t signer_id_size, + size_t *signer_id_len, + uint8_t *version, + size_t version_size, + size_t *version_len, + uint32_t *measurement_algo, + uint8_t *sw_type, + size_t sw_type_size, + size_t *sw_type_len, + uint8_t *measurement_value, + size_t measurement_value_size, + size_t *measurement_value_len, + bool *is_locked) +{ + psa_status_t status; + struct measured_boot_read_iovec_in_t read_iov_in = { + .index = index, + .sw_type_size = sw_type_size, + .version_size = version_size, + }; + + struct measured_boot_read_iovec_out_t read_iov_out; + + psa_invec in_vec[] = { + {.base = &read_iov_in, + .len = sizeof(struct measured_boot_read_iovec_in_t)}, + }; + + psa_outvec out_vec[] = { + {.base = &read_iov_out, + .len = sizeof(struct measured_boot_read_iovec_out_t)}, + {.base = signer_id, .len = signer_id_size}, + {.base = measurement_value, .len = measurement_value_size} + }; + + status = psa_call(RSS_MEASURED_BOOT_HANDLE, RSS_MEASURED_BOOT_READ, + in_vec, IOVEC_LEN(in_vec), + out_vec, IOVEC_LEN(out_vec)); + + if (status == PSA_SUCCESS) { + *is_locked = read_iov_out.is_locked; + *measurement_algo = read_iov_out.measurement_algo; + *sw_type_len = read_iov_out.sw_type_len; + *version_len = read_iov_out.version_len; + memcpy(sw_type, read_iov_out.sw_type, read_iov_out.sw_type_len); + memcpy(version, read_iov_out.version, read_iov_out.version_len); + *signer_id_len = out_vec[1].len; + *measurement_value_len = out_vec[2].len; + } + + return status; +} + +#else /* !PLAT_RSS_NOT_SUPPORTED */ + +psa_status_t +rss_measured_boot_extend_measurement(uint8_t index, + const uint8_t *signer_id, + size_t signer_id_size, + const uint8_t *version, + size_t version_size, + uint32_t measurement_algo, + const uint8_t *sw_type, + size_t sw_type_size, + const uint8_t *measurement_value, + size_t measurement_value_size, + bool lock_measurement) +{ + log_measurement(index, signer_id, signer_id_size, + version, version_size, sw_type, sw_type_size, + measurement_algo, measurement_value, + measurement_value_size, lock_measurement); + + return PSA_SUCCESS; +} + +psa_status_t rss_measured_boot_read_measurement(uint8_t index, + uint8_t *signer_id, + size_t signer_id_size, + size_t *signer_id_len, + uint8_t *version, + size_t version_size, + size_t *version_len, + uint32_t *measurement_algo, + uint8_t *sw_type, + size_t sw_type_size, + size_t *sw_type_len, + uint8_t *measurement_value, + size_t measurement_value_size, + size_t *measurement_value_len, + bool *is_locked) +{ + return PSA_SUCCESS; +} + +#endif /* !PLAT_RSS_NOT_SUPPORTED */ diff --git a/lib/psa/measured_boot_private.h b/lib/psa/measured_boot_private.h new file mode 100644 index 0000000..80d2c19 --- /dev/null +++ b/lib/psa/measured_boot_private.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + * + */ + +#ifndef PSA_MEASURED_BOOT_PRIVATE_H +#define PSA_MEASURED_BOOT_PRIVATE_H + +#include <stdint.h> + +/* Measured boot message types that distinguish its services */ +#define RSS_MEASURED_BOOT_READ 1001U +#define RSS_MEASURED_BOOT_EXTEND 1002U + +struct measured_boot_read_iovec_in_t { + uint8_t index; + uint8_t sw_type_size; + uint8_t version_size; +}; + +struct measured_boot_read_iovec_out_t { + uint8_t is_locked; + uint32_t measurement_algo; + uint8_t sw_type[SW_TYPE_MAX_SIZE]; + uint8_t sw_type_len; + uint8_t version[VERSION_MAX_SIZE]; + uint8_t version_len; +}; + +struct measured_boot_extend_iovec_t { + uint8_t index; + uint8_t lock_measurement; + uint32_t measurement_algo; + uint8_t sw_type[SW_TYPE_MAX_SIZE]; + uint8_t sw_type_size; +}; + +#endif /* PSA_MEASURED_BOOT_PRIVATE_H */ diff --git a/lib/psa/rss_platform.c b/lib/psa/rss_platform.c new file mode 100644 index 0000000..7d90bfc --- /dev/null +++ b/lib/psa/rss_platform.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + * + */ + +#include <psa/client.h> +#include <psa_manifest/sid.h> +#include <rss_crypto_defs.h> +#include <rss_platform_api.h> + +psa_status_t +rss_platform_nv_counter_increment(uint32_t counter_id) +{ + struct psa_invec in_vec[1]; + + in_vec[0].base = &counter_id; + in_vec[0].len = sizeof(counter_id); + + return psa_call(RSS_PLATFORM_SERVICE_HANDLE, + RSS_PLATFORM_API_ID_NV_INCREMENT, + in_vec, 1, NULL, 0); +} + +psa_status_t +rss_platform_nv_counter_read(uint32_t counter_id, + uint32_t size, uint8_t *val) +{ + struct psa_invec in_vec[1]; + struct psa_outvec out_vec[1]; + + in_vec[0].base = &counter_id; + in_vec[0].len = sizeof(counter_id); + + out_vec[0].base = val; + out_vec[0].len = size; + + return psa_call(RSS_PLATFORM_SERVICE_HANDLE, + RSS_PLATFORM_API_ID_NV_READ, + in_vec, 1, out_vec, 1); +} + +psa_status_t +rss_platform_key_read(enum rss_key_id_builtin_t key, uint8_t *data, + size_t data_size, size_t *data_length) +{ + psa_status_t status; + + struct rss_crypto_pack_iovec iov = { + .function_id = RSS_CRYPTO_EXPORT_PUBLIC_KEY_SID, + .key_id = key, + }; + + psa_invec in_vec[] = { + {.base = &iov, .len = sizeof(struct rss_crypto_pack_iovec)}, + }; + psa_outvec out_vec[] = { + {.base = data, .len = data_size} + }; + + status = psa_call(RSS_CRYPTO_HANDLE, PSA_IPC_CALL, + in_vec, IOVEC_LEN(in_vec), + out_vec, IOVEC_LEN(out_vec)); + + *data_length = out_vec[0].len; + + return status; +} diff --git a/lib/psci/aarch32/psci_helpers.S b/lib/psci/aarch32/psci_helpers.S new file mode 100644 index 0000000..4e1013c --- /dev/null +++ b/lib/psci/aarch32/psci_helpers.S @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <lib/psci/psci.h> +#include <platform_def.h> + + .globl psci_do_pwrdown_cache_maintenance + .globl psci_do_pwrup_cache_maintenance + .globl psci_power_down_wfi + +/* ----------------------------------------------------------------------- + * void psci_do_pwrdown_cache_maintenance(unsigned int power level); + * + * This function performs cache maintenance for the specified power + * level. The levels of cache affected are determined by the power + * level which is passed as the argument i.e. level 0 results + * in a flush of the L1 cache. Both the L1 and L2 caches are flushed + * for a higher power level. + * + * Additionally, this function also ensures that stack memory is correctly + * flushed out to avoid coherency issues due to a change in its memory + * attributes after the data cache is disabled. + * ----------------------------------------------------------------------- + */ +func psci_do_pwrdown_cache_maintenance + push {r4, lr} + + /* ---------------------------------------------- + * Turn OFF cache and do stack maintenance + * prior to cpu operations . This sequence is + * different from AArch64 because in AArch32 the + * assembler routines for cpu operations utilize + * the stack whereas in AArch64 it doesn't. + * ---------------------------------------------- + */ + mov r4, r0 + bl do_stack_maintenance + + /* --------------------------------------------- + * Invoke CPU-specifc power down operations for + * the appropriate level + * --------------------------------------------- + */ + mov r0, r4 + pop {r4, lr} + b prepare_cpu_pwr_dwn +endfunc psci_do_pwrdown_cache_maintenance + + +/* ----------------------------------------------------------------------- + * void psci_do_pwrup_cache_maintenance(void); + * + * This function performs cache maintenance after this cpu is powered up. + * Currently, this involves managing the used stack memory before turning + * on the data cache. + * ----------------------------------------------------------------------- + */ +func psci_do_pwrup_cache_maintenance + /* r12 is pushed to meet the 8 byte stack alignment requirement */ + push {r12, lr} + + /* --------------------------------------------- + * Ensure any inflight stack writes have made it + * to main memory. + * --------------------------------------------- + */ + dmb st + + /* --------------------------------------------- + * Calculate and store the size of the used + * stack memory in r1. Calculate and store the + * stack base address in r0. + * --------------------------------------------- + */ + bl plat_get_my_stack + mov r1, sp + sub r1, r0, r1 + mov r0, sp + bl inv_dcache_range + + /* --------------------------------------------- + * Enable the data cache. + * --------------------------------------------- + */ + ldcopr r0, SCTLR + orr r0, r0, #SCTLR_C_BIT + stcopr r0, SCTLR + isb + + pop {r12, pc} +endfunc psci_do_pwrup_cache_maintenance + + /* --------------------------------------------- + * void do_stack_maintenance(void) + * Do stack maintenance by flushing the used + * stack to the main memory and invalidating the + * remainder. + * --------------------------------------------- + */ +func do_stack_maintenance + push {r4, lr} + bl plat_get_my_stack + + /* Turn off the D-cache */ + ldcopr r1, SCTLR + bic r1, #SCTLR_C_BIT + stcopr r1, SCTLR + isb + + /* --------------------------------------------- + * Calculate and store the size of the used + * stack memory in r1. + * --------------------------------------------- + */ + mov r4, r0 + mov r1, sp + sub r1, r0, r1 + mov r0, sp + bl flush_dcache_range + + /* --------------------------------------------- + * Calculate and store the size of the unused + * stack memory in r1. Calculate and store the + * stack base address in r0. + * --------------------------------------------- + */ + sub r0, r4, #PLATFORM_STACK_SIZE + sub r1, sp, r0 + bl inv_dcache_range + + pop {r4, pc} +endfunc do_stack_maintenance + +/* ----------------------------------------------------------------------- + * This function is called to indicate to the power controller that it + * is safe to power down this cpu. It should not exit the wfi and will + * be released from reset upon power up. + * ----------------------------------------------------------------------- + */ +func psci_power_down_wfi + dsb sy // ensure write buffer empty +1: + wfi + b 1b +endfunc psci_power_down_wfi diff --git a/lib/psci/aarch64/psci_helpers.S b/lib/psci/aarch64/psci_helpers.S new file mode 100644 index 0000000..3b77ab2 --- /dev/null +++ b/lib/psci/aarch64/psci_helpers.S @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2014-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <assert_macros.S> +#include <lib/psci/psci.h> +#include <platform_def.h> + + .globl psci_do_pwrdown_cache_maintenance + .globl psci_do_pwrup_cache_maintenance + .globl psci_power_down_wfi + +/* ----------------------------------------------------------------------- + * void psci_do_pwrdown_cache_maintenance(unsigned int power level); + * + * This function performs cache maintenance for the specified power + * level. The levels of cache affected are determined by the power + * level which is passed as the argument i.e. level 0 results + * in a flush of the L1 cache. Both the L1 and L2 caches are flushed + * for a higher power level. + * + * Additionally, this function also ensures that stack memory is correctly + * flushed out to avoid coherency issues due to a change in its memory + * attributes after the data cache is disabled. + * ----------------------------------------------------------------------- + */ +func psci_do_pwrdown_cache_maintenance + stp x29, x30, [sp,#-16]! + stp x19, x20, [sp,#-16]! + + /* --------------------------------------------- + * Invoke CPU-specific power down operations for + * the appropriate level + * --------------------------------------------- + */ + bl prepare_cpu_pwr_dwn + + /* --------------------------------------------- + * Do stack maintenance by flushing the used + * stack to the main memory and invalidating the + * remainder. + * --------------------------------------------- + */ + bl plat_get_my_stack + + /* --------------------------------------------- + * Calculate and store the size of the used + * stack memory in x1. + * --------------------------------------------- + */ + mov x19, x0 + mov x1, sp + sub x1, x0, x1 + mov x0, sp + bl flush_dcache_range + + /* --------------------------------------------- + * Calculate and store the size of the unused + * stack memory in x1. Calculate and store the + * stack base address in x0. + * --------------------------------------------- + */ + sub x0, x19, #PLATFORM_STACK_SIZE + sub x1, sp, x0 + bl inv_dcache_range + + ldp x19, x20, [sp], #16 + ldp x29, x30, [sp], #16 + ret +endfunc psci_do_pwrdown_cache_maintenance + + +/* ----------------------------------------------------------------------- + * void psci_do_pwrup_cache_maintenance(void); + * + * This function performs cache maintenance after this cpu is powered up. + * Currently, this involves managing the used stack memory before turning + * on the data cache. + * ----------------------------------------------------------------------- + */ +func psci_do_pwrup_cache_maintenance + stp x29, x30, [sp,#-16]! + + /* --------------------------------------------- + * Ensure any inflight stack writes have made it + * to main memory. + * --------------------------------------------- + */ + dmb st + + /* --------------------------------------------- + * Calculate and store the size of the used + * stack memory in x1. Calculate and store the + * stack base address in x0. + * --------------------------------------------- + */ + bl plat_get_my_stack + mov x1, sp + sub x1, x0, x1 + mov x0, sp + bl inv_dcache_range + + /* --------------------------------------------- + * Enable the data cache. + * --------------------------------------------- + */ + mrs x0, sctlr_el3 + orr x0, x0, #SCTLR_C_BIT + msr sctlr_el3, x0 + isb + + ldp x29, x30, [sp], #16 + ret +endfunc psci_do_pwrup_cache_maintenance + +/* ----------------------------------------------------------------------- + * void psci_power_down_wfi(void); + * This function is called to indicate to the power controller that it + * is safe to power down this cpu. It should not exit the wfi and will + * be released from reset upon power up. + * ----------------------------------------------------------------------- + */ +func psci_power_down_wfi +#if ERRATA_A510_2684597 + bl apply_cpu_pwr_dwn_errata +#endif + dsb sy // ensure write buffer empty +1: + wfi + b 1b +endfunc psci_power_down_wfi diff --git a/lib/psci/aarch64/runtime_errata.S b/lib/psci/aarch64/runtime_errata.S new file mode 100644 index 0000000..89e3e12 --- /dev/null +++ b/lib/psci/aarch64/runtime_errata.S @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <cortex_a510.h> +#include <cpu_macros.S> + +/* + * void apply_cpu_pwr_dwn_errata(void); + * + * This function applies various CPU errata during power down. + */ + .globl apply_cpu_pwr_dwn_errata +func apply_cpu_pwr_dwn_errata + mov x19, x30 + bl cpu_get_rev_var + mov x18, x0 + +#if ERRATA_A510_2684597 + bl erratum_cortex_a510_2684597_wa +#endif + + ret x19 +endfunc apply_cpu_pwr_dwn_errata diff --git a/lib/psci/psci_common.c b/lib/psci/psci_common.c new file mode 100644 index 0000000..f9de432 --- /dev/null +++ b/lib/psci/psci_common.c @@ -0,0 +1,1285 @@ +/* + * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <string.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/bl_common.h> +#include <common/debug.h> +#include <context.h> +#include <drivers/delay_timer.h> +#include <lib/el3_runtime/context_mgmt.h> +#include <lib/utils.h> +#include <plat/common/platform.h> + +#include "psci_private.h" + +/* + * SPD power management operations, expected to be supplied by the registered + * SPD on successful SP initialization + */ +const spd_pm_ops_t *psci_spd_pm; + +/* + * PSCI requested local power state map. This array is used to store the local + * power states requested by a CPU for power levels from level 1 to + * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power + * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a + * CPU are the same. + * + * During state coordination, the platform is passed an array containing the + * local states requested for a particular non cpu power domain by each cpu + * within the domain. + * + * TODO: Dense packing of the requested states will cause cache thrashing + * when multiple power domains write to it. If we allocate the requested + * states at each power level in a cache-line aligned per-domain memory, + * the cache thrashing can be avoided. + */ +static plat_local_state_t + psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT]; + +unsigned int psci_plat_core_count; + +/******************************************************************************* + * Arrays that hold the platform's power domain tree information for state + * management of power domains. + * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain + * which is an ancestor of a CPU power domain. + * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain + ******************************************************************************/ +non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS] +#if USE_COHERENT_MEM +__section(".tzfw_coherent_mem") +#endif +; + +/* Lock for PSCI state coordination */ +DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); + +cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT]; + +/******************************************************************************* + * Pointer to functions exported by the platform to complete power mgmt. ops + ******************************************************************************/ +const plat_psci_ops_t *psci_plat_pm_ops; + +/****************************************************************************** + * Check that the maximum power level supported by the platform makes sense + *****************************************************************************/ +CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) && + (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL), + assert_platform_max_pwrlvl_check); + +#if PSCI_OS_INIT_MODE +/******************************************************************************* + * The power state coordination mode used in CPU_SUSPEND. + * Defaults to platform-coordinated mode. + ******************************************************************************/ +suspend_mode_t psci_suspend_mode = PLAT_COORD; +#endif + +/* + * The plat_local_state used by the platform is one of these types: RUN, + * RETENTION and OFF. The platform can define further sub-states for each type + * apart from RUN. This categorization is done to verify the sanity of the + * psci_power_state passed by the platform and to print debug information. The + * categorization is done on the basis of the following conditions: + * + * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN. + * + * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is + * STATE_TYPE_RETN. + * + * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is + * STATE_TYPE_OFF. + */ +typedef enum plat_local_state_type { + STATE_TYPE_RUN = 0, + STATE_TYPE_RETN, + STATE_TYPE_OFF +} plat_local_state_type_t; + +/* Function used to categorize plat_local_state. */ +static plat_local_state_type_t find_local_state_type(plat_local_state_t state) +{ + if (state != 0U) { + if (state > PLAT_MAX_RET_STATE) { + return STATE_TYPE_OFF; + } else { + return STATE_TYPE_RETN; + } + } else { + return STATE_TYPE_RUN; + } +} + +/****************************************************************************** + * Check that the maximum retention level supported by the platform is less + * than the maximum off level. + *****************************************************************************/ +CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE, + assert_platform_max_off_and_retn_state_check); + +/****************************************************************************** + * This function ensures that the power state parameter in a CPU_SUSPEND request + * is valid. If so, it returns the requested states for each power level. + *****************************************************************************/ +int psci_validate_power_state(unsigned int power_state, + psci_power_state_t *state_info) +{ + /* Check SBZ bits in power state are zero */ + if (psci_check_power_state(power_state) != 0U) + return PSCI_E_INVALID_PARAMS; + + assert(psci_plat_pm_ops->validate_power_state != NULL); + + /* Validate the power_state using platform pm_ops */ + return psci_plat_pm_ops->validate_power_state(power_state, state_info); +} + +/****************************************************************************** + * This function retrieves the `psci_power_state_t` for system suspend from + * the platform. + *****************************************************************************/ +void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info) +{ + /* + * Assert that the required pm_ops hook is implemented to ensure that + * the capability detected during psci_setup() is valid. + */ + assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL); + + /* + * Query the platform for the power_state required for system suspend + */ + psci_plat_pm_ops->get_sys_suspend_power_state(state_info); +} + +#if PSCI_OS_INIT_MODE +/******************************************************************************* + * This function verifies that all the other cores at the 'end_pwrlvl' have been + * idled and the current CPU is the last running CPU at the 'end_pwrlvl'. + * Returns 1 (true) if the current CPU is the last ON CPU or 0 (false) + * otherwise. + ******************************************************************************/ +static bool psci_is_last_cpu_to_idle_at_pwrlvl(unsigned int end_pwrlvl) +{ + unsigned int my_idx, lvl, parent_idx; + unsigned int cpu_start_idx, ncpus, cpu_idx; + plat_local_state_t local_state; + + if (end_pwrlvl == PSCI_CPU_PWR_LVL) { + return true; + } + + my_idx = plat_my_core_pos(); + + for (lvl = PSCI_CPU_PWR_LVL; lvl <= end_pwrlvl; lvl++) { + parent_idx = psci_cpu_pd_nodes[my_idx].parent_node; + } + + cpu_start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; + ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; + + for (cpu_idx = cpu_start_idx; cpu_idx < cpu_start_idx + ncpus; + cpu_idx++) { + local_state = psci_get_cpu_local_state_by_idx(cpu_idx); + if (cpu_idx == my_idx) { + assert(is_local_state_run(local_state) != 0); + continue; + } + + if (is_local_state_run(local_state) != 0) { + return false; + } + } + + return true; +} +#endif + +/******************************************************************************* + * This function verifies that all the other cores in the system have been + * turned OFF and the current CPU is the last running CPU in the system. + * Returns true, if the current CPU is the last ON CPU or false otherwise. + ******************************************************************************/ +bool psci_is_last_on_cpu(void) +{ + unsigned int cpu_idx, my_idx = plat_my_core_pos(); + + for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) { + if (cpu_idx == my_idx) { + assert(psci_get_aff_info_state() == AFF_STATE_ON); + continue; + } + + if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) { + VERBOSE("core=%u other than current core=%u %s\n", + cpu_idx, my_idx, "running in the system"); + return false; + } + } + + return true; +} + +/******************************************************************************* + * This function verifies that all cores in the system have been turned ON. + * Returns true, if all CPUs are ON or false otherwise. + ******************************************************************************/ +static bool psci_are_all_cpus_on(void) +{ + unsigned int cpu_idx; + + for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) { + if (psci_get_aff_info_state_by_idx(cpu_idx) == AFF_STATE_OFF) { + return false; + } + } + + return true; +} + +/******************************************************************************* + * Routine to return the maximum power level to traverse to after a cpu has + * been physically powered up. It is expected to be called immediately after + * reset from assembler code. + ******************************************************************************/ +static unsigned int get_power_on_target_pwrlvl(void) +{ + unsigned int pwrlvl; + + /* + * Assume that this cpu was suspended and retrieve its target power + * level. If it is invalid then it could only have been turned off + * earlier. PLAT_MAX_PWR_LVL will be the highest power level a + * cpu can be turned off to. + */ + pwrlvl = psci_get_suspend_pwrlvl(); + if (pwrlvl == PSCI_INVALID_PWR_LVL) + pwrlvl = PLAT_MAX_PWR_LVL; + assert(pwrlvl < PSCI_INVALID_PWR_LVL); + return pwrlvl; +} + +/****************************************************************************** + * Helper function to update the requested local power state array. This array + * does not store the requested state for the CPU power level. Hence an + * assertion is added to prevent us from accessing the CPU power level. + *****************************************************************************/ +static void psci_set_req_local_pwr_state(unsigned int pwrlvl, + unsigned int cpu_idx, + plat_local_state_t req_pwr_state) +{ + assert(pwrlvl > PSCI_CPU_PWR_LVL); + if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && + (cpu_idx < psci_plat_core_count)) { + psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state; + } +} + +/****************************************************************************** + * This function initializes the psci_req_local_pwr_states. + *****************************************************************************/ +void __init psci_init_req_local_pwr_states(void) +{ + /* Initialize the requested state of all non CPU power domains as OFF */ + unsigned int pwrlvl; + unsigned int core; + + for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) { + for (core = 0; core < psci_plat_core_count; core++) { + psci_req_local_pwr_states[pwrlvl][core] = + PLAT_MAX_OFF_STATE; + } + } +} + +/****************************************************************************** + * Helper function to return a reference to an array containing the local power + * states requested by each cpu for a power domain at 'pwrlvl'. The size of the + * array will be the number of cpu power domains of which this power domain is + * an ancestor. These requested states will be used to determine a suitable + * target state for this power domain during psci state coordination. An + * assertion is added to prevent us from accessing the CPU power level. + *****************************************************************************/ +static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl, + unsigned int cpu_idx) +{ + assert(pwrlvl > PSCI_CPU_PWR_LVL); + + if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && + (cpu_idx < psci_plat_core_count)) { + return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx]; + } else + return NULL; +} + +#if PSCI_OS_INIT_MODE +/****************************************************************************** + * Helper function to save a copy of the psci_req_local_pwr_states (prev) for a + * CPU (cpu_idx), and update psci_req_local_pwr_states with the new requested + * local power states (state_info). + *****************************************************************************/ +void psci_update_req_local_pwr_states(unsigned int end_pwrlvl, + unsigned int cpu_idx, + psci_power_state_t *state_info, + plat_local_state_t *prev) +{ + unsigned int lvl; +#ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL + unsigned int max_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; +#else + unsigned int max_pwrlvl = PLAT_MAX_PWR_LVL; +#endif + plat_local_state_t req_state; + + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= max_pwrlvl; lvl++) { + /* Save the previous requested local power state */ + prev[lvl - 1U] = *psci_get_req_local_pwr_states(lvl, cpu_idx); + + /* Update the new requested local power state */ + if (lvl <= end_pwrlvl) { + req_state = state_info->pwr_domain_state[lvl]; + } else { + req_state = state_info->pwr_domain_state[end_pwrlvl]; + } + psci_set_req_local_pwr_state(lvl, cpu_idx, req_state); + } +} + +/****************************************************************************** + * Helper function to restore the previously saved requested local power states + * (prev) for a CPU (cpu_idx) to psci_req_local_pwr_states. + *****************************************************************************/ +void psci_restore_req_local_pwr_states(unsigned int cpu_idx, + plat_local_state_t *prev) +{ + unsigned int lvl; +#ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL + unsigned int max_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; +#else + unsigned int max_pwrlvl = PLAT_MAX_PWR_LVL; +#endif + + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= max_pwrlvl; lvl++) { + /* Restore the previous requested local power state */ + psci_set_req_local_pwr_state(lvl, cpu_idx, prev[lvl - 1U]); + } +} +#endif + +/* + * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent + * memory. + * + * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory, + * it's accessed by both cached and non-cached participants. To serve the common + * minimum, perform a cache flush before read and after write so that non-cached + * participants operate on latest data in main memory. + * + * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent + * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent. + * In both cases, no cache operations are required. + */ + +/* + * Retrieve local state of non-CPU power domain node from a non-cached CPU, + * after any required cache maintenance operation. + */ +static plat_local_state_t get_non_cpu_pd_node_local_state( + unsigned int parent_idx) +{ +#if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + flush_dcache_range( + (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], + sizeof(psci_non_cpu_pd_nodes[parent_idx])); +#endif + return psci_non_cpu_pd_nodes[parent_idx].local_state; +} + +/* + * Update local state of non-CPU power domain node from a cached CPU; perform + * any required cache maintenance operation afterwards. + */ +static void set_non_cpu_pd_node_local_state(unsigned int parent_idx, + plat_local_state_t state) +{ + psci_non_cpu_pd_nodes[parent_idx].local_state = state; +#if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + flush_dcache_range( + (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], + sizeof(psci_non_cpu_pd_nodes[parent_idx])); +#endif +} + +/****************************************************************************** + * Helper function to return the current local power state of each power domain + * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This + * function will be called after a cpu is powered on to find the local state + * each power domain has emerged from. + *****************************************************************************/ +void psci_get_target_local_pwr_states(unsigned int end_pwrlvl, + psci_power_state_t *target_state) +{ + unsigned int parent_idx, lvl; + plat_local_state_t *pd_state = target_state->pwr_domain_state; + + pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state(); + parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; + + /* Copy the local power state from node to state_info */ + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { + pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx); + parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; + } + + /* Set the the higher levels to RUN */ + for (; lvl <= PLAT_MAX_PWR_LVL; lvl++) + target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; +} + +/****************************************************************************** + * Helper function to set the target local power state that each power domain + * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will + * enter. This function will be called after coordination of requested power + * states has been done for each power level. + *****************************************************************************/ +void psci_set_target_local_pwr_states(unsigned int end_pwrlvl, + const psci_power_state_t *target_state) +{ + unsigned int parent_idx, lvl; + const plat_local_state_t *pd_state = target_state->pwr_domain_state; + + psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]); + + /* + * Need to flush as local_state might be accessed with Data Cache + * disabled during power on + */ + psci_flush_cpu_data(psci_svc_cpu_data.local_state); + + parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; + + /* Copy the local_state from state_info */ + for (lvl = 1U; lvl <= end_pwrlvl; lvl++) { + set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]); + parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; + } +} + +/******************************************************************************* + * PSCI helper function to get the parent nodes corresponding to a cpu_index. + ******************************************************************************/ +void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx, + unsigned int end_lvl, + unsigned int *node_index) +{ + unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node; + unsigned int i; + unsigned int *node = node_index; + + for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) { + *node = parent_node; + node++; + parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node; + } +} + +/****************************************************************************** + * This function is invoked post CPU power up and initialization. It sets the + * affinity info state, target power state and requested power state for the + * current CPU and all its ancestor power domains to RUN. + *****************************************************************************/ +void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl) +{ + unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl; + parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; + + /* Reset the local_state to RUN for the non cpu power domains. */ + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { + set_non_cpu_pd_node_local_state(parent_idx, + PSCI_LOCAL_STATE_RUN); + psci_set_req_local_pwr_state(lvl, + cpu_idx, + PSCI_LOCAL_STATE_RUN); + parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; + } + + /* Set the affinity info state to ON */ + psci_set_aff_info_state(AFF_STATE_ON); + + psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN); + psci_flush_cpu_data(psci_svc_cpu_data); +} + +/****************************************************************************** + * This function is used in platform-coordinated mode. + * + * This function is passed the local power states requested for each power + * domain (state_info) between the current CPU domain and its ancestors until + * the target power level (end_pwrlvl). It updates the array of requested power + * states with this information. + * + * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it + * retrieves the states requested by all the cpus of which the power domain at + * that level is an ancestor. It passes this information to the platform to + * coordinate and return the target power state. If the target state for a level + * is RUN then subsequent levels are not considered. At the CPU level, state + * coordination is not required. Hence, the requested and the target states are + * the same. + * + * The 'state_info' is updated with the target state for each level between the + * CPU and the 'end_pwrlvl' and returned to the caller. + * + * This function will only be invoked with data cache enabled and while + * powering down a core. + *****************************************************************************/ +void psci_do_state_coordination(unsigned int end_pwrlvl, + psci_power_state_t *state_info) +{ + unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos(); + unsigned int start_idx; + unsigned int ncpus; + plat_local_state_t target_state, *req_states; + + assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); + parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; + + /* For level 0, the requested state will be equivalent + to target state */ + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { + + /* First update the requested power state */ + psci_set_req_local_pwr_state(lvl, cpu_idx, + state_info->pwr_domain_state[lvl]); + + /* Get the requested power states for this power level */ + start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; + req_states = psci_get_req_local_pwr_states(lvl, start_idx); + + /* + * Let the platform coordinate amongst the requested states at + * this power level and return the target local power state. + */ + ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; + target_state = plat_get_target_pwr_state(lvl, + req_states, + ncpus); + + state_info->pwr_domain_state[lvl] = target_state; + + /* Break early if the negotiated target power state is RUN */ + if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0) + break; + + parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; + } + + /* + * This is for cases when we break out of the above loop early because + * the target power state is RUN at a power level < end_pwlvl. + * We update the requested power state from state_info and then + * set the target state as RUN. + */ + for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) { + psci_set_req_local_pwr_state(lvl, cpu_idx, + state_info->pwr_domain_state[lvl]); + state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; + + } +} + +#if PSCI_OS_INIT_MODE +/****************************************************************************** + * This function is used in OS-initiated mode. + * + * This function is passed the local power states requested for each power + * domain (state_info) between the current CPU domain and its ancestors until + * the target power level (end_pwrlvl), and ensures the requested power states + * are valid. It updates the array of requested power states with this + * information. + * + * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it + * retrieves the states requested by all the cpus of which the power domain at + * that level is an ancestor. It passes this information to the platform to + * coordinate and return the target power state. If the requested state does + * not match the target state, the request is denied. + * + * The 'state_info' is not modified. + * + * This function will only be invoked with data cache enabled and while + * powering down a core. + *****************************************************************************/ +int psci_validate_state_coordination(unsigned int end_pwrlvl, + psci_power_state_t *state_info) +{ + int rc = PSCI_E_SUCCESS; + unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos(); + unsigned int start_idx; + unsigned int ncpus; + plat_local_state_t target_state, *req_states; + plat_local_state_t prev[PLAT_MAX_PWR_LVL]; + + assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); + parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; + + /* + * Save a copy of the previous requested local power states and update + * the new requested local power states. + */ + psci_update_req_local_pwr_states(end_pwrlvl, cpu_idx, state_info, prev); + + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { + /* Get the requested power states for this power level */ + start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; + req_states = psci_get_req_local_pwr_states(lvl, start_idx); + + /* + * Let the platform coordinate amongst the requested states at + * this power level and return the target local power state. + */ + ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; + target_state = plat_get_target_pwr_state(lvl, + req_states, + ncpus); + + /* + * Verify that the requested power state matches the target + * local power state. + */ + if (state_info->pwr_domain_state[lvl] != target_state) { + if (target_state == PSCI_LOCAL_STATE_RUN) { + rc = PSCI_E_DENIED; + } else { + rc = PSCI_E_INVALID_PARAMS; + } + goto exit; + } + } + + /* + * Verify that the current core is the last running core at the + * specified power level. + */ + lvl = state_info->last_at_pwrlvl; + if (!psci_is_last_cpu_to_idle_at_pwrlvl(lvl)) { + rc = PSCI_E_DENIED; + } + +exit: + if (rc != PSCI_E_SUCCESS) { + /* Restore the previous requested local power states. */ + psci_restore_req_local_pwr_states(cpu_idx, prev); + return rc; + } + + return rc; +} +#endif + +/****************************************************************************** + * This function validates a suspend request by making sure that if a standby + * state is requested then no power level is turned off and the highest power + * level is placed in a standby/retention state. + * + * It also ensures that the state level X will enter is not shallower than the + * state level X + 1 will enter. + * + * This validation will be enabled only for DEBUG builds as the platform is + * expected to perform these validations as well. + *****************************************************************************/ +int psci_validate_suspend_req(const psci_power_state_t *state_info, + unsigned int is_power_down_state) +{ + unsigned int max_off_lvl, target_lvl, max_retn_lvl; + plat_local_state_t state; + plat_local_state_type_t req_state_type, deepest_state_type; + int i; + + /* Find the target suspend power level */ + target_lvl = psci_find_target_suspend_lvl(state_info); + if (target_lvl == PSCI_INVALID_PWR_LVL) + return PSCI_E_INVALID_PARAMS; + + /* All power domain levels are in a RUN state to begin with */ + deepest_state_type = STATE_TYPE_RUN; + + for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) { + state = state_info->pwr_domain_state[i]; + req_state_type = find_local_state_type(state); + + /* + * While traversing from the highest power level to the lowest, + * the state requested for lower levels has to be the same or + * deeper i.e. equal to or greater than the state at the higher + * levels. If this condition is true, then the requested state + * becomes the deepest state encountered so far. + */ + if (req_state_type < deepest_state_type) + return PSCI_E_INVALID_PARAMS; + deepest_state_type = req_state_type; + } + + /* Find the highest off power level */ + max_off_lvl = psci_find_max_off_lvl(state_info); + + /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */ + max_retn_lvl = PSCI_INVALID_PWR_LVL; + if (target_lvl != max_off_lvl) + max_retn_lvl = target_lvl; + + /* + * If this is not a request for a power down state then max off level + * has to be invalid and max retention level has to be a valid power + * level. + */ + if ((is_power_down_state == 0U) && + ((max_off_lvl != PSCI_INVALID_PWR_LVL) || + (max_retn_lvl == PSCI_INVALID_PWR_LVL))) + return PSCI_E_INVALID_PARAMS; + + return PSCI_E_SUCCESS; +} + +/****************************************************************************** + * This function finds the highest power level which will be powered down + * amongst all the power levels specified in the 'state_info' structure + *****************************************************************************/ +unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info) +{ + int i; + + for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { + if (is_local_state_off(state_info->pwr_domain_state[i]) != 0) + return (unsigned int) i; + } + + return PSCI_INVALID_PWR_LVL; +} + +/****************************************************************************** + * This functions finds the level of the highest power domain which will be + * placed in a low power state during a suspend operation. + *****************************************************************************/ +unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info) +{ + int i; + + for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { + if (is_local_state_run(state_info->pwr_domain_state[i]) == 0) + return (unsigned int) i; + } + + return PSCI_INVALID_PWR_LVL; +} + +/******************************************************************************* + * This function is passed the highest level in the topology tree that the + * operation should be applied to and a list of node indexes. It picks up locks + * from the node index list in order of increasing power domain level in the + * range specified. + ******************************************************************************/ +void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl, + const unsigned int *parent_nodes) +{ + unsigned int parent_idx; + unsigned int level; + + /* No locking required for level 0. Hence start locking from level 1 */ + for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) { + parent_idx = parent_nodes[level - 1U]; + psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]); + } +} + +/******************************************************************************* + * This function is passed the highest level in the topology tree that the + * operation should be applied to and a list of node indexes. It releases the + * locks in order of decreasing power domain level in the range specified. + ******************************************************************************/ +void psci_release_pwr_domain_locks(unsigned int end_pwrlvl, + const unsigned int *parent_nodes) +{ + unsigned int parent_idx; + unsigned int level; + + /* Unlock top down. No unlocking required for level 0. */ + for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) { + parent_idx = parent_nodes[level - 1U]; + psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]); + } +} + +/******************************************************************************* + * This function determines the full entrypoint information for the requested + * PSCI entrypoint on power on/resume and returns it. + ******************************************************************************/ +#ifdef __aarch64__ +static int psci_get_ns_ep_info(entry_point_info_t *ep, + uintptr_t entrypoint, + u_register_t context_id) +{ + u_register_t ep_attr, sctlr; + unsigned int daif, ee, mode; + u_register_t ns_scr_el3 = read_scr_el3(); + u_register_t ns_sctlr_el1 = read_sctlr_el1(); + + sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? + read_sctlr_el2() : ns_sctlr_el1; + ee = 0; + + ep_attr = NON_SECURE | EP_ST_DISABLE; + if ((sctlr & SCTLR_EE_BIT) != 0U) { + ep_attr |= EP_EE_BIG; + ee = 1; + } + SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); + + ep->pc = entrypoint; + zeromem(&ep->args, sizeof(ep->args)); + ep->args.arg0 = context_id; + + /* + * Figure out whether the cpu enters the non-secure address space + * in aarch32 or aarch64 + */ + if ((ns_scr_el3 & SCR_RW_BIT) != 0U) { + + /* + * Check whether a Thumb entry point has been provided for an + * aarch64 EL + */ + if ((entrypoint & 0x1UL) != 0UL) + return PSCI_E_INVALID_ADDRESS; + + mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1; + + ep->spsr = SPSR_64((uint64_t)mode, MODE_SP_ELX, + DISABLE_ALL_EXCEPTIONS); + } else { + + mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? + MODE32_hyp : MODE32_svc; + + /* + * TODO: Choose async. exception bits if HYP mode is not + * implemented according to the values of SCR.{AW, FW} bits + */ + daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT; + + ep->spsr = SPSR_MODE32((uint64_t)mode, entrypoint & 0x1, ee, + daif); + } + + return PSCI_E_SUCCESS; +} +#else /* !__aarch64__ */ +static int psci_get_ns_ep_info(entry_point_info_t *ep, + uintptr_t entrypoint, + u_register_t context_id) +{ + u_register_t ep_attr; + unsigned int aif, ee, mode; + u_register_t scr = read_scr(); + u_register_t ns_sctlr, sctlr; + + /* Switch to non secure state */ + write_scr(scr | SCR_NS_BIT); + isb(); + ns_sctlr = read_sctlr(); + + sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr; + + /* Return to original state */ + write_scr(scr); + isb(); + ee = 0; + + ep_attr = NON_SECURE | EP_ST_DISABLE; + if (sctlr & SCTLR_EE_BIT) { + ep_attr |= EP_EE_BIG; + ee = 1; + } + SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); + + ep->pc = entrypoint; + zeromem(&ep->args, sizeof(ep->args)); + ep->args.arg0 = context_id; + + mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc; + + /* + * TODO: Choose async. exception bits if HYP mode is not + * implemented according to the values of SCR.{AW, FW} bits + */ + aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT; + + ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif); + + return PSCI_E_SUCCESS; +} + +#endif /* __aarch64__ */ + +/******************************************************************************* + * This function validates the entrypoint with the platform layer if the + * appropriate pm_ops hook is exported by the platform and returns the + * 'entry_point_info'. + ******************************************************************************/ +int psci_validate_entry_point(entry_point_info_t *ep, + uintptr_t entrypoint, + u_register_t context_id) +{ + int rc; + + /* Validate the entrypoint using platform psci_ops */ + if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) { + rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint); + if (rc != PSCI_E_SUCCESS) + return PSCI_E_INVALID_ADDRESS; + } + + /* + * Verify and derive the re-entry information for + * the non-secure world from the non-secure state from + * where this call originated. + */ + rc = psci_get_ns_ep_info(ep, entrypoint, context_id); + return rc; +} + +/******************************************************************************* + * Generic handler which is called when a cpu is physically powered on. It + * traverses the node information and finds the highest power level powered + * off and performs generic, architectural, platform setup and state management + * to power on that power level and power levels below it. + * e.g. For a cpu that's been powered on, it will call the platform specific + * code to enable the gic cpu interface and for a cluster it will enable + * coherency at the interconnect level in addition to gic cpu interface. + ******************************************************************************/ +void psci_warmboot_entrypoint(void) +{ + unsigned int end_pwrlvl; + unsigned int cpu_idx = plat_my_core_pos(); + unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; + psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; + + /* Init registers that never change for the lifetime of TF-A */ + cm_manage_extensions_el3(); + + /* + * Verify that we have been explicitly turned ON or resumed from + * suspend. + */ + if (psci_get_aff_info_state() == AFF_STATE_OFF) { + ERROR("Unexpected affinity info state.\n"); + panic(); + } + + /* + * Get the maximum power domain level to traverse to after this cpu + * has been physically powered up. + */ + end_pwrlvl = get_power_on_target_pwrlvl(); + + /* Get the parent nodes */ + psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); + + /* + * This function acquires the lock corresponding to each power level so + * that by the time all locks are taken, the system topology is snapshot + * and state management can be done safely. + */ + psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); + + psci_get_target_local_pwr_states(end_pwrlvl, &state_info); + +#if ENABLE_PSCI_STAT + plat_psci_stat_accounting_stop(&state_info); +#endif + + /* + * This CPU could be resuming from suspend or it could have just been + * turned on. To distinguish between these 2 cases, we examine the + * affinity state of the CPU: + * - If the affinity state is ON_PENDING then it has just been + * turned on. + * - Else it is resuming from suspend. + * + * Depending on the type of warm reset identified, choose the right set + * of power management handler and perform the generic, architecture + * and platform specific handling. + */ + if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING) + psci_cpu_on_finish(cpu_idx, &state_info); + else + psci_cpu_suspend_finish(cpu_idx, &state_info); + + /* + * Generic management: Now we just need to retrieve the + * information that we had stashed away during the cpu_on + * call to set this cpu on its way. + */ + cm_prepare_el3_exit_ns(); + + /* + * Set the requested and target state of this CPU and all the higher + * power domains which are ancestors of this CPU to run. + */ + psci_set_pwr_domains_to_run(end_pwrlvl); + +#if ENABLE_PSCI_STAT + /* + * Update PSCI stats. + * Caches are off when writing stats data on the power down path. + * Since caches are now enabled, it's necessary to do cache + * maintenance before reading that same data. + */ + psci_stats_update_pwr_up(end_pwrlvl, &state_info); +#endif + + /* + * This loop releases the lock corresponding to each power level + * in the reverse order to which they were acquired. + */ + psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); +} + +/******************************************************************************* + * This function initializes the set of hooks that PSCI invokes as part of power + * management operation. The power management hooks are expected to be provided + * by the SPD, after it finishes all its initialization + ******************************************************************************/ +void psci_register_spd_pm_hook(const spd_pm_ops_t *pm) +{ + assert(pm != NULL); + psci_spd_pm = pm; + + if (pm->svc_migrate != NULL) + psci_caps |= define_psci_cap(PSCI_MIG_AARCH64); + + if (pm->svc_migrate_info != NULL) + psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64) + | define_psci_cap(PSCI_MIG_INFO_TYPE); +} + +/******************************************************************************* + * This function invokes the migrate info hook in the spd_pm_ops. It performs + * the necessary return value validation. If the Secure Payload is UP and + * migrate capable, it returns the mpidr of the CPU on which the Secure payload + * is resident through the mpidr parameter. Else the value of the parameter on + * return is undefined. + ******************************************************************************/ +int psci_spd_migrate_info(u_register_t *mpidr) +{ + int rc; + + if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL)) + return PSCI_E_NOT_SUPPORTED; + + rc = psci_spd_pm->svc_migrate_info(mpidr); + + assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) || + (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED)); + + return rc; +} + + +/******************************************************************************* + * This function prints the state of all power domains present in the + * system + ******************************************************************************/ +void psci_print_power_domain_map(void) +{ +#if LOG_LEVEL >= LOG_LEVEL_INFO + unsigned int idx; + plat_local_state_t state; + plat_local_state_type_t state_type; + + /* This array maps to the PSCI_STATE_X definitions in psci.h */ + static const char * const psci_state_type_str[] = { + "ON", + "RETENTION", + "OFF", + }; + + INFO("PSCI Power Domain Map:\n"); + for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count); + idx++) { + state_type = find_local_state_type( + psci_non_cpu_pd_nodes[idx].local_state); + INFO(" Domain Node : Level %u, parent_node %u," + " State %s (0x%x)\n", + psci_non_cpu_pd_nodes[idx].level, + psci_non_cpu_pd_nodes[idx].parent_node, + psci_state_type_str[state_type], + psci_non_cpu_pd_nodes[idx].local_state); + } + + for (idx = 0; idx < psci_plat_core_count; idx++) { + state = psci_get_cpu_local_state_by_idx(idx); + state_type = find_local_state_type(state); + INFO(" CPU Node : MPID 0x%llx, parent_node %u," + " State %s (0x%x)\n", + (unsigned long long)psci_cpu_pd_nodes[idx].mpidr, + psci_cpu_pd_nodes[idx].parent_node, + psci_state_type_str[state_type], + psci_get_cpu_local_state_by_idx(idx)); + } +#endif +} + +/****************************************************************************** + * Return whether any secondaries were powered up with CPU_ON call. A CPU that + * have ever been powered up would have set its MPDIR value to something other + * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to + * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is + * meaningful only when called on the primary CPU during early boot. + *****************************************************************************/ +int psci_secondaries_brought_up(void) +{ + unsigned int idx, n_valid = 0U; + + for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) { + if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR) + n_valid++; + } + + assert(n_valid > 0U); + + return (n_valid > 1U) ? 1 : 0; +} + +/******************************************************************************* + * Initiate power down sequence, by calling power down operations registered for + * this CPU. + ******************************************************************************/ +void psci_pwrdown_cpu(unsigned int power_level) +{ +#if HW_ASSISTED_COHERENCY + /* + * With hardware-assisted coherency, the CPU drivers only initiate the + * power down sequence, without performing cache-maintenance operations + * in software. Data caches enabled both before and after this call. + */ + prepare_cpu_pwr_dwn(power_level); +#else + /* + * Without hardware-assisted coherency, the CPU drivers disable data + * caches, then perform cache-maintenance operations in software. + * + * This also calls prepare_cpu_pwr_dwn() to initiate power down + * sequence, but that function will return with data caches disabled. + * We must ensure that the stack memory is flushed out to memory before + * we start popping from it again. + */ + psci_do_pwrdown_cache_maintenance(power_level); +#endif +} + +/******************************************************************************* + * This function invokes the callback 'stop_func()' with the 'mpidr' of each + * online PE. Caller can pass suitable method to stop a remote core. + * + * 'wait_ms' is the timeout value in milliseconds for the other cores to + * transition to power down state. Passing '0' makes it non-blocking. + * + * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the + * given timeout. + ******************************************************************************/ +int psci_stop_other_cores(unsigned int wait_ms, + void (*stop_func)(u_register_t mpidr)) +{ + unsigned int idx, this_cpu_idx; + + this_cpu_idx = plat_my_core_pos(); + + /* Invoke stop_func for each core */ + for (idx = 0U; idx < psci_plat_core_count; idx++) { + /* skip current CPU */ + if (idx == this_cpu_idx) { + continue; + } + + /* Check if the CPU is ON */ + if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) { + (*stop_func)(psci_cpu_pd_nodes[idx].mpidr); + } + } + + /* Need to wait for other cores to shutdown */ + if (wait_ms != 0U) { + while ((wait_ms-- != 0U) && (!psci_is_last_on_cpu())) { + mdelay(1U); + } + + if (!psci_is_last_on_cpu()) { + WARN("Failed to stop all cores!\n"); + psci_print_power_domain_map(); + return PSCI_E_DENIED; + } + } + + return PSCI_E_SUCCESS; +} + +/******************************************************************************* + * This function verifies that all the other cores in the system have been + * turned OFF and the current CPU is the last running CPU in the system. + * Returns true if the current CPU is the last ON CPU or false otherwise. + * + * This API has following differences with psci_is_last_on_cpu + * 1. PSCI states are locked + ******************************************************************************/ +bool psci_is_last_on_cpu_safe(void) +{ + unsigned int this_core = plat_my_core_pos(); + unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; + + psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes); + + psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); + + if (!psci_is_last_on_cpu()) { + psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); + return false; + } + + psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); + + return true; +} + +/******************************************************************************* + * This function verifies that all cores in the system have been turned ON. + * Returns true, if all CPUs are ON or false otherwise. + * + * This API has following differences with psci_are_all_cpus_on + * 1. PSCI states are locked + ******************************************************************************/ +bool psci_are_all_cpus_on_safe(void) +{ + unsigned int this_core = plat_my_core_pos(); + unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; + + psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes); + + psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); + + if (!psci_are_all_cpus_on()) { + psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); + return false; + } + + psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); + + return true; +} diff --git a/lib/psci/psci_lib.mk b/lib/psci/psci_lib.mk new file mode 100644 index 0000000..c71580f --- /dev/null +++ b/lib/psci/psci_lib.mk @@ -0,0 +1,36 @@ +# +# Copyright (c) 2016-2023, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +PSCI_LIB_SOURCES := lib/el3_runtime/cpu_data_array.c \ + lib/el3_runtime/${ARCH}/cpu_data.S \ + lib/el3_runtime/${ARCH}/context_mgmt.c \ + lib/cpus/${ARCH}/cpu_helpers.S \ + lib/cpus/errata_report.c \ + lib/locks/exclusive/${ARCH}/spinlock.S \ + lib/psci/psci_off.c \ + lib/psci/psci_on.c \ + lib/psci/psci_suspend.c \ + lib/psci/psci_common.c \ + lib/psci/psci_main.c \ + lib/psci/psci_setup.c \ + lib/psci/psci_system_off.c \ + lib/psci/psci_mem_protect.c \ + lib/psci/${ARCH}/psci_helpers.S + +ifeq (${ARCH}, aarch64) +PSCI_LIB_SOURCES += lib/el3_runtime/aarch64/context.S \ + lib/psci/aarch64/runtime_errata.S +endif + +ifeq (${USE_COHERENT_MEM}, 1) +PSCI_LIB_SOURCES += lib/locks/bakery/bakery_lock_coherent.c +else +PSCI_LIB_SOURCES += lib/locks/bakery/bakery_lock_normal.c +endif + +ifeq (${ENABLE_PSCI_STAT}, 1) +PSCI_LIB_SOURCES += lib/psci/psci_stat.c +endif diff --git a/lib/psci/psci_main.c b/lib/psci/psci_main.c new file mode 100644 index 0000000..a015531 --- /dev/null +++ b/lib/psci/psci_main.c @@ -0,0 +1,606 @@ +/* + * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <string.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/pmf/pmf.h> +#include <lib/runtime_instr.h> +#include <lib/smccc.h> +#include <plat/common/platform.h> +#include <services/arm_arch_svc.h> + +#include "psci_private.h" + +/******************************************************************************* + * PSCI frontend api for servicing SMCs. Described in the PSCI spec. + ******************************************************************************/ +int psci_cpu_on(u_register_t target_cpu, + uintptr_t entrypoint, + u_register_t context_id) + +{ + int rc; + entry_point_info_t ep; + + /* Validate the target CPU */ + if (!is_valid_mpidr(target_cpu)) + return PSCI_E_INVALID_PARAMS; + + /* Validate the entry point and get the entry_point_info */ + rc = psci_validate_entry_point(&ep, entrypoint, context_id); + if (rc != PSCI_E_SUCCESS) + return rc; + + /* + * To turn this cpu on, specify which power + * levels need to be turned on + */ + return psci_cpu_on_start(target_cpu, &ep); +} + +unsigned int psci_version(void) +{ + return PSCI_MAJOR_VER | PSCI_MINOR_VER; +} + +int psci_cpu_suspend(unsigned int power_state, + uintptr_t entrypoint, + u_register_t context_id) +{ + int rc; + unsigned int target_pwrlvl, is_power_down_state; + entry_point_info_t ep; + psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; + plat_local_state_t cpu_pd_state; +#if PSCI_OS_INIT_MODE + unsigned int cpu_idx = plat_my_core_pos(); + plat_local_state_t prev[PLAT_MAX_PWR_LVL]; +#endif + + /* Validate the power_state parameter */ + rc = psci_validate_power_state(power_state, &state_info); + if (rc != PSCI_E_SUCCESS) { + assert(rc == PSCI_E_INVALID_PARAMS); + return rc; + } + + /* + * Get the value of the state type bit from the power state parameter. + */ + is_power_down_state = psci_get_pstate_type(power_state); + + /* Sanity check the requested suspend levels */ + assert(psci_validate_suspend_req(&state_info, is_power_down_state) + == PSCI_E_SUCCESS); + + target_pwrlvl = psci_find_target_suspend_lvl(&state_info); + if (target_pwrlvl == PSCI_INVALID_PWR_LVL) { + ERROR("Invalid target power level for suspend operation\n"); + panic(); + } + + /* Fast path for CPU standby.*/ + if (is_cpu_standby_req(is_power_down_state, target_pwrlvl)) { + if (psci_plat_pm_ops->cpu_standby == NULL) + return PSCI_E_INVALID_PARAMS; + + /* + * Set the state of the CPU power domain to the platform + * specific retention state and enter the standby state. + */ + cpu_pd_state = state_info.pwr_domain_state[PSCI_CPU_PWR_LVL]; + psci_set_cpu_local_state(cpu_pd_state); + +#if PSCI_OS_INIT_MODE + /* + * If in OS-initiated mode, save a copy of the previous + * requested local power states and update the new requested + * local power states for this CPU. + */ + if (psci_suspend_mode == OS_INIT) { + psci_update_req_local_pwr_states(target_pwrlvl, cpu_idx, + &state_info, prev); + } +#endif + +#if ENABLE_PSCI_STAT + plat_psci_stat_accounting_start(&state_info); +#endif + +#if ENABLE_RUNTIME_INSTRUMENTATION + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_ENTER_HW_LOW_PWR, + PMF_NO_CACHE_MAINT); +#endif + + psci_plat_pm_ops->cpu_standby(cpu_pd_state); + + /* Upon exit from standby, set the state back to RUN. */ + psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN); + +#if PSCI_OS_INIT_MODE + /* + * If in OS-initiated mode, restore the previous requested + * local power states for this CPU. + */ + if (psci_suspend_mode == OS_INIT) { + psci_restore_req_local_pwr_states(cpu_idx, prev); + } +#endif + +#if ENABLE_RUNTIME_INSTRUMENTATION + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_EXIT_HW_LOW_PWR, + PMF_NO_CACHE_MAINT); +#endif + +#if ENABLE_PSCI_STAT + plat_psci_stat_accounting_stop(&state_info); + + /* Update PSCI stats */ + psci_stats_update_pwr_up(PSCI_CPU_PWR_LVL, &state_info); +#endif + + return PSCI_E_SUCCESS; + } + + /* + * If a power down state has been requested, we need to verify entry + * point and program entry information. + */ + if (is_power_down_state != 0U) { + rc = psci_validate_entry_point(&ep, entrypoint, context_id); + if (rc != PSCI_E_SUCCESS) + return rc; + } + + /* + * Do what is needed to enter the power down state. Upon success, + * enter the final wfi which will power down this CPU. This function + * might return if the power down was abandoned for any reason, e.g. + * arrival of an interrupt + */ + rc = psci_cpu_suspend_start(&ep, + target_pwrlvl, + &state_info, + is_power_down_state); + + return rc; +} + + +int psci_system_suspend(uintptr_t entrypoint, u_register_t context_id) +{ + int rc; + psci_power_state_t state_info; + entry_point_info_t ep; + + /* Check if the current CPU is the last ON CPU in the system */ + if (!psci_is_last_on_cpu()) + return PSCI_E_DENIED; + + /* Validate the entry point and get the entry_point_info */ + rc = psci_validate_entry_point(&ep, entrypoint, context_id); + if (rc != PSCI_E_SUCCESS) + return rc; + + /* Query the psci_power_state for system suspend */ + psci_query_sys_suspend_pwrstate(&state_info); + + /* + * Check if platform allows suspend to Highest power level + * (System level) + */ + if (psci_find_target_suspend_lvl(&state_info) < PLAT_MAX_PWR_LVL) + return PSCI_E_DENIED; + + /* Ensure that the psci_power_state makes sense */ + assert(psci_validate_suspend_req(&state_info, PSTATE_TYPE_POWERDOWN) + == PSCI_E_SUCCESS); + assert(is_local_state_off( + state_info.pwr_domain_state[PLAT_MAX_PWR_LVL]) != 0); + + /* + * Do what is needed to enter the system suspend state. This function + * might return if the power down was abandoned for any reason, e.g. + * arrival of an interrupt + */ + rc = psci_cpu_suspend_start(&ep, + PLAT_MAX_PWR_LVL, + &state_info, + PSTATE_TYPE_POWERDOWN); + + return rc; +} + +int psci_cpu_off(void) +{ + int rc; + unsigned int target_pwrlvl = PLAT_MAX_PWR_LVL; + + /* + * Do what is needed to power off this CPU and possible higher power + * levels if it able to do so. Upon success, enter the final wfi + * which will power down this CPU. + */ + rc = psci_do_cpu_off(target_pwrlvl); + + /* + * The only error cpu_off can return is E_DENIED. So check if that's + * indeed the case. + */ + assert(rc == PSCI_E_DENIED); + + return rc; +} + +int psci_affinity_info(u_register_t target_affinity, + unsigned int lowest_affinity_level) +{ + unsigned int target_idx; + + /* Validate the target affinity */ + if (!is_valid_mpidr(target_affinity)) + return PSCI_E_INVALID_PARAMS; + + /* We dont support level higher than PSCI_CPU_PWR_LVL */ + if (lowest_affinity_level > PSCI_CPU_PWR_LVL) + return PSCI_E_INVALID_PARAMS; + + /* Calculate the cpu index of the target */ + target_idx = (unsigned int) plat_core_pos_by_mpidr(target_affinity); + + /* + * Generic management: + * Perform cache maintanence ahead of reading the target CPU state to + * ensure that the data is not stale. + * There is a theoretical edge case where the cache may contain stale + * data for the target CPU data - this can occur under the following + * conditions: + * - the target CPU is in another cluster from the current + * - the target CPU was the last CPU to shutdown on its cluster + * - the cluster was removed from coherency as part of the CPU shutdown + * + * In this case the cache maintenace that was performed as part of the + * target CPUs shutdown was not seen by the current CPU's cluster. And + * so the cache may contain stale data for the target CPU. + */ + flush_cpu_data_by_index(target_idx, + psci_svc_cpu_data.aff_info_state); + + return psci_get_aff_info_state_by_idx(target_idx); +} + +int psci_migrate(u_register_t target_cpu) +{ + int rc; + u_register_t resident_cpu_mpidr; + + /* Validate the target cpu */ + if (!is_valid_mpidr(target_cpu)) + return PSCI_E_INVALID_PARAMS; + + rc = psci_spd_migrate_info(&resident_cpu_mpidr); + if (rc != PSCI_TOS_UP_MIG_CAP) + return (rc == PSCI_TOS_NOT_UP_MIG_CAP) ? + PSCI_E_DENIED : PSCI_E_NOT_SUPPORTED; + + /* + * Migrate should only be invoked on the CPU where + * the Secure OS is resident. + */ + if (resident_cpu_mpidr != read_mpidr_el1()) + return PSCI_E_NOT_PRESENT; + + /* Check the validity of the specified target cpu */ + if (!is_valid_mpidr(target_cpu)) + return PSCI_E_INVALID_PARAMS; + + assert((psci_spd_pm != NULL) && (psci_spd_pm->svc_migrate != NULL)); + + rc = psci_spd_pm->svc_migrate(read_mpidr_el1(), target_cpu); + assert((rc == PSCI_E_SUCCESS) || (rc == PSCI_E_INTERN_FAIL)); + + return rc; +} + +int psci_migrate_info_type(void) +{ + u_register_t resident_cpu_mpidr; + + return psci_spd_migrate_info(&resident_cpu_mpidr); +} + +u_register_t psci_migrate_info_up_cpu(void) +{ + u_register_t resident_cpu_mpidr; + int rc; + + /* + * Return value of this depends upon what + * psci_spd_migrate_info() returns. + */ + rc = psci_spd_migrate_info(&resident_cpu_mpidr); + if ((rc != PSCI_TOS_NOT_UP_MIG_CAP) && (rc != PSCI_TOS_UP_MIG_CAP)) + return (u_register_t)(register_t) PSCI_E_INVALID_PARAMS; + + return resident_cpu_mpidr; +} + +int psci_node_hw_state(u_register_t target_cpu, + unsigned int power_level) +{ + int rc; + + /* Validate target_cpu */ + if (!is_valid_mpidr(target_cpu)) + return PSCI_E_INVALID_PARAMS; + + /* Validate power_level against PLAT_MAX_PWR_LVL */ + if (power_level > PLAT_MAX_PWR_LVL) + return PSCI_E_INVALID_PARAMS; + + /* + * Dispatch this call to platform to query power controller, and pass on + * to the caller what it returns + */ + assert(psci_plat_pm_ops->get_node_hw_state != NULL); + rc = psci_plat_pm_ops->get_node_hw_state(target_cpu, power_level); + assert(((rc >= HW_ON) && (rc <= HW_STANDBY)) + || (rc == PSCI_E_NOT_SUPPORTED) + || (rc == PSCI_E_INVALID_PARAMS)); + return rc; +} + +int psci_features(unsigned int psci_fid) +{ + unsigned int local_caps = psci_caps; + + if (psci_fid == SMCCC_VERSION) + return PSCI_E_SUCCESS; + + /* Check if it is a 64 bit function */ + if (((psci_fid >> FUNCID_CC_SHIFT) & FUNCID_CC_MASK) == SMC_64) + local_caps &= PSCI_CAP_64BIT_MASK; + + /* Check for invalid fid */ + if (!(is_std_svc_call(psci_fid) && is_valid_fast_smc(psci_fid) + && is_psci_fid(psci_fid))) + return PSCI_E_NOT_SUPPORTED; + + + /* Check if the psci fid is supported or not */ + if ((local_caps & define_psci_cap(psci_fid)) == 0U) + return PSCI_E_NOT_SUPPORTED; + + /* Format the feature flags */ + if ((psci_fid == PSCI_CPU_SUSPEND_AARCH32) || + (psci_fid == PSCI_CPU_SUSPEND_AARCH64)) { + unsigned int ret = ((FF_PSTATE << FF_PSTATE_SHIFT) | + (FF_SUPPORTS_OS_INIT_MODE << FF_MODE_SUPPORT_SHIFT)); + return (int)ret; + } + + /* Return 0 for all other fid's */ + return PSCI_E_SUCCESS; +} + +#if PSCI_OS_INIT_MODE +int psci_set_suspend_mode(unsigned int mode) +{ + if (psci_suspend_mode == mode) { + return PSCI_E_SUCCESS; + } + + if (mode == PLAT_COORD) { + /* Check if the current CPU is the last ON CPU in the system */ + if (!psci_is_last_on_cpu_safe()) { + return PSCI_E_DENIED; + } + } + + if (mode == OS_INIT) { + /* + * Check if all CPUs in the system are ON or if the current + * CPU is the last ON CPU in the system. + */ + if (!(psci_are_all_cpus_on_safe() || + psci_is_last_on_cpu_safe())) { + return PSCI_E_DENIED; + } + } + + psci_suspend_mode = mode; + psci_flush_dcache_range((uintptr_t)&psci_suspend_mode, + sizeof(psci_suspend_mode)); + + return PSCI_E_SUCCESS; +} +#endif + +/******************************************************************************* + * PSCI top level handler for servicing SMCs. + ******************************************************************************/ +u_register_t psci_smc_handler(uint32_t smc_fid, + u_register_t x1, + u_register_t x2, + u_register_t x3, + u_register_t x4, + void *cookie, + void *handle, + u_register_t flags) +{ + u_register_t ret; + + if (is_caller_secure(flags)) + return (u_register_t)SMC_UNK; + + /* Check the fid against the capabilities */ + if ((psci_caps & define_psci_cap(smc_fid)) == 0U) + return (u_register_t)SMC_UNK; + + if (((smc_fid >> FUNCID_CC_SHIFT) & FUNCID_CC_MASK) == SMC_32) { + /* 32-bit PSCI function, clear top parameter bits */ + + uint32_t r1 = (uint32_t)x1; + uint32_t r2 = (uint32_t)x2; + uint32_t r3 = (uint32_t)x3; + + switch (smc_fid) { + case PSCI_VERSION: + ret = (u_register_t)psci_version(); + break; + + case PSCI_CPU_OFF: + ret = (u_register_t)psci_cpu_off(); + break; + + case PSCI_CPU_SUSPEND_AARCH32: + ret = (u_register_t)psci_cpu_suspend(r1, r2, r3); + break; + + case PSCI_CPU_ON_AARCH32: + ret = (u_register_t)psci_cpu_on(r1, r2, r3); + break; + + case PSCI_AFFINITY_INFO_AARCH32: + ret = (u_register_t)psci_affinity_info(r1, r2); + break; + + case PSCI_MIG_AARCH32: + ret = (u_register_t)psci_migrate(r1); + break; + + case PSCI_MIG_INFO_TYPE: + ret = (u_register_t)psci_migrate_info_type(); + break; + + case PSCI_MIG_INFO_UP_CPU_AARCH32: + ret = psci_migrate_info_up_cpu(); + break; + + case PSCI_NODE_HW_STATE_AARCH32: + ret = (u_register_t)psci_node_hw_state(r1, r2); + break; + + case PSCI_SYSTEM_SUSPEND_AARCH32: + ret = (u_register_t)psci_system_suspend(r1, r2); + break; + + case PSCI_SYSTEM_OFF: + psci_system_off(); + /* We should never return from psci_system_off() */ + break; + + case PSCI_SYSTEM_RESET: + psci_system_reset(); + /* We should never return from psci_system_reset() */ + break; + + case PSCI_FEATURES: + ret = (u_register_t)psci_features(r1); + break; + +#if PSCI_OS_INIT_MODE + case PSCI_SET_SUSPEND_MODE: + ret = (u_register_t)psci_set_suspend_mode(r1); + break; +#endif + +#if ENABLE_PSCI_STAT + case PSCI_STAT_RESIDENCY_AARCH32: + ret = psci_stat_residency(r1, r2); + break; + + case PSCI_STAT_COUNT_AARCH32: + ret = psci_stat_count(r1, r2); + break; +#endif + case PSCI_MEM_PROTECT: + ret = psci_mem_protect(r1); + break; + + case PSCI_MEM_CHK_RANGE_AARCH32: + ret = psci_mem_chk_range(r1, r2); + break; + + case PSCI_SYSTEM_RESET2_AARCH32: + /* We should never return from psci_system_reset2() */ + ret = psci_system_reset2(r1, r2); + break; + + default: + WARN("Unimplemented PSCI Call: 0x%x\n", smc_fid); + ret = (u_register_t)SMC_UNK; + break; + } + } else { + /* 64-bit PSCI function */ + + switch (smc_fid) { + case PSCI_CPU_SUSPEND_AARCH64: + ret = (u_register_t) + psci_cpu_suspend((unsigned int)x1, x2, x3); + break; + + case PSCI_CPU_ON_AARCH64: + ret = (u_register_t)psci_cpu_on(x1, x2, x3); + break; + + case PSCI_AFFINITY_INFO_AARCH64: + ret = (u_register_t) + psci_affinity_info(x1, (unsigned int)x2); + break; + + case PSCI_MIG_AARCH64: + ret = (u_register_t)psci_migrate(x1); + break; + + case PSCI_MIG_INFO_UP_CPU_AARCH64: + ret = psci_migrate_info_up_cpu(); + break; + + case PSCI_NODE_HW_STATE_AARCH64: + ret = (u_register_t)psci_node_hw_state( + x1, (unsigned int) x2); + break; + + case PSCI_SYSTEM_SUSPEND_AARCH64: + ret = (u_register_t)psci_system_suspend(x1, x2); + break; + +#if ENABLE_PSCI_STAT + case PSCI_STAT_RESIDENCY_AARCH64: + ret = psci_stat_residency(x1, (unsigned int) x2); + break; + + case PSCI_STAT_COUNT_AARCH64: + ret = psci_stat_count(x1, (unsigned int) x2); + break; +#endif + + case PSCI_MEM_CHK_RANGE_AARCH64: + ret = psci_mem_chk_range(x1, x2); + break; + + case PSCI_SYSTEM_RESET2_AARCH64: + /* We should never return from psci_system_reset2() */ + ret = psci_system_reset2((uint32_t) x1, x2); + break; + + default: + WARN("Unimplemented PSCI Call: 0x%x\n", smc_fid); + ret = (u_register_t)SMC_UNK; + break; + } + } + + return ret; +} diff --git a/lib/psci/psci_mem_protect.c b/lib/psci/psci_mem_protect.c new file mode 100644 index 0000000..385dcd2 --- /dev/null +++ b/lib/psci/psci_mem_protect.c @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <limits.h> + +#include <lib/utils.h> + +#include "psci_private.h" + +u_register_t psci_mem_protect(unsigned int enable) +{ + int val; + + assert(psci_plat_pm_ops->read_mem_protect != NULL); + assert(psci_plat_pm_ops->write_mem_protect != NULL); + + if (psci_plat_pm_ops->read_mem_protect(&val) < 0) + return (u_register_t) PSCI_E_NOT_SUPPORTED; + if (psci_plat_pm_ops->write_mem_protect(enable) < 0) + return (u_register_t) PSCI_E_NOT_SUPPORTED; + + return (val != 0) ? 1U : 0U; +} + +u_register_t psci_mem_chk_range(uintptr_t base, u_register_t length) +{ + int ret; + + assert(psci_plat_pm_ops->mem_protect_chk != NULL); + + if ((length == 0U) || check_uptr_overflow(base, length - 1U)) + return (u_register_t) PSCI_E_DENIED; + + ret = psci_plat_pm_ops->mem_protect_chk(base, length); + return (ret < 0) ? + (u_register_t) PSCI_E_DENIED : (u_register_t) PSCI_E_SUCCESS; +} diff --git a/lib/psci/psci_off.c b/lib/psci/psci_off.c new file mode 100644 index 0000000..f83753f --- /dev/null +++ b/lib/psci/psci_off.c @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2023, NVIDIA Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <string.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/pmf/pmf.h> +#include <lib/runtime_instr.h> +#include <plat/common/platform.h> + +#include "psci_private.h" + +/****************************************************************************** + * Construct the psci_power_state to request power OFF at all power levels. + ******************************************************************************/ +static void psci_set_power_off_state(psci_power_state_t *state_info) +{ + unsigned int lvl; + + for (lvl = PSCI_CPU_PWR_LVL; lvl <= PLAT_MAX_PWR_LVL; lvl++) + state_info->pwr_domain_state[lvl] = PLAT_MAX_OFF_STATE; +} + +/****************************************************************************** + * Top level handler which is called when a cpu wants to power itself down. + * It's assumed that along with turning the cpu power domain off, power + * domains at higher levels will be turned off as far as possible. It finds + * the highest level where a domain has to be powered off by traversing the + * node information and then performs generic, architectural, platform setup + * and state management required to turn OFF that power domain and domains + * below it. e.g. For a cpu that's to be powered OFF, it could mean programming + * the power controller whereas for a cluster that's to be powered off, it will + * call the platform specific code which will disable coherency at the + * interconnect level if the cpu is the last in the cluster and also the + * program the power controller. + ******************************************************************************/ +int psci_do_cpu_off(unsigned int end_pwrlvl) +{ + int rc = PSCI_E_SUCCESS; + unsigned int idx = plat_my_core_pos(); + psci_power_state_t state_info; + unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; + + /* + * This function must only be called on platforms where the + * CPU_OFF platform hooks have been implemented. + */ + assert(psci_plat_pm_ops->pwr_domain_off != NULL); + + /* Construct the psci_power_state for CPU_OFF */ + psci_set_power_off_state(&state_info); + + /* + * Call the platform provided early CPU_OFF handler to allow + * platforms to perform any housekeeping activities before + * actually powering the CPU off. PSCI_E_DENIED indicates that + * the CPU off sequence should be aborted at this time. + */ + if (psci_plat_pm_ops->pwr_domain_off_early) { + rc = psci_plat_pm_ops->pwr_domain_off_early(&state_info); + if (rc == PSCI_E_DENIED) { + return rc; + } + } + + /* + * Get the parent nodes here, this is important to do before we + * initiate the power down sequence as after that point the core may + * have exited coherency and its cache may be disabled, any access to + * shared memory after that (such as the parent node lookup in + * psci_cpu_pd_nodes) can cause coherency issues on some platforms. + */ + psci_get_parent_pwr_domain_nodes(idx, end_pwrlvl, parent_nodes); + + /* + * This function acquires the lock corresponding to each power + * level so that by the time all locks are taken, the system topology + * is snapshot and state management can be done safely. + */ + psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); + + /* + * Call the cpu off handler registered by the Secure Payload Dispatcher + * to let it do any bookkeeping. Assume that the SPD always reports an + * E_DENIED error if SP refuse to power down + */ + if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_off != NULL)) { + rc = psci_spd_pm->svc_off(0); + if (rc != 0) + goto exit; + } + + /* + * This function is passed the requested state info and + * it returns the negotiated state info for each power level upto + * the end level specified. + */ + psci_do_state_coordination(end_pwrlvl, &state_info); + + /* Update the target state in the power domain nodes */ + psci_set_target_local_pwr_states(end_pwrlvl, &state_info); + +#if ENABLE_PSCI_STAT + /* Update the last cpu for each level till end_pwrlvl */ + psci_stats_update_pwr_down(end_pwrlvl, &state_info); +#endif + +#if ENABLE_RUNTIME_INSTRUMENTATION + + /* + * Flush cache line so that even if CPU power down happens + * the timestamp update is reflected in memory. + */ + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_ENTER_CFLUSH, + PMF_CACHE_MAINT); +#endif + + /* + * Arch. management. Initiate power down sequence. + */ + psci_pwrdown_cpu(psci_find_max_off_lvl(&state_info)); + +#if ENABLE_RUNTIME_INSTRUMENTATION + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_EXIT_CFLUSH, + PMF_NO_CACHE_MAINT); +#endif + + /* + * Plat. management: Perform platform specific actions to turn this + * cpu off e.g. exit cpu coherency, program the power controller etc. + */ + psci_plat_pm_ops->pwr_domain_off(&state_info); + +#if ENABLE_PSCI_STAT + plat_psci_stat_accounting_start(&state_info); +#endif + +exit: + /* + * Release the locks corresponding to each power level in the + * reverse order to which they were acquired. + */ + psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); + + /* + * Check if all actions needed to safely power down this cpu have + * successfully completed. + */ + if (rc == PSCI_E_SUCCESS) { + /* + * Set the affinity info state to OFF. When caches are disabled, + * this writes directly to main memory, so cache maintenance is + * required to ensure that later cached reads of aff_info_state + * return AFF_STATE_OFF. A dsbish() ensures ordering of the + * update to the affinity info state prior to cache line + * invalidation. + */ + psci_flush_cpu_data(psci_svc_cpu_data.aff_info_state); + psci_set_aff_info_state(AFF_STATE_OFF); + psci_dsbish(); + psci_inv_cpu_data(psci_svc_cpu_data.aff_info_state); + +#if ENABLE_RUNTIME_INSTRUMENTATION + + /* + * Update the timestamp with cache off. We assume this + * timestamp can only be read from the current CPU and the + * timestamp cache line will be flushed before return to + * normal world on wakeup. + */ + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_ENTER_HW_LOW_PWR, + PMF_NO_CACHE_MAINT); +#endif + + if (psci_plat_pm_ops->pwr_domain_pwr_down_wfi != NULL) { + /* This function must not return */ + psci_plat_pm_ops->pwr_domain_pwr_down_wfi(&state_info); + } else { + /* + * Enter a wfi loop which will allow the power + * controller to physically power down this cpu. + */ + psci_power_down_wfi(); + } + } + + return rc; +} diff --git a/lib/psci/psci_on.c b/lib/psci/psci_on.c new file mode 100644 index 0000000..b279774 --- /dev/null +++ b/lib/psci/psci_on.c @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stddef.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/bl_common.h> +#include <common/debug.h> +#include <lib/el3_runtime/context_mgmt.h> +#include <lib/el3_runtime/pubsub_events.h> +#include <plat/common/platform.h> + +#include "psci_private.h" + +/* + * Helper functions for the CPU level spinlocks + */ +static inline void psci_spin_lock_cpu(unsigned int idx) +{ + spin_lock(&psci_cpu_pd_nodes[idx].cpu_lock); +} + +static inline void psci_spin_unlock_cpu(unsigned int idx) +{ + spin_unlock(&psci_cpu_pd_nodes[idx].cpu_lock); +} + +/******************************************************************************* + * This function checks whether a cpu which has been requested to be turned on + * is OFF to begin with. + ******************************************************************************/ +static int cpu_on_validate_state(aff_info_state_t aff_state) +{ + if (aff_state == AFF_STATE_ON) + return PSCI_E_ALREADY_ON; + + if (aff_state == AFF_STATE_ON_PENDING) + return PSCI_E_ON_PENDING; + + assert(aff_state == AFF_STATE_OFF); + return PSCI_E_SUCCESS; +} + +/******************************************************************************* + * Generic handler which is called to physically power on a cpu identified by + * its mpidr. It performs the generic, architectural, platform setup and state + * management to power on the target cpu e.g. it will ensure that + * enough information is stashed for it to resume execution in the non-secure + * security state. + * + * The state of all the relevant power domains are changed after calling the + * platform handler as it can return error. + ******************************************************************************/ +int psci_cpu_on_start(u_register_t target_cpu, + const entry_point_info_t *ep) +{ + int rc; + aff_info_state_t target_aff_state; + unsigned int target_idx = (unsigned int)plat_core_pos_by_mpidr(target_cpu); + + /* + * This function must only be called on platforms where the + * CPU_ON platform hooks have been implemented. + */ + assert((psci_plat_pm_ops->pwr_domain_on != NULL) && + (psci_plat_pm_ops->pwr_domain_on_finish != NULL)); + + /* Protect against multiple CPUs trying to turn ON the same target CPU */ + psci_spin_lock_cpu(target_idx); + + /* + * Generic management: Ensure that the cpu is off to be + * turned on. + * Perform cache maintanence ahead of reading the target CPU state to + * ensure that the data is not stale. + * There is a theoretical edge case where the cache may contain stale + * data for the target CPU data - this can occur under the following + * conditions: + * - the target CPU is in another cluster from the current + * - the target CPU was the last CPU to shutdown on its cluster + * - the cluster was removed from coherency as part of the CPU shutdown + * + * In this case the cache maintenace that was performed as part of the + * target CPUs shutdown was not seen by the current CPU's cluster. And + * so the cache may contain stale data for the target CPU. + */ + flush_cpu_data_by_index(target_idx, + psci_svc_cpu_data.aff_info_state); + rc = cpu_on_validate_state(psci_get_aff_info_state_by_idx(target_idx)); + if (rc != PSCI_E_SUCCESS) + goto exit; + + /* + * Call the cpu on handler registered by the Secure Payload Dispatcher + * to let it do any bookeeping. If the handler encounters an error, it's + * expected to assert within + */ + if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_on != NULL)) + psci_spd_pm->svc_on(target_cpu); + + /* + * Set the Affinity info state of the target cpu to ON_PENDING. + * Flush aff_info_state as it will be accessed with caches + * turned OFF. + */ + psci_set_aff_info_state_by_idx(target_idx, AFF_STATE_ON_PENDING); + flush_cpu_data_by_index(target_idx, + psci_svc_cpu_data.aff_info_state); + + /* + * The cache line invalidation by the target CPU after setting the + * state to OFF (see psci_do_cpu_off()), could cause the update to + * aff_info_state to be invalidated. Retry the update if the target + * CPU aff_info_state is not ON_PENDING. + */ + target_aff_state = psci_get_aff_info_state_by_idx(target_idx); + if (target_aff_state != AFF_STATE_ON_PENDING) { + assert(target_aff_state == AFF_STATE_OFF); + psci_set_aff_info_state_by_idx(target_idx, AFF_STATE_ON_PENDING); + flush_cpu_data_by_index(target_idx, + psci_svc_cpu_data.aff_info_state); + + assert(psci_get_aff_info_state_by_idx(target_idx) == + AFF_STATE_ON_PENDING); + } + + /* + * Perform generic, architecture and platform specific handling. + */ + /* + * Plat. management: Give the platform the current state + * of the target cpu to allow it to perform the necessary + * steps to power on. + */ + rc = psci_plat_pm_ops->pwr_domain_on(target_cpu); + assert((rc == PSCI_E_SUCCESS) || (rc == PSCI_E_INTERN_FAIL)); + + if (rc == PSCI_E_SUCCESS) + /* Store the re-entry information for the non-secure world. */ + cm_init_context_by_index(target_idx, ep); + else { + /* Restore the state on error. */ + psci_set_aff_info_state_by_idx(target_idx, AFF_STATE_OFF); + flush_cpu_data_by_index(target_idx, + psci_svc_cpu_data.aff_info_state); + } + +exit: + psci_spin_unlock_cpu(target_idx); + return rc; +} + +/******************************************************************************* + * The following function finish an earlier power on request. They + * are called by the common finisher routine in psci_common.c. The `state_info` + * is the psci_power_state from which this CPU has woken up from. + ******************************************************************************/ +void psci_cpu_on_finish(unsigned int cpu_idx, const psci_power_state_t *state_info) +{ + /* + * Plat. management: Perform the platform specific actions + * for this cpu e.g. enabling the gic or zeroing the mailbox + * register. The actual state of this cpu has already been + * changed. + */ + psci_plat_pm_ops->pwr_domain_on_finish(state_info); + +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + /* + * Arch. management: Enable data cache and manage stack memory + */ + psci_do_pwrup_cache_maintenance(); +#endif + + /* + * Plat. management: Perform any platform specific actions which + * can only be done with the cpu and the cluster guaranteed to + * be coherent. + */ + if (psci_plat_pm_ops->pwr_domain_on_finish_late != NULL) + psci_plat_pm_ops->pwr_domain_on_finish_late(state_info); + + /* + * All the platform specific actions for turning this cpu + * on have completed. Perform enough arch.initialization + * to run in the non-secure address space. + */ + psci_arch_setup(); + + /* + * Lock the CPU spin lock to make sure that the context initialization + * is done. Since the lock is only used in this function to create + * a synchronization point with cpu_on_start(), it can be released + * immediately. + */ + psci_spin_lock_cpu(cpu_idx); + psci_spin_unlock_cpu(cpu_idx); + + /* Ensure we have been explicitly woken up by another cpu */ + assert(psci_get_aff_info_state() == AFF_STATE_ON_PENDING); + + /* + * Call the cpu on finish handler registered by the Secure Payload + * Dispatcher to let it do any bookeeping. If the handler encounters an + * error, it's expected to assert within + */ + if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_on_finish != NULL)) + psci_spd_pm->svc_on_finish(0); + + PUBLISH_EVENT(psci_cpu_on_finish); + + /* Populate the mpidr field within the cpu node array */ + /* This needs to be done only once */ + psci_cpu_pd_nodes[cpu_idx].mpidr = read_mpidr() & MPIDR_AFFINITY_MASK; +} diff --git a/lib/psci/psci_private.h b/lib/psci/psci_private.h new file mode 100644 index 0000000..2eb4a9b --- /dev/null +++ b/lib/psci/psci_private.h @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PSCI_PRIVATE_H +#define PSCI_PRIVATE_H + +#include <stdbool.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/bl_common.h> +#include <lib/bakery_lock.h> +#include <lib/el3_runtime/cpu_data.h> +#include <lib/psci/psci.h> +#include <lib/spinlock.h> + +/* + * The PSCI capability which are provided by the generic code but does not + * depend on the platform or spd capabilities. + */ +#define PSCI_GENERIC_CAP \ + (define_psci_cap(PSCI_VERSION) | \ + define_psci_cap(PSCI_AFFINITY_INFO_AARCH64) | \ + define_psci_cap(PSCI_FEATURES)) + +/* + * The PSCI capabilities mask for 64 bit functions. + */ +#define PSCI_CAP_64BIT_MASK \ + (define_psci_cap(PSCI_CPU_SUSPEND_AARCH64) | \ + define_psci_cap(PSCI_CPU_ON_AARCH64) | \ + define_psci_cap(PSCI_AFFINITY_INFO_AARCH64) | \ + define_psci_cap(PSCI_MIG_AARCH64) | \ + define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64) | \ + define_psci_cap(PSCI_NODE_HW_STATE_AARCH64) | \ + define_psci_cap(PSCI_SYSTEM_SUSPEND_AARCH64) | \ + define_psci_cap(PSCI_STAT_RESIDENCY_AARCH64) | \ + define_psci_cap(PSCI_STAT_COUNT_AARCH64) | \ + define_psci_cap(PSCI_SYSTEM_RESET2_AARCH64) | \ + define_psci_cap(PSCI_MEM_CHK_RANGE_AARCH64)) + +/* Internally PSCI uses a uint16_t for various cpu indexes so + * define a limit to number of CPUs that can be initialised. + */ +#define PSCI_MAX_CPUS_INDEX 0xFFFFU + +/* Invalid parent */ +#define PSCI_PARENT_NODE_INVALID 0xFFFFFFFFU + +/* + * Helper functions to get/set the fields of PSCI per-cpu data. + */ +static inline void psci_set_aff_info_state(aff_info_state_t aff_state) +{ + set_cpu_data(psci_svc_cpu_data.aff_info_state, aff_state); +} + +static inline aff_info_state_t psci_get_aff_info_state(void) +{ + return get_cpu_data(psci_svc_cpu_data.aff_info_state); +} + +static inline aff_info_state_t psci_get_aff_info_state_by_idx(unsigned int idx) +{ + return get_cpu_data_by_index(idx, + psci_svc_cpu_data.aff_info_state); +} + +static inline void psci_set_aff_info_state_by_idx(unsigned int idx, + aff_info_state_t aff_state) +{ + set_cpu_data_by_index(idx, + psci_svc_cpu_data.aff_info_state, aff_state); +} + +static inline unsigned int psci_get_suspend_pwrlvl(void) +{ + return get_cpu_data(psci_svc_cpu_data.target_pwrlvl); +} + +static inline void psci_set_suspend_pwrlvl(unsigned int target_lvl) +{ + set_cpu_data(psci_svc_cpu_data.target_pwrlvl, target_lvl); +} + +static inline void psci_set_cpu_local_state(plat_local_state_t state) +{ + set_cpu_data(psci_svc_cpu_data.local_state, state); +} + +static inline plat_local_state_t psci_get_cpu_local_state(void) +{ + return get_cpu_data(psci_svc_cpu_data.local_state); +} + +static inline plat_local_state_t psci_get_cpu_local_state_by_idx( + unsigned int idx) +{ + return get_cpu_data_by_index(idx, + psci_svc_cpu_data.local_state); +} + +/* Helper function to identify a CPU standby request in PSCI Suspend call */ +static inline bool is_cpu_standby_req(unsigned int is_power_down_state, + unsigned int retn_lvl) +{ + return (is_power_down_state == 0U) && (retn_lvl == 0U); +} + +/******************************************************************************* + * The following two data structures implement the power domain tree. The tree + * is used to track the state of all the nodes i.e. power domain instances + * described by the platform. The tree consists of nodes that describe CPU power + * domains i.e. leaf nodes and all other power domains which are parents of a + * CPU power domain i.e. non-leaf nodes. + ******************************************************************************/ +typedef struct non_cpu_pwr_domain_node { + /* + * Index of the first CPU power domain node level 0 which has this node + * as its parent. + */ + unsigned int cpu_start_idx; + + /* + * Number of CPU power domains which are siblings of the domain indexed + * by 'cpu_start_idx' i.e. all the domains in the range 'cpu_start_idx + * -> cpu_start_idx + ncpus' have this node as their parent. + */ + unsigned int ncpus; + + /* + * Index of the parent power domain node. + * TODO: Figure out whether to whether using pointer is more efficient. + */ + unsigned int parent_node; + + plat_local_state_t local_state; + + unsigned char level; + + /* For indexing the psci_lock array*/ + uint16_t lock_index; +} non_cpu_pd_node_t; + +typedef struct cpu_pwr_domain_node { + u_register_t mpidr; + + /* + * Index of the parent power domain node. + * TODO: Figure out whether to whether using pointer is more efficient. + */ + unsigned int parent_node; + + /* + * A CPU power domain does not require state coordination like its + * parent power domains. Hence this node does not include a bakery + * lock. A spinlock is required by the CPU_ON handler to prevent a race + * when multiple CPUs try to turn ON the same target CPU. + */ + spinlock_t cpu_lock; +} cpu_pd_node_t; + +#if PSCI_OS_INIT_MODE +/******************************************************************************* + * The supported power state coordination modes that can be used in CPU_SUSPEND. + ******************************************************************************/ +typedef enum suspend_mode { + PLAT_COORD = 0, + OS_INIT = 1 +} suspend_mode_t; +#endif + +/******************************************************************************* + * The following are helpers and declarations of locks. + ******************************************************************************/ +#if HW_ASSISTED_COHERENCY +/* + * On systems where participant CPUs are cache-coherent, we can use spinlocks + * instead of bakery locks. + */ +#define DEFINE_PSCI_LOCK(_name) spinlock_t _name +#define DECLARE_PSCI_LOCK(_name) extern DEFINE_PSCI_LOCK(_name) + +/* One lock is required per non-CPU power domain node */ +DECLARE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); + +/* + * On systems with hardware-assisted coherency, make PSCI cache operations NOP, + * as PSCI participants are cache-coherent, and there's no need for explicit + * cache maintenance operations or barriers to coordinate their state. + */ +static inline void psci_flush_dcache_range(uintptr_t __unused addr, + size_t __unused size) +{ + /* Empty */ +} + +#define psci_flush_cpu_data(member) +#define psci_inv_cpu_data(member) + +static inline void psci_dsbish(void) +{ + /* Empty */ +} + +static inline void psci_lock_get(non_cpu_pd_node_t *non_cpu_pd_node) +{ + spin_lock(&psci_locks[non_cpu_pd_node->lock_index]); +} + +static inline void psci_lock_release(non_cpu_pd_node_t *non_cpu_pd_node) +{ + spin_unlock(&psci_locks[non_cpu_pd_node->lock_index]); +} + +#else /* if HW_ASSISTED_COHERENCY == 0 */ +/* + * Use bakery locks for state coordination as not all PSCI participants are + * cache coherent. + */ +#define DEFINE_PSCI_LOCK(_name) DEFINE_BAKERY_LOCK(_name) +#define DECLARE_PSCI_LOCK(_name) DECLARE_BAKERY_LOCK(_name) + +/* One lock is required per non-CPU power domain node */ +DECLARE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); + +/* + * If not all PSCI participants are cache-coherent, perform cache maintenance + * and issue barriers wherever required to coordinate state. + */ +static inline void psci_flush_dcache_range(uintptr_t addr, size_t size) +{ + flush_dcache_range(addr, size); +} + +#define psci_flush_cpu_data(member) flush_cpu_data(member) +#define psci_inv_cpu_data(member) inv_cpu_data(member) + +static inline void psci_dsbish(void) +{ + dsbish(); +} + +static inline void psci_lock_get(non_cpu_pd_node_t *non_cpu_pd_node) +{ + bakery_lock_get(&psci_locks[non_cpu_pd_node->lock_index]); +} + +static inline void psci_lock_release(non_cpu_pd_node_t *non_cpu_pd_node) +{ + bakery_lock_release(&psci_locks[non_cpu_pd_node->lock_index]); +} + +#endif /* HW_ASSISTED_COHERENCY */ + +static inline void psci_lock_init(non_cpu_pd_node_t *non_cpu_pd_node, + uint16_t idx) +{ + non_cpu_pd_node[idx].lock_index = idx; +} + +/******************************************************************************* + * Data prototypes + ******************************************************************************/ +extern const plat_psci_ops_t *psci_plat_pm_ops; +extern non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS]; +extern cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT]; +extern unsigned int psci_caps; +extern unsigned int psci_plat_core_count; +#if PSCI_OS_INIT_MODE +extern suspend_mode_t psci_suspend_mode; +#endif + +/******************************************************************************* + * SPD's power management hooks registered with PSCI + ******************************************************************************/ +extern const spd_pm_ops_t *psci_spd_pm; + +/******************************************************************************* + * Function prototypes + ******************************************************************************/ +/* Private exported functions from psci_common.c */ +int psci_validate_power_state(unsigned int power_state, + psci_power_state_t *state_info); +void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info); +void psci_init_req_local_pwr_states(void); +#if PSCI_OS_INIT_MODE +void psci_update_req_local_pwr_states(unsigned int end_pwrlvl, + unsigned int cpu_idx, + psci_power_state_t *state_info, + plat_local_state_t *prev); +void psci_restore_req_local_pwr_states(unsigned int cpu_idx, + plat_local_state_t *prev); +#endif +void psci_get_target_local_pwr_states(unsigned int end_pwrlvl, + psci_power_state_t *target_state); +void psci_set_target_local_pwr_states(unsigned int end_pwrlvl, + const psci_power_state_t *target_state); +int psci_validate_entry_point(entry_point_info_t *ep, + uintptr_t entrypoint, u_register_t context_id); +void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx, + unsigned int end_lvl, + unsigned int *node_index); +void psci_do_state_coordination(unsigned int end_pwrlvl, + psci_power_state_t *state_info); +#if PSCI_OS_INIT_MODE +int psci_validate_state_coordination(unsigned int end_pwrlvl, + psci_power_state_t *state_info); +#endif +void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl, + const unsigned int *parent_nodes); +void psci_release_pwr_domain_locks(unsigned int end_pwrlvl, + const unsigned int *parent_nodes); +int psci_validate_suspend_req(const psci_power_state_t *state_info, + unsigned int is_power_down_state); +unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info); +unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info); +void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl); +void psci_print_power_domain_map(void); +bool psci_is_last_on_cpu(void); +int psci_spd_migrate_info(u_register_t *mpidr); + +/* + * CPU power down is directly called only when HW_ASSISTED_COHERENCY is + * available. Otherwise, this needs post-call stack maintenance, which is + * handled in assembly. + */ +void prepare_cpu_pwr_dwn(unsigned int power_level); + +/* This function applies various CPU errata during power down. */ +void apply_cpu_pwr_dwn_errata(void); + +/* Private exported functions from psci_on.c */ +int psci_cpu_on_start(u_register_t target_cpu, + const entry_point_info_t *ep); + +void psci_cpu_on_finish(unsigned int cpu_idx, const psci_power_state_t *state_info); + +/* Private exported functions from psci_off.c */ +int psci_do_cpu_off(unsigned int end_pwrlvl); + +/* Private exported functions from psci_suspend.c */ +int psci_cpu_suspend_start(const entry_point_info_t *ep, + unsigned int end_pwrlvl, + psci_power_state_t *state_info, + unsigned int is_power_down_state); + +void psci_cpu_suspend_finish(unsigned int cpu_idx, const psci_power_state_t *state_info); + +/* Private exported functions from psci_helpers.S */ +void psci_do_pwrdown_cache_maintenance(unsigned int pwr_level); +void psci_do_pwrup_cache_maintenance(void); + +/* Private exported functions from psci_system_off.c */ +void __dead2 psci_system_off(void); +void __dead2 psci_system_reset(void); +u_register_t psci_system_reset2(uint32_t reset_type, u_register_t cookie); + +/* Private exported functions from psci_stat.c */ +void psci_stats_update_pwr_down(unsigned int end_pwrlvl, + const psci_power_state_t *state_info); +void psci_stats_update_pwr_up(unsigned int end_pwrlvl, + const psci_power_state_t *state_info); +u_register_t psci_stat_residency(u_register_t target_cpu, + unsigned int power_state); +u_register_t psci_stat_count(u_register_t target_cpu, + unsigned int power_state); + +/* Private exported functions from psci_mem_protect.c */ +u_register_t psci_mem_protect(unsigned int enable); +u_register_t psci_mem_chk_range(uintptr_t base, u_register_t length); + +#endif /* PSCI_PRIVATE_H */ diff --git a/lib/psci/psci_setup.c b/lib/psci/psci_setup.c new file mode 100644 index 0000000..6bf1ff4 --- /dev/null +++ b/lib/psci/psci_setup.c @@ -0,0 +1,318 @@ +/* + * Copyright (c) 2013-2020, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stddef.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/bl_common.h> +#include <context.h> +#include <lib/cpus/errata.h> +#include <lib/el3_runtime/context_mgmt.h> +#include <plat/common/platform.h> + +#include "psci_private.h" + +/* + * Check that PLATFORM_CORE_COUNT fits into the number of cores + * that can be represented by PSCI_MAX_CPUS_INDEX. + */ +CASSERT(PLATFORM_CORE_COUNT <= (PSCI_MAX_CPUS_INDEX + 1U), assert_psci_cores_overflow); + +/******************************************************************************* + * Per cpu non-secure contexts used to program the architectural state prior + * return to the normal world. + * TODO: Use the memory allocator to set aside memory for the contexts instead + * of relying on platform defined constants. + ******************************************************************************/ +static cpu_context_t psci_ns_context[PLATFORM_CORE_COUNT]; + +/****************************************************************************** + * Define the psci capability variable. + *****************************************************************************/ +unsigned int psci_caps; + +/******************************************************************************* + * Function which initializes the 'psci_non_cpu_pd_nodes' or the + * 'psci_cpu_pd_nodes' corresponding to the power level. + ******************************************************************************/ +static void __init psci_init_pwr_domain_node(uint16_t node_idx, + unsigned int parent_idx, + unsigned char level) +{ + if (level > PSCI_CPU_PWR_LVL) { + assert(node_idx < PSCI_NUM_NON_CPU_PWR_DOMAINS); + + psci_non_cpu_pd_nodes[node_idx].level = level; + psci_lock_init(psci_non_cpu_pd_nodes, node_idx); + psci_non_cpu_pd_nodes[node_idx].parent_node = parent_idx; + psci_non_cpu_pd_nodes[node_idx].local_state = + PLAT_MAX_OFF_STATE; + } else { + psci_cpu_data_t *svc_cpu_data; + + assert(node_idx < PLATFORM_CORE_COUNT); + + psci_cpu_pd_nodes[node_idx].parent_node = parent_idx; + + /* Initialize with an invalid mpidr */ + psci_cpu_pd_nodes[node_idx].mpidr = PSCI_INVALID_MPIDR; + + svc_cpu_data = + &(_cpu_data_by_index(node_idx)->psci_svc_cpu_data); + + /* Set the Affinity Info for the cores as OFF */ + svc_cpu_data->aff_info_state = AFF_STATE_OFF; + + /* Invalidate the suspend level for the cpu */ + svc_cpu_data->target_pwrlvl = PSCI_INVALID_PWR_LVL; + + /* Set the power state to OFF state */ + svc_cpu_data->local_state = PLAT_MAX_OFF_STATE; + + psci_flush_dcache_range((uintptr_t)svc_cpu_data, + sizeof(*svc_cpu_data)); + + cm_set_context_by_index(node_idx, + (void *) &psci_ns_context[node_idx], + NON_SECURE); + } +} + +/******************************************************************************* + * This functions updates cpu_start_idx and ncpus field for each of the node in + * psci_non_cpu_pd_nodes[]. It does so by comparing the parent nodes of each of + * the CPUs and check whether they match with the parent of the previous + * CPU. The basic assumption for this work is that children of the same parent + * are allocated adjacent indices. The platform should ensure this though proper + * mapping of the CPUs to indices via plat_core_pos_by_mpidr() and + * plat_my_core_pos() APIs. + *******************************************************************************/ +static void __init psci_update_pwrlvl_limits(void) +{ + unsigned int cpu_idx; + int j; + unsigned int nodes_idx[PLAT_MAX_PWR_LVL] = {0}; + unsigned int temp_index[PLAT_MAX_PWR_LVL]; + + for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) { + psci_get_parent_pwr_domain_nodes(cpu_idx, + PLAT_MAX_PWR_LVL, + temp_index); + for (j = (int)PLAT_MAX_PWR_LVL - 1; j >= 0; j--) { + if (temp_index[j] != nodes_idx[j]) { + nodes_idx[j] = temp_index[j]; + psci_non_cpu_pd_nodes[nodes_idx[j]].cpu_start_idx + = cpu_idx; + } + psci_non_cpu_pd_nodes[nodes_idx[j]].ncpus++; + } + } +} + +/******************************************************************************* + * Core routine to populate the power domain tree. The tree descriptor passed by + * the platform is populated breadth-first and the first entry in the map + * informs the number of root power domains. The parent nodes of the root nodes + * will point to an invalid entry(-1). + ******************************************************************************/ +static unsigned int __init populate_power_domain_tree(const unsigned char + *topology) +{ + unsigned int i, j = 0U, num_nodes_at_lvl = 1U, num_nodes_at_next_lvl; + unsigned int node_index = 0U, num_children; + unsigned int parent_node_index = 0U; + int level = (int)PLAT_MAX_PWR_LVL; + + /* + * For each level the inputs are: + * - number of nodes at this level in plat_array i.e. num_nodes_at_level + * This is the sum of values of nodes at the parent level. + * - Index of first entry at this level in the plat_array i.e. + * parent_node_index. + * - Index of first free entry in psci_non_cpu_pd_nodes[] or + * psci_cpu_pd_nodes[] i.e. node_index depending upon the level. + */ + while (level >= (int) PSCI_CPU_PWR_LVL) { + num_nodes_at_next_lvl = 0U; + /* + * For each entry (parent node) at this level in the plat_array: + * - Find the number of children + * - Allocate a node in a power domain array for each child + * - Set the parent of the child to the parent_node_index - 1 + * - Increment parent_node_index to point to the next parent + * - Accumulate the number of children at next level. + */ + for (i = 0U; i < num_nodes_at_lvl; i++) { + assert(parent_node_index <= + PSCI_NUM_NON_CPU_PWR_DOMAINS); + num_children = topology[parent_node_index]; + + for (j = node_index; + j < (node_index + num_children); j++) + psci_init_pwr_domain_node((uint16_t)j, + parent_node_index - 1U, + (unsigned char)level); + + node_index = j; + num_nodes_at_next_lvl += num_children; + parent_node_index++; + } + + num_nodes_at_lvl = num_nodes_at_next_lvl; + level--; + + /* Reset the index for the cpu power domain array */ + if (level == (int) PSCI_CPU_PWR_LVL) + node_index = 0; + } + + /* Validate the sanity of array exported by the platform */ + assert(j <= PLATFORM_CORE_COUNT); + return j; +} + +/******************************************************************************* + * This function does the architectural setup and takes the warm boot + * entry-point `mailbox_ep` as an argument. The function also initializes the + * power domain topology tree by querying the platform. The power domain nodes + * higher than the CPU are populated in the array psci_non_cpu_pd_nodes[] and + * the CPU power domains are populated in psci_cpu_pd_nodes[]. The platform + * exports its static topology map through the + * populate_power_domain_topology_tree() API. The algorithm populates the + * psci_non_cpu_pd_nodes and psci_cpu_pd_nodes iteratively by using this + * topology map. On a platform that implements two clusters of 2 cpus each, + * and supporting 3 domain levels, the populated psci_non_cpu_pd_nodes would + * look like this: + * + * --------------------------------------------------- + * | system node | cluster 0 node | cluster 1 node | + * --------------------------------------------------- + * + * And populated psci_cpu_pd_nodes would look like this : + * <- cpus cluster0 -><- cpus cluster1 -> + * ------------------------------------------------ + * | CPU 0 | CPU 1 | CPU 2 | CPU 3 | + * ------------------------------------------------ + ******************************************************************************/ +int __init psci_setup(const psci_lib_args_t *lib_args) +{ + const unsigned char *topology_tree; + + assert(VERIFY_PSCI_LIB_ARGS_V1(lib_args)); + + /* Do the Architectural initialization */ + psci_arch_setup(); + + /* Query the topology map from the platform */ + topology_tree = plat_get_power_domain_tree_desc(); + + /* Populate the power domain arrays using the platform topology map */ + psci_plat_core_count = populate_power_domain_tree(topology_tree); + + /* Update the CPU limits for each node in psci_non_cpu_pd_nodes */ + psci_update_pwrlvl_limits(); + + /* Populate the mpidr field of cpu node for this CPU */ + psci_cpu_pd_nodes[plat_my_core_pos()].mpidr = + read_mpidr() & MPIDR_AFFINITY_MASK; + + psci_init_req_local_pwr_states(); + + /* + * Set the requested and target state of this CPU and all the higher + * power domain levels for this CPU to run. + */ + psci_set_pwr_domains_to_run(PLAT_MAX_PWR_LVL); + + (void) plat_setup_psci_ops((uintptr_t)lib_args->mailbox_ep, + &psci_plat_pm_ops); + assert(psci_plat_pm_ops != NULL); + + /* + * Flush `psci_plat_pm_ops` as it will be accessed by secondary CPUs + * during warm boot, possibly before data cache is enabled. + */ + psci_flush_dcache_range((uintptr_t)&psci_plat_pm_ops, + sizeof(psci_plat_pm_ops)); + + /* Initialize the psci capability */ + psci_caps = PSCI_GENERIC_CAP; + + if (psci_plat_pm_ops->pwr_domain_off != NULL) + psci_caps |= define_psci_cap(PSCI_CPU_OFF); + if ((psci_plat_pm_ops->pwr_domain_on != NULL) && + (psci_plat_pm_ops->pwr_domain_on_finish != NULL)) + psci_caps |= define_psci_cap(PSCI_CPU_ON_AARCH64); + if ((psci_plat_pm_ops->pwr_domain_suspend != NULL) && + (psci_plat_pm_ops->pwr_domain_suspend_finish != NULL)) { + if (psci_plat_pm_ops->validate_power_state != NULL) + psci_caps |= define_psci_cap(PSCI_CPU_SUSPEND_AARCH64); + if (psci_plat_pm_ops->get_sys_suspend_power_state != NULL) + psci_caps |= define_psci_cap(PSCI_SYSTEM_SUSPEND_AARCH64); +#if PSCI_OS_INIT_MODE + psci_caps |= define_psci_cap(PSCI_SET_SUSPEND_MODE); +#endif + } + if (psci_plat_pm_ops->system_off != NULL) + psci_caps |= define_psci_cap(PSCI_SYSTEM_OFF); + if (psci_plat_pm_ops->system_reset != NULL) + psci_caps |= define_psci_cap(PSCI_SYSTEM_RESET); + if (psci_plat_pm_ops->get_node_hw_state != NULL) + psci_caps |= define_psci_cap(PSCI_NODE_HW_STATE_AARCH64); + if ((psci_plat_pm_ops->read_mem_protect != NULL) && + (psci_plat_pm_ops->write_mem_protect != NULL)) + psci_caps |= define_psci_cap(PSCI_MEM_PROTECT); + if (psci_plat_pm_ops->mem_protect_chk != NULL) + psci_caps |= define_psci_cap(PSCI_MEM_CHK_RANGE_AARCH64); + if (psci_plat_pm_ops->system_reset2 != NULL) + psci_caps |= define_psci_cap(PSCI_SYSTEM_RESET2_AARCH64); + +#if ENABLE_PSCI_STAT + psci_caps |= define_psci_cap(PSCI_STAT_RESIDENCY_AARCH64); + psci_caps |= define_psci_cap(PSCI_STAT_COUNT_AARCH64); +#endif + + return 0; +} + +/******************************************************************************* + * This duplicates what the primary cpu did after a cold boot in BL1. The same + * needs to be done when a cpu is hotplugged in. This function could also over- + * ride any EL3 setup done by BL1 as this code resides in rw memory. + ******************************************************************************/ +void psci_arch_setup(void) +{ +#if (ARM_ARCH_MAJOR > 7) || defined(ARMV7_SUPPORTS_GENERIC_TIMER) + /* Program the counter frequency */ + write_cntfrq_el0(plat_get_syscnt_freq2()); +#endif + + /* Initialize the cpu_ops pointer. */ + init_cpu_ops(); + + /* Having initialized cpu_ops, we can now print errata status */ + print_errata_status(); + +#if ENABLE_PAUTH + /* Store APIAKey_EL1 key */ + set_cpu_data(apiakey[0], read_apiakeylo_el1()); + set_cpu_data(apiakey[1], read_apiakeyhi_el1()); +#endif /* ENABLE_PAUTH */ +} + +/****************************************************************************** + * PSCI Library interface to initialize the cpu context for the next non + * secure image during cold boot. The relevant registers in the cpu context + * need to be retrieved and programmed on return from this interface. + *****************************************************************************/ +void psci_prepare_next_non_secure_ctx(entry_point_info_t *next_image_info) +{ + assert(GET_SECURITY_STATE(next_image_info->h.attr) == NON_SECURE); + cm_init_my_context(next_image_info); + cm_prepare_el3_exit(NON_SECURE); +} diff --git a/lib/psci/psci_stat.c b/lib/psci/psci_stat.c new file mode 100644 index 0000000..bedb816 --- /dev/null +++ b/lib/psci/psci_stat.c @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2016-2019, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> + +#include <platform_def.h> + +#include <common/debug.h> +#include <plat/common/platform.h> + +#include "psci_private.h" + +#ifndef PLAT_MAX_PWR_LVL_STATES +#define PLAT_MAX_PWR_LVL_STATES 2U +#endif + +/* Following structure is used for PSCI STAT */ +typedef struct psci_stat { + u_register_t residency; + u_register_t count; +} psci_stat_t; + +/* + * Following is used to keep track of the last cpu + * that goes to power down in non cpu power domains. + */ +static int last_cpu_in_non_cpu_pd[PSCI_NUM_NON_CPU_PWR_DOMAINS] = { + [0 ... PSCI_NUM_NON_CPU_PWR_DOMAINS - 1U] = -1}; + +/* + * Following are used to store PSCI STAT values for + * CPU and non CPU power domains. + */ +static psci_stat_t psci_cpu_stat[PLATFORM_CORE_COUNT] + [PLAT_MAX_PWR_LVL_STATES]; +static psci_stat_t psci_non_cpu_stat[PSCI_NUM_NON_CPU_PWR_DOMAINS] + [PLAT_MAX_PWR_LVL_STATES]; + +/* + * This functions returns the index into the `psci_stat_t` array given the + * local power state and power domain level. If the platform implements the + * `get_pwr_lvl_state_idx` pm hook, then that will be used to return the index. + */ +static int get_stat_idx(plat_local_state_t local_state, unsigned int pwr_lvl) +{ + int idx; + + if (psci_plat_pm_ops->get_pwr_lvl_state_idx == NULL) { + assert(PLAT_MAX_PWR_LVL_STATES == 2U); + if (is_local_state_retn(local_state) != 0) + return 0; + + assert(is_local_state_off(local_state) != 0); + return 1; + } + + idx = psci_plat_pm_ops->get_pwr_lvl_state_idx(local_state, pwr_lvl); + assert((idx >= 0) && (idx < (int) PLAT_MAX_PWR_LVL_STATES)); + return idx; +} + +/******************************************************************************* + * This function is passed the target local power states for each power + * domain (state_info) between the current CPU domain and its ancestors until + * the target power level (end_pwrlvl). + * + * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it + * updates the `last_cpu_in_non_cpu_pd[]` with last power down cpu id. + * + * This function will only be invoked with data cache enabled and while + * powering down a core. + ******************************************************************************/ +void psci_stats_update_pwr_down(unsigned int end_pwrlvl, + const psci_power_state_t *state_info) +{ + unsigned int lvl, parent_idx; + unsigned int cpu_idx = plat_my_core_pos(); + + assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); + assert(state_info != NULL); + + parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; + + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { + + /* Break early if the target power state is RUN */ + if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0) + break; + + /* + * The power domain is entering a low power state, so this is + * the last CPU for this power domain + */ + last_cpu_in_non_cpu_pd[parent_idx] = (int)cpu_idx; + + parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; + } + +} + +/******************************************************************************* + * This function updates the PSCI STATS(residency time and count) for CPU + * and NON-CPU power domains. + * It is called with caches enabled and locks acquired(for NON-CPU domain) + ******************************************************************************/ +void psci_stats_update_pwr_up(unsigned int end_pwrlvl, + const psci_power_state_t *state_info) +{ + unsigned int lvl, parent_idx; + unsigned int cpu_idx = plat_my_core_pos(); + int stat_idx; + plat_local_state_t local_state; + u_register_t residency; + + assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); + assert(state_info != NULL); + + /* Get the index into the stats array */ + local_state = state_info->pwr_domain_state[PSCI_CPU_PWR_LVL]; + stat_idx = get_stat_idx(local_state, PSCI_CPU_PWR_LVL); + + /* Call into platform interface to calculate residency. */ + residency = plat_psci_stat_get_residency(PSCI_CPU_PWR_LVL, + state_info, cpu_idx); + + /* Update CPU stats. */ + psci_cpu_stat[cpu_idx][stat_idx].residency += residency; + psci_cpu_stat[cpu_idx][stat_idx].count++; + + /* + * Check what power domains above CPU were off + * prior to this CPU powering on. + */ + parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; + /* Return early if this is the first power up. */ + if (last_cpu_in_non_cpu_pd[parent_idx] == -1) + return; + + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { + local_state = state_info->pwr_domain_state[lvl]; + if (is_local_state_run(local_state) != 0) { + /* Break early */ + break; + } + + assert(last_cpu_in_non_cpu_pd[parent_idx] != -1); + + /* Call into platform interface to calculate residency. */ + residency = plat_psci_stat_get_residency(lvl, state_info, + (unsigned int)last_cpu_in_non_cpu_pd[parent_idx]); + + /* Initialize back to reset value */ + last_cpu_in_non_cpu_pd[parent_idx] = -1; + + /* Get the index into the stats array */ + stat_idx = get_stat_idx(local_state, lvl); + + /* Update non cpu stats */ + psci_non_cpu_stat[parent_idx][stat_idx].residency += residency; + psci_non_cpu_stat[parent_idx][stat_idx].count++; + + parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; + } + +} + +/******************************************************************************* + * This function returns the appropriate count and residency time of the + * local state for the highest power level expressed in the `power_state` + * for the node represented by `target_cpu`. + ******************************************************************************/ +static int psci_get_stat(u_register_t target_cpu, unsigned int power_state, + psci_stat_t *psci_stat) +{ + int rc; + unsigned int pwrlvl, lvl, parent_idx, target_idx; + int stat_idx; + psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; + plat_local_state_t local_state; + + /* Determine the cpu index */ + target_idx = (unsigned int) plat_core_pos_by_mpidr(target_cpu); + + /* Validate the power_state parameter */ + if (psci_plat_pm_ops->translate_power_state_by_mpidr == NULL) + rc = psci_validate_power_state(power_state, &state_info); + else + rc = psci_plat_pm_ops->translate_power_state_by_mpidr( + target_cpu, power_state, &state_info); + + if (rc != PSCI_E_SUCCESS) + return PSCI_E_INVALID_PARAMS; + + /* Find the highest power level */ + pwrlvl = psci_find_target_suspend_lvl(&state_info); + if (pwrlvl == PSCI_INVALID_PWR_LVL) { + ERROR("Invalid target power level for PSCI statistics operation\n"); + panic(); + } + + /* Get the index into the stats array */ + local_state = state_info.pwr_domain_state[pwrlvl]; + stat_idx = get_stat_idx(local_state, pwrlvl); + + if (pwrlvl > PSCI_CPU_PWR_LVL) { + /* Get the power domain index */ + parent_idx = SPECULATION_SAFE_VALUE(psci_cpu_pd_nodes[target_idx].parent_node); + for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl < pwrlvl; lvl++) + parent_idx = SPECULATION_SAFE_VALUE(psci_non_cpu_pd_nodes[parent_idx].parent_node); + + /* Get the non cpu power domain stats */ + *psci_stat = psci_non_cpu_stat[parent_idx][stat_idx]; + } else { + /* Get the cpu power domain stats */ + *psci_stat = psci_cpu_stat[target_idx][stat_idx]; + } + + return PSCI_E_SUCCESS; +} + +/* This is the top level function for PSCI_STAT_RESIDENCY SMC. */ +u_register_t psci_stat_residency(u_register_t target_cpu, + unsigned int power_state) +{ + psci_stat_t psci_stat; + + /* Validate the target cpu */ + if (!is_valid_mpidr(target_cpu)) + return 0; + + int rc = psci_get_stat(target_cpu, power_state, &psci_stat); + + if (rc == PSCI_E_SUCCESS) + return psci_stat.residency; + else + return 0; +} + +/* This is the top level function for PSCI_STAT_COUNT SMC. */ +u_register_t psci_stat_count(u_register_t target_cpu, + unsigned int power_state) +{ + psci_stat_t psci_stat; + + /* Validate the target cpu */ + if (!is_valid_mpidr(target_cpu)) + return 0; + + int rc = psci_get_stat(target_cpu, power_state, &psci_stat); + + if (rc == PSCI_E_SUCCESS) + return psci_stat.count; + else + return 0; +} diff --git a/lib/psci/psci_suspend.c b/lib/psci/psci_suspend.c new file mode 100644 index 0000000..cb12b83 --- /dev/null +++ b/lib/psci/psci_suspend.c @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2013-2022, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stddef.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/bl_common.h> +#include <common/debug.h> +#include <context.h> +#include <lib/el3_runtime/context_mgmt.h> +#include <lib/el3_runtime/cpu_data.h> +#include <lib/el3_runtime/pubsub_events.h> +#include <lib/pmf/pmf.h> +#include <lib/runtime_instr.h> +#include <plat/common/platform.h> + +#include "psci_private.h" + +/******************************************************************************* + * This function does generic and platform specific operations after a wake-up + * from standby/retention states at multiple power levels. + ******************************************************************************/ +static void psci_suspend_to_standby_finisher(unsigned int cpu_idx, + unsigned int end_pwrlvl) +{ + unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; + psci_power_state_t state_info; + + /* Get the parent nodes */ + psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); + + psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); + + /* + * Find out which retention states this CPU has exited from until the + * 'end_pwrlvl'. The exit retention state could be deeper than the entry + * state as a result of state coordination amongst other CPUs post wfi. + */ + psci_get_target_local_pwr_states(end_pwrlvl, &state_info); + +#if ENABLE_PSCI_STAT + plat_psci_stat_accounting_stop(&state_info); + psci_stats_update_pwr_up(end_pwrlvl, &state_info); +#endif + + /* + * Plat. management: Allow the platform to do operations + * on waking up from retention. + */ + psci_plat_pm_ops->pwr_domain_suspend_finish(&state_info); + + /* + * Set the requested and target state of this CPU and all the higher + * power domain levels for this CPU to run. + */ + psci_set_pwr_domains_to_run(end_pwrlvl); + + psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); +} + +/******************************************************************************* + * This function does generic and platform specific suspend to power down + * operations. + ******************************************************************************/ +static void psci_suspend_to_pwrdown_start(unsigned int end_pwrlvl, + const entry_point_info_t *ep, + const psci_power_state_t *state_info) +{ + unsigned int max_off_lvl = psci_find_max_off_lvl(state_info); + + PUBLISH_EVENT(psci_suspend_pwrdown_start); + +#if PSCI_OS_INIT_MODE +#ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL + end_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; +#else + end_pwrlvl = PLAT_MAX_PWR_LVL; +#endif +#endif + + /* Save PSCI target power level for the suspend finisher handler */ + psci_set_suspend_pwrlvl(end_pwrlvl); + + /* + * Flush the target power level as it might be accessed on power up with + * Data cache disabled. + */ + psci_flush_cpu_data(psci_svc_cpu_data.target_pwrlvl); + + /* + * Call the cpu suspend handler registered by the Secure Payload + * Dispatcher to let it do any book-keeping. If the handler encounters an + * error, it's expected to assert within + */ + if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend != NULL)) + psci_spd_pm->svc_suspend(max_off_lvl); + +#if !HW_ASSISTED_COHERENCY + /* + * Plat. management: Allow the platform to perform any early + * actions required to power down the CPU. This might be useful for + * HW_ASSISTED_COHERENCY = 0 platforms that can safely perform these + * actions with data caches enabled. + */ + if (psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early != NULL) + psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early(state_info); +#endif + + /* + * Store the re-entry information for the non-secure world. + */ + cm_init_my_context(ep); + +#if ENABLE_RUNTIME_INSTRUMENTATION + + /* + * Flush cache line so that even if CPU power down happens + * the timestamp update is reflected in memory. + */ + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_ENTER_CFLUSH, + PMF_CACHE_MAINT); +#endif + + /* + * Arch. management. Initiate power down sequence. + * TODO : Introduce a mechanism to query the cache level to flush + * and the cpu-ops power down to perform from the platform. + */ + psci_pwrdown_cpu(max_off_lvl); + +#if ENABLE_RUNTIME_INSTRUMENTATION + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_EXIT_CFLUSH, + PMF_NO_CACHE_MAINT); +#endif +} + +/******************************************************************************* + * Top level handler which is called when a cpu wants to suspend its execution. + * It is assumed that along with suspending the cpu power domain, power domains + * at higher levels until the target power level will be suspended as well. It + * coordinates with the platform to negotiate the target state for each of + * the power domain level till the target power domain level. It then performs + * generic, architectural, platform setup and state management required to + * suspend that power domain level and power domain levels below it. + * e.g. For a cpu that's to be suspended, it could mean programming the + * power controller whereas for a cluster that's to be suspended, it will call + * the platform specific code which will disable coherency at the interconnect + * level if the cpu is the last in the cluster and also the program the power + * controller. + * + * All the required parameter checks are performed at the beginning and after + * the state transition has been done, no further error is expected and it is + * not possible to undo any of the actions taken beyond that point. + ******************************************************************************/ +int psci_cpu_suspend_start(const entry_point_info_t *ep, + unsigned int end_pwrlvl, + psci_power_state_t *state_info, + unsigned int is_power_down_state) +{ + int rc = PSCI_E_SUCCESS; + bool skip_wfi = false; + unsigned int idx = plat_my_core_pos(); + unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; + + /* + * This function must only be called on platforms where the + * CPU_SUSPEND platform hooks have been implemented. + */ + assert((psci_plat_pm_ops->pwr_domain_suspend != NULL) && + (psci_plat_pm_ops->pwr_domain_suspend_finish != NULL)); + + /* Get the parent nodes */ + psci_get_parent_pwr_domain_nodes(idx, end_pwrlvl, parent_nodes); + + /* + * This function acquires the lock corresponding to each power + * level so that by the time all locks are taken, the system topology + * is snapshot and state management can be done safely. + */ + psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); + + /* + * We check if there are any pending interrupts after the delay + * introduced by lock contention to increase the chances of early + * detection that a wake-up interrupt has fired. + */ + if (read_isr_el1() != 0U) { + skip_wfi = true; + goto exit; + } + +#if PSCI_OS_INIT_MODE + if (psci_suspend_mode == OS_INIT) { + /* + * This function validates the requested state info for + * OS-initiated mode. + */ + rc = psci_validate_state_coordination(end_pwrlvl, state_info); + if (rc != PSCI_E_SUCCESS) { + skip_wfi = true; + goto exit; + } + } else { +#endif + /* + * This function is passed the requested state info and + * it returns the negotiated state info for each power level upto + * the end level specified. + */ + psci_do_state_coordination(end_pwrlvl, state_info); +#if PSCI_OS_INIT_MODE + } +#endif + +#if PSCI_OS_INIT_MODE + if (psci_plat_pm_ops->pwr_domain_validate_suspend != NULL) { + rc = psci_plat_pm_ops->pwr_domain_validate_suspend(state_info); + if (rc != PSCI_E_SUCCESS) { + skip_wfi = true; + goto exit; + } + } +#endif + + /* Update the target state in the power domain nodes */ + psci_set_target_local_pwr_states(end_pwrlvl, state_info); + +#if ENABLE_PSCI_STAT + /* Update the last cpu for each level till end_pwrlvl */ + psci_stats_update_pwr_down(end_pwrlvl, state_info); +#endif + + if (is_power_down_state != 0U) + psci_suspend_to_pwrdown_start(end_pwrlvl, ep, state_info); + + /* + * Plat. management: Allow the platform to perform the + * necessary actions to turn off this cpu e.g. set the + * platform defined mailbox with the psci entrypoint, + * program the power controller etc. + */ + + psci_plat_pm_ops->pwr_domain_suspend(state_info); + +#if ENABLE_PSCI_STAT + plat_psci_stat_accounting_start(state_info); +#endif + +exit: + /* + * Release the locks corresponding to each power level in the + * reverse order to which they were acquired. + */ + psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); + + if (skip_wfi) { + return rc; + } + + if (is_power_down_state != 0U) { +#if ENABLE_RUNTIME_INSTRUMENTATION + + /* + * Update the timestamp with cache off. We assume this + * timestamp can only be read from the current CPU and the + * timestamp cache line will be flushed before return to + * normal world on wakeup. + */ + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_ENTER_HW_LOW_PWR, + PMF_NO_CACHE_MAINT); +#endif + + /* The function calls below must not return */ + if (psci_plat_pm_ops->pwr_domain_pwr_down_wfi != NULL) + psci_plat_pm_ops->pwr_domain_pwr_down_wfi(state_info); + else + psci_power_down_wfi(); + } + +#if ENABLE_RUNTIME_INSTRUMENTATION + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_ENTER_HW_LOW_PWR, + PMF_NO_CACHE_MAINT); +#endif + + /* + * We will reach here if only retention/standby states have been + * requested at multiple power levels. This means that the cpu + * context will be preserved. + */ + wfi(); + +#if ENABLE_RUNTIME_INSTRUMENTATION + PMF_CAPTURE_TIMESTAMP(rt_instr_svc, + RT_INSTR_EXIT_HW_LOW_PWR, + PMF_NO_CACHE_MAINT); +#endif + + /* + * After we wake up from context retaining suspend, call the + * context retaining suspend finisher. + */ + psci_suspend_to_standby_finisher(idx, end_pwrlvl); + + return rc; +} + +/******************************************************************************* + * The following functions finish an earlier suspend request. They + * are called by the common finisher routine in psci_common.c. The `state_info` + * is the psci_power_state from which this CPU has woken up from. + ******************************************************************************/ +void psci_cpu_suspend_finish(unsigned int cpu_idx, const psci_power_state_t *state_info) +{ + unsigned int counter_freq; + unsigned int max_off_lvl; + + /* Ensure we have been woken up from a suspended state */ + assert((psci_get_aff_info_state() == AFF_STATE_ON) && + (is_local_state_off( + state_info->pwr_domain_state[PSCI_CPU_PWR_LVL]) != 0)); + + /* + * Plat. management: Perform the platform specific actions + * before we change the state of the cpu e.g. enabling the + * gic or zeroing the mailbox register. If anything goes + * wrong then assert as there is no way to recover from this + * situation. + */ + psci_plat_pm_ops->pwr_domain_suspend_finish(state_info); + +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + /* Arch. management: Enable the data cache, stack memory maintenance. */ + psci_do_pwrup_cache_maintenance(); +#endif + + /* Re-init the cntfrq_el0 register */ + counter_freq = plat_get_syscnt_freq2(); + write_cntfrq_el0(counter_freq); + +#if ENABLE_PAUTH + /* Store APIAKey_EL1 key */ + set_cpu_data(apiakey[0], read_apiakeylo_el1()); + set_cpu_data(apiakey[1], read_apiakeyhi_el1()); +#endif /* ENABLE_PAUTH */ + + /* + * Call the cpu suspend finish handler registered by the Secure Payload + * Dispatcher to let it do any bookeeping. If the handler encounters an + * error, it's expected to assert within + */ + if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend_finish != NULL)) { + max_off_lvl = psci_find_max_off_lvl(state_info); + assert(max_off_lvl != PSCI_INVALID_PWR_LVL); + psci_spd_pm->svc_suspend_finish(max_off_lvl); + } + + /* Invalidate the suspend level for the cpu */ + psci_set_suspend_pwrlvl(PSCI_INVALID_PWR_LVL); + + PUBLISH_EVENT(psci_suspend_pwrdown_finish); +} diff --git a/lib/psci/psci_system_off.c b/lib/psci/psci_system_off.c new file mode 100644 index 0000000..002392c --- /dev/null +++ b/lib/psci/psci_system_off.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2014-2020, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stddef.h> + +#include <arch_helpers.h> +#include <common/debug.h> +#include <drivers/console.h> +#include <plat/common/platform.h> + +#include "psci_private.h" + +void __dead2 psci_system_off(void) +{ + psci_print_power_domain_map(); + + assert(psci_plat_pm_ops->system_off != NULL); + + /* Notify the Secure Payload Dispatcher */ + if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_system_off != NULL)) { + psci_spd_pm->svc_system_off(); + } + + console_flush(); + + /* Call the platform specific hook */ + psci_plat_pm_ops->system_off(); + + /* This function does not return. We should never get here */ +} + +void __dead2 psci_system_reset(void) +{ + psci_print_power_domain_map(); + + assert(psci_plat_pm_ops->system_reset != NULL); + + /* Notify the Secure Payload Dispatcher */ + if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_system_reset != NULL)) { + psci_spd_pm->svc_system_reset(); + } + + console_flush(); + + /* Call the platform specific hook */ + psci_plat_pm_ops->system_reset(); + + /* This function does not return. We should never get here */ +} + +u_register_t psci_system_reset2(uint32_t reset_type, u_register_t cookie) +{ + unsigned int is_vendor; + + psci_print_power_domain_map(); + + assert(psci_plat_pm_ops->system_reset2 != NULL); + + is_vendor = (reset_type >> PSCI_RESET2_TYPE_VENDOR_SHIFT) & 1U; + if (is_vendor == 0U) { + /* + * Only WARM_RESET is allowed for architectural type resets. + */ + if (reset_type != PSCI_RESET2_SYSTEM_WARM_RESET) + return (u_register_t) PSCI_E_INVALID_PARAMS; + if ((psci_plat_pm_ops->write_mem_protect != NULL) && + (psci_plat_pm_ops->write_mem_protect(0) < 0)) { + return (u_register_t) PSCI_E_NOT_SUPPORTED; + } + } + + /* Notify the Secure Payload Dispatcher */ + if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_system_reset != NULL)) { + psci_spd_pm->svc_system_reset(); + } + console_flush(); + + return (u_register_t) + psci_plat_pm_ops->system_reset2((int) is_vendor, reset_type, + cookie); +} diff --git a/lib/romlib/Makefile b/lib/romlib/Makefile new file mode 100644 index 0000000..c3ddc5a --- /dev/null +++ b/lib/romlib/Makefile @@ -0,0 +1,98 @@ +# +# Copyright (c) 2018-2019, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +AS = $(CROSS_COMPILE)as +AR = $(CROSS_COMPILE)ar +LD = $(CROSS_COMPILE)ld +OC = $(CROSS_COMPILE)objcopy +CPP = $(CROSS_COMPILE)cpp +ROMLIB_GEN = ./romlib_generator.py +BUILD_DIR = $(BUILD_PLAT)/romlib +LIB_DIR = $(BUILD_PLAT)/lib +WRAPPER_DIR = $(BUILD_PLAT)/libwrapper +LIBS = -lmbedtls -lfdt -lc +INC = $(INCLUDES:-I%=-I../../%) +PPFLAGS = $(INC) $(DEFINES) -P -x assembler-with-cpp -D__LINKER__ -MD -MP -MT $(BUILD_DIR)/romlib.ld +OBJS = $(BUILD_DIR)/jmptbl.o $(BUILD_DIR)/init.o +MAPFILE = $(BUILD_PLAT)/romlib/romlib.map + +ifneq ($(PLAT_DIR),) + WRAPPER_SOURCES = $(shell $(ROMLIB_GEN) genwrappers -b $(WRAPPER_DIR) --list ../../$(PLAT_DIR)/jmptbl.i) + WRAPPER_OBJS = $(WRAPPER_SOURCES:.s=.o) +endif + +V ?= 0 +ifeq ($(V),0) + Q := @ +else + Q := +endif + +LDFLAGS := --gc-sections -O1 +ifeq ($(DEBUG),1) + LDFLAGS += -Map=$(MAPFILE) +endif + +ifeq (${ARM_ARCH_MINOR},0) + ASFLAGS = -march=armv8-a +else + ASFLAGS = -march=armv8.${ARM_ARCH_MINOR}-a +endif + +.PHONY: all clean distclean + +all: $(BUILD_DIR)/romlib.bin $(LIB_DIR)/libwrappers.a + +%.o: %.s + @echo " AS $@" + $(Q)$(AS) $(ASFLAGS) -o $@ $< + +$(BUILD_DIR)/%.o: %.s + @echo " AS $@" + $(Q)$(AS) $(ASFLAGS) -o $@ $< + +$(BUILD_DIR)/romlib.ld: romlib.ld.S + @echo " PP $@" + $(Q)$(CPP) $(PPFLAGS) -o $@ romlib.ld.S + +$(BUILD_DIR)/romlib.elf: $(OBJS) $(BUILD_DIR)/romlib.ld + @echo " LD $@" + $(Q)$(LD) -T $(BUILD_DIR)/romlib.ld -L$(LIB_DIR) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) + +$(BUILD_DIR)/romlib.bin: $(BUILD_DIR)/romlib.elf + @echo " BIN $@" + $(Q)$(OC) -O binary $(BUILD_DIR)/romlib.elf $@ + +$(WRAPPER_DIR)/jmpvar.s: $(BUILD_DIR)/romlib.elf + @echo " VAR $@" + $(Q)$(ROMLIB_GEN) genvar --output $@ $< + +$(LIB_DIR)/libwrappers.a: $(WRAPPER_DIR)/jmpvar.o $(WRAPPER_OBJS) + @echo " AR $@" + $(Q)$(AR) -rc $@ $(WRAPPER_DIR)/jmpvar.o $(WRAPPER_OBJS) + +$(BUILD_DIR)/jmptbl.i: ../../$(PLAT_DIR)/jmptbl.i + @echo " PRE $@" + $(Q)$(ROMLIB_GEN) pre --output $@ --deps $(BUILD_DIR)/jmptbl.d $< + +$(BUILD_DIR)/wrappers.stamp: $(BUILD_DIR)/jmptbl.i + @echo " WRP $<" + $(Q)$(ROMLIB_GEN) genwrappers --bti=$(ENABLE_BTI) -b $(WRAPPER_DIR) $< + @touch $@ + +$(WRAPPER_SOURCES): $(BUILD_DIR)/wrappers.stamp + +$(WRAPPER_OBJS): $(WRAPPER_SOURCES) $(BUILD_DIR)/wrappers.stamp + +$(BUILD_DIR)/jmptbl.s: $(BUILD_DIR)/jmptbl.i + @echo " TBL $@" + $(Q)$(ROMLIB_GEN) gentbl --output $@ --bti=$(ENABLE_BTI) $< + +clean: + @rm -f $(BUILD_DIR)/* + +-include $(BUILD_DIR)/romlib.d +-include $(BUILD_DIR)/jmptbl.d diff --git a/lib/romlib/gen_combined_bl1_romlib.sh b/lib/romlib/gen_combined_bl1_romlib.sh new file mode 100755 index 0000000..3941467 --- /dev/null +++ b/lib/romlib/gen_combined_bl1_romlib.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +set -e + +output="bl1_romlib.bin" + +# Set trap for removing temporary file +trap 'r=$?;rm -f $bin_path/$$.tmp;exit $r' EXIT HUP QUIT INT TERM + +# Read input parameters +for i +do + case $i in + -o) + output=$2 + shift 2 + ;; + --) + shift + break + ;; + -*) + echo usage: gen_combined_bl1_romlib.sh [-o output] path_to_build_directory >&2 + ;; + esac +done + + +bin_path=$1 +romlib_path=$1/romlib +bl1_file="$1/bl1/bl1.elf" +romlib_file="$1/romlib/romlib.elf" +bl1_end="" +romlib_begin="" + +# Get address of __BL1_ROM_END__ +bl1_end=`nm -a "$bl1_file" | +awk '$3 == "__BL1_ROM_END__" {print "0x"$1}'` + +# Get start address of romlib "text" section +romlib_begin=`nm -a "$romlib_file" | +awk '$3 == ".text" {print "0x"$1}'` + +# Character "U" will be read as "55" in hex when it is +# concatenated with bl1.bin. Generate combined BL1 and ROMLIB +# binary with filler bytes for juno +(cat $bin_path/bl1.bin + yes U | sed $(($romlib_begin - $bl1_end))q | tr -d '\n' + cat $bin_path/romlib/romlib.bin) > $bin_path/$$.tmp && +mv $bin_path/$$.tmp $bin_path/$output diff --git a/lib/romlib/init.s b/lib/romlib/init.s new file mode 100644 index 0000000..2bea15b --- /dev/null +++ b/lib/romlib/init.s @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + .globl rom_lib_init + .extern __DATA_RAM_START__, __DATA_ROM_START__, __DATA_RAM_END__ + .extern memset, memcpy + +rom_lib_init: + cmp w0, #1 + mov w0, #0 + b.le 1f + ret + +1: stp x29, x30, [sp, #-16]! + adrp x0, __DATA_RAM_START__ + adrp x1, __DATA_ROM_START__ + add x1, x1, :lo12:__DATA_ROM_START__ + adrp x2, __DATA_RAM_END__ + add x2, x2, :lo12:__DATA_RAM_END__ + sub x2, x2, x0 + bl memcpy + + adrp x0,__BSS_START__ + add x0, x0, :lo12:__BSS_START__ + mov x1, #0 + adrp x2, __BSS_END__ + add x2, x2, :lo12:__BSS_END__ + sub x2, x2, x0 + bl memset + ldp x29, x30, [sp], #16 + + mov w0, #1 + ret diff --git a/lib/romlib/jmptbl.i b/lib/romlib/jmptbl.i new file mode 100644 index 0000000..44751a2 --- /dev/null +++ b/lib/romlib/jmptbl.i @@ -0,0 +1,44 @@ +# +# Copyright (c) 2018-2020, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +# Format: +# lib function [patch] +# Add "patch" at the end of the line to patch a function. For example: +# mbedtls mbedtls_memory_buffer_alloc_init patch +# Holes can be introduced in the table by using a special keyword "reserved". +# Example: +# reserved reserved +# The jump table will contain an invalid instruction instead of branch + +rom rom_lib_init +fdt fdt_getprop_namelen +fdt fdt_setprop_inplace +fdt fdt_check_header +fdt fdt_node_offset_by_compatible +fdt fdt_setprop_inplace_namelen_partial +mbedtls mbedtls_asn1_get_alg +mbedtls mbedtls_asn1_get_alg_null +mbedtls mbedtls_asn1_get_bitstring_null +mbedtls mbedtls_asn1_get_bool +mbedtls mbedtls_asn1_get_int +mbedtls mbedtls_asn1_get_tag +mbedtls mbedtls_free +mbedtls mbedtls_md +mbedtls mbedtls_md_get_size +mbedtls mbedtls_memory_buffer_alloc_init +mbedtls mbedtls_oid_get_md_alg +mbedtls mbedtls_oid_get_numeric_string +mbedtls mbedtls_oid_get_pk_alg +mbedtls mbedtls_oid_get_sig_alg +mbedtls mbedtls_pk_free +mbedtls mbedtls_pk_init +mbedtls mbedtls_pk_parse_subpubkey +mbedtls mbedtls_pk_verify_ext +mbedtls mbedtls_platform_set_snprintf +mbedtls mbedtls_x509_get_rsassa_pss_params +mbedtls mbedtls_x509_get_sig_alg +mbedtls mbedtls_md_info_from_type +c exit +c atexit diff --git a/lib/romlib/romlib.ld.S b/lib/romlib/romlib.ld.S new file mode 100644 index 0000000..0d076b4 --- /dev/null +++ b/lib/romlib/romlib.ld.S @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <lib/xlat_tables/xlat_tables_defs.h> +#include <platform_def.h> + +MEMORY { + ROM (rx): ORIGIN = ROMLIB_RO_BASE, LENGTH = ROMLIB_RO_LIMIT - ROMLIB_RO_BASE + RAM (rwx): ORIGIN = ROMLIB_RW_BASE, LENGTH = ROMLIB_RW_END - ROMLIB_RW_BASE +} + +OUTPUT_FORMAT(PLATFORM_LINKER_FORMAT) +OUTPUT_ARCH(PLATFORM_LINKER_ARCH) +ENTRY(jmptbl) + +SECTIONS { + . = ROMLIB_RO_BASE; + + .text : { + *jmptbl.o(.text) + *(.text*) + *(.rodata*) + } >ROM + + __DATA_ROM_START__ = LOADADDR(.data); + + .data : { + __DATA_RAM_START__ = .; + + *(.data*) + + __DATA_RAM_END__ = .; + } >RAM AT>ROM + + __DATA_SIZE__ = SIZEOF(.data); + + .bss : { + __BSS_START__ = .; + + *(.bss*) + + __BSS_END__ = .; + } >RAM + + __BSS_SIZE__ = SIZEOF(.bss); +} diff --git a/lib/romlib/romlib_generator.py b/lib/romlib/romlib_generator.py new file mode 100755 index 0000000..0682dd4 --- /dev/null +++ b/lib/romlib/romlib_generator.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +This module contains a set of classes and a runner that can generate code for the romlib module +based on the templates in the 'templates' directory. +""" + +import argparse +import os +import re +import subprocess +import string +import sys + +class IndexFileParser: + """ + Parses the contents of the index file into the items and dependencies variables. It + also resolves included files in the index files recursively with circular inclusion detection. + """ + + def __init__(self): + self.items = [] + self.dependencies = {} + self.include_chain = [] + + def add_dependency(self, parent, dependency): + """ Adds a dependency into the dependencies variable. """ + if parent in self.dependencies: + self.dependencies[parent].append(dependency) + else: + self.dependencies[parent] = [dependency] + + def get_dependencies(self, parent): + """ Gets all the recursive dependencies of a parent file. """ + parent = os.path.normpath(parent) + if parent in self.dependencies: + direct_deps = self.dependencies[parent] + deps = direct_deps + for direct_dep in direct_deps: + deps += self.get_dependencies(direct_dep) + return deps + + return [] + + def parse(self, file_name): + """ Opens and parses index file. """ + file_name = os.path.normpath(file_name) + + if file_name not in self.include_chain: + self.include_chain.append(file_name) + self.dependencies[file_name] = [] + else: + raise Exception("Circular dependency detected: " + file_name) + + with open(file_name, "r") as index_file: + for line in index_file.readlines(): + line_elements = line.split() + + if line.startswith("#") or not line_elements: + # Comment or empty line + continue + + if line_elements[0] == "reserved": + # Reserved slot in the jump table + self.items.append({"type": "reserved"}) + elif line_elements[0] == "include" and len(line_elements) > 1: + # Include other index file + included_file = os.path.normpath(line_elements[1]) + self.add_dependency(file_name, included_file) + self.parse(included_file) + elif len(line_elements) > 1: + # Library function + library_name = line_elements[0] + function_name = line_elements[1] + patch = bool(len(line_elements) > 2 and line_elements[2] == "patch") + + self.items.append({"type": "function", "library_name": library_name, + "function_name": function_name, "patch": patch}) + else: + raise Exception("Invalid line: '" + line + "'") + + self.include_chain.pop() + +class RomlibApplication: + """ Base class of romlib applications. """ + TEMPLATE_DIR = os.path.dirname(os.path.realpath(__file__)) + "/templates/" + + def __init__(self, prog): + self.args = argparse.ArgumentParser(prog=prog, description=self.__doc__) + self.config = None + + def parse_arguments(self, argv): + """ Parses the arguments that should come from the command line arguments. """ + self.config = self.args.parse_args(argv) + + def build_template(self, name, mapping=None, remove_comment=False): + """ + Loads a template and builds it with the defined mapping. Template paths are always relative + to this script. + """ + + with open(self.TEMPLATE_DIR + name, "r") as template_file: + if remove_comment: + # Removing copyright comment to make the generated code more readable when the + # template is inserted multiple times into the output. + template_lines = template_file.readlines() + end_of_comment_line = 0 + for index, line in enumerate(template_lines): + if line.find("*/") != -1: + end_of_comment_line = index + break + template_data = "".join(template_lines[end_of_comment_line + 1:]) + else: + template_data = template_file.read() + + template = string.Template(template_data) + return template.substitute(mapping) + +class IndexPreprocessor(RomlibApplication): + """ Removes empty and comment lines from the index file and resolves includes. """ + + def __init__(self, prog): + RomlibApplication.__init__(self, prog) + + self.args.add_argument("-o", "--output", help="Output file", metavar="output", + default="jmpvar.s") + self.args.add_argument("--deps", help="Dependency file") + self.args.add_argument("file", help="Input file") + + def main(self): + """ + After parsing the input index file it generates a clean output with all includes resolved. + Using --deps option it also outputs the dependencies in makefile format like gcc's with -M. + """ + + index_file_parser = IndexFileParser() + index_file_parser.parse(self.config.file) + + with open(self.config.output, "w") as output_file: + for item in index_file_parser.items: + if item["type"] == "function": + patch = "\tpatch" if item["patch"] else "" + output_file.write( + item["library_name"] + "\t" + item["function_name"] + patch + "\n") + else: + output_file.write("reserved\n") + + if self.config.deps: + with open(self.config.deps, "w") as deps_file: + deps = [self.config.file] + index_file_parser.get_dependencies(self.config.file) + deps_file.write(self.config.output + ": " + " \\\n".join(deps) + "\n") + +class TableGenerator(RomlibApplication): + """ Generates the jump table by parsing the index file. """ + + def __init__(self, prog): + RomlibApplication.__init__(self, prog) + + self.args.add_argument("-o", "--output", help="Output file", metavar="output", + default="jmpvar.s") + self.args.add_argument("--bti", help="Branch Target Identification", type=int) + self.args.add_argument("file", help="Input file") + + def main(self): + """ + Inserts the jmptbl definition and the jump entries into the output file. Also can insert + BTI related code before entries if --bti option set. It can output a dependency file of the + included index files. This can be directly included in makefiles. + """ + + index_file_parser = IndexFileParser() + index_file_parser.parse(self.config.file) + + with open(self.config.output, "w") as output_file: + output_file.write(self.build_template("jmptbl_header.S")) + bti = "_bti" if self.config.bti == 1 else "" + + for item in index_file_parser.items: + template_name = "jmptbl_entry_" + item["type"] + bti + ".S" + output_file.write(self.build_template(template_name, item, True)) + +class WrapperGenerator(RomlibApplication): + """ + Generates a wrapper function for each entry in the index file except for the ones that contain + the keyword patch. The generated wrapper file is called <lib>_<fn_name>.s. + """ + + def __init__(self, prog): + RomlibApplication.__init__(self, prog) + + self.args.add_argument("-b", help="Build directory", default=".", metavar="build") + self.args.add_argument("--bti", help="Branch Target Identification", type=int) + self.args.add_argument("--list", help="Only list assembly files", action="store_true") + self.args.add_argument("file", help="Input file") + + def main(self): + """ + Iterates through the items in the parsed index file and builds the template for each entry. + """ + + index_file_parser = IndexFileParser() + index_file_parser.parse(self.config.file) + + bti = "_bti" if self.config.bti == 1 else "" + function_offset = 0 + files = [] + + for item_index in range(0, len(index_file_parser.items)): + item = index_file_parser.items[item_index] + + if item["type"] == "reserved" or item["patch"]: + continue + + asm = self.config.b + "/" + item["function_name"] + ".s" + if self.config.list: + # Only listing files + files.append(asm) + else: + with open(asm, "w") as asm_file: + # The jump instruction is 4 bytes but BTI requires and extra instruction so + # this makes it 8 bytes per entry. + function_offset = item_index * (8 if self.config.bti else 4) + + item["function_offset"] = function_offset + asm_file.write(self.build_template("wrapper" + bti + ".S", item)) + + if self.config.list: + print(" ".join(files)) + +class VariableGenerator(RomlibApplication): + """ Generates the jump table global variable with the absolute address in ROM. """ + + def __init__(self, prog): + RomlibApplication.__init__(self, prog) + + self.args.add_argument("-o", "--output", help="Output file", metavar="output", + default="jmpvar.s") + self.args.add_argument("file", help="Input file") + + def main(self): + """ + Runs nm -a command on the input file and inserts the address of the .text section into the + template as the ROM address of the jmp_table. + """ + symbols = subprocess.check_output(["nm", "-a", self.config.file]) + + matching_symbol = re.search("([0-9A-Fa-f]+) . \\.text", str(symbols)) + if not matching_symbol: + raise Exception("No '.text' section was found in %s" % self.config.file) + + mapping = {"jmptbl_address": matching_symbol.group(1)} + + with open(self.config.output, "w") as output_file: + output_file.write(self.build_template("jmptbl_glob_var.S", mapping)) + +if __name__ == "__main__": + APPS = {"genvar": VariableGenerator, "pre": IndexPreprocessor, + "gentbl": TableGenerator, "genwrappers": WrapperGenerator} + + if len(sys.argv) < 2 or sys.argv[1] not in APPS: + print("usage: romlib_generator.py [%s] [args]" % "|".join(APPS.keys()), file=sys.stderr) + sys.exit(1) + + APP = APPS[sys.argv[1]]("romlib_generator.py " + sys.argv[1]) + APP.parse_arguments(sys.argv[2:]) + try: + APP.main() + sys.exit(0) + except FileNotFoundError as file_not_found_error: + print(file_not_found_error, file=sys.stderr) + except subprocess.CalledProcessError as called_process_error: + print(called_process_error.output, file=sys.stderr) + + sys.exit(1) diff --git a/lib/romlib/templates/jmptbl_entry_function.S b/lib/romlib/templates/jmptbl_entry_function.S new file mode 100644 index 0000000..a0f8456 --- /dev/null +++ b/lib/romlib/templates/jmptbl_entry_function.S @@ -0,0 +1,6 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + b ${function_name} diff --git a/lib/romlib/templates/jmptbl_entry_function_bti.S b/lib/romlib/templates/jmptbl_entry_function_bti.S new file mode 100644 index 0000000..d96ee94 --- /dev/null +++ b/lib/romlib/templates/jmptbl_entry_function_bti.S @@ -0,0 +1,7 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + bti j + b ${function_name} diff --git a/lib/romlib/templates/jmptbl_entry_reserved.S b/lib/romlib/templates/jmptbl_entry_reserved.S new file mode 100644 index 0000000..a9b5f18 --- /dev/null +++ b/lib/romlib/templates/jmptbl_entry_reserved.S @@ -0,0 +1,6 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + b . diff --git a/lib/romlib/templates/jmptbl_entry_reserved_bti.S b/lib/romlib/templates/jmptbl_entry_reserved_bti.S new file mode 100644 index 0000000..a9f0375 --- /dev/null +++ b/lib/romlib/templates/jmptbl_entry_reserved_bti.S @@ -0,0 +1,7 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + bti j + b . diff --git a/lib/romlib/templates/jmptbl_glob_var.S b/lib/romlib/templates/jmptbl_glob_var.S new file mode 100644 index 0000000..d306512 --- /dev/null +++ b/lib/romlib/templates/jmptbl_glob_var.S @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + .data + .globl jmptbl + .align 4 +jmptbl: .quad 0x${jmptbl_address} diff --git a/lib/romlib/templates/jmptbl_header.S b/lib/romlib/templates/jmptbl_header.S new file mode 100644 index 0000000..72b8ce5 --- /dev/null +++ b/lib/romlib/templates/jmptbl_header.S @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + .text + .globl jmptbl +jmptbl: diff --git a/lib/romlib/templates/wrapper.S b/lib/romlib/templates/wrapper.S new file mode 100644 index 0000000..734a68a --- /dev/null +++ b/lib/romlib/templates/wrapper.S @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + .globl ${function_name} +${function_name}: + ldr x17, =jmptbl + mov x16, #${function_offset} + ldr x17, [x17] + add x16, x16, x17 + br x16 diff --git a/lib/romlib/templates/wrapper_bti.S b/lib/romlib/templates/wrapper_bti.S new file mode 100644 index 0000000..ba9b11c --- /dev/null +++ b/lib/romlib/templates/wrapper_bti.S @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2019, Arm Limited. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + .globl ${function_name} +${function_name}: + bti jc + ldr x17, =jmptbl + mov x16, #${function_offset} + ldr x17, [x17] + add x16, x16, x17 + br x16 diff --git a/lib/semihosting/aarch32/semihosting_call.S b/lib/semihosting/aarch32/semihosting_call.S new file mode 100644 index 0000000..42d4718 --- /dev/null +++ b/lib/semihosting/aarch32/semihosting_call.S @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + + .globl semihosting_call + +func semihosting_call + svc #0x123456 + bx lr +endfunc semihosting_call diff --git a/lib/semihosting/aarch64/semihosting_call.S b/lib/semihosting/aarch64/semihosting_call.S new file mode 100644 index 0000000..6f60f3e --- /dev/null +++ b/lib/semihosting/aarch64/semihosting_call.S @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2013-2014, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> + + .globl semihosting_call + +func semihosting_call + hlt #0xf000 + ret +endfunc semihosting_call diff --git a/lib/semihosting/semihosting.c b/lib/semihosting/semihosting.c new file mode 100644 index 0000000..adc6b78 --- /dev/null +++ b/lib/semihosting/semihosting.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include <lib/semihosting.h> + +#ifndef SEMIHOSTING_SUPPORTED +#define SEMIHOSTING_SUPPORTED 1 +#endif + +long semihosting_call(unsigned long operation, uintptr_t system_block_address); + +typedef struct { + const char *file_name; + unsigned long mode; + size_t name_length; +} smh_file_open_block_t; + +typedef struct { + long handle; + uintptr_t buffer; + size_t length; +} smh_file_read_write_block_t; + +typedef struct { + long handle; + ssize_t location; +} smh_file_seek_block_t; + +typedef struct { + char *command_line; + size_t command_length; +} smh_system_block_t; + +long semihosting_connection_supported(void) +{ + return SEMIHOSTING_SUPPORTED; +} + +long semihosting_file_open(const char *file_name, size_t mode) +{ + smh_file_open_block_t open_block; + + open_block.file_name = file_name; + open_block.mode = mode; + open_block.name_length = strlen(file_name); + + return semihosting_call(SEMIHOSTING_SYS_OPEN, (uintptr_t)&open_block); +} + +long semihosting_file_seek(long file_handle, ssize_t offset) +{ + smh_file_seek_block_t seek_block; + long result; + + seek_block.handle = file_handle; + seek_block.location = offset; + + result = semihosting_call(SEMIHOSTING_SYS_SEEK, (uintptr_t)&seek_block); + + if (result < 0) { + result = semihosting_call(SEMIHOSTING_SYS_ERRNO, 0); + } else { + result = 0; + } + + return result; +} + +long semihosting_file_read(long file_handle, size_t *length, uintptr_t buffer) +{ + smh_file_read_write_block_t read_block; + long result = -EINVAL; + + if ((length == NULL) || (buffer == (uintptr_t)NULL)) { + return result; + } + + read_block.handle = file_handle; + read_block.buffer = buffer; + read_block.length = *length; + + result = semihosting_call(SEMIHOSTING_SYS_READ, (uintptr_t)&read_block); + + if (result == *length) { + return -EINVAL; + } else if (result < *length) { + *length -= result; + return 0; + } else { + return result; + } +} + +long semihosting_file_write(long file_handle, size_t *length, + const uintptr_t buffer) +{ + smh_file_read_write_block_t write_block; + long result = -EINVAL; + + if ((length == NULL) || (buffer == (uintptr_t)NULL)) { + return -EINVAL; + } + + write_block.handle = file_handle; + write_block.buffer = (uintptr_t)buffer; /* cast away const */ + write_block.length = *length; + + result = semihosting_call(SEMIHOSTING_SYS_WRITE, + (uintptr_t)&write_block); + + *length = result; + + return (result == 0) ? 0 : -EINVAL; +} + +long semihosting_file_close(long file_handle) +{ + return semihosting_call(SEMIHOSTING_SYS_CLOSE, (uintptr_t)&file_handle); +} + +long semihosting_file_length(long file_handle) +{ + return semihosting_call(SEMIHOSTING_SYS_FLEN, (uintptr_t)&file_handle); +} + +char semihosting_read_char(void) +{ + return semihosting_call(SEMIHOSTING_SYS_READC, 0); +} + +void semihosting_write_char(char character) +{ + semihosting_call(SEMIHOSTING_SYS_WRITEC, (uintptr_t)&character); +} + +void semihosting_write_string(char *string) +{ + semihosting_call(SEMIHOSTING_SYS_WRITE0, (uintptr_t)string); +} + +long semihosting_system(char *command_line) +{ + smh_system_block_t system_block; + + system_block.command_line = command_line; + system_block.command_length = strlen(command_line); + + return semihosting_call(SEMIHOSTING_SYS_SYSTEM, + (uintptr_t)&system_block); +} + +long semihosting_get_flen(const char *file_name) +{ + long file_handle; + long length; + + assert(semihosting_connection_supported() != 0); + + file_handle = semihosting_file_open(file_name, FOPEN_MODE_RB); + if (file_handle == -1) { + return file_handle; + } + + /* Find the length of the file */ + length = semihosting_file_length(file_handle); + + return (semihosting_file_close(file_handle) != 0) ? -1 : length; +} + +long semihosting_download_file(const char *file_name, + size_t buf_size, + uintptr_t buf) +{ + long ret = -EINVAL; + size_t length; + long file_handle; + + /* Null pointer check */ + if (buf == 0U) { + return ret; + } + + assert(semihosting_connection_supported() != 0); + + file_handle = semihosting_file_open(file_name, FOPEN_MODE_RB); + if (file_handle == -1) { + return ret; + } + + /* Find the actual length of the file */ + length = semihosting_file_length(file_handle); + if (length == (size_t)(-1)) { + goto semihosting_fail; + } + + /* Signal error if we do not have enough space for the file */ + if (length > buf_size) { + goto semihosting_fail; + } + + /* + * A successful read will return 0 in which case we pass back + * the actual number of bytes read. Else we pass a negative + * value indicating an error. + */ + ret = semihosting_file_read(file_handle, &length, buf); + if (ret != 0) { + goto semihosting_fail; + } else { + ret = (long)length; + } + +semihosting_fail: + semihosting_file_close(file_handle); + return ret; +} + +void semihosting_exit(uint32_t reason, uint32_t subcode) +{ +#ifdef __aarch64__ + uint64_t parameters[] = {reason, subcode}; + + (void)semihosting_call(SEMIHOSTING_SYS_EXIT, (uintptr_t)¶meters); +#else + /* The subcode is not supported on AArch32. */ + (void)semihosting_call(SEMIHOSTING_SYS_EXIT, reason); +#endif +} diff --git a/lib/stack_protector/aarch32/asm_stack_protector.S b/lib/stack_protector/aarch32/asm_stack_protector.S new file mode 100644 index 0000000..0ffcf8a --- /dev/null +++ b/lib/stack_protector/aarch32/asm_stack_protector.S @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> + + .globl update_stack_protector_canary + +/* ----------------------------------------------------------------------- + * void update_stack_protector_canary(void) + * + * Change the value of the canary used for stack smashing attacks protection. + * Note: This must be called when it is safe to call C code, but this cannot be + * called by C code. Doing this will make the check fail when the calling + * function returns. + * ----------------------------------------------------------------------- + */ + +func update_stack_protector_canary + /* Use r4 as it is callee-saved */ + mov r4, lr + bl plat_get_stack_protector_canary + + /* Update the canary with the returned value */ + ldr r1, =__stack_chk_guard + str r0, [r1] + bx r4 +endfunc update_stack_protector_canary + + diff --git a/lib/stack_protector/aarch64/asm_stack_protector.S b/lib/stack_protector/aarch64/asm_stack_protector.S new file mode 100644 index 0000000..a002617 --- /dev/null +++ b/lib/stack_protector/aarch64/asm_stack_protector.S @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <assert_macros.S> + + .globl update_stack_protector_canary + +/* ----------------------------------------------------------------------- + * void update_stack_protector_canary(void) + * + * Change the value of the canary used for stack smashing attacks protection. + * Note: This must be called when it is safe to call C code, but this cannot be + * called by C code. Doing this will make the check fail when the calling + * function returns. + * ----------------------------------------------------------------------- + */ + +func update_stack_protector_canary + /* Use x19 as it is callee-saved */ + mov x19, x30 + bl plat_get_stack_protector_canary + + /* Update the canary with the returned value */ + adrp x1, __stack_chk_guard + str x0, [x1, #:lo12:__stack_chk_guard] + ret x19 +endfunc update_stack_protector_canary + + diff --git a/lib/stack_protector/stack_protector.c b/lib/stack_protector/stack_protector.c new file mode 100644 index 0000000..d0f2400 --- /dev/null +++ b/lib/stack_protector/stack_protector.c @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdint.h> + +#include <common/debug.h> +#include <plat/common/platform.h> + +/* + * Canary value used by the compiler runtime checks to detect stack corruption. + * + * Force the canary to be in .data to allow predictable memory layout relatively + * to the stacks. + */ +u_register_t __attribute__((section(".data.stack_protector_canary"))) + __stack_chk_guard = (u_register_t) 3288484550995823360ULL; + +/* + * Function called when the stack's canary check fails, which means the stack + * was corrupted. It must not return. + */ +void __dead2 __stack_chk_fail(void) +{ +#if DEBUG + ERROR("Stack corruption detected\n"); +#endif + panic(); +} + diff --git a/lib/stack_protector/stack_protector.mk b/lib/stack_protector/stack_protector.mk new file mode 100644 index 0000000..4f13bc3 --- /dev/null +++ b/lib/stack_protector/stack_protector.mk @@ -0,0 +1,28 @@ +# +# Copyright (c) 2017-2019, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +# Boolean macro to be used in C code +STACK_PROTECTOR_ENABLED := 0 + +ifeq (${ENABLE_STACK_PROTECTOR},0) + ENABLE_STACK_PROTECTOR := none +endif + +ifeq (${ENABLE_STACK_PROTECTOR},none) + TF_CFLAGS += -fno-stack-protector +else + STACK_PROTECTOR_ENABLED := 1 + BL_COMMON_SOURCES += lib/stack_protector/stack_protector.c \ + lib/stack_protector/${ARCH}/asm_stack_protector.S + + ifeq (${ENABLE_STACK_PROTECTOR},default) + TF_CFLAGS += -fstack-protector + else + TF_CFLAGS += -fstack-protector-${ENABLE_STACK_PROTECTOR} + endif +endif + +$(eval $(call add_define,STACK_PROTECTOR_ENABLED)) diff --git a/lib/transfer_list/transfer_list.c b/lib/transfer_list/transfer_list.c new file mode 100644 index 0000000..e38bf74 --- /dev/null +++ b/lib/transfer_list/transfer_list.c @@ -0,0 +1,474 @@ +/* + * Copyright (c) 2023, Linaro Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <inttypes.h> +#include <string.h> + +#include <common/debug.h> +#include <lib/transfer_list.h> +#include <lib/utils_def.h> + +void transfer_list_dump(struct transfer_list_header *tl) +{ + struct transfer_list_entry *te = NULL; + int i = 0; + + if (!tl) { + return; + } + NOTICE("Dump transfer list:\n"); + NOTICE("signature 0x%x\n", tl->signature); + NOTICE("checksum 0x%x\n", tl->checksum); + NOTICE("version 0x%x\n", tl->version); + NOTICE("hdr_size 0x%x\n", tl->hdr_size); + NOTICE("alignment 0x%x\n", tl->alignment); + NOTICE("size 0x%x\n", tl->size); + NOTICE("max_size 0x%x\n", tl->max_size); + while (true) { + te = transfer_list_next(tl, te); + if (!te) { + break; + } + NOTICE("Entry %d:\n", i++); + NOTICE("tag_id 0x%x\n", te->tag_id); + NOTICE("hdr_size 0x%x\n", te->hdr_size); + NOTICE("data_size 0x%x\n", te->data_size); + NOTICE("data_addr 0x%lx\n", + (unsigned long)transfer_list_entry_data(te)); + } +} + +/******************************************************************************* + * Creating a transfer list in a reserved memory region specified + * Compliant to 2.4.5 of Firmware handoff specification (v0.9) + * Return pointer to the created transfer list or NULL on error + ******************************************************************************/ +struct transfer_list_header *transfer_list_init(void *addr, size_t max_size) +{ + struct transfer_list_header *tl = addr; + + if (!addr || max_size == 0) { + return NULL; + } + + if (!is_aligned((uintptr_t)addr, 1 << TRANSFER_LIST_INIT_MAX_ALIGN) || + !is_aligned(max_size, 1 << TRANSFER_LIST_INIT_MAX_ALIGN) || + max_size < sizeof(*tl)) { + return NULL; + } + + memset(tl, 0, max_size); + tl->signature = TRANSFER_LIST_SIGNATURE; + tl->version = TRANSFER_LIST_VERSION; + tl->hdr_size = sizeof(*tl); + tl->alignment = TRANSFER_LIST_INIT_MAX_ALIGN; // initial max align + tl->size = sizeof(*tl); // initial size is the size of header + tl->max_size = max_size; + + transfer_list_update_checksum(tl); + + return tl; +} + +/******************************************************************************* + * Relocating a transfer list to a reserved memory region specified + * Compliant to 2.4.6 of Firmware handoff specification (v0.9) + * Return true on success or false on error + ******************************************************************************/ +struct transfer_list_header *transfer_list_relocate( + struct transfer_list_header *tl, + void *addr, size_t max_size) +{ + uintptr_t new_addr, align_mask, align_off; + struct transfer_list_header *new_tl; + uint32_t new_max_size; + + if (!tl || !addr || max_size == 0) { + return NULL; + } + + align_mask = (1 << tl->alignment) - 1; + align_off = (uintptr_t)tl & align_mask; + new_addr = ((uintptr_t)addr & ~align_mask) + align_off; + + if (new_addr < (uintptr_t)addr) { + new_addr += (1 << tl->alignment); + } + + new_max_size = max_size - (new_addr - (uintptr_t)addr); + + // the new space is not sufficient for the tl + if (tl->size > new_max_size) { + return NULL; + } + + new_tl = (struct transfer_list_header *)new_addr; + memmove(new_tl, tl, tl->size); + new_tl->max_size = new_max_size; + + transfer_list_update_checksum(new_tl); + + return new_tl; +} + +/******************************************************************************* + * Verifying the header of a transfer list + * Compliant to 2.4.1 of Firmware handoff specification (v0.9) + * Return transfer list operation status code + ******************************************************************************/ +enum transfer_list_ops transfer_list_check_header( + const struct transfer_list_header *tl) +{ + if (!tl) { + return TL_OPS_NON; + } + + if (tl->signature != TRANSFER_LIST_SIGNATURE) { + ERROR("Bad transfer list signature %#"PRIx32"\n", + tl->signature); + return TL_OPS_NON; + } + + if (!tl->max_size) { + ERROR("Bad transfer list max size %#"PRIx32"\n", + tl->max_size); + return TL_OPS_NON; + } + + if (tl->size > tl->max_size) { + ERROR("Bad transfer list size %#"PRIx32"\n", tl->size); + return TL_OPS_NON; + } + + if (tl->hdr_size != sizeof(struct transfer_list_header)) { + ERROR("Bad transfer list header size %#"PRIx32"\n", tl->hdr_size); + return TL_OPS_NON; + } + + if (!transfer_list_verify_checksum(tl)) { + ERROR("Bad transfer list checksum %#"PRIx32"\n", tl->checksum); + return TL_OPS_NON; + } + + if (tl->version == 0) { + ERROR("Transfer list version is invalid\n"); + return TL_OPS_NON; + } else if (tl->version == TRANSFER_LIST_VERSION) { + INFO("Transfer list version is valid for all operations\n"); + return TL_OPS_ALL; + } else if (tl->version > TRANSFER_LIST_VERSION) { + INFO("Transfer list version is valid for read-only\n"); + return TL_OPS_RO; + } + + INFO("Old transfer list version is detected\n"); + return TL_OPS_CUS; +} + +/******************************************************************************* + * Enumerate the next transfer entry + * Return pointer to the next transfer entry or NULL on error + ******************************************************************************/ +struct transfer_list_entry *transfer_list_next(struct transfer_list_header *tl, + struct transfer_list_entry *last) +{ + struct transfer_list_entry *te = NULL; + uintptr_t tl_ev = 0; + uintptr_t va = 0; + uintptr_t ev = 0; + size_t sz = 0; + + if (!tl) { + return NULL; + } + + tl_ev = (uintptr_t)tl + tl->size; + + if (last) { + va = (uintptr_t)last; + // check if the total size overflow + if (add_overflow(last->hdr_size, + last->data_size, &sz)) { + return NULL; + } + // roundup to the next entry + if (add_with_round_up_overflow(va, sz, + TRANSFER_LIST_GRANULE, &va)) { + return NULL; + } + } else { + va = (uintptr_t)tl + tl->hdr_size; + } + + te = (struct transfer_list_entry *)va; + + if (va + sizeof(*te) > tl_ev || te->hdr_size < sizeof(*te) || + add_overflow(te->hdr_size, te->data_size, &sz) || + add_overflow(va, sz, &ev) || + ev > tl_ev) { + return NULL; + } + + return te; +} + +/******************************************************************************* + * Calculate the byte sum of a transfer list + * Return byte sum of the transfer list + ******************************************************************************/ +static uint8_t calc_byte_sum(const struct transfer_list_header *tl) +{ + uint8_t *b = (uint8_t *)tl; + uint8_t cs = 0; + size_t n = 0; + + if (!tl) { + return 0; + } + + for (n = 0; n < tl->size; n++) { + cs += b[n]; + } + + return cs; +} + +/******************************************************************************* + * Update the checksum of a transfer list + * Return updated checksum of the transfer list + ******************************************************************************/ +void transfer_list_update_checksum(struct transfer_list_header *tl) +{ + uint8_t cs; + + if (!tl) { + return; + } + + cs = calc_byte_sum(tl); + cs -= tl->checksum; + cs = 256 - cs; + tl->checksum = cs; + assert(transfer_list_verify_checksum(tl)); +} + +/******************************************************************************* + * Verify the checksum of a transfer list + * Return true if verified or false if not + ******************************************************************************/ +bool transfer_list_verify_checksum(const struct transfer_list_header *tl) +{ + return !calc_byte_sum(tl); +} + +/******************************************************************************* + * Update the data size of a transfer entry + * Return true on success or false on error + ******************************************************************************/ +bool transfer_list_set_data_size(struct transfer_list_header *tl, + struct transfer_list_entry *te, + uint32_t new_data_size) +{ + uintptr_t tl_old_ev, new_ev = 0, old_ev = 0, ru_new_ev; + struct transfer_list_entry *dummy_te = NULL; + size_t gap = 0; + size_t mov_dis = 0; + size_t sz = 0; + + if (!tl || !te) { + return false; + } + tl_old_ev = (uintptr_t)tl + tl->size; + + // calculate the old and new end of TE + // both must be roundup to align with TRANSFER_LIST_GRANULE + if (add_overflow(te->hdr_size, te->data_size, &sz) || + add_with_round_up_overflow((uintptr_t)te, sz, + TRANSFER_LIST_GRANULE, &old_ev)) { + return false; + } + if (add_overflow(te->hdr_size, new_data_size, &sz) || + add_with_round_up_overflow((uintptr_t)te, sz, + TRANSFER_LIST_GRANULE, &new_ev)) { + return false; + } + + if (new_ev > old_ev) { + // move distance should be roundup + // to meet the requirement of TE data max alignment + // ensure that the increased size doesn't exceed + // the max size of TL + mov_dis = new_ev - old_ev; + if (round_up_overflow(mov_dis, 1 << tl->alignment, + &mov_dis) || tl->size + mov_dis > tl->max_size) { + return false; + } + ru_new_ev = old_ev + mov_dis; + memmove((void *)ru_new_ev, (void *)old_ev, tl_old_ev - old_ev); + tl->size += mov_dis; + gap = ru_new_ev - new_ev; + } else { + gap = old_ev - new_ev; + } + + if (gap >= sizeof(*dummy_te)) { + // create a dummy TE to fill up the gap + dummy_te = (struct transfer_list_entry *)new_ev; + dummy_te->tag_id = TL_TAG_EMPTY; + dummy_te->reserved0 = 0; + dummy_te->hdr_size = sizeof(*dummy_te); + dummy_te->data_size = gap - sizeof(*dummy_te); + } + + te->data_size = new_data_size; + + transfer_list_update_checksum(tl); + return true; +} + +/******************************************************************************* + * Remove a specified transfer entry from a transfer list + * Return true on success or false on error + ******************************************************************************/ +bool transfer_list_rem(struct transfer_list_header *tl, + struct transfer_list_entry *te) +{ + if (!tl || !te || (uintptr_t)te > (uintptr_t)tl + tl->size) { + return false; + } + te->tag_id = TL_TAG_EMPTY; + te->reserved0 = 0; + transfer_list_update_checksum(tl); + return true; +} + +/******************************************************************************* + * Add a new transfer entry into a transfer list + * Compliant to 2.4.3 of Firmware handoff specification (v0.9) + * Return pointer to the added transfer entry or NULL on error + ******************************************************************************/ +struct transfer_list_entry *transfer_list_add(struct transfer_list_header *tl, + uint16_t tag_id, + uint32_t data_size, + const void *data) +{ + uintptr_t max_tl_ev, tl_ev, ev; + struct transfer_list_entry *te = NULL; + uint8_t *te_data = NULL; + size_t sz = 0; + + if (!tl) { + return NULL; + } + + max_tl_ev = (uintptr_t)tl + tl->max_size; + tl_ev = (uintptr_t)tl + tl->size; + ev = tl_ev; + + // skip the step 1 (optional step) + // new TE will be added into the tail + if (add_overflow(sizeof(*te), data_size, &sz) || + add_with_round_up_overflow(ev, sz, + TRANSFER_LIST_GRANULE, &ev) || ev > max_tl_ev) { + return NULL; + } + + te = (struct transfer_list_entry *)tl_ev; + te->tag_id = tag_id; + te->reserved0 = 0; + te->hdr_size = sizeof(*te); + te->data_size = data_size; + tl->size += ev - tl_ev; + + if (data) { + // get TE data pointer + te_data = transfer_list_entry_data(te); + if (!te_data) { + return NULL; + } + memmove(te_data, data, data_size); + } + + transfer_list_update_checksum(tl); + + return te; +} + +/******************************************************************************* + * Add a new transfer entry into a transfer list with specified new data + * alignment requirement + * Compliant to 2.4.4 of Firmware handoff specification (v0.9) + * Return pointer to the added transfer entry or NULL on error + ******************************************************************************/ +struct transfer_list_entry *transfer_list_add_with_align( + struct transfer_list_header *tl, + uint16_t tag_id, uint32_t data_size, + const void *data, uint8_t alignment) +{ + struct transfer_list_entry *te = NULL; + uintptr_t tl_ev, ev, new_tl_ev; + size_t dummy_te_data_sz = 0; + + if (!tl) { + return NULL; + } + + tl_ev = (uintptr_t)tl + tl->size; + ev = tl_ev + sizeof(struct transfer_list_entry); + + if (!is_aligned(ev, 1 << alignment)) { + // TE data address is not aligned to the new alignment + // fill the gap with an empty TE as a placeholder before + // adding the desire TE + new_tl_ev = round_up(ev, 1 << alignment) - + sizeof(struct transfer_list_entry); + dummy_te_data_sz = new_tl_ev - tl_ev - + sizeof(struct transfer_list_entry); + if (!transfer_list_add(tl, TL_TAG_EMPTY, dummy_te_data_sz, + NULL)) { + return NULL; + } + } + + te = transfer_list_add(tl, tag_id, data_size, data); + + if (alignment > tl->alignment) { + tl->alignment = alignment; + transfer_list_update_checksum(tl); + } + + return te; +} + +/******************************************************************************* + * Search for an existing transfer entry with the specified tag id from a + * transfer list + * Return pointer to the found transfer entry or NULL on error + ******************************************************************************/ +struct transfer_list_entry *transfer_list_find(struct transfer_list_header *tl, + uint16_t tag_id) +{ + struct transfer_list_entry *te = NULL; + + do { + te = transfer_list_next(tl, te); + } while (te && (te->tag_id != tag_id || te->reserved0 != 0)); + + return te; +} + +/******************************************************************************* + * Retrieve the data pointer of a specified transfer entry + * Return pointer to the transfer entry data or NULL on error + ******************************************************************************/ +void *transfer_list_entry_data(struct transfer_list_entry *entry) +{ + if (!entry) { + return NULL; + } + return (uint8_t *)entry + entry->hdr_size; +} diff --git a/lib/transfer_list/transfer_list.mk b/lib/transfer_list/transfer_list.mk new file mode 100644 index 0000000..42574e8 --- /dev/null +++ b/lib/transfer_list/transfer_list.mk @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +ifeq (${TRANSFER_LIST},1) + +ifeq (${ARCH},aarch32) +$(eval $(call add_define,TRANSFER_LIST_AARCH32)) +endif + +TRANSFER_LIST_SOURCES += $(addprefix lib/transfer_list/, \ + transfer_list.c) + +BL31_SOURCES += $(TRANSFER_LIST_SOURCES) +BL2_SOURCES += $(TRANSFER_LIST_SOURCES) + +endif # TRANSFER_LIST + diff --git a/lib/utils/mem_region.c b/lib/utils/mem_region.c new file mode 100644 index 0000000..e9f0a12 --- /dev/null +++ b/lib/utils/mem_region.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2017-2020, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> + +#include <common/debug.h> +#include <lib/utils.h> +#include <lib/xlat_tables/xlat_tables_compat.h> + +/* + * All the regions defined in mem_region_t must have the following properties + * + * - Any contiguous regions must be merged into a single entry. + * - The number of bytes of each region must be greater than zero. + * - The calculation of the highest address within the region (base + nbytes-1) + * doesn't produce an overflow. + * + * These conditions must be fulfilled by the caller and they aren't checked + * at runtime. + */ + +/* + * zero_normalmem all the regions defined in tbl. + * It assumes that MMU is enabled and the memory is Normal memory. + * tbl must be a valid pointer to a memory mem_region_t array, + * nregions is the size of the array. + */ +void clear_mem_regions(mem_region_t *tbl, size_t nregions) +{ + size_t i; + + assert(tbl != NULL); + assert(nregions > 0U); + + for (i = 0; i < nregions; i++) { + assert(tbl->nbytes > 0); + assert(!check_uptr_overflow(tbl->base, tbl->nbytes-1)); + zero_normalmem((void *) (tbl->base), tbl->nbytes); + tbl++; + } +} + +#if defined(PLAT_XLAT_TABLES_DYNAMIC) +/* + * zero_normalmem all the regions defined in regions. + * It assumes that MMU is enabled and the memory is Normal memory. + * regions must be a valid pointer to a memory mem_region_t array, + * nregions is the size of the array. va is the virtual address + * where we want to map the physical pages that are going to + * be cleared, and chunk is the amount of memory mapped and + * cleared in every iteration. + */ +void clear_map_dyn_mem_regions(struct mem_region *regions, + size_t nregions, + uintptr_t va, + size_t chunk) +{ + uintptr_t begin; + int r; + size_t size; + const unsigned int attr = MT_MEMORY | MT_RW | MT_NS; + + assert(regions != NULL); + assert(nregions != 0U); + assert(chunk != 0U); + + for (unsigned int i = 0U; i < nregions; i++) { + begin = regions[i].base; + size = regions[i].nbytes; + if (((begin & (chunk-1U)) != 0U) || + ((size & (chunk-1U)) != 0U)) { + INFO("PSCI: Not correctly aligned region\n"); + panic(); + } + + while (size > 0U) { + r = mmap_add_dynamic_region(begin, va, chunk, attr); + if (r != 0) { + INFO("PSCI: %s failed with %d\n", + "mmap_add_dynamic_region", r); + panic(); + } + + zero_normalmem((void *)va, chunk); + + r = mmap_remove_dynamic_region(va, chunk); + if (r != 0) { + INFO("PSCI: %s failed with %d\n", + "mmap_remove_dynamic_region", r); + panic(); + } + + begin += chunk; + size -= chunk; + } + } +} +#endif + +/* + * This function checks that a region (addr + nbytes-1) of memory is totally + * covered by one of the regions defined in tbl. + * tbl must be a valid pointer to a memory mem_region_t array, nregions + * is the size of the array and the region described by addr and nbytes must + * not generate an overflow. + * Returns: + * -1 means that the region is not covered by any of the regions + * described in tbl. + * 0 the region (addr + nbytes-1) is covered by one of the regions described + * in tbl + */ +int mem_region_in_array_chk(mem_region_t *tbl, size_t nregions, + uintptr_t addr, size_t nbytes) +{ + uintptr_t region_start, region_end, start, end; + size_t i; + + assert(tbl != NULL); + assert(nbytes != 0U); + assert(!check_uptr_overflow(addr, nbytes-1)); + + region_start = addr; + region_end = addr + (nbytes - 1U); + for (i = 0U; i < nregions; i++) { + assert(tbl->nbytes > 0); + assert(!check_uptr_overflow(tbl->base, tbl->nbytes-1)); + start = tbl->base; + end = start + (tbl->nbytes - 1); + if ((region_start >= start) && (region_end <= end)) { + return 0; + } + tbl++; + } + + return -1; +} diff --git a/lib/xlat_mpu/aarch64/enable_mpu.S b/lib/xlat_mpu/aarch64/enable_mpu.S new file mode 100644 index 0000000..0dda979 --- /dev/null +++ b/lib/xlat_mpu/aarch64/enable_mpu.S @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <assert_macros.S> +#include <lib/xlat_tables/xlat_tables_v2.h> +#include <platform_def.h> + + .global enable_mpu_direct_el2 + + /* void enable_mmu_direct_el2(unsigned int flags) */ +func enable_mpu_direct_el2 +#if ENABLE_ASSERTIONS + mrs x1, sctlr_el2 + tst x1, #SCTLR_M_BIT + ASM_ASSERT(eq) +#endif + mov x7, x0 + adrp x0, mmu_cfg_params + add x0, x0, :lo12:mmu_cfg_params + + /* (MAIRs are already set up) */ + + /* TCR */ + ldr x2, [x0, #(MMU_CFG_TCR << 3)] + msr tcr_el2, x2 + + /* + * Ensure all translation table writes have drained into memory, the TLB + * invalidation is complete, and translation register writes are + * committed before enabling the MMU + */ + dsb ish + isb + + /* Set and clear required fields of SCTLR */ + mrs x4, sctlr_el2 + mov_imm x5, SCTLR_WXN_BIT | SCTLR_C_BIT | SCTLR_M_BIT + orr x4, x4, x5 + + /* Additionally, amend SCTLR fields based on flags */ + bic x5, x4, #SCTLR_C_BIT + tst x7, #DISABLE_DCACHE + csel x4, x5, x4, ne + + msr sctlr_el2, x4 + isb + + ret +endfunc enable_mpu_direct_el2 diff --git a/lib/xlat_mpu/aarch64/xlat_mpu_arch.c b/lib/xlat_mpu/aarch64/xlat_mpu_arch.c new file mode 100644 index 0000000..5a2120b --- /dev/null +++ b/lib/xlat_mpu/aarch64/xlat_mpu_arch.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> + +#include "../xlat_mpu_private.h" +#include <arch.h> +#include <arch_features.h> +#include <lib/cassert.h> +#include <lib/utils_def.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +#include <fvp_r_arch_helpers.h> + +#warning "xlat_mpu library is currently experimental and its API may change in future." + +#if ENABLE_ASSERTIONS +/* + * Return minimum virtual address space size supported by the architecture + */ +uintptr_t xlat_get_min_virt_addr_space_size(void) +{ + uintptr_t ret; + + if (is_armv8_4_ttst_present()) { + ret = MIN_VIRT_ADDR_SPACE_SIZE_TTST; + } else { + ret = MIN_VIRT_ADDR_SPACE_SIZE; + } + return ret; +} +#endif /* ENABLE_ASSERTIONS*/ + +bool is_mpu_enabled_ctx(const xlat_ctx_t *ctx) +{ + if (ctx->xlat_regime == EL1_EL0_REGIME) { + assert(xlat_arch_current_el() >= 1U); + return (read_sctlr_el1() & SCTLR_M_BIT) != 0U; + } else { + assert(xlat_arch_current_el() >= 2U); + return (read_sctlr_el2() & SCTLR_M_BIT) != 0U; + } +} + +bool is_dcache_enabled(void) +{ + unsigned int el = get_current_el(); + + if (el == 1U) { + return (read_sctlr_el1() & SCTLR_C_BIT) != 0U; + } else { /* must be EL2 */ + return (read_sctlr_el2() & SCTLR_C_BIT) != 0U; + } +} + +unsigned int xlat_arch_current_el(void) +{ + unsigned int el = (unsigned int)GET_EL(read_CurrentEl()); + + assert(el > 0U); + + return el; +} + diff --git a/lib/xlat_mpu/ro_xlat_mpu.mk b/lib/xlat_mpu/ro_xlat_mpu.mk new file mode 100644 index 0000000..d3af3bd --- /dev/null +++ b/lib/xlat_mpu/ro_xlat_mpu.mk @@ -0,0 +1,14 @@ +# +# Copyright (c) 2021, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +ifeq (${USE_DEBUGFS}, 1) + $(error "Debugfs requires functionality from the dynamic translation \ + library and is incompatible with ALLOW_RO_XLAT_TABLES.") +endif + +ifeq (${ARCH},aarch32) + $(error "The xlat_mpu library does not currently support AArch32.") +endif diff --git a/lib/xlat_mpu/xlat_mpu.mk b/lib/xlat_mpu/xlat_mpu.mk new file mode 100644 index 0000000..31bef04 --- /dev/null +++ b/lib/xlat_mpu/xlat_mpu.mk @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +XLAT_MPU_LIB_V1_SRCS := $(addprefix lib/xlat_mpu/, \ + ${ARCH}/enable_mpu.S \ + ${ARCH}/xlat_mpu_arch.c \ + xlat_mpu_context.c \ + xlat_mpu_core.c \ + xlat_mpu_utils.c) + +XLAT_MPU_LIB_V1 := 1 +$(eval $(call add_define,XLAT_MPU_LIB_V1)) + +ifeq (${ALLOW_XLAT_MPU}, 1) + include lib/xlat_mpu_v2/ro_xlat_mpu.mk +endif diff --git a/lib/xlat_mpu/xlat_mpu_context.c b/lib/xlat_mpu/xlat_mpu_context.c new file mode 100644 index 0000000..783f21d --- /dev/null +++ b/lib/xlat_mpu/xlat_mpu_context.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> + +#include <common/debug.h> + +#include "lib/xlat_mpu/xlat_mpu.h" +#include <lib/xlat_tables/xlat_tables_defs.h> +#include <lib/xlat_tables/xlat_tables_v2.h> +#include "xlat_mpu_private.h" + +#include <fvp_r_arch_helpers.h> +#include <platform_def.h> + +#warning "xlat_mpu library is currently experimental and its API may change in future." + + +/* + * MMU configuration register values for the active translation context. Used + * from the MMU assembly helpers. + */ +uint64_t mmu_cfg_params[MMU_CFG_PARAM_MAX]; + +/* + * Allocate and initialise the default translation context for the BL image + * currently executing. + */ +REGISTER_XLAT_CONTEXT(tf, MAX_MMAP_REGIONS, MAX_XLAT_TABLES, + PLAT_VIRT_ADDR_SPACE_SIZE, PLAT_PHY_ADDR_SPACE_SIZE); + +void mmap_add(const mmap_region_t *mm) +{ + mmap_add_ctx(&tf_xlat_ctx, mm); +} + +void __init init_xlat_tables(void) +{ + assert(tf_xlat_ctx.xlat_regime == EL_REGIME_INVALID); + + unsigned int current_el = xlat_arch_current_el(); + + if (current_el == 1U) { + tf_xlat_ctx.xlat_regime = EL1_EL0_REGIME; + } else { + assert(current_el == 2U); + tf_xlat_ctx.xlat_regime = EL2_REGIME; + } + /* Note: If EL3 is supported in future v8-R64, add EL3 assignment */ + init_xlat_tables_ctx(&tf_xlat_ctx); +} + +int xlat_get_mem_attributes(uintptr_t base_va, uint32_t *attr) +{ + return xlat_get_mem_attributes_ctx(&tf_xlat_ctx, base_va, attr); +} + +void enable_mpu_el2(unsigned int flags) +{ + /* EL2 is strictly MPU on v8-R64, so no need for setup_mpu_cfg() */ + enable_mpu_direct_el2(flags); +} diff --git a/lib/xlat_mpu/xlat_mpu_core.c b/lib/xlat_mpu/xlat_mpu_core.c new file mode 100644 index 0000000..33f381d --- /dev/null +++ b/lib/xlat_mpu/xlat_mpu_core.c @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <errno.h> +#include <stdbool.h> +#include <stdint.h> +#include <string.h> + +#include <arch_features.h> +#include <common/debug.h> +#include <lib/utils_def.h> +#include <lib/xlat_tables/xlat_tables_defs.h> +#include <lib/xlat_tables/xlat_tables_v2.h> +#include "xlat_mpu_private.h" + +#include <fvp_r_arch_helpers.h> +#include <platform_def.h> + +#warning "xlat_mpu library is currently experimental and its API may change in future." + + +/* Helper function that cleans the data cache only if it is enabled. */ +static inline __attribute__((unused)) + void xlat_clean_dcache_range(uintptr_t addr, size_t size) +{ + if (is_dcache_enabled()) { + clean_dcache_range(addr, size); + } +} + + + +/* Calculate region-attributes byte for PRBAR part of MPU-region descriptor: */ +uint64_t prbar_attr_value(uint32_t attr) +{ + uint64_t retValue = UL(0); + uint64_t extract; /* temp var holding bit extracted from attr */ + + /* Extract and stuff SH: */ + extract = (uint64_t) ((attr >> MT_SHAREABILITY_SHIFT) + & MT_SHAREABILITY_MASK); + retValue |= (extract << PRBAR_SH_SHIFT); + + /* Extract and stuff AP: */ + extract = (uint64_t) ((attr >> MT_PERM_SHIFT) & MT_PERM_MASK); + if (extract == 0U) { + retValue |= (UL(2) << PRBAR_AP_SHIFT); + } else /* extract == 1 */ { + retValue |= (UL(0) << PRBAR_AP_SHIFT); + } + + /* Extract and stuff XN: */ + extract = (uint64_t) ((attr >> MT_EXECUTE_SHIFT) & MT_EXECUTE_MASK); + retValue |= (extract << PRBAR_XN_SHIFT); + /* However, also don't execute in peripheral space: */ + extract = (uint64_t) ((attr >> MT_TYPE_SHIFT) & MT_TYPE_MASK); + if (extract == 0U) { + retValue |= (UL(1) << PRBAR_XN_SHIFT); + } + return retValue; +} + +/* Calculate region-attributes byte for PRLAR part of MPU-region descriptor: */ +uint64_t prlar_attr_value(uint32_t attr) +{ + uint64_t retValue = UL(0); + uint64_t extract; /* temp var holding bit extracted from attr */ + + /* Extract and stuff AttrIndx: */ + extract = (uint64_t) ((attr >> MT_TYPE_SHIFT) + & MT_TYPE_MASK); + switch (extract) { + case UL(0): + retValue |= (UL(1) << PRLAR_ATTR_SHIFT); + break; + case UL(2): + /* 0, so OR in nothing */ + break; + case UL(3): + retValue |= (UL(2) << PRLAR_ATTR_SHIFT); + break; + default: + retValue |= (extract << PRLAR_ATTR_SHIFT); + break; + } + + /* Stuff EN: */ + retValue |= (UL(1) << PRLAR_EN_SHIFT); + + /* Force NS to 0 (Secure); v8-R64 only supports Secure: */ + extract = ~(1U << PRLAR_NS_SHIFT); + retValue &= extract; + + return retValue; +} + +/* + * Function that writes an MPU "translation" into the MPU registers. If not + * possible (e.g., if no more MPU regions available) boot is aborted. + */ +static void mpu_map_region(mmap_region_t *mm) +{ + uint64_t prenr_el2_value = 0UL; + uint64_t prbar_attrs = 0UL; + uint64_t prlar_attrs = 0UL; + int region_to_use = 0; + + /* If all MPU regions in use, then abort boot: */ + prenr_el2_value = read_prenr_el2(); + assert(prenr_el2_value != 0xffffffff); + + /* Find and select first-available MPU region (PRENR has an enable bit + * for each MPU region, 1 for in-use or 0 for unused): + */ + for (region_to_use = 0; region_to_use < N_MPU_REGIONS; + region_to_use++) { + if (((prenr_el2_value >> region_to_use) & 1) == 0) { + break; + } + } + write_prselr_el2((uint64_t) (region_to_use)); + isb(); + + /* Set base and limit addresses: */ + write_prbar_el2(mm->base_pa & PRBAR_PRLAR_ADDR_MASK); + write_prlar_el2((mm->base_pa + mm->size - 1UL) + & PRBAR_PRLAR_ADDR_MASK); + dsbsy(); + isb(); + + /* Set attributes: */ + prbar_attrs = prbar_attr_value(mm->attr); + write_prbar_el2(read_prbar_el2() | prbar_attrs); + prlar_attrs = prlar_attr_value(mm->attr); + write_prlar_el2(read_prlar_el2() | prlar_attrs); + dsbsy(); + isb(); + + /* Mark this MPU region as used: */ + prenr_el2_value |= (1 << region_to_use); + write_prenr_el2(prenr_el2_value); + isb(); +} + +/* + * Function that verifies that a region can be mapped. + * Returns: + * 0: Success, the mapping is allowed. + * EINVAL: Invalid values were used as arguments. + * ERANGE: The memory limits were surpassed. + * ENOMEM: There is not enough memory in the mmap array. + * EPERM: Region overlaps another one in an invalid way. + */ +static int mmap_add_region_check(const xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + unsigned long long base_pa = mm->base_pa; + uintptr_t base_va = mm->base_va; + size_t size = mm->size; + + unsigned long long end_pa = base_pa + size - 1U; + uintptr_t end_va = base_va + size - 1U; + + if (base_pa != base_va) { + return -EINVAL; /* MPU does not perform address translation */ + } + if ((base_pa % 64ULL) != 0ULL) { + return -EINVAL; /* MPU requires 64-byte alignment */ + } + /* Check for overflows */ + if ((base_pa > end_pa) || (base_va > end_va)) { + return -ERANGE; + } + if (end_pa > ctx->pa_max_address) { + return -ERANGE; + } + /* Check that there is space in the ctx->mmap array */ + if (ctx->mmap[ctx->mmap_num - 1].size != 0U) { + return -ENOMEM; + } + /* Check for PAs and VAs overlaps with all other regions */ + for (const mmap_region_t *mm_cursor = ctx->mmap; + mm_cursor->size != 0U; ++mm_cursor) { + + uintptr_t mm_cursor_end_va = + mm_cursor->base_va + mm_cursor->size - 1U; + + /* + * Check if one of the regions is completely inside the other + * one. + */ + bool fully_overlapped_va = + ((base_va >= mm_cursor->base_va) && + (end_va <= mm_cursor_end_va)) || + ((mm_cursor->base_va >= base_va) && + (mm_cursor_end_va <= end_va)); + + /* + * Full VA overlaps are only allowed if both regions are + * identity mapped (zero offset) or have the same VA to PA + * offset. Also, make sure that it's not the exact same area. + * This can only be done with static regions. + */ + if (fully_overlapped_va) { + +#if PLAT_XLAT_TABLES_DYNAMIC + if (((mm->attr & MT_DYNAMIC) != 0U) || + ((mm_cursor->attr & MT_DYNAMIC) != 0U)) { + return -EPERM; + } +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + if ((mm_cursor->base_va - mm_cursor->base_pa) + != (base_va - base_pa)) { + return -EPERM; + } + if ((base_va == mm_cursor->base_va) && + (size == mm_cursor->size)) { + return -EPERM; + } + } else { + /* + * If the regions do not have fully overlapping VAs, + * then they must have fully separated VAs and PAs. + * Partial overlaps are not allowed + */ + + unsigned long long mm_cursor_end_pa = + mm_cursor->base_pa + mm_cursor->size - 1U; + + bool separated_pa = (end_pa < mm_cursor->base_pa) || + (base_pa > mm_cursor_end_pa); + bool separated_va = (end_va < mm_cursor->base_va) || + (base_va > mm_cursor_end_va); + + if (!separated_va || !separated_pa) { + return -EPERM; + } + } + } + + return 0; +} + +void mmap_add_region_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + mmap_region_t *mm_cursor = ctx->mmap, *mm_destination; + const mmap_region_t *mm_end = ctx->mmap + ctx->mmap_num; + const mmap_region_t *mm_last; + unsigned long long end_pa = mm->base_pa + mm->size - 1U; + uintptr_t end_va = mm->base_va + mm->size - 1U; + int ret; + + /* Ignore empty regions */ + if (mm->size == 0U) { + return; + } + + /* Static regions must be added before initializing the xlat tables. */ + assert(!ctx->initialized); + + ret = mmap_add_region_check(ctx, mm); + if (ret != 0) { + ERROR("mmap_add_region_check() failed. error %d\n", ret); + assert(false); + return; + } + + /* + * Find the last entry marker in the mmap + */ + mm_last = ctx->mmap; + while ((mm_last->size != 0U) && (mm_last < mm_end)) { + ++mm_last; + } + + /* + * Check if we have enough space in the memory mapping table. + * This shouldn't happen as we have checked in mmap_add_region_check + * that there is free space. + */ + assert(mm_last->size == 0U); + + /* Make room for new region by moving other regions up by one place */ + mm_destination = mm_cursor + 1; + (void)memmove(mm_destination, mm_cursor, + (uintptr_t)mm_last - (uintptr_t)mm_cursor); + + /* + * Check we haven't lost the empty sentinel from the end of the array. + * This shouldn't happen as we have checked in mmap_add_region_check + * that there is free space. + */ + assert(mm_end->size == 0U); + + *mm_cursor = *mm; + + if (end_pa > ctx->max_pa) { + ctx->max_pa = end_pa; + } + if (end_va > ctx->max_va) { + ctx->max_va = end_va; + } +} + +void mmap_add_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + const mmap_region_t *mm_cursor = mm; + + while (mm_cursor->granularity != 0U) { + mmap_add_region_ctx(ctx, mm_cursor); + mm_cursor++; + } +} + +void __init init_xlat_tables_ctx(xlat_ctx_t *ctx) +{ + uint64_t mair = UL(0); + + assert(ctx != NULL); + assert(!ctx->initialized); + assert((ctx->xlat_regime == EL2_REGIME) || + (ctx->xlat_regime == EL1_EL0_REGIME)); + /* Note: Add EL3_REGIME if EL3 is supported in future v8-R64 cores. */ + assert(!is_mpu_enabled_ctx(ctx)); + + mmap_region_t *mm = ctx->mmap; + + assert(ctx->va_max_address >= + (xlat_get_min_virt_addr_space_size() - 1U)); + assert(ctx->va_max_address <= (MAX_VIRT_ADDR_SPACE_SIZE - 1U)); + assert(IS_POWER_OF_TWO(ctx->va_max_address + 1U)); + + xlat_mmap_print(mm); + + /* All tables must be zeroed before mapping any region. */ + + for (unsigned int i = 0U; i < ctx->base_table_entries; i++) + ctx->base_table[i] = INVALID_DESC; + + /* Also mark all MPU regions as invalid in the MPU hardware itself: */ + write_prenr_el2(0); + /* Sufficient for current, max-32-region implementations. */ + dsbsy(); + isb(); + while (mm->size != 0U) { + if (read_prenr_el2() == ALL_MPU_EL2_REGIONS_USED) { + ERROR("Not enough MPU regions to map region:\n" + " VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x\n", + mm->base_va, mm->base_pa, mm->size, mm->attr); + panic(); + } else { +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + xlat_clean_dcache_range((uintptr_t)mm->base_va, + mm->size); +#endif + mpu_map_region(mm); + } + mm++; + } + + ctx->initialized = true; + + xlat_tables_print(ctx); + + /* Set attributes in the right indices of the MAIR */ + mair = MAIR_ATTR_SET(ATTR_DEVICE, ATTR_DEVICE_INDEX); + mair |= MAIR_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, + ATTR_IWBWA_OWBWA_NTR_INDEX); + mair |= MAIR_ATTR_SET(ATTR_NON_CACHEABLE, + ATTR_NON_CACHEABLE_INDEX); + write_mair_el2(mair); + dsbsy(); + isb(); +} + +/* + * Function to wipe clean and disable all MPU regions. This function expects + * that the MPU has already been turned off, and caching concerns addressed, + * but it nevertheless also explicitly turns off the MPU. + */ +void clear_all_mpu_regions(void) +{ + uint64_t sctlr_el2_value = 0UL; + uint64_t region_n = 0UL; + + /* + * MPU should already be disabled, but explicitly disable it + * nevertheless: + */ + sctlr_el2_value = read_sctlr_el2() & ~(1UL); + write_sctlr_el2(sctlr_el2_value); + + /* Disable all regions: */ + write_prenr_el2(0UL); + + /* Sequence through all regions, zeroing them out and turning off: */ + for (region_n = 0UL; region_n < N_MPU_REGIONS; region_n++) { + write_prselr_el2(region_n); + isb(); + write_prbar_el2((uint64_t) 0); + write_prlar_el2((uint64_t) 0); + dsbsy(); + isb(); + } +} diff --git a/lib/xlat_mpu/xlat_mpu_private.h b/lib/xlat_mpu/xlat_mpu_private.h new file mode 100644 index 0000000..14800e9 --- /dev/null +++ b/lib/xlat_mpu/xlat_mpu_private.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef XLAT_MPU_PRIVATE_H +#define XLAT_MPU_PRIVATE_H + +#include <stdbool.h> + +#include <lib/xlat_tables/xlat_tables_defs.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +#include <platform_def.h> + +#if PLAT_XLAT_TABLES_DYNAMIC +/* + * Private shifts and masks to access fields of an mmap attribute + */ +/* Dynamic or static */ +#define MT_DYN_SHIFT U(31) + +/* + * Memory mapping private attributes + * + * Private attributes not exposed in the public header. + */ + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +/* Calculate region-attributes byte for PRBAR part of MPU-region descriptor: */ +uint64_t prbar_attr_value(uint32_t attr); +/* Calculate region-attributes byte for PRLAR part of MPU-region descriptor: */ +uint64_t prlar_attr_value(uint32_t attr); +/* Calculates the attr value for a given PRBAR and PRLAR entry value: */ +uint32_t region_attr(uint64_t prbar_attr, uint64_t prlar_attr); + +#define PRBAR_PRLAR_ADDR_MASK UL(0xffffffffffc0) + /* mask for PRBAR & PRLAR MPU-region field */ +/* MPU region attribute bit fields: */ +#define PRBAR_SH_SHIFT UL(4) +#define PRBAR_SH_MASK UL(0x3) +#define PRBAR_AP_SHIFT UL(2) +#define PRBAR_AP_MASK UL(0x3) +#define PRBAR_XN_SHIFT UL(1) +#define PRBAR_XN_MASK UL(0x3) +#define PRLAR_NS_SHIFT UL(4) +#define PRLAR_NS_MASK UL(0x3) +#define PRBAR_ATTR_SHIFT UL(0) +#define PRBAR_ATTR_MASK UL(0x3f) +#define PRLAR_ATTR_SHIFT UL(1) +#define PRLAR_ATTR_MASK UL(0x7) +#define PRLAR_EN_SHIFT UL(0) +#define PRLAR_EN_MASK UL(0x1) +/* Aspects of the source attributes not defined elsewhere: */ +#define MT_PERM_MASK UL(0x1) +#define MT_SEC_MASK UL(0x1) +#define MT_EXECUTE_MASK UL(0x3) +#define MT_TYPE_SHIFT UL(0) + +extern uint64_t mmu_cfg_params[MMU_CFG_PARAM_MAX]; + +/* + * Return the execute-never mask that will prevent instruction fetch at the + * given translation regime. + */ +uint64_t xlat_arch_regime_get_xn_desc(int xlat_regime); + +/* Print VA, PA, size and attributes of all regions in the mmap array. */ +void xlat_mmap_print(const mmap_region_t *mmap); + +/* + * Print the current state of the translation tables by reading them from + * memory. + */ +void xlat_tables_print(xlat_ctx_t *ctx); + +/* + * Returns a block/page table descriptor for the given level and attributes. + */ +uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr, + unsigned long long addr_pa, unsigned int level); + +/* + * Architecture-specific initialization code. + */ + +/* Returns the current Exception Level. The returned EL must be 1 or higher. */ +unsigned int xlat_arch_current_el(void); + +/* + * Returns true if the MMU of the translation regime managed by the given + * xlat_ctx_t is enabled, false otherwise. + */ +bool is_mpu_enabled_ctx(const xlat_ctx_t *ctx); + +/* + * Returns minimum virtual address space size supported by the architecture + */ +uintptr_t xlat_get_min_virt_addr_space_size(void); + +#endif /* XLAT_MPU_PRIVATE_H */ diff --git a/lib/xlat_mpu/xlat_mpu_utils.c b/lib/xlat_mpu/xlat_mpu_utils.c new file mode 100644 index 0000000..dc5f956 --- /dev/null +++ b/lib/xlat_mpu/xlat_mpu_utils.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <errno.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> + +#include <common/debug.h> +#include <lib/utils_def.h> +#include <lib/xlat_tables/xlat_tables_defs.h> +#include <lib/xlat_tables/xlat_tables_v2.h> +#include "xlat_mpu_private.h" + +#include <fvp_r_arch_helpers.h> +#include <platform_def.h> + +#warning "xlat_mpu library is currently experimental and its API may change in future." + + +void xlat_mmap_print(__unused const mmap_region_t *mmap) +{ + /* Empty */ +} + +#if LOG_LEVEL < LOG_LEVEL_VERBOSE + +void xlat_tables_print(__unused xlat_ctx_t *ctx) +{ + /* Empty */ +} + +#else /* if LOG_LEVEL >= LOG_LEVEL_VERBOSE */ + +static void xlat_tables_print_internal(__unused xlat_ctx_t *ctx) +{ + int region_to_use = 0; + uintptr_t region_base; + size_t region_size; + uint64_t prenr_el2_value = 0U; + + /* + * Keep track of how many invalid descriptors are counted in a row. + * Whenever multiple invalid descriptors are found, only the first one + * is printed, and a line is added to inform about how many descriptors + * have been omitted. + */ + + /* + * TODO: Remove this WARN() and comment when these API calls are more + * completely implemented and tested! + */ + WARN("%s in this early version of xlat_mpu library may not produce reliable results!", + __func__); + + /* + * Sequence through all regions and print those in-use (PRENR has an + * enable bit for each MPU region, 1 for in-use or 0 for unused): + */ + prenr_el2_value = read_prenr_el2(); + for (region_to_use = 0; region_to_use < N_MPU_REGIONS; + region_to_use++) { + if (((prenr_el2_value >> region_to_use) & 1U) == 0U) { + continue; + } + region_base = read_prbar_el2() & PRBAR_PRLAR_ADDR_MASK; + region_size = read_prlar_el2() & PRBAR_PRLAR_ADDR_MASK; + printf("Address: 0x%llx, size: 0x%llx ", + (long long) region_base, + (long long) region_size); + } +} + +void xlat_tables_print(__unused xlat_ctx_t *ctx) +{ + xlat_tables_print_internal(ctx); +} + +#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ diff --git a/lib/xlat_tables/aarch32/nonlpae_tables.c b/lib/xlat_tables/aarch32/nonlpae_tables.c new file mode 100644 index 0000000..f582404 --- /dev/null +++ b/lib/xlat_tables/aarch32/nonlpae_tables.c @@ -0,0 +1,566 @@ +/* + * Copyright (c) 2016-2017, Linaro Limited. All rights reserved. + * Copyright (c) 2014-2020, Arm Limited. All rights reserved. + * Copyright (c) 2014, STMicroelectronics International N.V. + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdio.h> +#include <string.h> + +#include <platform_def.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/cassert.h> +#include <lib/utils.h> +#include <lib/xlat_tables/xlat_tables.h> + +#include "../xlat_tables_private.h" + +#ifdef ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING +#error "ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING flag is set. \ +This module is to be used when LPAE is not supported" +#endif + +CASSERT(PLAT_VIRT_ADDR_SPACE_SIZE == (1ULL << 32), invalid_vaddr_space_size); +CASSERT(PLAT_PHY_ADDR_SPACE_SIZE == (1ULL << 32), invalid_paddr_space_size); + +#define MMU32B_UNSET_DESC ~0UL +#define MMU32B_INVALID_DESC 0UL + +#define MT_UNKNOWN ~0U + +/* + * MMU related values + */ + +/* Sharable */ +#define MMU32B_TTB_S (1U << 1) + +/* Not Outer Sharable */ +#define MMU32B_TTB_NOS (1U << 5) + +/* Normal memory, Inner Non-cacheable */ +#define MMU32B_TTB_IRGN_NC 0U + +/* Normal memory, Inner Write-Back Write-Allocate Cacheable */ +#define MMU32B_TTB_IRGN_WBWA (1U << 6) + +/* Normal memory, Inner Write-Through Cacheable */ +#define MMU32B_TTB_IRGN_WT 1U + +/* Normal memory, Inner Write-Back no Write-Allocate Cacheable */ +#define MMU32B_TTB_IRGN_WB (1U | (1U << 6)) + +/* Normal memory, Outer Write-Back Write-Allocate Cacheable */ +#define MMU32B_TTB_RNG_WBWA (1U << 3) + +#define MMU32B_DEFAULT_ATTRS \ + (MMU32B_TTB_S | MMU32B_TTB_NOS | \ + MMU32B_TTB_IRGN_WBWA | MMU32B_TTB_RNG_WBWA) + +/* armv7 memory mapping attributes: section mapping */ +#define SECTION_SECURE (0U << 19) +#define SECTION_NOTSECURE (1U << 19) +#define SECTION_SHARED (1U << 16) +#define SECTION_NOTGLOBAL (1U << 17) +#define SECTION_ACCESS_FLAG (1U << 10) +#define SECTION_UNPRIV (1U << 11) +#define SECTION_RO (1U << 15) +#define SECTION_TEX(tex) ((((tex) >> 2) << 12) | \ + ((((tex) >> 1) & 0x1) << 3) | \ + (((tex) & 0x1) << 2)) +#define SECTION_DEVICE SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX) +#define SECTION_NORMAL SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX) +#define SECTION_NORMAL_CACHED \ + SECTION_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX) + +#define SECTION_XN (1U << 4) +#define SECTION_PXN (1U << 0) +#define SECTION_SECTION (2U << 0) + +#define SECTION_PT_NOTSECURE (1U << 3) +#define SECTION_PT_PT (1U << 0) + +#define SMALL_PAGE_SMALL_PAGE (1U << 1) +#define SMALL_PAGE_SHARED (1U << 10) +#define SMALL_PAGE_NOTGLOBAL (1U << 11) +#define SMALL_PAGE_TEX(tex) ((((tex) >> 2) << 6) | \ + ((((tex) >> 1) & 0x1) << 3) | \ + (((tex) & 0x1) << 2)) +#define SMALL_PAGE_DEVICE \ + SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX) +#define SMALL_PAGE_NORMAL \ + SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX) +#define SMALL_PAGE_NORMAL_CACHED \ + SMALL_PAGE_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX) +#define SMALL_PAGE_ACCESS_FLAG (1U << 4) +#define SMALL_PAGE_UNPRIV (1U << 5) +#define SMALL_PAGE_RO (1U << 9) +#define SMALL_PAGE_XN (1U << 0) + +/* The TEX, C and B bits concatenated */ +#define MMU32B_ATTR_DEVICE_INDEX 0U +#define MMU32B_ATTR_IWBWA_OWBWA_INDEX 1U + +#define MMU32B_PRRR_IDX(idx, tr, nos) (((tr) << (2 * (idx))) | \ + ((uint32_t)(nos) << ((idx) + 24))) +#define MMU32B_NMRR_IDX(idx, ir, or) (((ir) << (2 * (idx))) | \ + ((uint32_t)(or) << (2 * (idx) + 16))) +#define MMU32B_PRRR_DS0 (1U << 16) +#define MMU32B_PRRR_DS1 (1U << 17) +#define MMU32B_PRRR_NS0 (1U << 18) +#define MMU32B_PRRR_NS1 (1U << 19) + +#define DACR_DOMAIN(num, perm) ((perm) << ((num) * 2)) +#define DACR_DOMAIN_PERM_NO_ACCESS 0U +#define DACR_DOMAIN_PERM_CLIENT 1U +#define DACR_DOMAIN_PERM_MANAGER 3U + +#define NUM_1MB_IN_4GB (1UL << 12) +#define NUM_4K_IN_1MB (1UL << 8) + +#define ONE_MB_SHIFT 20 + +/* mmu 32b integration */ +#define MMU32B_L1_TABLE_SIZE (NUM_1MB_IN_4GB * 4) +#define MMU32B_L2_TABLE_SIZE (NUM_4K_IN_1MB * 4) +#define MMU32B_L1_TABLE_ALIGN (1U << 14) +#define MMU32B_L2_TABLE_ALIGN (1U << 10) + +static unsigned int next_xlat; +static unsigned long long xlat_max_pa; +static uintptr_t xlat_max_va; + +static uint32_t mmu_l1_base[NUM_1MB_IN_4GB] + __aligned(MMU32B_L1_TABLE_ALIGN) __attribute__((section(".xlat_table"))); + +static uint32_t mmu_l2_base[MAX_XLAT_TABLES][NUM_4K_IN_1MB] + __aligned(MMU32B_L2_TABLE_ALIGN) __attribute__((section(".xlat_table"))); + +/* + * Array of all memory regions stored in order of ascending base address. + * The list is terminated by the first entry with size == 0. + */ +static mmap_region_t mmap[MAX_MMAP_REGIONS + 1]; + +void print_mmap(void) +{ +#if LOG_LEVEL >= LOG_LEVEL_VERBOSE + mmap_region_t *mm = mmap; + + printf("init xlat - l1:%p l2:%p (%d)\n", + (void *)mmu_l1_base, (void *)mmu_l2_base, MAX_XLAT_TABLES); + printf("mmap:\n"); + while (mm->size) { + printf(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", + (void *)mm->base_va, mm->base_pa, + mm->size, mm->attr); + ++mm; + }; + printf("\n"); +#endif +} + +void mmap_add(const mmap_region_t *mm) +{ + const mmap_region_t *mm_cursor = mm; + + while ((mm_cursor->size != 0U) || (mm_cursor->attr != 0U)) { + mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va, + mm_cursor->size, mm_cursor->attr); + mm_cursor++; + } +} + +void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, + size_t size, unsigned int attr) +{ + mmap_region_t *mm = mmap; + const mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1U; + unsigned long long end_pa = base_pa + size - 1U; + uintptr_t end_va = base_va + size - 1U; + + assert(IS_PAGE_ALIGNED(base_pa)); + assert(IS_PAGE_ALIGNED(base_va)); + assert(IS_PAGE_ALIGNED(size)); + + if (size == 0U) { + return; + } + + assert(base_pa < end_pa); /* Check for overflows */ + assert(base_va < end_va); + + assert((base_va + (uintptr_t)size - (uintptr_t)1) <= + (PLAT_VIRT_ADDR_SPACE_SIZE - 1U)); + assert((base_pa + (unsigned long long)size - 1ULL) <= + (PLAT_PHY_ADDR_SPACE_SIZE - 1U)); + +#if ENABLE_ASSERTIONS + + /* Check for PAs and VAs overlaps with all other regions */ + for (mm = mmap; mm->size; ++mm) { + + uintptr_t mm_end_va = mm->base_va + mm->size - 1U; + + /* + * Check if one of the regions is completely inside the other + * one. + */ + bool fully_overlapped_va = + ((base_va >= mm->base_va) && (end_va <= mm_end_va)) || + ((mm->base_va >= base_va) && (mm_end_va <= end_va)); + + /* + * Full VA overlaps are only allowed if both regions are + * identity mapped (zero offset) or have the same VA to PA + * offset. Also, make sure that it's not the exact same area. + */ + if (fully_overlapped_va) { + assert((mm->base_va - mm->base_pa) == + (base_va - base_pa)); + assert((base_va != mm->base_va) || (size != mm->size)); + } else { + /* + * If the regions do not have fully overlapping VAs, + * then they must have fully separated VAs and PAs. + * Partial overlaps are not allowed + */ + + unsigned long long mm_end_pa = + mm->base_pa + mm->size - 1; + + bool separated_pa = (end_pa < mm->base_pa) || + (base_pa > mm_end_pa); + bool separated_va = (end_va < mm->base_va) || + (base_va > mm_end_va); + + assert(separated_va && separated_pa); + } + } + + mm = mmap; /* Restore pointer to the start of the array */ + +#endif /* ENABLE_ASSERTIONS */ + + /* Find correct place in mmap to insert new region */ + while ((mm->base_va < base_va) && (mm->size != 0U)) { + ++mm; + } + + /* + * If a section is contained inside another one with the same base + * address, it must be placed after the one it is contained in: + * + * 1st |-----------------------| + * 2nd |------------| + * 3rd |------| + * + * This is required for mmap_region_attr() to get the attributes of the + * small region correctly. + */ + while ((mm->base_va == base_va) && (mm->size > size)) { + ++mm; + } + + /* Make room for new region by moving other regions up by one place */ + (void)memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm); + + /* Check we haven't lost the empty sentinel from the end of the array */ + assert(mm_last->size == 0U); + + mm->base_pa = base_pa; + mm->base_va = base_va; + mm->size = size; + mm->attr = attr; + + if (end_pa > xlat_max_pa) { + xlat_max_pa = end_pa; + } + if (end_va > xlat_max_va) { + xlat_max_va = end_va; + } +} + +/* map all memory as shared/global/domain0/no-usr access */ +static uint32_t mmap_desc(unsigned attr, unsigned int addr_pa, + unsigned int level) +{ + uint32_t desc; + + switch (level) { + case 1U: + assert((addr_pa & (MMU32B_L1_TABLE_ALIGN - 1)) == 0U); + + desc = SECTION_SECTION | SECTION_SHARED; + + desc |= (attr & MT_NS) != 0U ? SECTION_NOTSECURE : 0U; + + desc |= SECTION_ACCESS_FLAG; + desc |= (attr & MT_RW) != 0U ? 0U : SECTION_RO; + + desc |= (attr & MT_MEMORY) != 0U ? + SECTION_NORMAL_CACHED : SECTION_DEVICE; + + if (((attr & MT_RW) != 0U) || ((attr & MT_MEMORY) == 0U)) { + desc |= SECTION_XN; + } + break; + case 2U: + assert((addr_pa & (MMU32B_L2_TABLE_ALIGN - 1)) == 0U); + + desc = SMALL_PAGE_SMALL_PAGE | SMALL_PAGE_SHARED; + + desc |= SMALL_PAGE_ACCESS_FLAG; + desc |= (attr & MT_RW) != 0U ? 0U : SMALL_PAGE_RO; + + desc |= (attr & MT_MEMORY) != 0U ? + SMALL_PAGE_NORMAL_CACHED : SMALL_PAGE_DEVICE; + + if (((attr & MT_RW) != 0U) || ((attr & MT_MEMORY) == 0U)) { + desc |= SMALL_PAGE_XN; + } + break; + default: + panic(); + } +#if LOG_LEVEL >= LOG_LEVEL_VERBOSE + /* dump only the non-lpae level 2 tables */ + if (level == 2U) { + printf(attr & MT_MEMORY ? "MEM" : "dev"); + printf(attr & MT_RW ? "-rw" : "-RO"); + printf(attr & MT_NS ? "-NS" : "-S"); + } +#endif + return desc | addr_pa; +} + +static unsigned int mmap_region_attr(const mmap_region_t *mm, uintptr_t base_va, + size_t size, unsigned int *attr) +{ + /* Don't assume that the area is contained in the first region */ + unsigned int ret = MT_UNKNOWN; + + /* + * Get attributes from last (innermost) region that contains the + * requested area. Don't stop as soon as one region doesn't contain it + * because there may be other internal regions that contain this area: + * + * |-----------------------------1-----------------------------| + * |----2----| |-------3-------| |----5----| + * |--4--| + * + * |---| <- Area we want the attributes of. + * + * In this example, the area is contained in regions 1, 3 and 4 but not + * in region 2. The loop shouldn't stop at region 2 as inner regions + * have priority over outer regions, it should stop at region 5. + */ + for ( ; ; ++mm) { + + if (mm->size == 0U) { + return ret; /* Reached end of list */ + } + + if (mm->base_va > (base_va + size - 1U)) { + return ret; /* Next region is after area so end */ + } + + if ((mm->base_va + mm->size - 1U) < base_va) { + continue; /* Next region has already been overtaken */ + } + + if ((ret == 0U) && (mm->attr == *attr)) { + continue; /* Region doesn't override attribs so skip */ + } + + if ((mm->base_va > base_va) || + ((mm->base_va + mm->size - 1U) < + (base_va + size - 1U))) { + return MT_UNKNOWN; /* Region doesn't fully cover area */ + } + + *attr = mm->attr; + ret = 0U; + } +} + +static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm, + unsigned int base_va, + uint32_t *table, + unsigned int level) +{ + unsigned int level_size_shift = (level == 1U) ? + ONE_MB_SHIFT : FOUR_KB_SHIFT; + unsigned int level_size = 1U << level_size_shift; + unsigned int level_index_mask = (level == 1U) ? + (NUM_1MB_IN_4GB - 1) << ONE_MB_SHIFT : + (NUM_4K_IN_1MB - 1) << FOUR_KB_SHIFT; + + assert((level == 1U) || (level == 2U)); + + VERBOSE("init xlat table at %p (level%1u)\n", (void *)table, level); + + do { + uint32_t desc = MMU32B_UNSET_DESC; + + if (mm->base_va + mm->size <= base_va) { + /* Area now after the region so skip it */ + ++mm; + continue; + } +#if LOG_LEVEL >= LOG_LEVEL_VERBOSE + /* dump only non-lpae level 2 tables content */ + if (level == 2U) { + printf(" 0x%lx %x " + 6 - 2 * level, + base_va, level_size); + } +#endif + if (mm->base_va >= base_va + level_size) { + /* Next region is after area so nothing to map yet */ + desc = MMU32B_INVALID_DESC; + } else if ((mm->base_va <= base_va) && + (mm->base_va + mm->size) >= + (base_va + level_size)) { + /* Next region covers all of area */ + unsigned int attr = mm->attr; + unsigned int r = mmap_region_attr(mm, base_va, + level_size, &attr); + + if (r == 0U) { + desc = mmap_desc(attr, + base_va - mm->base_va + mm->base_pa, + level); + } + } + + if (desc == MMU32B_UNSET_DESC) { + uintptr_t xlat_table; + + /* + * Area not covered by a region so need finer table + * Reuse next level table if any (assert attrib matching). + * Otherwise allocate a xlat table. + */ + if (*table) { + assert((*table & 3) == SECTION_PT_PT); + assert(((*table & SECTION_PT_NOTSECURE) == 0U) + == ((mm->attr & MT_NS) == 0U)); + + xlat_table = (*table) & + ~(MMU32B_L1_TABLE_ALIGN - 1); + desc = *table; + } else { + xlat_table = (uintptr_t)mmu_l2_base + + next_xlat * MMU32B_L2_TABLE_SIZE; + next_xlat++; + assert(next_xlat <= MAX_XLAT_TABLES); + (void)memset((char *)xlat_table, 0, + MMU32B_L2_TABLE_SIZE); + + desc = xlat_table | SECTION_PT_PT; + desc |= (mm->attr & MT_NS) != 0U ? + SECTION_PT_NOTSECURE : 0; + } + /* Recurse to fill in new table */ + mm = init_xlation_table_inner(mm, base_va, + (uint32_t *)xlat_table, + level + 1); + } +#if LOG_LEVEL >= LOG_LEVEL_VERBOSE + /* dump only non-lpae level 2 tables content */ + if (level == 2U) { + printf("\n"); + } +#endif + *table++ = desc; + base_va += level_size; + } while ((mm->size != 0U) && ((base_va & level_index_mask) != 0U)); + + return mm; +} + +void init_xlat_tables(void) +{ + print_mmap(); + + assert(((unsigned int)mmu_l1_base & (MMU32B_L1_TABLE_ALIGN - 1)) == 0U); + assert(((unsigned int)mmu_l2_base & (MMU32B_L2_TABLE_ALIGN - 1)) == 0U); + + (void)memset(mmu_l1_base, 0, MMU32B_L1_TABLE_SIZE); + + init_xlation_table_inner(mmap, 0, (uint32_t *)mmu_l1_base, 1); + + VERBOSE("init xlat - max_va=%p, max_pa=%llx\n", + (void *)xlat_max_va, xlat_max_pa); + assert(xlat_max_pa <= (PLAT_VIRT_ADDR_SPACE_SIZE - 1)); +} + +/******************************************************************************* + * Function for enabling the MMU in Secure PL1, assuming that the + * page-tables have already been created. + ******************************************************************************/ +void enable_mmu_svc_mon(unsigned int flags) +{ + unsigned int prrr; + unsigned int nmrr; + unsigned int sctlr; + + assert(IS_IN_SECURE()); + assert((read_sctlr() & SCTLR_M_BIT) == 0U); + + /* Enable Access flag (simplified access permissions) and TEX remap */ + write_sctlr(read_sctlr() | SCTLR_AFE_BIT | SCTLR_TRE_BIT); + + prrr = MMU32B_PRRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 1, 0) + | MMU32B_PRRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 2, 1); + nmrr = MMU32B_NMRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 0, 0) + | MMU32B_NMRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 1, 1); + + prrr |= MMU32B_PRRR_NS1 | MMU32B_PRRR_DS1; + + write_prrr(prrr); + write_nmrr(nmrr); + + /* Program Domain access control register: domain 0 only */ + write_dacr(DACR_DOMAIN(0, DACR_DOMAIN_PERM_CLIENT)); + + /* Invalidate TLBs at the current exception level */ + tlbiall(); + + /* set MMU base xlat table entry (use only TTBR0) */ + write_ttbr0((uint32_t)mmu_l1_base | MMU32B_DEFAULT_ATTRS); + write_ttbr1(0U); + + /* + * Ensure all translation table writes have drained + * into memory, the TLB invalidation is complete, + * and translation register writes are committed + * before enabling the MMU + */ + dsb(); + isb(); + + sctlr = read_sctlr(); + sctlr |= SCTLR_M_BIT; +#ifdef ARMV7_SUPPORTS_VIRTUALIZATION + sctlr |= SCTLR_WXN_BIT; +#endif + + if ((flags & DISABLE_DCACHE) != 0U) { + sctlr &= ~SCTLR_C_BIT; + } else { + sctlr |= SCTLR_C_BIT; + } + + write_sctlr(sctlr); + + /* Ensure the MMU enable takes effect immediately */ + isb(); +} diff --git a/lib/xlat_tables/aarch32/xlat_tables.c b/lib/xlat_tables/aarch32/xlat_tables.c new file mode 100644 index 0000000..6d098be --- /dev/null +++ b/lib/xlat_tables/aarch32/xlat_tables.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2016-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> + +#include <platform_def.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <lib/utils.h> +#include <lib/xlat_tables/xlat_tables_arch.h> +#include <lib/xlat_tables/xlat_tables.h> + +#include "../xlat_tables_private.h" + +#if (ARM_ARCH_MAJOR == 7) && !defined(ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING) +#error ARMv7 target does not support LPAE MMU descriptors +#endif + +#define XLAT_TABLE_LEVEL_BASE \ + GET_XLAT_TABLE_LEVEL_BASE(PLAT_VIRT_ADDR_SPACE_SIZE) + +#define NUM_BASE_LEVEL_ENTRIES \ + GET_NUM_BASE_LEVEL_ENTRIES(PLAT_VIRT_ADDR_SPACE_SIZE) + +static uint64_t base_xlation_table[NUM_BASE_LEVEL_ENTRIES] + __aligned(NUM_BASE_LEVEL_ENTRIES * sizeof(uint64_t)); + +#if ENABLE_ASSERTIONS +static unsigned long long get_max_supported_pa(void) +{ + /* Physical address space size for long descriptor format. */ + return (1ULL << 40) - 1ULL; +} +#endif /* ENABLE_ASSERTIONS */ + +unsigned int xlat_arch_current_el(void) +{ + /* + * If EL3 is in AArch32 mode, all secure PL1 modes (Monitor, System, + * SVC, Abort, UND, IRQ and FIQ modes) execute at EL3. + */ + return 3U; +} + +uint64_t xlat_arch_get_xn_desc(unsigned int el __unused) +{ + return UPPER_ATTRS(XN); +} + +void init_xlat_tables(void) +{ + unsigned long long max_pa; + uintptr_t max_va; + + assert(PLAT_VIRT_ADDR_SPACE_SIZE >= MIN_VIRT_ADDR_SPACE_SIZE); + assert(PLAT_VIRT_ADDR_SPACE_SIZE <= MAX_VIRT_ADDR_SPACE_SIZE); + assert(IS_POWER_OF_TWO(PLAT_VIRT_ADDR_SPACE_SIZE)); + + print_mmap(); + init_xlation_table(0U, base_xlation_table, XLAT_TABLE_LEVEL_BASE, + &max_va, &max_pa); + + assert(max_va <= (PLAT_VIRT_ADDR_SPACE_SIZE - 1U)); + assert(max_pa <= (PLAT_PHY_ADDR_SPACE_SIZE - 1U)); + assert((PLAT_PHY_ADDR_SPACE_SIZE - 1U) <= get_max_supported_pa()); +} + +void enable_mmu_svc_mon(unsigned int flags) +{ + unsigned int mair0, ttbcr, sctlr; + uint64_t ttbr0; + + assert(IS_IN_SECURE()); + assert((read_sctlr() & SCTLR_M_BIT) == 0U); + + /* Set attributes in the right indices of the MAIR */ + mair0 = MAIR0_ATTR_SET(ATTR_DEVICE, ATTR_DEVICE_INDEX); + mair0 |= MAIR0_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, + ATTR_IWBWA_OWBWA_NTR_INDEX); + mair0 |= MAIR0_ATTR_SET(ATTR_NON_CACHEABLE, + ATTR_NON_CACHEABLE_INDEX); + write_mair0(mair0); + + /* Invalidate TLBs at the current exception level */ + tlbiall(); + + /* + * Set TTBCR bits as well. Set TTBR0 table properties. Disable TTBR1. + */ + int t0sz = 32 - __builtin_ctzll(PLAT_VIRT_ADDR_SPACE_SIZE); + + if ((flags & XLAT_TABLE_NC) != 0U) { + /* Inner & outer non-cacheable non-shareable. */ + ttbcr = TTBCR_EAE_BIT | + TTBCR_SH0_NON_SHAREABLE | TTBCR_RGN0_OUTER_NC | + TTBCR_RGN0_INNER_NC | (uint32_t) t0sz; + } else { + /* Inner & outer WBWA & shareable. */ + ttbcr = TTBCR_EAE_BIT | + TTBCR_SH0_INNER_SHAREABLE | TTBCR_RGN0_OUTER_WBA | + TTBCR_RGN0_INNER_WBA | (uint32_t) t0sz; + } + ttbcr |= TTBCR_EPD1_BIT; + write_ttbcr(ttbcr); + + /* Set TTBR0 bits as well */ + ttbr0 = (uintptr_t) base_xlation_table; + write64_ttbr0(ttbr0); + write64_ttbr1(0U); + + /* + * Ensure all translation table writes have drained + * into memory, the TLB invalidation is complete, + * and translation register writes are committed + * before enabling the MMU + */ + dsbish(); + isb(); + + sctlr = read_sctlr(); + sctlr |= SCTLR_WXN_BIT | SCTLR_M_BIT; + + if ((flags & DISABLE_DCACHE) != 0U) + sctlr &= ~SCTLR_C_BIT; + else + sctlr |= SCTLR_C_BIT; + + write_sctlr(sctlr); + + /* Ensure the MMU enable takes effect immediately */ + isb(); +} + +void enable_mmu_direct_svc_mon(unsigned int flags) +{ + enable_mmu_svc_mon(flags); +} diff --git a/lib/xlat_tables/aarch64/xlat_tables.c b/lib/xlat_tables/aarch64/xlat_tables.c new file mode 100644 index 0000000..4dbfc11 --- /dev/null +++ b/lib/xlat_tables/aarch64/xlat_tables.c @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2014-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdint.h> + +#include <platform_def.h> + +#include <arch.h> +#include <arch_features.h> +#include <common/bl_common.h> +#include <lib/utils.h> +#include <lib/xlat_tables/xlat_tables.h> +#include <lib/xlat_tables/xlat_tables_arch.h> +#include <plat/common/common_def.h> + +#include "../xlat_tables_private.h" + +#define XLAT_TABLE_LEVEL_BASE \ + GET_XLAT_TABLE_LEVEL_BASE(PLAT_VIRT_ADDR_SPACE_SIZE) + +#define NUM_BASE_LEVEL_ENTRIES \ + GET_NUM_BASE_LEVEL_ENTRIES(PLAT_VIRT_ADDR_SPACE_SIZE) + +static uint64_t base_xlation_table[NUM_BASE_LEVEL_ENTRIES] + __aligned(NUM_BASE_LEVEL_ENTRIES * sizeof(uint64_t)); + +static unsigned long long tcr_ps_bits; + +static unsigned long long calc_physical_addr_size_bits( + unsigned long long max_addr) +{ + /* Physical address can't exceed 48 bits */ + assert((max_addr & ADDR_MASK_48_TO_63) == 0U); + + /* 48 bits address */ + if ((max_addr & ADDR_MASK_44_TO_47) != 0U) + return TCR_PS_BITS_256TB; + + /* 44 bits address */ + if ((max_addr & ADDR_MASK_42_TO_43) != 0U) + return TCR_PS_BITS_16TB; + + /* 42 bits address */ + if ((max_addr & ADDR_MASK_40_TO_41) != 0U) + return TCR_PS_BITS_4TB; + + /* 40 bits address */ + if ((max_addr & ADDR_MASK_36_TO_39) != 0U) + return TCR_PS_BITS_1TB; + + /* 36 bits address */ + if ((max_addr & ADDR_MASK_32_TO_35) != 0U) + return TCR_PS_BITS_64GB; + + return TCR_PS_BITS_4GB; +} + +#if ENABLE_ASSERTIONS +/* + * Physical Address ranges supported in the AArch64 Memory Model. Value 0b110 is + * supported in ARMv8.2 onwards. + */ +static const unsigned int pa_range_bits_arr[] = { + PARANGE_0000, PARANGE_0001, PARANGE_0010, PARANGE_0011, PARANGE_0100, + PARANGE_0101, PARANGE_0110 +}; + +static unsigned long long get_max_supported_pa(void) +{ + u_register_t pa_range = read_id_aa64mmfr0_el1() & + ID_AA64MMFR0_EL1_PARANGE_MASK; + + /* All other values are reserved */ + assert(pa_range < ARRAY_SIZE(pa_range_bits_arr)); + + return (1ULL << pa_range_bits_arr[pa_range]) - 1ULL; +} + +/* + * Return minimum virtual address space size supported by the architecture + */ +static uintptr_t xlat_get_min_virt_addr_space_size(void) +{ + uintptr_t ret; + + if (is_armv8_4_ttst_present()) + ret = MIN_VIRT_ADDR_SPACE_SIZE_TTST; + else + ret = MIN_VIRT_ADDR_SPACE_SIZE; + + return ret; +} +#endif /* ENABLE_ASSERTIONS */ + +unsigned int xlat_arch_current_el(void) +{ + unsigned int el = (unsigned int)GET_EL(read_CurrentEl()); + + assert(el > 0U); + + return el; +} + +uint64_t xlat_arch_get_xn_desc(unsigned int el) +{ + if (el == 3U) { + return UPPER_ATTRS(XN); + } else { + assert(el == 1U); + return UPPER_ATTRS(PXN); + } +} + +void init_xlat_tables(void) +{ + unsigned long long max_pa; + uintptr_t max_va; + + assert(PLAT_VIRT_ADDR_SPACE_SIZE >= + (xlat_get_min_virt_addr_space_size() - 1U)); + assert(PLAT_VIRT_ADDR_SPACE_SIZE <= MAX_VIRT_ADDR_SPACE_SIZE); + assert(IS_POWER_OF_TWO(PLAT_VIRT_ADDR_SPACE_SIZE)); + + print_mmap(); + init_xlation_table(0U, base_xlation_table, XLAT_TABLE_LEVEL_BASE, + &max_va, &max_pa); + + assert(max_va <= (PLAT_VIRT_ADDR_SPACE_SIZE - 1U)); + assert(max_pa <= (PLAT_PHY_ADDR_SPACE_SIZE - 1U)); + assert((PLAT_PHY_ADDR_SPACE_SIZE - 1U) <= get_max_supported_pa()); + + tcr_ps_bits = calc_physical_addr_size_bits(max_pa); +} + +/******************************************************************************* + * Macro generating the code for the function enabling the MMU in the given + * exception level, assuming that the pagetables have already been created. + * + * _el: Exception level at which the function will run + * _tcr_extra: Extra bits to set in the TCR register. This mask will + * be OR'ed with the default TCR value. + * _tlbi_fct: Function to invalidate the TLBs at the current + * exception level + ******************************************************************************/ +#define DEFINE_ENABLE_MMU_EL(_el, _tcr_extra, _tlbi_fct) \ + void enable_mmu_el##_el(unsigned int flags) \ + { \ + uint64_t mair, tcr, ttbr; \ + uint32_t sctlr; \ + \ + assert(IS_IN_EL(_el)); \ + assert((read_sctlr_el##_el() & SCTLR_M_BIT) == 0U); \ + \ + /* Set attributes in the right indices of the MAIR */ \ + mair = MAIR_ATTR_SET(ATTR_DEVICE, ATTR_DEVICE_INDEX); \ + mair |= MAIR_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, \ + ATTR_IWBWA_OWBWA_NTR_INDEX); \ + mair |= MAIR_ATTR_SET(ATTR_NON_CACHEABLE, \ + ATTR_NON_CACHEABLE_INDEX); \ + write_mair_el##_el(mair); \ + \ + /* Invalidate TLBs at the current exception level */ \ + _tlbi_fct(); \ + \ + /* Set TCR bits as well. */ \ + /* Set T0SZ to (64 - width of virtual address space) */ \ + int t0sz = 64 - __builtin_ctzll(PLAT_VIRT_ADDR_SPACE_SIZE);\ + \ + if ((flags & XLAT_TABLE_NC) != 0U) { \ + /* Inner & outer non-cacheable non-shareable. */\ + tcr = TCR_SH_NON_SHAREABLE | \ + TCR_RGN_OUTER_NC | TCR_RGN_INNER_NC | \ + ((uint64_t)t0sz << TCR_T0SZ_SHIFT); \ + } else { \ + /* Inner & outer WBWA & shareable. */ \ + tcr = TCR_SH_INNER_SHAREABLE | \ + TCR_RGN_OUTER_WBA | TCR_RGN_INNER_WBA | \ + ((uint64_t)t0sz << TCR_T0SZ_SHIFT); \ + } \ + tcr |= _tcr_extra; \ + write_tcr_el##_el(tcr); \ + \ + /* Set TTBR bits as well */ \ + ttbr = (uint64_t) base_xlation_table; \ + write_ttbr0_el##_el(ttbr); \ + \ + /* Ensure all translation table writes have drained */ \ + /* into memory, the TLB invalidation is complete, */ \ + /* and translation register writes are committed */ \ + /* before enabling the MMU */ \ + dsbish(); \ + isb(); \ + \ + sctlr = read_sctlr_el##_el(); \ + sctlr |= SCTLR_WXN_BIT | SCTLR_M_BIT; \ + \ + if ((flags & DISABLE_DCACHE) != 0U) \ + sctlr &= ~SCTLR_C_BIT; \ + else \ + sctlr |= SCTLR_C_BIT; \ + \ + write_sctlr_el##_el(sctlr); \ + \ + /* Ensure the MMU enable takes effect immediately */ \ + isb(); \ + } \ + \ + void enable_mmu_direct_el##_el(unsigned int flags) \ + { \ + enable_mmu_el##_el(flags); \ + } + +/* Define EL1 and EL3 variants of the function enabling the MMU */ +DEFINE_ENABLE_MMU_EL(1, + /* + * TCR_EL1.EPD1: Disable translation table walk for addresses + * that are translated using TTBR1_EL1. + */ + TCR_EPD1_BIT | (tcr_ps_bits << TCR_EL1_IPS_SHIFT), + tlbivmalle1) +DEFINE_ENABLE_MMU_EL(3, + TCR_EL3_RES1 | (tcr_ps_bits << TCR_EL3_PS_SHIFT), + tlbialle3) diff --git a/lib/xlat_tables/xlat_tables_common.c b/lib/xlat_tables/xlat_tables_common.c new file mode 100644 index 0000000..8b44df2 --- /dev/null +++ b/lib/xlat_tables/xlat_tables_common.c @@ -0,0 +1,417 @@ +/* + * Copyright (c) 2016-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> +#include <string.h> + +#include <platform_def.h> + +#include <arch.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/cassert.h> +#include <lib/utils.h> +#include <lib/xlat_tables/xlat_tables.h> +#include <plat/common/common_def.h> + +#include "xlat_tables_private.h" + +#if LOG_LEVEL >= LOG_LEVEL_VERBOSE +#define LVL0_SPACER "" +#define LVL1_SPACER " " +#define LVL2_SPACER " " +#define LVL3_SPACER " " +#define get_level_spacer(level) \ + (((level) == U(0)) ? LVL0_SPACER : \ + (((level) == U(1)) ? LVL1_SPACER : \ + (((level) == U(2)) ? LVL2_SPACER : LVL3_SPACER))) +#define debug_print(...) printf(__VA_ARGS__) +#else +#define debug_print(...) ((void)0) +#endif + +#define UNSET_DESC ~0ULL +#define MT_UNKNOWN ~0U + +static uint64_t xlat_tables[MAX_XLAT_TABLES][XLAT_TABLE_ENTRIES] + __aligned(XLAT_TABLE_SIZE) __section(".xlat_table"); + +static unsigned int next_xlat; +static unsigned long long xlat_max_pa; +static uintptr_t xlat_max_va; + +static uint64_t execute_never_mask; +static uint64_t ap1_mask; + +/* + * Array of all memory regions stored in order of ascending base address. + * The list is terminated by the first entry with size == 0. + */ +static mmap_region_t mmap[MAX_MMAP_REGIONS + 1]; + + +void print_mmap(void) +{ +#if LOG_LEVEL >= LOG_LEVEL_VERBOSE + debug_print("mmap:\n"); + mmap_region_t *mm = mmap; + while (mm->size != 0U) { + debug_print(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", + (void *)mm->base_va, mm->base_pa, + mm->size, mm->attr); + ++mm; + }; + debug_print("\n"); +#endif +} + +void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, + size_t size, unsigned int attr) +{ + mmap_region_t *mm = mmap; + const mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1U; + unsigned long long end_pa = base_pa + size - 1U; + uintptr_t end_va = base_va + size - 1U; + + assert(IS_PAGE_ALIGNED(base_pa)); + assert(IS_PAGE_ALIGNED(base_va)); + assert(IS_PAGE_ALIGNED(size)); + + if (size == 0U) + return; + + assert(base_pa < end_pa); /* Check for overflows */ + assert(base_va < end_va); + + assert((base_va + (uintptr_t)size - (uintptr_t)1) <= + (PLAT_VIRT_ADDR_SPACE_SIZE - 1U)); + assert((base_pa + (unsigned long long)size - 1ULL) <= + (PLAT_PHY_ADDR_SPACE_SIZE - 1U)); + +#if ENABLE_ASSERTIONS + + /* Check for PAs and VAs overlaps with all other regions */ + for (mm = mmap; mm->size; ++mm) { + + uintptr_t mm_end_va = mm->base_va + mm->size - 1U; + + /* + * Check if one of the regions is completely inside the other + * one. + */ + bool fully_overlapped_va = + ((base_va >= mm->base_va) && (end_va <= mm_end_va)) || + ((mm->base_va >= base_va) && (mm_end_va <= end_va)); + + /* + * Full VA overlaps are only allowed if both regions are + * identity mapped (zero offset) or have the same VA to PA + * offset. Also, make sure that it's not the exact same area. + */ + if (fully_overlapped_va) { + assert((mm->base_va - mm->base_pa) == + (base_va - base_pa)); + assert((base_va != mm->base_va) || (size != mm->size)); + } else { + /* + * If the regions do not have fully overlapping VAs, + * then they must have fully separated VAs and PAs. + * Partial overlaps are not allowed + */ + + unsigned long long mm_end_pa = + mm->base_pa + mm->size - 1; + + bool separated_pa = (end_pa < mm->base_pa) || + (base_pa > mm_end_pa); + bool separated_va = (end_va < mm->base_va) || + (base_va > mm_end_va); + + assert(separated_va && separated_pa); + } + } + + mm = mmap; /* Restore pointer to the start of the array */ + +#endif /* ENABLE_ASSERTIONS */ + + /* Find correct place in mmap to insert new region */ + while ((mm->base_va < base_va) && (mm->size != 0U)) + ++mm; + + /* + * If a section is contained inside another one with the same base + * address, it must be placed after the one it is contained in: + * + * 1st |-----------------------| + * 2nd |------------| + * 3rd |------| + * + * This is required for mmap_region_attr() to get the attributes of the + * small region correctly. + */ + while ((mm->base_va == base_va) && (mm->size > size)) + ++mm; + + /* Make room for new region by moving other regions up by one place */ + (void)memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm); + + /* Check we haven't lost the empty sentinel from the end of the array */ + assert(mm_last->size == 0U); + + mm->base_pa = base_pa; + mm->base_va = base_va; + mm->size = size; + mm->attr = attr; + + if (end_pa > xlat_max_pa) + xlat_max_pa = end_pa; + if (end_va > xlat_max_va) + xlat_max_va = end_va; +} + +void mmap_add(const mmap_region_t *mm) +{ + const mmap_region_t *mm_cursor = mm; + + while ((mm_cursor->size != 0U) || (mm_cursor->attr != 0U)) { + mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va, + mm_cursor->size, mm_cursor->attr); + mm_cursor++; + } +} + +static uint64_t mmap_desc(unsigned int attr, unsigned long long addr_pa, + unsigned int level) +{ + uint64_t desc; + int mem_type; + + /* Make sure that the granularity is fine enough to map this address. */ + assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0U); + + desc = addr_pa; + /* + * There are different translation table descriptors for level 3 and the + * rest. + */ + desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; + desc |= ((attr & MT_NS) != 0U) ? LOWER_ATTRS(NS) : 0U; + desc |= ((attr & MT_RW) != 0U) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); + /* + * Always set the access flag, as this library assumes access flag + * faults aren't managed. + */ + desc |= LOWER_ATTRS(ACCESS_FLAG); + desc |= ap1_mask; + + /* + * Deduce shareability domain and executability of the memory region + * from the memory type. + * + * Data accesses to device memory and non-cacheable normal memory are + * coherent for all observers in the system, and correspondingly are + * always treated as being Outer Shareable. Therefore, for these 2 types + * of memory, it is not strictly needed to set the shareability field + * in the translation tables. + */ + mem_type = MT_TYPE(attr); + if (mem_type == MT_DEVICE) { + desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH); + /* + * Always map device memory as execute-never. + * This is to avoid the possibility of a speculative instruction + * fetch, which could be an issue if this memory region + * corresponds to a read-sensitive peripheral. + */ + desc |= execute_never_mask; + + } else { /* Normal memory */ + /* + * Always map read-write normal memory as execute-never. + * This library assumes that it is used by software that does + * not self-modify its code, therefore R/W memory is reserved + * for data storage, which must not be executable. + * + * Note that setting the XN bit here is for consistency only. + * The function that enables the MMU sets the SCTLR_ELx.WXN bit, + * which makes any writable memory region to be treated as + * execute-never, regardless of the value of the XN bit in the + * translation table. + * + * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER + * attribute to figure out the value of the XN bit. + */ + if (((attr & MT_RW) != 0U) || ((attr & MT_EXECUTE_NEVER) != 0U)) { + desc |= execute_never_mask; + } + + if (mem_type == MT_MEMORY) { + desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH); + } else { + assert(mem_type == MT_NON_CACHEABLE); + desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH); + } + } + + debug_print((mem_type == MT_MEMORY) ? "MEM" : + ((mem_type == MT_NON_CACHEABLE) ? "NC" : "DEV")); + debug_print(((attr & MT_RW) != 0U) ? "-RW" : "-RO"); + debug_print(((attr & MT_NS) != 0U) ? "-NS" : "-S"); + debug_print(((attr & MT_EXECUTE_NEVER) != 0U) ? "-XN" : "-EXEC"); + return desc; +} + +/* + * Look for the innermost region that contains the area at `base_va` with size + * `size`. Populate *attr with the attributes of this region. + * + * On success, this function returns 0. + * If there are partial overlaps (meaning that a smaller size is needed) or if + * the region can't be found in the given area, it returns MT_UNKNOWN. In this + * case the value pointed by attr should be ignored by the caller. + */ +static unsigned int mmap_region_attr(const mmap_region_t *mm, uintptr_t base_va, + size_t size, unsigned int *attr) +{ + /* Don't assume that the area is contained in the first region */ + unsigned int ret = MT_UNKNOWN; + + /* + * Get attributes from last (innermost) region that contains the + * requested area. Don't stop as soon as one region doesn't contain it + * because there may be other internal regions that contain this area: + * + * |-----------------------------1-----------------------------| + * |----2----| |-------3-------| |----5----| + * |--4--| + * + * |---| <- Area we want the attributes of. + * + * In this example, the area is contained in regions 1, 3 and 4 but not + * in region 2. The loop shouldn't stop at region 2 as inner regions + * have priority over outer regions, it should stop at region 5. + */ + for ( ; ; ++mm) { + + if (mm->size == 0U) + return ret; /* Reached end of list */ + + if (mm->base_va > (base_va + size - 1U)) + return ret; /* Next region is after area so end */ + + if ((mm->base_va + mm->size - 1U) < base_va) + continue; /* Next region has already been overtaken */ + + if ((ret == 0U) && (mm->attr == *attr)) + continue; /* Region doesn't override attribs so skip */ + + if ((mm->base_va > base_va) || + ((mm->base_va + mm->size - 1U) < (base_va + size - 1U))) + return MT_UNKNOWN; /* Region doesn't fully cover area */ + + *attr = mm->attr; + ret = 0U; + } + return ret; +} + +static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm, + uintptr_t base_va, + uint64_t *table, + unsigned int level) +{ + assert((level >= XLAT_TABLE_LEVEL_MIN) && + (level <= XLAT_TABLE_LEVEL_MAX)); + + unsigned int level_size_shift = + L0_XLAT_ADDRESS_SHIFT - level * XLAT_TABLE_ENTRIES_SHIFT; + u_register_t level_size = (u_register_t)1 << level_size_shift; + u_register_t level_index_mask = + ((u_register_t)XLAT_TABLE_ENTRIES_MASK) << level_size_shift; + + debug_print("New xlat table:\n"); + + do { + uint64_t desc = UNSET_DESC; + + if (mm->size == 0U) { + /* Done mapping regions; finish zeroing the table */ + desc = INVALID_DESC; + } else if ((mm->base_va + mm->size - 1U) < base_va) { + /* This area is after the region so get next region */ + ++mm; + continue; + } + + debug_print("%s VA:%p size:0x%llx ", get_level_spacer(level), + (void *)base_va, (unsigned long long)level_size); + + if (mm->base_va > (base_va + level_size - 1U)) { + /* Next region is after this area. Nothing to map yet */ + desc = INVALID_DESC; + /* Make sure that the current level allows block descriptors */ + } else if (level >= XLAT_BLOCK_LEVEL_MIN) { + /* + * Try to get attributes of this area. It will fail if + * there are partially overlapping regions. On success, + * it will return the innermost region's attributes. + */ + unsigned int attr; + unsigned int r = mmap_region_attr(mm, base_va, + level_size, &attr); + + if (r == 0U) { + desc = mmap_desc(attr, + base_va - mm->base_va + mm->base_pa, + level); + } + } + + if (desc == UNSET_DESC) { + /* Area not covered by a region so need finer table */ + uint64_t *new_table = xlat_tables[next_xlat]; + + next_xlat++; + assert(next_xlat <= MAX_XLAT_TABLES); + desc = TABLE_DESC | (uintptr_t)new_table; + + /* Recurse to fill in new table */ + mm = init_xlation_table_inner(mm, base_va, + new_table, level + 1U); + } + + debug_print("\n"); + + *table++ = desc; + base_va += level_size; + } while ((base_va & level_index_mask) && + ((base_va - 1U) < (PLAT_VIRT_ADDR_SPACE_SIZE - 1U))); + + return mm; +} + +void init_xlation_table(uintptr_t base_va, uint64_t *table, + unsigned int level, uintptr_t *max_va, + unsigned long long *max_pa) +{ + unsigned int el = xlat_arch_current_el(); + + execute_never_mask = xlat_arch_get_xn_desc(el); + + if (el == 3U) { + ap1_mask = LOWER_ATTRS(AP_ONE_VA_RANGE_RES1); + } else { + assert(el == 1U); + ap1_mask = 0ULL; + } + + init_xlation_table_inner(mmap, base_va, table, level); + *max_va = xlat_max_va; + *max_pa = xlat_max_pa; +} diff --git a/lib/xlat_tables/xlat_tables_private.h b/lib/xlat_tables/xlat_tables_private.h new file mode 100644 index 0000000..cfecc39 --- /dev/null +++ b/lib/xlat_tables/xlat_tables_private.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016-2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef XLAT_TABLES_PRIVATE_H +#define XLAT_TABLES_PRIVATE_H + +#include <platform_def.h> + +#include <lib/cassert.h> +#include <lib/xlat_tables/xlat_tables_arch.h> + +#if HW_ASSISTED_COHERENCY +#error xlat tables v2 must be used with HW_ASSISTED_COHERENCY +#endif + +CASSERT(CHECK_PHY_ADDR_SPACE_SIZE(PLAT_PHY_ADDR_SPACE_SIZE), + assert_valid_phy_addr_space_size); + +/* Alias to retain compatibility with the old #define name */ +#define XLAT_BLOCK_LEVEL_MIN MIN_LVL_BLOCK_DESC + +void print_mmap(void); + +/* Returns the current Exception Level. The returned EL must be 1 or higher. */ +unsigned int xlat_arch_current_el(void); + +/* + * Returns the bit mask that has to be ORed to the rest of a translation table + * descriptor so that execution of code is prohibited at the given Exception + * Level. + */ +uint64_t xlat_arch_get_xn_desc(unsigned int el); + +void init_xlation_table(uintptr_t base_va, uint64_t *table, + unsigned int level, uintptr_t *max_va, + unsigned long long *max_pa); + +#endif /* XLAT_TABLES_PRIVATE_H */ diff --git a/lib/xlat_tables_v2/aarch32/enable_mmu.S b/lib/xlat_tables_v2/aarch32/enable_mmu.S new file mode 100644 index 0000000..22c2876 --- /dev/null +++ b/lib/xlat_tables_v2/aarch32/enable_mmu.S @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <assert_macros.S> +#include <lib/xlat_tables/xlat_tables_v2.h> + + .global enable_mmu_direct_svc_mon + .global enable_mmu_direct_hyp + + /* void enable_mmu_direct_svc_mon(unsigned int flags) */ +func enable_mmu_direct_svc_mon + /* Assert that MMU is turned off */ +#if ENABLE_ASSERTIONS + ldcopr r1, SCTLR + tst r1, #SCTLR_M_BIT + ASM_ASSERT(eq) +#endif + + /* Invalidate TLB entries */ + TLB_INVALIDATE(r0, TLBIALL) + + mov r3, r0 + ldr r0, =mmu_cfg_params + + /* MAIR0. Only the lower 32 bits are used. */ + ldr r1, [r0, #(MMU_CFG_MAIR << 3)] + stcopr r1, MAIR0 + + /* TTBCR. Only the lower 32 bits are used. */ + ldr r2, [r0, #(MMU_CFG_TCR << 3)] + stcopr r2, TTBCR + + /* TTBR0 */ + ldr r1, [r0, #(MMU_CFG_TTBR0 << 3)] + ldr r2, [r0, #((MMU_CFG_TTBR0 << 3) + 4)] + stcopr16 r1, r2, TTBR0_64 + + /* TTBR1 is unused right now; set it to 0. */ + mov r1, #0 + mov r2, #0 + stcopr16 r1, r2, TTBR1_64 + + /* + * Ensure all translation table writes have drained into memory, the TLB + * invalidation is complete, and translation register writes are + * committed before enabling the MMU + */ + dsb ish + isb + + /* Enable enable MMU by honoring flags */ + ldcopr r1, SCTLR + ldr r2, =(SCTLR_WXN_BIT | SCTLR_C_BIT | SCTLR_M_BIT) + orr r1, r1, r2 + + /* Clear C bit if requested */ + tst r3, #DISABLE_DCACHE + bicne r1, r1, #SCTLR_C_BIT + + stcopr r1, SCTLR + isb + + bx lr +endfunc enable_mmu_direct_svc_mon + + + /* void enable_mmu_direct_hyp(unsigned int flags) */ +func enable_mmu_direct_hyp + /* Assert that MMU is turned off */ +#if ENABLE_ASSERTIONS + ldcopr r1, HSCTLR + tst r1, #HSCTLR_M_BIT + ASM_ASSERT(eq) +#endif + + /* Invalidate TLB entries */ + TLB_INVALIDATE(r0, TLBIALL) + + mov r3, r0 + ldr r0, =mmu_cfg_params + + /* HMAIR0 */ + ldr r1, [r0, #(MMU_CFG_MAIR << 3)] + stcopr r1, HMAIR0 + + /* HTCR */ + ldr r2, [r0, #(MMU_CFG_TCR << 3)] + stcopr r2, HTCR + + /* HTTBR */ + ldr r1, [r0, #(MMU_CFG_TTBR0 << 3)] + ldr r2, [r0, #((MMU_CFG_TTBR0 << 3) + 4)] + stcopr16 r1, r2, HTTBR_64 + + /* + * Ensure all translation table writes have drained into memory, the TLB + * invalidation is complete, and translation register writes are + * committed before enabling the MMU + */ + dsb ish + isb + + /* Enable enable MMU by honoring flags */ + ldcopr r1, HSCTLR + ldr r2, =(HSCTLR_WXN_BIT | HSCTLR_C_BIT | HSCTLR_M_BIT) + orr r1, r1, r2 + + /* Clear C bit if requested */ + tst r3, #DISABLE_DCACHE + bicne r1, r1, #HSCTLR_C_BIT + + stcopr r1, HSCTLR + isb + + bx lr +endfunc enable_mmu_direct_hyp diff --git a/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c b/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c new file mode 100644 index 0000000..920754b --- /dev/null +++ b/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdbool.h> + +#include <platform_def.h> + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/cassert.h> +#include <lib/utils_def.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +#include "../xlat_tables_private.h" + +#if (ARM_ARCH_MAJOR == 7) && !defined(ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING) +#error ARMv7 target does not support LPAE MMU descriptors +#endif + +/* + * Returns true if the provided granule size is supported, false otherwise. + */ +bool xlat_arch_is_granule_size_supported(size_t size) +{ + /* + * The library uses the long descriptor translation table format, which + * supports 4 KiB pages only. + */ + return size == PAGE_SIZE_4KB; +} + +size_t xlat_arch_get_max_supported_granule_size(void) +{ + return PAGE_SIZE_4KB; +} + +/* + * Determine the physical address space encoded in the 'attr' parameter. + * + * The physical address will fall into one of two spaces; secure or + * nonsecure. + */ +uint32_t xlat_arch_get_pas(uint32_t attr) +{ + uint32_t pas = MT_PAS(attr); + + if (pas == MT_NS) { + return LOWER_ATTRS(NS); + } else { /* MT_SECURE */ + return 0U; + } +} + +#if ENABLE_ASSERTIONS +unsigned long long xlat_arch_get_max_supported_pa(void) +{ + /* Physical address space size for long descriptor format. */ + return (1ULL << 40) - 1ULL; +} + +/* + * Return minimum virtual address space size supported by the architecture + */ +uintptr_t xlat_get_min_virt_addr_space_size(void) +{ + return MIN_VIRT_ADDR_SPACE_SIZE; +} +#endif /* ENABLE_ASSERTIONS*/ + +bool is_mmu_enabled_ctx(const xlat_ctx_t *ctx) +{ + if (ctx->xlat_regime == EL1_EL0_REGIME) { + assert(xlat_arch_current_el() == 1U); + return (read_sctlr() & SCTLR_M_BIT) != 0U; + } else { + assert(ctx->xlat_regime == EL2_REGIME); + assert(xlat_arch_current_el() == 2U); + return (read_hsctlr() & HSCTLR_M_BIT) != 0U; + } +} + +bool is_dcache_enabled(void) +{ + if (IS_IN_EL2()) { + return (read_hsctlr() & HSCTLR_C_BIT) != 0U; + } else { + return (read_sctlr() & SCTLR_C_BIT) != 0U; + } +} + +uint64_t xlat_arch_regime_get_xn_desc(int xlat_regime) +{ + if (xlat_regime == EL1_EL0_REGIME) { + return UPPER_ATTRS(XN) | UPPER_ATTRS(PXN); + } else { + assert(xlat_regime == EL2_REGIME); + return UPPER_ATTRS(XN); + } +} + +void xlat_arch_tlbi_va(uintptr_t va, int xlat_regime) +{ + /* + * Ensure the translation table write has drained into memory before + * invalidating the TLB entry. + */ + dsbishst(); + + if (xlat_regime == EL1_EL0_REGIME) { + tlbimvaais(TLBI_ADDR(va)); + } else { + assert(xlat_regime == EL2_REGIME); + tlbimvahis(TLBI_ADDR(va)); + } +} + +void xlat_arch_tlbi_va_sync(void) +{ + /* Invalidate all entries from branch predictors. */ + bpiallis(); + + /* + * A TLB maintenance instruction can complete at any time after + * it is issued, but is only guaranteed to be complete after the + * execution of DSB by the PE that executed the TLB maintenance + * instruction. After the TLB invalidate instruction is + * complete, no new memory accesses using the invalidated TLB + * entries will be observed by any observer of the system + * domain. See section D4.8.2 of the ARMv8 (issue k), paragraph + * "Ordering and completion of TLB maintenance instructions". + */ + dsbish(); + + /* + * The effects of a completed TLB maintenance instruction are + * only guaranteed to be visible on the PE that executed the + * instruction after the execution of an ISB instruction by the + * PE that executed the TLB maintenance instruction. + */ + isb(); +} + +unsigned int xlat_arch_current_el(void) +{ + if (IS_IN_HYP()) { + return 2U; + } else { + assert(IS_IN_SVC() || IS_IN_MON()); + /* + * If EL3 is in AArch32 mode, all secure PL1 modes (Monitor, + * System, SVC, Abort, UND, IRQ and FIQ modes) execute at EL3. + * + * The PL1&0 translation regime in AArch32 behaves like the + * EL1&0 regime in AArch64 except for the XN bits, but we set + * and unset them at the same time, so there's no difference in + * practice. + */ + return 1U; + } +} + +/******************************************************************************* + * Function for enabling the MMU in PL1 or PL2, assuming that the page tables + * have already been created. + ******************************************************************************/ +void setup_mmu_cfg(uint64_t *params, unsigned int flags, + const uint64_t *base_table, unsigned long long max_pa, + uintptr_t max_va, __unused int xlat_regime) +{ + uint64_t mair, ttbr0; + uint32_t ttbcr; + + /* Set attributes in the right indices of the MAIR */ + mair = MAIR0_ATTR_SET(ATTR_DEVICE, ATTR_DEVICE_INDEX); + mair |= MAIR0_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, + ATTR_IWBWA_OWBWA_NTR_INDEX); + mair |= MAIR0_ATTR_SET(ATTR_NON_CACHEABLE, + ATTR_NON_CACHEABLE_INDEX); + + /* + * Configure the control register for stage 1 of the PL1&0 or EL2 + * translation regimes. + */ + + /* Use the Long-descriptor translation table format. */ + ttbcr = TTBCR_EAE_BIT; + + if (xlat_regime == EL1_EL0_REGIME) { + assert(IS_IN_SVC() || IS_IN_MON()); + /* + * Disable translation table walk for addresses that are + * translated using TTBR1. Therefore, only TTBR0 is used. + */ + ttbcr |= TTBCR_EPD1_BIT; + } else { + assert(xlat_regime == EL2_REGIME); + assert(IS_IN_HYP()); + + /* + * Set HTCR bits as well. Set HTTBR table properties + * as Inner & outer WBWA & shareable. + */ + ttbcr |= HTCR_RES1 | + HTCR_SH0_INNER_SHAREABLE | HTCR_RGN0_OUTER_WBA | + HTCR_RGN0_INNER_WBA; + } + + /* + * Limit the input address ranges and memory region sizes translated + * using TTBR0 to the given virtual address space size, if smaller than + * 32 bits. + */ + if (max_va != UINT32_MAX) { + uintptr_t virtual_addr_space_size = max_va + 1U; + + assert(virtual_addr_space_size >= + xlat_get_min_virt_addr_space_size()); + assert(IS_POWER_OF_TWO(virtual_addr_space_size)); + + /* + * __builtin_ctzll(0) is undefined but here we are guaranteed + * that virtual_addr_space_size is in the range [1, UINT32_MAX]. + */ + int t0sz = 32 - __builtin_ctzll(virtual_addr_space_size); + + ttbcr |= (uint32_t) t0sz; + } + + /* + * Set the cacheability and shareability attributes for memory + * associated with translation table walks using TTBR0. + */ + if ((flags & XLAT_TABLE_NC) != 0U) { + /* Inner & outer non-cacheable non-shareable. */ + ttbcr |= TTBCR_SH0_NON_SHAREABLE | TTBCR_RGN0_OUTER_NC | + TTBCR_RGN0_INNER_NC; + } else { + /* Inner & outer WBWA & shareable. */ + ttbcr |= TTBCR_SH0_INNER_SHAREABLE | TTBCR_RGN0_OUTER_WBA | + TTBCR_RGN0_INNER_WBA; + } + + /* Set TTBR0 bits as well */ + ttbr0 = (uint64_t)(uintptr_t) base_table; + + if (is_armv8_2_ttcnp_present()) { + /* Enable CnP bit so as to share page tables with all PEs. */ + ttbr0 |= TTBR_CNP_BIT; + } + + /* Now populate MMU configuration */ + params[MMU_CFG_MAIR] = mair; + params[MMU_CFG_TCR] = (uint64_t) ttbcr; + params[MMU_CFG_TTBR0] = ttbr0; +} diff --git a/lib/xlat_tables_v2/aarch64/enable_mmu.S b/lib/xlat_tables_v2/aarch64/enable_mmu.S new file mode 100644 index 0000000..247a796 --- /dev/null +++ b/lib/xlat_tables_v2/aarch64/enable_mmu.S @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <asm_macros.S> +#include <assert_macros.S> +#include <lib/xlat_tables/xlat_tables_v2.h> + + .global enable_mmu_direct_el1 + .global enable_mmu_direct_el2 + .global enable_mmu_direct_el3 + + /* Macros to read and write to system register for a given EL. */ + .macro _msr reg_name, el, gp_reg + msr \reg_name\()_el\()\el, \gp_reg + .endm + + .macro _mrs gp_reg, reg_name, el + mrs \gp_reg, \reg_name\()_el\()\el + .endm + + .macro tlbi_invalidate_all el + .if \el == 1 + TLB_INVALIDATE(vmalle1) + .elseif \el == 2 + TLB_INVALIDATE(alle2) + .elseif \el == 3 + TLB_INVALIDATE(alle3) + .else + .error "EL must be 1, 2 or 3" + .endif + .endm + + /* void enable_mmu_direct_el<x>(unsigned int flags) */ + .macro define_mmu_enable_func el + func enable_mmu_direct_\()el\el +#if ENABLE_ASSERTIONS + _mrs x1, sctlr, \el + tst x1, #SCTLR_M_BIT + ASM_ASSERT(eq) +#endif + /* Invalidate all TLB entries */ + tlbi_invalidate_all \el + + mov x7, x0 + adrp x0, mmu_cfg_params + add x0, x0, :lo12:mmu_cfg_params + + /* MAIR */ + ldr x1, [x0, #(MMU_CFG_MAIR << 3)] + _msr mair, \el, x1 + + /* TCR */ + ldr x2, [x0, #(MMU_CFG_TCR << 3)] + _msr tcr, \el, x2 + + /* TTBR */ + ldr x3, [x0, #(MMU_CFG_TTBR0 << 3)] + _msr ttbr0, \el, x3 + + /* + * Ensure all translation table writes have drained into memory, the TLB + * invalidation is complete, and translation register writes are + * committed before enabling the MMU + */ + dsb ish + isb + + /* Set and clear required fields of SCTLR */ + _mrs x4, sctlr, \el + mov_imm x5, SCTLR_WXN_BIT | SCTLR_C_BIT | SCTLR_M_BIT + orr x4, x4, x5 + + /* Additionally, amend SCTLR fields based on flags */ + bic x5, x4, #SCTLR_C_BIT + tst x7, #DISABLE_DCACHE + csel x4, x5, x4, ne + + _msr sctlr, \el, x4 + isb + + ret + endfunc enable_mmu_direct_\()el\el + .endm + + /* + * Define MMU-enabling functions for EL1, EL2 and EL3: + * + * enable_mmu_direct_el1 + * enable_mmu_direct_el2 + * enable_mmu_direct_el3 + */ + define_mmu_enable_func 1 + define_mmu_enable_func 2 + define_mmu_enable_func 3 diff --git a/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c b/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c new file mode 100644 index 0000000..bb6a35c --- /dev/null +++ b/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> + +#include <arch.h> +#include <arch_features.h> +#include <arch_helpers.h> +#include <lib/cassert.h> +#include <lib/utils_def.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +#include "../xlat_tables_private.h" + +/* + * Returns true if the provided granule size is supported, false otherwise. + */ +bool xlat_arch_is_granule_size_supported(size_t size) +{ + unsigned int tgranx; + + if (size == PAGE_SIZE_4KB) { + tgranx = read_id_aa64mmfr0_el0_tgran4_field(); + /* MSB of TGRAN4 field will be '1' for unsupported feature */ + return (tgranx < 8U); + } else if (size == PAGE_SIZE_16KB) { + tgranx = read_id_aa64mmfr0_el0_tgran16_field(); + return (tgranx >= ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED); + } else if (size == PAGE_SIZE_64KB) { + tgranx = read_id_aa64mmfr0_el0_tgran64_field(); + /* MSB of TGRAN64 field will be '1' for unsupported feature */ + return (tgranx < 8U); + } else { + return false; + } +} + +size_t xlat_arch_get_max_supported_granule_size(void) +{ + if (xlat_arch_is_granule_size_supported(PAGE_SIZE_64KB)) { + return PAGE_SIZE_64KB; + } else if (xlat_arch_is_granule_size_supported(PAGE_SIZE_16KB)) { + return PAGE_SIZE_16KB; + } else { + assert(xlat_arch_is_granule_size_supported(PAGE_SIZE_4KB)); + return PAGE_SIZE_4KB; + } +} + +/* + * Determine the physical address space encoded in the 'attr' parameter. + * + * The physical address will fall into one of four spaces; secure, + * nonsecure, root, or realm if RME is enabled, or one of two spaces; + * secure and nonsecure otherwise. + */ +uint32_t xlat_arch_get_pas(uint32_t attr) +{ + uint32_t pas = MT_PAS(attr); + + switch (pas) { +#if ENABLE_RME + /* TTD.NSE = 1 and TTD.NS = 1 for Realm PAS */ + case MT_REALM: + return LOWER_ATTRS(EL3_S1_NSE | NS); + /* TTD.NSE = 1 and TTD.NS = 0 for Root PAS */ + case MT_ROOT: + return LOWER_ATTRS(EL3_S1_NSE); +#endif + case MT_NS: + return LOWER_ATTRS(NS); + default: /* MT_SECURE */ + return 0U; + } +} + +unsigned long long tcr_physical_addr_size_bits(unsigned long long max_addr) +{ + /* Physical address can't exceed 48 bits */ + assert((max_addr & ADDR_MASK_48_TO_63) == 0U); + + /* 48 bits address */ + if ((max_addr & ADDR_MASK_44_TO_47) != 0U) + return TCR_PS_BITS_256TB; + + /* 44 bits address */ + if ((max_addr & ADDR_MASK_42_TO_43) != 0U) + return TCR_PS_BITS_16TB; + + /* 42 bits address */ + if ((max_addr & ADDR_MASK_40_TO_41) != 0U) + return TCR_PS_BITS_4TB; + + /* 40 bits address */ + if ((max_addr & ADDR_MASK_36_TO_39) != 0U) + return TCR_PS_BITS_1TB; + + /* 36 bits address */ + if ((max_addr & ADDR_MASK_32_TO_35) != 0U) + return TCR_PS_BITS_64GB; + + return TCR_PS_BITS_4GB; +} + +#if ENABLE_ASSERTIONS +/* + * Physical Address ranges supported in the AArch64 Memory Model. Value 0b110 is + * supported in ARMv8.2 onwards. + */ +static const unsigned int pa_range_bits_arr[] = { + PARANGE_0000, PARANGE_0001, PARANGE_0010, PARANGE_0011, PARANGE_0100, + PARANGE_0101, PARANGE_0110 +}; + +unsigned long long xlat_arch_get_max_supported_pa(void) +{ + u_register_t pa_range = read_id_aa64mmfr0_el1() & + ID_AA64MMFR0_EL1_PARANGE_MASK; + + /* All other values are reserved */ + assert(pa_range < ARRAY_SIZE(pa_range_bits_arr)); + + return (1ULL << pa_range_bits_arr[pa_range]) - 1ULL; +} + +/* + * Return minimum virtual address space size supported by the architecture + */ +uintptr_t xlat_get_min_virt_addr_space_size(void) +{ + uintptr_t ret; + + if (is_armv8_4_ttst_present()) + ret = MIN_VIRT_ADDR_SPACE_SIZE_TTST; + else + ret = MIN_VIRT_ADDR_SPACE_SIZE; + + return ret; +} +#endif /* ENABLE_ASSERTIONS*/ + +bool is_mmu_enabled_ctx(const xlat_ctx_t *ctx) +{ + if (ctx->xlat_regime == EL1_EL0_REGIME) { + assert(xlat_arch_current_el() >= 1U); + return (read_sctlr_el1() & SCTLR_M_BIT) != 0U; + } else if (ctx->xlat_regime == EL2_REGIME) { + assert(xlat_arch_current_el() >= 2U); + return (read_sctlr_el2() & SCTLR_M_BIT) != 0U; + } else { + assert(ctx->xlat_regime == EL3_REGIME); + assert(xlat_arch_current_el() >= 3U); + return (read_sctlr_el3() & SCTLR_M_BIT) != 0U; + } +} + +bool is_dcache_enabled(void) +{ + unsigned int el = get_current_el_maybe_constant(); + + if (el == 1U) { + return (read_sctlr_el1() & SCTLR_C_BIT) != 0U; + } else if (el == 2U) { + return (read_sctlr_el2() & SCTLR_C_BIT) != 0U; + } else { + return (read_sctlr_el3() & SCTLR_C_BIT) != 0U; + } +} + +uint64_t xlat_arch_regime_get_xn_desc(int xlat_regime) +{ + if (xlat_regime == EL1_EL0_REGIME) { + return UPPER_ATTRS(UXN) | UPPER_ATTRS(PXN); + } else { + assert((xlat_regime == EL2_REGIME) || + (xlat_regime == EL3_REGIME)); + return UPPER_ATTRS(XN); + } +} + +void xlat_arch_tlbi_va(uintptr_t va, int xlat_regime) +{ + /* + * Ensure the translation table write has drained into memory before + * invalidating the TLB entry. + */ + dsbishst(); + + /* + * This function only supports invalidation of TLB entries for the EL3 + * and EL1&0 translation regimes. + * + * Also, it is architecturally UNDEFINED to invalidate TLBs of a higher + * exception level (see section D4.9.2 of the ARM ARM rev B.a). + */ + if (xlat_regime == EL1_EL0_REGIME) { + assert(xlat_arch_current_el() >= 1U); + tlbivaae1is(TLBI_ADDR(va)); + } else if (xlat_regime == EL2_REGIME) { + assert(xlat_arch_current_el() >= 2U); + tlbivae2is(TLBI_ADDR(va)); + } else { + assert(xlat_regime == EL3_REGIME); + assert(xlat_arch_current_el() >= 3U); + tlbivae3is(TLBI_ADDR(va)); + } +} + +void xlat_arch_tlbi_va_sync(void) +{ + /* + * A TLB maintenance instruction can complete at any time after + * it is issued, but is only guaranteed to be complete after the + * execution of DSB by the PE that executed the TLB maintenance + * instruction. After the TLB invalidate instruction is + * complete, no new memory accesses using the invalidated TLB + * entries will be observed by any observer of the system + * domain. See section D4.8.2 of the ARMv8 (issue k), paragraph + * "Ordering and completion of TLB maintenance instructions". + */ + dsbish(); + + /* + * The effects of a completed TLB maintenance instruction are + * only guaranteed to be visible on the PE that executed the + * instruction after the execution of an ISB instruction by the + * PE that executed the TLB maintenance instruction. + */ + isb(); +} + +unsigned int xlat_arch_current_el(void) +{ + unsigned int el = (unsigned int)GET_EL(read_CurrentEl()); + + assert(el > 0U); + + return el; +} + +void setup_mmu_cfg(uint64_t *params, unsigned int flags, + const uint64_t *base_table, unsigned long long max_pa, + uintptr_t max_va, int xlat_regime) +{ + uint64_t mair, ttbr0, tcr; + uintptr_t virtual_addr_space_size; + + /* Set attributes in the right indices of the MAIR. */ + mair = MAIR_ATTR_SET(ATTR_DEVICE, ATTR_DEVICE_INDEX); + mair |= MAIR_ATTR_SET(ATTR_IWBWA_OWBWA_NTR, ATTR_IWBWA_OWBWA_NTR_INDEX); + mair |= MAIR_ATTR_SET(ATTR_NON_CACHEABLE, ATTR_NON_CACHEABLE_INDEX); + + /* + * Limit the input address ranges and memory region sizes translated + * using TTBR0 to the given virtual address space size. + */ + assert(max_va < ((uint64_t)UINTPTR_MAX)); + + virtual_addr_space_size = (uintptr_t)max_va + 1U; + + assert(virtual_addr_space_size >= + xlat_get_min_virt_addr_space_size()); + assert(virtual_addr_space_size <= MAX_VIRT_ADDR_SPACE_SIZE); + assert(IS_POWER_OF_TWO(virtual_addr_space_size)); + + /* + * __builtin_ctzll(0) is undefined but here we are guaranteed that + * virtual_addr_space_size is in the range [1,UINTPTR_MAX]. + */ + int t0sz = 64 - __builtin_ctzll(virtual_addr_space_size); + + tcr = (uint64_t)t0sz << TCR_T0SZ_SHIFT; + + /* + * Set the cacheability and shareability attributes for memory + * associated with translation table walks. + */ + if ((flags & XLAT_TABLE_NC) != 0U) { + /* Inner & outer non-cacheable non-shareable. */ + tcr |= TCR_SH_NON_SHAREABLE | + TCR_RGN_OUTER_NC | TCR_RGN_INNER_NC; + } else { + /* Inner & outer WBWA & shareable. */ + tcr |= TCR_SH_INNER_SHAREABLE | + TCR_RGN_OUTER_WBA | TCR_RGN_INNER_WBA; + } + + /* + * It is safer to restrict the max physical address accessible by the + * hardware as much as possible. + */ + unsigned long long tcr_ps_bits = tcr_physical_addr_size_bits(max_pa); + + if (xlat_regime == EL1_EL0_REGIME) { + /* + * TCR_EL1.EPD1: Disable translation table walk for addresses + * that are translated using TTBR1_EL1. + */ + tcr |= TCR_EPD1_BIT | (tcr_ps_bits << TCR_EL1_IPS_SHIFT); + } else if (xlat_regime == EL2_REGIME) { + tcr |= TCR_EL2_RES1 | (tcr_ps_bits << TCR_EL2_PS_SHIFT); + } else { + assert(xlat_regime == EL3_REGIME); + tcr |= TCR_EL3_RES1 | (tcr_ps_bits << TCR_EL3_PS_SHIFT); + } + + /* Set TTBR bits as well */ + ttbr0 = (uint64_t) base_table; + + if (is_armv8_2_ttcnp_present()) { + /* Enable CnP bit so as to share page tables with all PEs. */ + ttbr0 |= TTBR_CNP_BIT; + } + + params[MMU_CFG_MAIR] = mair; + params[MMU_CFG_TCR] = tcr; + params[MMU_CFG_TTBR0] = ttbr0; +} diff --git a/lib/xlat_tables_v2/ro_xlat_tables.mk b/lib/xlat_tables_v2/ro_xlat_tables.mk new file mode 100644 index 0000000..20bb015 --- /dev/null +++ b/lib/xlat_tables_v2/ro_xlat_tables.mk @@ -0,0 +1,41 @@ +# +# Copyright (c) 2020-2022, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +ifeq (${USE_DEBUGFS}, 1) + $(error "Debugfs requires functionality from the dynamic translation \ + library and is incompatible with ALLOW_RO_XLAT_TABLES.") +endif + +ifeq (${ARCH},aarch32) + ifeq (${RESET_TO_SP_MIN},1) + $(error "RESET_TO_SP_MIN requires functionality from the dynamic \ + translation library and is incompatible with \ + ALLOW_RO_XLAT_TABLES.") + endif +else # if AArch64 + ifeq (${PLAT},tegra) + $(error "Tegra requires functionality from the dynamic translation \ + library and is incompatible with ALLOW_RO_XLAT_TABLES.") + endif + ifeq (${RESET_TO_BL31},1) + $(error "RESET_TO_BL31 requires functionality from the dynamic \ + translation library and is incompatible with \ + ALLOW_RO_XLAT_TABLES.") + endif + ifeq (${SPD},trusty) + $(error "Trusty requires functionality from the dynamic translation \ + library and is incompatible with ALLOW_RO_XLAT_TABLES.") + endif + ifeq (${SPM_MM},1) + $(error "SPM_MM requires functionality to change memory region \ + attributes, which is not possible once the translation tables \ + have been made read-only.") + endif + ifeq (${SPMC_AT_EL3},1) + $(error "EL3 SPMC requires functionality from the dynamic translation \ + library and is incompatible with ALLOW_RO_XLAT_TABLES.") + endif +endif diff --git a/lib/xlat_tables_v2/xlat_tables.mk b/lib/xlat_tables_v2/xlat_tables.mk new file mode 100644 index 0000000..b4ad34b --- /dev/null +++ b/lib/xlat_tables_v2/xlat_tables.mk @@ -0,0 +1,19 @@ +# +# Copyright (c) 2017-2020, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +XLAT_TABLES_LIB_SRCS := $(addprefix lib/xlat_tables_v2/, \ + ${ARCH}/enable_mmu.S \ + ${ARCH}/xlat_tables_arch.c \ + xlat_tables_context.c \ + xlat_tables_core.c \ + xlat_tables_utils.c) + +XLAT_TABLES_LIB_V2 := 1 +$(eval $(call add_define,XLAT_TABLES_LIB_V2)) + +ifeq (${ALLOW_RO_XLAT_TABLES}, 1) + include lib/xlat_tables_v2/ro_xlat_tables.mk +endif diff --git a/lib/xlat_tables_v2/xlat_tables_context.c b/lib/xlat_tables_v2/xlat_tables_context.c new file mode 100644 index 0000000..ae9244a --- /dev/null +++ b/lib/xlat_tables_v2/xlat_tables_context.c @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2017-2020, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch_helpers.h> +#include <assert.h> + +#include <platform_def.h> + +#include <common/debug.h> +#include <lib/xlat_tables/xlat_tables_defs.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +#include "xlat_tables_private.h" + +/* + * MMU configuration register values for the active translation context. Used + * from the MMU assembly helpers. + */ +uint64_t mmu_cfg_params[MMU_CFG_PARAM_MAX]; + +/* + * Allocate and initialise the default translation context for the BL image + * currently executing. + */ +REGISTER_XLAT_CONTEXT(tf, MAX_MMAP_REGIONS, MAX_XLAT_TABLES, + PLAT_VIRT_ADDR_SPACE_SIZE, PLAT_PHY_ADDR_SPACE_SIZE); + +void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, size_t size, + unsigned int attr) +{ + mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr); + + mmap_add_region_ctx(&tf_xlat_ctx, &mm); +} + +void mmap_add(const mmap_region_t *mm) +{ + mmap_add_ctx(&tf_xlat_ctx, mm); +} + +void mmap_add_region_alloc_va(unsigned long long base_pa, uintptr_t *base_va, + size_t size, unsigned int attr) +{ + mmap_region_t mm = MAP_REGION_ALLOC_VA(base_pa, size, attr); + + mmap_add_region_alloc_va_ctx(&tf_xlat_ctx, &mm); + + *base_va = mm.base_va; +} + +void mmap_add_alloc_va(mmap_region_t *mm) +{ + while (mm->granularity != 0U) { + assert(mm->base_va == 0U); + mmap_add_region_alloc_va_ctx(&tf_xlat_ctx, mm); + mm++; + } +} + +#if PLAT_XLAT_TABLES_DYNAMIC + +int mmap_add_dynamic_region(unsigned long long base_pa, uintptr_t base_va, + size_t size, unsigned int attr) +{ + mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr); + + return mmap_add_dynamic_region_ctx(&tf_xlat_ctx, &mm); +} + +int mmap_add_dynamic_region_alloc_va(unsigned long long base_pa, + uintptr_t *base_va, size_t size, + unsigned int attr) +{ + mmap_region_t mm = MAP_REGION_ALLOC_VA(base_pa, size, attr); + + int rc = mmap_add_dynamic_region_alloc_va_ctx(&tf_xlat_ctx, &mm); + + *base_va = mm.base_va; + + return rc; +} + + +int mmap_remove_dynamic_region(uintptr_t base_va, size_t size) +{ + return mmap_remove_dynamic_region_ctx(&tf_xlat_ctx, + base_va, size); +} + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +void __init init_xlat_tables(void) +{ + assert(tf_xlat_ctx.xlat_regime == EL_REGIME_INVALID); + + unsigned int current_el = xlat_arch_current_el(); + + if (current_el == 1U) { + tf_xlat_ctx.xlat_regime = EL1_EL0_REGIME; + } else if (current_el == 2U) { + tf_xlat_ctx.xlat_regime = EL2_REGIME; + } else { + assert(current_el == 3U); + tf_xlat_ctx.xlat_regime = EL3_REGIME; + } + + init_xlat_tables_ctx(&tf_xlat_ctx); +} + +int xlat_get_mem_attributes(uintptr_t base_va, uint32_t *attr) +{ + return xlat_get_mem_attributes_ctx(&tf_xlat_ctx, base_va, attr); +} + +int xlat_change_mem_attributes(uintptr_t base_va, size_t size, uint32_t attr) +{ + return xlat_change_mem_attributes_ctx(&tf_xlat_ctx, base_va, size, attr); +} + +#if PLAT_RO_XLAT_TABLES +/* Change the memory attributes of the descriptors which resolve the address + * range that belongs to the translation tables themselves, which are by default + * mapped as part of read-write data in the BL image's memory. + * + * Since the translation tables map themselves via these level 3 (page) + * descriptors, any change applied to them with the MMU on would introduce a + * chicken and egg problem because of the break-before-make sequence. + * Eventually, it would reach the descriptor that resolves the very table it + * belongs to and the invalidation (break step) would cause the subsequent write + * (make step) to it to generate an MMU fault. Therefore, the MMU is disabled + * before making the change. + * + * No assumption is made about what data this function needs, therefore all the + * caches are flushed in order to ensure coherency. A future optimization would + * be to only flush the required data to main memory. + */ +int xlat_make_tables_readonly(void) +{ + assert(tf_xlat_ctx.initialized == true); +#ifdef __aarch64__ + if (tf_xlat_ctx.xlat_regime == EL1_EL0_REGIME) { + disable_mmu_el1(); + } else if (tf_xlat_ctx.xlat_regime == EL3_REGIME) { + disable_mmu_el3(); + } else { + assert(tf_xlat_ctx.xlat_regime == EL2_REGIME); + return -1; + } + + /* Flush all caches. */ + dcsw_op_all(DCCISW); +#else /* !__aarch64__ */ + assert(tf_xlat_ctx.xlat_regime == EL1_EL0_REGIME); + /* On AArch32, we flush the caches before disabling the MMU. The reason + * for this is that the dcsw_op_all AArch32 function pushes some + * registers onto the stack under the assumption that it is writing to + * cache, which is not true with the MMU off. This would result in the + * stack becoming corrupted and a wrong/junk value for the LR being + * restored at the end of the routine. + */ + dcsw_op_all(DC_OP_CISW); + disable_mmu_secure(); +#endif + + int rc = xlat_change_mem_attributes_ctx(&tf_xlat_ctx, + (uintptr_t)tf_xlat_ctx.tables, + tf_xlat_ctx.tables_num * XLAT_TABLE_SIZE, + MT_RO_DATA | MT_SECURE); + +#ifdef __aarch64__ + if (tf_xlat_ctx.xlat_regime == EL1_EL0_REGIME) { + enable_mmu_el1(0U); + } else { + assert(tf_xlat_ctx.xlat_regime == EL3_REGIME); + enable_mmu_el3(0U); + } +#else /* !__aarch64__ */ + enable_mmu_svc_mon(0U); +#endif + + if (rc == 0) { + tf_xlat_ctx.readonly_tables = true; + } + + return rc; +} +#endif /* PLAT_RO_XLAT_TABLES */ + +/* + * If dynamic allocation of new regions is disabled then by the time we call the + * function enabling the MMU, we'll have registered all the memory regions to + * map for the system's lifetime. Therefore, at this point we know the maximum + * physical address that will ever be mapped. + * + * If dynamic allocation is enabled then we can't make any such assumption + * because the maximum physical address could get pushed while adding a new + * region. Therefore, in this case we have to assume that the whole address + * space size might be mapped. + */ +#if PLAT_XLAT_TABLES_DYNAMIC +#define MAX_PHYS_ADDR tf_xlat_ctx.pa_max_address +#else +#define MAX_PHYS_ADDR tf_xlat_ctx.max_pa +#endif + +#ifdef __aarch64__ + +void enable_mmu_el1(unsigned int flags) +{ + setup_mmu_cfg((uint64_t *)&mmu_cfg_params, flags, + tf_xlat_ctx.base_table, MAX_PHYS_ADDR, + tf_xlat_ctx.va_max_address, EL1_EL0_REGIME); + enable_mmu_direct_el1(flags); +} + +void enable_mmu_el2(unsigned int flags) +{ + setup_mmu_cfg((uint64_t *)&mmu_cfg_params, flags, + tf_xlat_ctx.base_table, MAX_PHYS_ADDR, + tf_xlat_ctx.va_max_address, EL2_REGIME); + enable_mmu_direct_el2(flags); +} + +void enable_mmu_el3(unsigned int flags) +{ + setup_mmu_cfg((uint64_t *)&mmu_cfg_params, flags, + tf_xlat_ctx.base_table, MAX_PHYS_ADDR, + tf_xlat_ctx.va_max_address, EL3_REGIME); + enable_mmu_direct_el3(flags); +} + +void enable_mmu(unsigned int flags) +{ + switch (get_current_el_maybe_constant()) { + case 1: + enable_mmu_el1(flags); + break; + case 2: + enable_mmu_el2(flags); + break; + case 3: + enable_mmu_el3(flags); + break; + default: + panic(); + } +} + +#else /* !__aarch64__ */ + +void enable_mmu_svc_mon(unsigned int flags) +{ + setup_mmu_cfg((uint64_t *)&mmu_cfg_params, flags, + tf_xlat_ctx.base_table, MAX_PHYS_ADDR, + tf_xlat_ctx.va_max_address, EL1_EL0_REGIME); + enable_mmu_direct_svc_mon(flags); +} + +void enable_mmu_hyp(unsigned int flags) +{ + setup_mmu_cfg((uint64_t *)&mmu_cfg_params, flags, + tf_xlat_ctx.base_table, MAX_PHYS_ADDR, + tf_xlat_ctx.va_max_address, EL2_REGIME); + enable_mmu_direct_hyp(flags); +} + +#endif /* __aarch64__ */ diff --git a/lib/xlat_tables_v2/xlat_tables_core.c b/lib/xlat_tables_v2/xlat_tables_core.c new file mode 100644 index 0000000..3a9c058 --- /dev/null +++ b/lib/xlat_tables_v2/xlat_tables_core.c @@ -0,0 +1,1244 @@ +/* + * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <errno.h> +#include <stdbool.h> +#include <stdint.h> +#include <string.h> + +#include <platform_def.h> + +#include <arch_features.h> +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/utils_def.h> +#include <lib/xlat_tables/xlat_tables_defs.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +#include "xlat_tables_private.h" + +/* Helper function that cleans the data cache only if it is enabled. */ +static inline __attribute__((unused)) void xlat_clean_dcache_range(uintptr_t addr, size_t size) +{ + if (is_dcache_enabled()) + clean_dcache_range(addr, size); +} + +#if PLAT_XLAT_TABLES_DYNAMIC + +/* + * The following functions assume that they will be called using subtables only. + * The base table can't be unmapped, so it is not needed to do any special + * handling for it. + */ + +/* + * Returns the index of the array corresponding to the specified translation + * table. + */ +static int xlat_table_get_index(const xlat_ctx_t *ctx, const uint64_t *table) +{ + for (int i = 0; i < ctx->tables_num; i++) + if (ctx->tables[i] == table) + return i; + + /* + * Maybe we were asked to get the index of the base level table, which + * should never happen. + */ + assert(false); + + return -1; +} + +/* Returns a pointer to an empty translation table. */ +static uint64_t *xlat_table_get_empty(const xlat_ctx_t *ctx) +{ + for (int i = 0; i < ctx->tables_num; i++) + if (ctx->tables_mapped_regions[i] == 0) + return ctx->tables[i]; + + return NULL; +} + +/* Increments region count for a given table. */ +static void xlat_table_inc_regions_count(const xlat_ctx_t *ctx, + const uint64_t *table) +{ + int idx = xlat_table_get_index(ctx, table); + + ctx->tables_mapped_regions[idx]++; +} + +/* Decrements region count for a given table. */ +static void xlat_table_dec_regions_count(const xlat_ctx_t *ctx, + const uint64_t *table) +{ + int idx = xlat_table_get_index(ctx, table); + + ctx->tables_mapped_regions[idx]--; +} + +/* Returns 0 if the specified table isn't empty, otherwise 1. */ +static bool xlat_table_is_empty(const xlat_ctx_t *ctx, const uint64_t *table) +{ + return ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)] == 0; +} + +#else /* PLAT_XLAT_TABLES_DYNAMIC */ + +/* Returns a pointer to the first empty translation table. */ +static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx) +{ + assert(ctx->next_table < ctx->tables_num); + + return ctx->tables[ctx->next_table++]; +} + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +/* + * Returns a block/page table descriptor for the given level and attributes. + */ +uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr, + unsigned long long addr_pa, unsigned int level) +{ + uint64_t desc; + uint32_t mem_type; + uint32_t shareability_type; + + /* Make sure that the granularity is fine enough to map this address. */ + assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0U); + + desc = addr_pa; + /* + * There are different translation table descriptors for level 3 and the + * rest. + */ + desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; + /* + * Always set the access flag, as this library assumes access flag + * faults aren't managed. + */ + desc |= LOWER_ATTRS(ACCESS_FLAG); + + /* Determine the physical address space this region belongs to. */ + desc |= xlat_arch_get_pas(attr); + + /* + * Deduce other fields of the descriptor based on the MT_RW memory + * region attributes. + */ + desc |= ((attr & MT_RW) != 0U) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); + + /* + * Do not allow unprivileged access when the mapping is for a privileged + * EL. For translation regimes that do not have mappings for access for + * lower exception levels, set AP[2] to AP_NO_ACCESS_UNPRIVILEGED. + */ + if (ctx->xlat_regime == EL1_EL0_REGIME) { + if ((attr & MT_USER) != 0U) { + /* EL0 mapping requested, so we give User access */ + desc |= LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED); + } else { + /* EL1 mapping requested, no User access granted */ + desc |= LOWER_ATTRS(AP_NO_ACCESS_UNPRIVILEGED); + } + } else { + assert((ctx->xlat_regime == EL2_REGIME) || + (ctx->xlat_regime == EL3_REGIME)); + desc |= LOWER_ATTRS(AP_ONE_VA_RANGE_RES1); + } + + /* + * Deduce shareability domain and executability of the memory region + * from the memory type of the attributes (MT_TYPE). + * + * Data accesses to device memory and non-cacheable normal memory are + * coherent for all observers in the system, and correspondingly are + * always treated as being Outer Shareable. Therefore, for these 2 types + * of memory, it is not strictly needed to set the shareability field + * in the translation tables. + */ + mem_type = MT_TYPE(attr); + if (mem_type == MT_DEVICE) { + desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH); + /* + * Always map device memory as execute-never. + * This is to avoid the possibility of a speculative instruction + * fetch, which could be an issue if this memory region + * corresponds to a read-sensitive peripheral. + */ + desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime); + + } else { /* Normal memory */ + /* + * Always map read-write normal memory as execute-never. + * This library assumes that it is used by software that does + * not self-modify its code, therefore R/W memory is reserved + * for data storage, which must not be executable. + * + * Note that setting the XN bit here is for consistency only. + * The function that enables the MMU sets the SCTLR_ELx.WXN bit, + * which makes any writable memory region to be treated as + * execute-never, regardless of the value of the XN bit in the + * translation table. + * + * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER + * attribute to figure out the value of the XN bit. The actual + * XN bit(s) to set in the descriptor depends on the context's + * translation regime and the policy applied in + * xlat_arch_regime_get_xn_desc(). + */ + if (((attr & MT_RW) != 0U) || ((attr & MT_EXECUTE_NEVER) != 0U)) { + desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime); + } + + shareability_type = MT_SHAREABILITY(attr); + if (mem_type == MT_MEMORY) { + desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX); + if (shareability_type == MT_SHAREABILITY_NSH) { + desc |= LOWER_ATTRS(NSH); + } else if (shareability_type == MT_SHAREABILITY_OSH) { + desc |= LOWER_ATTRS(OSH); + } else { + desc |= LOWER_ATTRS(ISH); + } + + /* Check if Branch Target Identification is enabled */ +#if ENABLE_BTI + /* Set GP bit for block and page code entries + * if BTI mechanism is implemented. + */ + if (is_armv8_5_bti_present() && + ((attr & (MT_TYPE_MASK | MT_RW | + MT_EXECUTE_NEVER)) == MT_CODE)) { + desc |= GP; + } +#endif + } else { + assert(mem_type == MT_NON_CACHEABLE); + desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH); + } + } + + return desc; +} + +/* + * Enumeration of actions that can be made when mapping table entries depending + * on the previous value in that entry and information about the region being + * mapped. + */ +typedef enum { + + /* Do nothing */ + ACTION_NONE, + + /* Write a block (or page, if in level 3) entry. */ + ACTION_WRITE_BLOCK_ENTRY, + + /* + * Create a new table and write a table entry pointing to it. Recurse + * into it for further processing. + */ + ACTION_CREATE_NEW_TABLE, + + /* + * There is a table descriptor in this entry, read it and recurse into + * that table for further processing. + */ + ACTION_RECURSE_INTO_TABLE, + +} action_t; + +/* + * Function that returns the first VA of the table affected by the specified + * mmap region. + */ +static uintptr_t xlat_tables_find_start_va(mmap_region_t *mm, + const uintptr_t table_base_va, + const unsigned int level) +{ + uintptr_t table_idx_va; + + if (mm->base_va > table_base_va) { + /* Find the first index of the table affected by the region. */ + table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level); + } else { + /* Start from the beginning of the table. */ + table_idx_va = table_base_va; + } + + return table_idx_va; +} + +/* + * Function that returns table index for the given VA and level arguments. + */ +static inline unsigned int xlat_tables_va_to_index(const uintptr_t table_base_va, + const uintptr_t va, + const unsigned int level) +{ + return (unsigned int)((va - table_base_va) >> XLAT_ADDR_SHIFT(level)); +} + +#if PLAT_XLAT_TABLES_DYNAMIC + +/* + * From the given arguments, it decides which action to take when unmapping the + * specified region. + */ +static action_t xlat_tables_unmap_region_action(const mmap_region_t *mm, + const uintptr_t table_idx_va, const uintptr_t table_idx_end_va, + const unsigned int level, const uint64_t desc_type) +{ + action_t action; + uintptr_t region_end_va = mm->base_va + mm->size - 1U; + + if ((mm->base_va <= table_idx_va) && + (region_end_va >= table_idx_end_va)) { + /* Region covers all block */ + + if (level == 3U) { + /* + * Last level, only page descriptors allowed, + * erase it. + */ + assert(desc_type == PAGE_DESC); + + action = ACTION_WRITE_BLOCK_ENTRY; + } else { + /* + * Other levels can have table descriptors. If + * so, recurse into it and erase descriptors + * inside it as needed. If there is a block + * descriptor, just erase it. If an invalid + * descriptor is found, this table isn't + * actually mapped, which shouldn't happen. + */ + if (desc_type == TABLE_DESC) { + action = ACTION_RECURSE_INTO_TABLE; + } else { + assert(desc_type == BLOCK_DESC); + action = ACTION_WRITE_BLOCK_ENTRY; + } + } + + } else if ((mm->base_va <= table_idx_end_va) || + (region_end_va >= table_idx_va)) { + /* + * Region partially covers block. + * + * It can't happen in level 3. + * + * There must be a table descriptor here, if not there + * was a problem when mapping the region. + */ + assert(level < 3U); + assert(desc_type == TABLE_DESC); + + action = ACTION_RECURSE_INTO_TABLE; + } else { + /* The region doesn't cover the block at all */ + action = ACTION_NONE; + } + + return action; +} +/* + * Recursive function that writes to the translation tables and unmaps the + * specified region. + */ +static void xlat_tables_unmap_region(xlat_ctx_t *ctx, mmap_region_t *mm, + const uintptr_t table_base_va, + uint64_t *const table_base, + const unsigned int table_entries, + const unsigned int level) +{ + assert((level >= ctx->base_level) && (level <= XLAT_TABLE_LEVEL_MAX)); + + uint64_t *subtable; + uint64_t desc; + + uintptr_t table_idx_va; + uintptr_t table_idx_end_va; /* End VA of this entry */ + + uintptr_t region_end_va = mm->base_va + mm->size - 1U; + + unsigned int table_idx; + + table_idx_va = xlat_tables_find_start_va(mm, table_base_va, level); + table_idx = xlat_tables_va_to_index(table_base_va, table_idx_va, level); + + while (table_idx < table_entries) { + + table_idx_end_va = table_idx_va + XLAT_BLOCK_SIZE(level) - 1U; + + desc = table_base[table_idx]; + uint64_t desc_type = desc & DESC_MASK; + + action_t action = xlat_tables_unmap_region_action(mm, + table_idx_va, table_idx_end_va, level, + desc_type); + + if (action == ACTION_WRITE_BLOCK_ENTRY) { + + table_base[table_idx] = INVALID_DESC; + xlat_arch_tlbi_va(table_idx_va, ctx->xlat_regime); + + } else if (action == ACTION_RECURSE_INTO_TABLE) { + + subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); + + /* Recurse to write into subtable */ + xlat_tables_unmap_region(ctx, mm, table_idx_va, + subtable, XLAT_TABLE_ENTRIES, + level + 1U); +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + xlat_clean_dcache_range((uintptr_t)subtable, + XLAT_TABLE_ENTRIES * sizeof(uint64_t)); +#endif + /* + * If the subtable is now empty, remove its reference. + */ + if (xlat_table_is_empty(ctx, subtable)) { + table_base[table_idx] = INVALID_DESC; + xlat_arch_tlbi_va(table_idx_va, + ctx->xlat_regime); + } + + } else { + assert(action == ACTION_NONE); + } + + table_idx++; + table_idx_va += XLAT_BLOCK_SIZE(level); + + /* If reached the end of the region, exit */ + if (region_end_va <= table_idx_va) + break; + } + + if (level > ctx->base_level) + xlat_table_dec_regions_count(ctx, table_base); +} + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +/* + * From the given arguments, it decides which action to take when mapping the + * specified region. + */ +static action_t xlat_tables_map_region_action(const mmap_region_t *mm, + unsigned int desc_type, unsigned long long dest_pa, + uintptr_t table_entry_base_va, unsigned int level) +{ + uintptr_t mm_end_va = mm->base_va + mm->size - 1U; + uintptr_t table_entry_end_va = + table_entry_base_va + XLAT_BLOCK_SIZE(level) - 1U; + + /* + * The descriptor types allowed depend on the current table level. + */ + + if ((mm->base_va <= table_entry_base_va) && + (mm_end_va >= table_entry_end_va)) { + + /* + * Table entry is covered by region + * -------------------------------- + * + * This means that this table entry can describe the whole + * translation with this granularity in principle. + */ + + if (level == 3U) { + /* + * Last level, only page descriptors are allowed. + */ + if (desc_type == PAGE_DESC) { + /* + * There's another region mapped here, don't + * overwrite. + */ + return ACTION_NONE; + } else { + assert(desc_type == INVALID_DESC); + return ACTION_WRITE_BLOCK_ENTRY; + } + + } else { + + /* + * Other levels. Table descriptors are allowed. Block + * descriptors too, but they have some limitations. + */ + + if (desc_type == TABLE_DESC) { + /* There's already a table, recurse into it. */ + return ACTION_RECURSE_INTO_TABLE; + + } else if (desc_type == INVALID_DESC) { + /* + * There's nothing mapped here, create a new + * entry. + * + * Check if the destination granularity allows + * us to use a block descriptor or we need a + * finer table for it. + * + * Also, check if the current level allows block + * descriptors. If not, create a table instead. + */ + if (((dest_pa & XLAT_BLOCK_MASK(level)) != 0U) + || (level < MIN_LVL_BLOCK_DESC) || + (mm->granularity < XLAT_BLOCK_SIZE(level))) + return ACTION_CREATE_NEW_TABLE; + else + return ACTION_WRITE_BLOCK_ENTRY; + + } else { + /* + * There's another region mapped here, don't + * overwrite. + */ + assert(desc_type == BLOCK_DESC); + + return ACTION_NONE; + } + } + + } else if ((mm->base_va <= table_entry_end_va) || + (mm_end_va >= table_entry_base_va)) { + + /* + * Region partially covers table entry + * ----------------------------------- + * + * This means that this table entry can't describe the whole + * translation, a finer table is needed. + + * There cannot be partial block overlaps in level 3. If that + * happens, some of the preliminary checks when adding the + * mmap region failed to detect that PA and VA must at least be + * aligned to PAGE_SIZE. + */ + assert(level < 3U); + + if (desc_type == INVALID_DESC) { + /* + * The block is not fully covered by the region. Create + * a new table, recurse into it and try to map the + * region with finer granularity. + */ + return ACTION_CREATE_NEW_TABLE; + + } else { + assert(desc_type == TABLE_DESC); + /* + * The block is not fully covered by the region, but + * there is already a table here. Recurse into it and + * try to map with finer granularity. + * + * PAGE_DESC for level 3 has the same value as + * TABLE_DESC, but this code can't run on a level 3 + * table because there can't be overlaps in level 3. + */ + return ACTION_RECURSE_INTO_TABLE; + } + } else { + + /* + * This table entry is outside of the region specified in the + * arguments, don't write anything to it. + */ + return ACTION_NONE; + } +} + +/* + * Recursive function that writes to the translation tables and maps the + * specified region. On success, it returns the VA of the last byte that was + * successfully mapped. On error, it returns the VA of the next entry that + * should have been mapped. + */ +static uintptr_t xlat_tables_map_region(xlat_ctx_t *ctx, mmap_region_t *mm, + uintptr_t table_base_va, + uint64_t *const table_base, + unsigned int table_entries, + unsigned int level) +{ + assert((level >= ctx->base_level) && (level <= XLAT_TABLE_LEVEL_MAX)); + + uintptr_t mm_end_va = mm->base_va + mm->size - 1U; + + uintptr_t table_idx_va; + unsigned long long table_idx_pa; + + uint64_t *subtable; + uint64_t desc; + + unsigned int table_idx; + + table_idx_va = xlat_tables_find_start_va(mm, table_base_va, level); + table_idx = xlat_tables_va_to_index(table_base_va, table_idx_va, level); + +#if PLAT_XLAT_TABLES_DYNAMIC + if (level > ctx->base_level) + xlat_table_inc_regions_count(ctx, table_base); +#endif + + while (table_idx < table_entries) { + + desc = table_base[table_idx]; + + table_idx_pa = mm->base_pa + table_idx_va - mm->base_va; + + action_t action = xlat_tables_map_region_action(mm, + (uint32_t)(desc & DESC_MASK), table_idx_pa, + table_idx_va, level); + + if (action == ACTION_WRITE_BLOCK_ENTRY) { + + table_base[table_idx] = + xlat_desc(ctx, (uint32_t)mm->attr, table_idx_pa, + level); + + } else if (action == ACTION_CREATE_NEW_TABLE) { + uintptr_t end_va; + + subtable = xlat_table_get_empty(ctx); + if (subtable == NULL) { + /* Not enough free tables to map this region */ + return table_idx_va; + } + + /* Point to new subtable from this one. */ + table_base[table_idx] = + TABLE_DESC | (uintptr_t)subtable; + + /* Recurse to write into subtable */ + end_va = xlat_tables_map_region(ctx, mm, table_idx_va, + subtable, XLAT_TABLE_ENTRIES, + level + 1U); +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + xlat_clean_dcache_range((uintptr_t)subtable, + XLAT_TABLE_ENTRIES * sizeof(uint64_t)); +#endif + if (end_va != + (table_idx_va + XLAT_BLOCK_SIZE(level) - 1U)) + return end_va; + + } else if (action == ACTION_RECURSE_INTO_TABLE) { + uintptr_t end_va; + + subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); + /* Recurse to write into subtable */ + end_va = xlat_tables_map_region(ctx, mm, table_idx_va, + subtable, XLAT_TABLE_ENTRIES, + level + 1U); +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + xlat_clean_dcache_range((uintptr_t)subtable, + XLAT_TABLE_ENTRIES * sizeof(uint64_t)); +#endif + if (end_va != + (table_idx_va + XLAT_BLOCK_SIZE(level) - 1U)) + return end_va; + + } else { + + assert(action == ACTION_NONE); + + } + + table_idx++; + table_idx_va += XLAT_BLOCK_SIZE(level); + + /* If reached the end of the region, exit */ + if (mm_end_va <= table_idx_va) + break; + } + + return table_idx_va - 1U; +} + +/* + * Function that verifies that a region can be mapped. + * Returns: + * 0: Success, the mapping is allowed. + * EINVAL: Invalid values were used as arguments. + * ERANGE: The memory limits were surpassed. + * ENOMEM: There is not enough memory in the mmap array. + * EPERM: Region overlaps another one in an invalid way. + */ +static int mmap_add_region_check(const xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + unsigned long long base_pa = mm->base_pa; + uintptr_t base_va = mm->base_va; + size_t size = mm->size; + size_t granularity = mm->granularity; + + unsigned long long end_pa = base_pa + size - 1U; + uintptr_t end_va = base_va + size - 1U; + + if (!IS_PAGE_ALIGNED(base_pa) || !IS_PAGE_ALIGNED(base_va) || + !IS_PAGE_ALIGNED(size)) + return -EINVAL; + + if ((granularity != XLAT_BLOCK_SIZE(1U)) && + (granularity != XLAT_BLOCK_SIZE(2U)) && + (granularity != XLAT_BLOCK_SIZE(3U))) { + return -EINVAL; + } + + /* Check for overflows */ + if ((base_pa > end_pa) || (base_va > end_va)) + return -ERANGE; + + if (end_va > ctx->va_max_address) + return -ERANGE; + + if (end_pa > ctx->pa_max_address) + return -ERANGE; + + /* Check that there is space in the ctx->mmap array */ + if (ctx->mmap[ctx->mmap_num - 1].size != 0U) + return -ENOMEM; + + /* Check for PAs and VAs overlaps with all other regions */ + for (const mmap_region_t *mm_cursor = ctx->mmap; + mm_cursor->size != 0U; ++mm_cursor) { + + uintptr_t mm_cursor_end_va = mm_cursor->base_va + + mm_cursor->size - 1U; + + /* + * Check if one of the regions is completely inside the other + * one. + */ + bool fully_overlapped_va = + ((base_va >= mm_cursor->base_va) && + (end_va <= mm_cursor_end_va)) || + ((mm_cursor->base_va >= base_va) && + (mm_cursor_end_va <= end_va)); + + /* + * Full VA overlaps are only allowed if both regions are + * identity mapped (zero offset) or have the same VA to PA + * offset. Also, make sure that it's not the exact same area. + * This can only be done with static regions. + */ + if (fully_overlapped_va) { + +#if PLAT_XLAT_TABLES_DYNAMIC + if (((mm->attr & MT_DYNAMIC) != 0U) || + ((mm_cursor->attr & MT_DYNAMIC) != 0U)) + return -EPERM; +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + if ((mm_cursor->base_va - mm_cursor->base_pa) != + (base_va - base_pa)) + return -EPERM; + + if ((base_va == mm_cursor->base_va) && + (size == mm_cursor->size)) + return -EPERM; + + } else { + /* + * If the regions do not have fully overlapping VAs, + * then they must have fully separated VAs and PAs. + * Partial overlaps are not allowed + */ + + unsigned long long mm_cursor_end_pa = + mm_cursor->base_pa + mm_cursor->size - 1U; + + bool separated_pa = (end_pa < mm_cursor->base_pa) || + (base_pa > mm_cursor_end_pa); + bool separated_va = (end_va < mm_cursor->base_va) || + (base_va > mm_cursor_end_va); + + if (!separated_va || !separated_pa) + return -EPERM; + } + } + + return 0; +} + +void mmap_add_region_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + mmap_region_t *mm_cursor = ctx->mmap, *mm_destination; + const mmap_region_t *mm_end = ctx->mmap + ctx->mmap_num; + const mmap_region_t *mm_last; + unsigned long long end_pa = mm->base_pa + mm->size - 1U; + uintptr_t end_va = mm->base_va + mm->size - 1U; + int ret; + + /* Ignore empty regions */ + if (mm->size == 0U) + return; + + /* Static regions must be added before initializing the xlat tables. */ + assert(!ctx->initialized); + + ret = mmap_add_region_check(ctx, mm); + if (ret != 0) { + ERROR("mmap_add_region_check() failed. error %d\n", ret); + assert(false); + return; + } + + /* + * Find correct place in mmap to insert new region. + * + * 1 - Lower region VA end first. + * 2 - Smaller region size first. + * + * VA 0 0xFF + * + * 1st |------| + * 2nd |------------| + * 3rd |------| + * 4th |---| + * 5th |---| + * 6th |----------| + * 7th |-------------------------------------| + * + * This is required for overlapping regions only. It simplifies adding + * regions with the loop in xlat_tables_init_internal because the outer + * ones won't overwrite block or page descriptors of regions added + * previously. + * + * Overlapping is only allowed for static regions. + */ + + while (((mm_cursor->base_va + mm_cursor->size - 1U) < end_va) + && (mm_cursor->size != 0U)) { + ++mm_cursor; + } + + while (((mm_cursor->base_va + mm_cursor->size - 1U) == end_va) && + (mm_cursor->size != 0U) && (mm_cursor->size < mm->size)) { + ++mm_cursor; + } + + /* + * Find the last entry marker in the mmap + */ + mm_last = ctx->mmap; + while ((mm_last->size != 0U) && (mm_last < mm_end)) { + ++mm_last; + } + + /* + * Check if we have enough space in the memory mapping table. + * This shouldn't happen as we have checked in mmap_add_region_check + * that there is free space. + */ + assert(mm_last->size == 0U); + + /* Make room for new region by moving other regions up by one place */ + mm_destination = mm_cursor + 1; + (void)memmove(mm_destination, mm_cursor, + (uintptr_t)mm_last - (uintptr_t)mm_cursor); + + /* + * Check we haven't lost the empty sentinel from the end of the array. + * This shouldn't happen as we have checked in mmap_add_region_check + * that there is free space. + */ + assert(mm_end->size == 0U); + + *mm_cursor = *mm; + + if (end_pa > ctx->max_pa) + ctx->max_pa = end_pa; + if (end_va > ctx->max_va) + ctx->max_va = end_va; +} + +/* + * Determine the table level closest to the initial lookup level that + * can describe this translation. Then, align base VA to the next block + * at the determined level. + */ +static void mmap_alloc_va_align_ctx(xlat_ctx_t *ctx, mmap_region_t *mm) +{ + /* + * By or'ing the size and base PA the alignment will be the one + * corresponding to the smallest boundary of the two of them. + * + * There are three different cases. For example (for 4 KiB page size): + * + * +--------------+------------------++--------------+ + * | PA alignment | Size multiple of || VA alignment | + * +--------------+------------------++--------------+ + * | 2 MiB | 2 MiB || 2 MiB | (1) + * | 2 MiB | 4 KiB || 4 KiB | (2) + * | 4 KiB | 2 MiB || 4 KiB | (3) + * +--------------+------------------++--------------+ + * + * - In (1), it is possible to take advantage of the alignment of the PA + * and the size of the region to use a level 2 translation table + * instead of a level 3 one. + * + * - In (2), the size is smaller than a block entry of level 2, so it is + * needed to use a level 3 table to describe the region or the library + * will map more memory than the desired one. + * + * - In (3), even though the region has the size of one level 2 block + * entry, it isn't possible to describe the translation with a level 2 + * block entry because of the alignment of the base PA. + * + * Only bits 47:21 of a level 2 block descriptor are used by the MMU, + * bits 20:0 of the resulting address are 0 in this case. Because of + * this, the PA generated as result of this translation is aligned to + * 2 MiB. The PA that was requested to be mapped is aligned to 4 KiB, + * though, which means that the resulting translation is incorrect. + * The only way to prevent this is by using a finer granularity. + */ + unsigned long long align_check; + + align_check = mm->base_pa | (unsigned long long)mm->size; + + /* + * Assume it is always aligned to level 3. There's no need to check that + * level because its block size is PAGE_SIZE. The checks to verify that + * the addresses and size are aligned to PAGE_SIZE are inside + * mmap_add_region. + */ + for (unsigned int level = ctx->base_level; level <= 2U; ++level) { + + if ((align_check & XLAT_BLOCK_MASK(level)) != 0U) + continue; + + mm->base_va = round_up(mm->base_va, XLAT_BLOCK_SIZE(level)); + return; + } +} + +void mmap_add_region_alloc_va_ctx(xlat_ctx_t *ctx, mmap_region_t *mm) +{ + mm->base_va = ctx->max_va + 1UL; + + assert(mm->size > 0U); + + mmap_alloc_va_align_ctx(ctx, mm); + + /* Detect overflows. More checks are done in mmap_add_region_check(). */ + assert(mm->base_va > ctx->max_va); + + mmap_add_region_ctx(ctx, mm); +} + +void mmap_add_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + const mmap_region_t *mm_cursor = mm; + + while (mm_cursor->granularity != 0U) { + mmap_add_region_ctx(ctx, mm_cursor); + mm_cursor++; + } +} + +#if PLAT_XLAT_TABLES_DYNAMIC + +int mmap_add_dynamic_region_ctx(xlat_ctx_t *ctx, mmap_region_t *mm) +{ + mmap_region_t *mm_cursor = ctx->mmap; + const mmap_region_t *mm_last = mm_cursor + ctx->mmap_num; + unsigned long long end_pa = mm->base_pa + mm->size - 1U; + uintptr_t end_va = mm->base_va + mm->size - 1U; + int ret; + + /* Nothing to do */ + if (mm->size == 0U) + return 0; + + /* Now this region is a dynamic one */ + mm->attr |= MT_DYNAMIC; + + ret = mmap_add_region_check(ctx, mm); + if (ret != 0) + return ret; + + /* + * Find the adequate entry in the mmap array in the same way done for + * static regions in mmap_add_region_ctx(). + */ + + while (((mm_cursor->base_va + mm_cursor->size - 1U) < end_va) + && (mm_cursor->size != 0U)) { + ++mm_cursor; + } + + while (((mm_cursor->base_va + mm_cursor->size - 1U) == end_va) && + (mm_cursor->size != 0U) && (mm_cursor->size < mm->size)) { + ++mm_cursor; + } + + /* Make room for new region by moving other regions up by one place */ + (void)memmove(mm_cursor + 1U, mm_cursor, + (uintptr_t)mm_last - (uintptr_t)mm_cursor); + + /* + * Check we haven't lost the empty sentinel from the end of the array. + * This shouldn't happen as we have checked in mmap_add_region_check + * that there is free space. + */ + assert(mm_last->size == 0U); + + *mm_cursor = *mm; + + /* + * Update the translation tables if the xlat tables are initialized. If + * not, this region will be mapped when they are initialized. + */ + if (ctx->initialized) { + end_va = xlat_tables_map_region(ctx, mm_cursor, + 0U, ctx->base_table, ctx->base_table_entries, + ctx->base_level); +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + xlat_clean_dcache_range((uintptr_t)ctx->base_table, + ctx->base_table_entries * sizeof(uint64_t)); +#endif + /* Failed to map, remove mmap entry, unmap and return error. */ + if (end_va != (mm_cursor->base_va + mm_cursor->size - 1U)) { + (void)memmove(mm_cursor, mm_cursor + 1U, + (uintptr_t)mm_last - (uintptr_t)mm_cursor); + + /* + * Check if the mapping function actually managed to map + * anything. If not, just return now. + */ + if (mm->base_va >= end_va) + return -ENOMEM; + + /* + * Something went wrong after mapping some table + * entries, undo every change done up to this point. + */ + mmap_region_t unmap_mm = { + .base_pa = 0U, + .base_va = mm->base_va, + .size = end_va - mm->base_va, + .attr = 0U + }; + xlat_tables_unmap_region(ctx, &unmap_mm, 0U, + ctx->base_table, ctx->base_table_entries, + ctx->base_level); +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + xlat_clean_dcache_range((uintptr_t)ctx->base_table, + ctx->base_table_entries * sizeof(uint64_t)); +#endif + return -ENOMEM; + } + + /* + * Make sure that all entries are written to the memory. There + * is no need to invalidate entries when mapping dynamic regions + * because new table/block/page descriptors only replace old + * invalid descriptors, that aren't TLB cached. + */ + dsbishst(); + } + + if (end_pa > ctx->max_pa) + ctx->max_pa = end_pa; + if (end_va > ctx->max_va) + ctx->max_va = end_va; + + return 0; +} + +int mmap_add_dynamic_region_alloc_va_ctx(xlat_ctx_t *ctx, mmap_region_t *mm) +{ + mm->base_va = ctx->max_va + 1UL; + + if (mm->size == 0U) + return 0; + + mmap_alloc_va_align_ctx(ctx, mm); + + /* Detect overflows. More checks are done in mmap_add_region_check(). */ + if (mm->base_va < ctx->max_va) { + return -ENOMEM; + } + + return mmap_add_dynamic_region_ctx(ctx, mm); +} + +/* + * Removes the region with given base Virtual Address and size from the given + * context. + * + * Returns: + * 0: Success. + * EINVAL: Invalid values were used as arguments (region not found). + * EPERM: Tried to remove a static region. + */ +int mmap_remove_dynamic_region_ctx(xlat_ctx_t *ctx, uintptr_t base_va, + size_t size) +{ + mmap_region_t *mm = ctx->mmap; + const mmap_region_t *mm_last = mm + ctx->mmap_num; + int update_max_va_needed = 0; + int update_max_pa_needed = 0; + + /* Check sanity of mmap array. */ + assert(mm[ctx->mmap_num].size == 0U); + + while (mm->size != 0U) { + if ((mm->base_va == base_va) && (mm->size == size)) + break; + ++mm; + } + + /* Check that the region was found */ + if (mm->size == 0U) + return -EINVAL; + + /* If the region is static it can't be removed */ + if ((mm->attr & MT_DYNAMIC) == 0U) + return -EPERM; + + /* Check if this region is using the top VAs or PAs. */ + if ((mm->base_va + mm->size - 1U) == ctx->max_va) + update_max_va_needed = 1; + if ((mm->base_pa + mm->size - 1U) == ctx->max_pa) + update_max_pa_needed = 1; + + /* Update the translation tables if needed */ + if (ctx->initialized) { + xlat_tables_unmap_region(ctx, mm, 0U, ctx->base_table, + ctx->base_table_entries, + ctx->base_level); +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + xlat_clean_dcache_range((uintptr_t)ctx->base_table, + ctx->base_table_entries * sizeof(uint64_t)); +#endif + xlat_arch_tlbi_va_sync(); + } + + /* Remove this region by moving the rest down by one place. */ + (void)memmove(mm, mm + 1U, (uintptr_t)mm_last - (uintptr_t)mm); + + /* Check if we need to update the max VAs and PAs */ + if (update_max_va_needed == 1) { + ctx->max_va = 0U; + mm = ctx->mmap; + while (mm->size != 0U) { + if ((mm->base_va + mm->size - 1U) > ctx->max_va) + ctx->max_va = mm->base_va + mm->size - 1U; + ++mm; + } + } + + if (update_max_pa_needed == 1) { + ctx->max_pa = 0U; + mm = ctx->mmap; + while (mm->size != 0U) { + if ((mm->base_pa + mm->size - 1U) > ctx->max_pa) + ctx->max_pa = mm->base_pa + mm->size - 1U; + ++mm; + } + } + + return 0; +} + +void xlat_setup_dynamic_ctx(xlat_ctx_t *ctx, unsigned long long pa_max, + uintptr_t va_max, struct mmap_region *mmap, + unsigned int mmap_num, uint64_t **tables, + unsigned int tables_num, uint64_t *base_table, + int xlat_regime, int *mapped_regions) +{ + ctx->xlat_regime = xlat_regime; + + ctx->pa_max_address = pa_max; + ctx->va_max_address = va_max; + + ctx->mmap = mmap; + ctx->mmap_num = mmap_num; + memset(ctx->mmap, 0, sizeof(struct mmap_region) * mmap_num); + + ctx->tables = (void *) tables; + ctx->tables_num = tables_num; + + uintptr_t va_space_size = va_max + 1; + ctx->base_level = GET_XLAT_TABLE_LEVEL_BASE(va_space_size); + ctx->base_table = base_table; + ctx->base_table_entries = GET_NUM_BASE_LEVEL_ENTRIES(va_space_size); + + ctx->tables_mapped_regions = mapped_regions; + + ctx->max_pa = 0; + ctx->max_va = 0; + ctx->initialized = 0; +} + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +void __init init_xlat_tables_ctx(xlat_ctx_t *ctx) +{ + assert(ctx != NULL); + assert(!ctx->initialized); + assert((ctx->xlat_regime == EL3_REGIME) || + (ctx->xlat_regime == EL2_REGIME) || + (ctx->xlat_regime == EL1_EL0_REGIME)); + assert(!is_mmu_enabled_ctx(ctx)); + + mmap_region_t *mm = ctx->mmap; + + assert(ctx->va_max_address >= + (xlat_get_min_virt_addr_space_size() - 1U)); + assert(ctx->va_max_address <= (MAX_VIRT_ADDR_SPACE_SIZE - 1U)); + assert(IS_POWER_OF_TWO(ctx->va_max_address + 1U)); + + xlat_mmap_print(mm); + + /* All tables must be zeroed before mapping any region. */ + + for (unsigned int i = 0U; i < ctx->base_table_entries; i++) + ctx->base_table[i] = INVALID_DESC; + + for (int j = 0; j < ctx->tables_num; j++) { +#if PLAT_XLAT_TABLES_DYNAMIC + ctx->tables_mapped_regions[j] = 0; +#endif + for (unsigned int i = 0U; i < XLAT_TABLE_ENTRIES; i++) + ctx->tables[j][i] = INVALID_DESC; + } + + while (mm->size != 0U) { + uintptr_t end_va = xlat_tables_map_region(ctx, mm, 0U, + ctx->base_table, ctx->base_table_entries, + ctx->base_level); +#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) + xlat_clean_dcache_range((uintptr_t)ctx->base_table, + ctx->base_table_entries * sizeof(uint64_t)); +#endif + if (end_va != (mm->base_va + mm->size - 1U)) { + ERROR("Not enough memory to map region:\n" + " VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x\n", + mm->base_va, mm->base_pa, mm->size, mm->attr); + panic(); + } + + mm++; + } + + assert(ctx->pa_max_address <= xlat_arch_get_max_supported_pa()); + assert(ctx->max_va <= ctx->va_max_address); + assert(ctx->max_pa <= ctx->pa_max_address); + + ctx->initialized = true; + + xlat_tables_print(ctx); +} diff --git a/lib/xlat_tables_v2/xlat_tables_private.h b/lib/xlat_tables_v2/xlat_tables_private.h new file mode 100644 index 0000000..42c9a43 --- /dev/null +++ b/lib/xlat_tables_v2/xlat_tables_private.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef XLAT_TABLES_PRIVATE_H +#define XLAT_TABLES_PRIVATE_H + +#include <stdbool.h> + +#include <platform_def.h> + +#include <lib/xlat_tables/xlat_tables_defs.h> + +#if PLAT_XLAT_TABLES_DYNAMIC +/* + * Private shifts and masks to access fields of an mmap attribute + */ +/* Dynamic or static */ +#define MT_DYN_SHIFT U(31) + +/* + * Memory mapping private attributes + * + * Private attributes not exposed in the public header. + */ + +/* + * Regions mapped before the MMU can't be unmapped dynamically (they are + * static) and regions mapped with MMU enabled can be unmapped. This + * behaviour can't be overridden. + * + * Static regions can overlap each other, dynamic regions can't. + */ +#define MT_STATIC (U(0) << MT_DYN_SHIFT) +#define MT_DYNAMIC (U(1) << MT_DYN_SHIFT) + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +extern uint64_t mmu_cfg_params[MMU_CFG_PARAM_MAX]; + +/* Determine the physical address space encoded in the 'attr' parameter. */ +uint32_t xlat_arch_get_pas(uint32_t attr); + +/* + * Return the execute-never mask that will prevent instruction fetch at the + * given translation regime. + */ +uint64_t xlat_arch_regime_get_xn_desc(int xlat_regime); + +/* + * Invalidate all TLB entries that match the given virtual address. This + * operation applies to all PEs in the same Inner Shareable domain as the PE + * that executes this function. This functions must be called for every + * translation table entry that is modified. It only affects the specified + * translation regime. + * + * Note, however, that it is architecturally UNDEFINED to invalidate TLB entries + * pertaining to a higher exception level, e.g. invalidating EL3 entries from + * S-EL1. + */ +void xlat_arch_tlbi_va(uintptr_t va, int xlat_regime); + +/* + * This function has to be called at the end of any code that uses the function + * xlat_arch_tlbi_va(). + */ +void xlat_arch_tlbi_va_sync(void); + +/* Print VA, PA, size and attributes of all regions in the mmap array. */ +void xlat_mmap_print(const mmap_region_t *mmap); + +/* + * Print the current state of the translation tables by reading them from + * memory. + */ +void xlat_tables_print(xlat_ctx_t *ctx); + +/* + * Returns a block/page table descriptor for the given level and attributes. + */ +uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr, + unsigned long long addr_pa, unsigned int level); + +/* + * Architecture-specific initialization code. + */ + +/* Returns the current Exception Level. The returned EL must be 1 or higher. */ +unsigned int xlat_arch_current_el(void); + +/* + * Return the maximum physical address supported by the hardware. + * This value depends on the execution state (AArch32/AArch64). + */ +unsigned long long xlat_arch_get_max_supported_pa(void); + +/* + * Returns true if the MMU of the translation regime managed by the given + * xlat_ctx_t is enabled, false otherwise. + */ +bool is_mmu_enabled_ctx(const xlat_ctx_t *ctx); + +/* + * Returns minimum virtual address space size supported by the architecture + */ +uintptr_t xlat_get_min_virt_addr_space_size(void); + +#endif /* XLAT_TABLES_PRIVATE_H */ diff --git a/lib/xlat_tables_v2/xlat_tables_utils.c b/lib/xlat_tables_v2/xlat_tables_utils.c new file mode 100644 index 0000000..f3a53cc --- /dev/null +++ b/lib/xlat_tables_v2/xlat_tables_utils.c @@ -0,0 +1,592 @@ +/* + * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <errno.h> +#include <inttypes.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> + +#include <platform_def.h> + +#include <arch_helpers.h> +#include <common/debug.h> +#include <lib/utils_def.h> +#include <lib/xlat_tables/xlat_tables_defs.h> +#include <lib/xlat_tables/xlat_tables_v2.h> + +#include "xlat_tables_private.h" + +#if LOG_LEVEL < LOG_LEVEL_VERBOSE + +void xlat_mmap_print(__unused const mmap_region_t *mmap) +{ + /* Empty */ +} + +void xlat_tables_print(__unused xlat_ctx_t *ctx) +{ + /* Empty */ +} + +#else /* if LOG_LEVEL >= LOG_LEVEL_VERBOSE */ + +void xlat_mmap_print(const mmap_region_t *mmap) +{ + printf("mmap:\n"); + const mmap_region_t *mm = mmap; + + while (mm->size != 0U) { + printf(" VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x granularity:0x%zx\n", + mm->base_va, mm->base_pa, mm->size, mm->attr, + mm->granularity); + ++mm; + }; + printf("\n"); +} + +/* Print the attributes of the specified block descriptor. */ +static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc) +{ + uint64_t mem_type_index = ATTR_INDEX_GET(desc); + int xlat_regime = ctx->xlat_regime; + + if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { + printf("MEM"); + } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) { + printf("NC"); + } else { + assert(mem_type_index == ATTR_DEVICE_INDEX); + printf("DEV"); + } + + if ((xlat_regime == EL3_REGIME) || (xlat_regime == EL2_REGIME)) { + /* For EL3 and EL2 only check the AP[2] and XN bits. */ + printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW"); + printf(((desc & UPPER_ATTRS(XN)) != 0ULL) ? "-XN" : "-EXEC"); + } else { + assert(xlat_regime == EL1_EL0_REGIME); + /* + * For EL0 and EL1: + * - In AArch64 PXN and UXN can be set independently but in + * AArch32 there is no UXN (XN affects both privilege levels). + * For consistency, we set them simultaneously in both cases. + * - RO and RW permissions must be the same in EL1 and EL0. If + * EL0 can access that memory region, so can EL1, with the + * same permissions. + */ +#if ENABLE_ASSERTIONS + uint64_t xn_mask = xlat_arch_regime_get_xn_desc(EL1_EL0_REGIME); + uint64_t xn_perm = desc & xn_mask; + + assert((xn_perm == xn_mask) || (xn_perm == 0ULL)); +#endif + printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW"); + /* Only check one of PXN and UXN, the other one is the same. */ + printf(((desc & UPPER_ATTRS(PXN)) != 0ULL) ? "-XN" : "-EXEC"); + /* + * Privileged regions can only be accessed from EL1, user + * regions can be accessed from EL1 and EL0. + */ + printf(((desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED)) != 0ULL) + ? "-USER" : "-PRIV"); + } + +#if ENABLE_RME + switch (desc & LOWER_ATTRS(EL3_S1_NSE | NS)) { + case 0ULL: + printf("-S"); + break; + case LOWER_ATTRS(NS): + printf("-NS"); + break; + case LOWER_ATTRS(EL3_S1_NSE): + printf("-RT"); + break; + default: /* LOWER_ATTRS(EL3_S1_NSE | NS) */ + printf("-RL"); + } +#else + printf(((LOWER_ATTRS(NS) & desc) != 0ULL) ? "-NS" : "-S"); +#endif + +#ifdef __aarch64__ + /* Check Guarded Page bit */ + if ((desc & GP) != 0ULL) { + printf("-GP"); + } +#endif +} + +static const char * const level_spacers[] = { + "[LV0] ", + " [LV1] ", + " [LV2] ", + " [LV3] " +}; + +static const char *invalid_descriptors_ommited = + "%s(%d invalid descriptors omitted)\n"; + +/* + * Recursive function that reads the translation tables passed as an argument + * and prints their status. + */ +static void xlat_tables_print_internal(xlat_ctx_t *ctx, uintptr_t table_base_va, + const uint64_t *table_base, unsigned int table_entries, + unsigned int level) +{ + assert(level <= XLAT_TABLE_LEVEL_MAX); + + uint64_t desc; + uintptr_t table_idx_va = table_base_va; + unsigned int table_idx = 0U; + size_t level_size = XLAT_BLOCK_SIZE(level); + + /* + * Keep track of how many invalid descriptors are counted in a row. + * Whenever multiple invalid descriptors are found, only the first one + * is printed, and a line is added to inform about how many descriptors + * have been omitted. + */ + int invalid_row_count = 0; + + while (table_idx < table_entries) { + + desc = table_base[table_idx]; + + if ((desc & DESC_MASK) == INVALID_DESC) { + + if (invalid_row_count == 0) { + printf("%sVA:0x%lx size:0x%zx\n", + level_spacers[level], + table_idx_va, level_size); + } + invalid_row_count++; + + } else { + + if (invalid_row_count > 1) { + printf(invalid_descriptors_ommited, + level_spacers[level], + invalid_row_count - 1); + } + invalid_row_count = 0; + + /* + * Check if this is a table or a block. Tables are only + * allowed in levels other than 3, but DESC_PAGE has the + * same value as DESC_TABLE, so we need to check. + */ + if (((desc & DESC_MASK) == TABLE_DESC) && + (level < XLAT_TABLE_LEVEL_MAX)) { + /* + * Do not print any PA for a table descriptor, + * as it doesn't directly map physical memory + * but instead points to the next translation + * table in the translation table walk. + */ + printf("%sVA:0x%lx size:0x%zx\n", + level_spacers[level], + table_idx_va, level_size); + + uintptr_t addr_inner = desc & TABLE_ADDR_MASK; + + xlat_tables_print_internal(ctx, table_idx_va, + (uint64_t *)addr_inner, + XLAT_TABLE_ENTRIES, level + 1U); + } else { + printf("%sVA:0x%lx PA:0x%" PRIx64 " size:0x%zx ", + level_spacers[level], table_idx_va, + (uint64_t)(desc & TABLE_ADDR_MASK), + level_size); + xlat_desc_print(ctx, desc); + printf("\n"); + } + } + + table_idx++; + table_idx_va += level_size; + } + + if (invalid_row_count > 1) { + printf(invalid_descriptors_ommited, + level_spacers[level], invalid_row_count - 1); + } +} + +void xlat_tables_print(xlat_ctx_t *ctx) +{ + const char *xlat_regime_str; + int used_page_tables; + + if (ctx->xlat_regime == EL1_EL0_REGIME) { + xlat_regime_str = "1&0"; + } else if (ctx->xlat_regime == EL2_REGIME) { + xlat_regime_str = "2"; + } else { + assert(ctx->xlat_regime == EL3_REGIME); + xlat_regime_str = "3"; + } + VERBOSE("Translation tables state:\n"); + VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str); + VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address); + VERBOSE(" Max allowed VA: 0x%lx\n", ctx->va_max_address); + VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa); + VERBOSE(" Max mapped VA: 0x%lx\n", ctx->max_va); + + VERBOSE(" Initial lookup level: %u\n", ctx->base_level); + VERBOSE(" Entries @initial lookup level: %u\n", + ctx->base_table_entries); + +#if PLAT_XLAT_TABLES_DYNAMIC + used_page_tables = 0; + for (int i = 0; i < ctx->tables_num; ++i) { + if (ctx->tables_mapped_regions[i] != 0) + ++used_page_tables; + } +#else + used_page_tables = ctx->next_table; +#endif + VERBOSE(" Used %d sub-tables out of %d (spare: %d)\n", + used_page_tables, ctx->tables_num, + ctx->tables_num - used_page_tables); + + xlat_tables_print_internal(ctx, 0U, ctx->base_table, + ctx->base_table_entries, ctx->base_level); +} + +#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ + +/* + * Do a translation table walk to find the block or page descriptor that maps + * virtual_addr. + * + * On success, return the address of the descriptor within the translation + * table. Its lookup level is stored in '*out_level'. + * On error, return NULL. + * + * xlat_table_base + * Base address for the initial lookup level. + * xlat_table_base_entries + * Number of entries in the translation table for the initial lookup level. + * virt_addr_space_size + * Size in bytes of the virtual address space. + */ +static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr, + void *xlat_table_base, + unsigned int xlat_table_base_entries, + unsigned long long virt_addr_space_size, + unsigned int *out_level) +{ + unsigned int start_level; + uint64_t *table; + unsigned int entries; + + start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size); + + table = xlat_table_base; + entries = xlat_table_base_entries; + + for (unsigned int level = start_level; + level <= XLAT_TABLE_LEVEL_MAX; + ++level) { + uint64_t idx, desc, desc_type; + + idx = XLAT_TABLE_IDX(virtual_addr, level); + if (idx >= entries) { + WARN("Missing xlat table entry at address 0x%lx\n", + virtual_addr); + return NULL; + } + + desc = table[idx]; + desc_type = desc & DESC_MASK; + + if (desc_type == INVALID_DESC) { + VERBOSE("Invalid entry (memory not mapped)\n"); + return NULL; + } + + if (level == XLAT_TABLE_LEVEL_MAX) { + /* + * Only page descriptors allowed at the final lookup + * level. + */ + assert(desc_type == PAGE_DESC); + *out_level = level; + return &table[idx]; + } + + if (desc_type == BLOCK_DESC) { + *out_level = level; + return &table[idx]; + } + + assert(desc_type == TABLE_DESC); + table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); + entries = XLAT_TABLE_ENTRIES; + } + + /* + * This shouldn't be reached, the translation table walk should end at + * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop. + */ + assert(false); + + return NULL; +} + + +static int xlat_get_mem_attributes_internal(const xlat_ctx_t *ctx, + uintptr_t base_va, uint32_t *attributes, uint64_t **table_entry, + unsigned long long *addr_pa, unsigned int *table_level) +{ + uint64_t *entry; + uint64_t desc; + unsigned int level; + unsigned long long virt_addr_space_size; + + /* + * Sanity-check arguments. + */ + assert(ctx != NULL); + assert(ctx->initialized); + assert((ctx->xlat_regime == EL1_EL0_REGIME) || + (ctx->xlat_regime == EL2_REGIME) || + (ctx->xlat_regime == EL3_REGIME)); + + virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1ULL; + assert(virt_addr_space_size > 0U); + + entry = find_xlat_table_entry(base_va, + ctx->base_table, + ctx->base_table_entries, + virt_addr_space_size, + &level); + if (entry == NULL) { + WARN("Address 0x%lx is not mapped.\n", base_va); + return -EINVAL; + } + + if (addr_pa != NULL) { + *addr_pa = *entry & TABLE_ADDR_MASK; + } + + if (table_entry != NULL) { + *table_entry = entry; + } + + if (table_level != NULL) { + *table_level = level; + } + + desc = *entry; + +#if LOG_LEVEL >= LOG_LEVEL_VERBOSE + VERBOSE("Attributes: "); + xlat_desc_print(ctx, desc); + printf("\n"); +#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ + + assert(attributes != NULL); + *attributes = 0U; + + uint64_t attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; + + if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { + *attributes |= MT_MEMORY; + } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) { + *attributes |= MT_NON_CACHEABLE; + } else { + assert(attr_index == ATTR_DEVICE_INDEX); + *attributes |= MT_DEVICE; + } + + uint64_t ap2_bit = (desc >> AP2_SHIFT) & 1U; + + if (ap2_bit == AP2_RW) + *attributes |= MT_RW; + + if (ctx->xlat_regime == EL1_EL0_REGIME) { + uint64_t ap1_bit = (desc >> AP1_SHIFT) & 1U; + + if (ap1_bit == AP1_ACCESS_UNPRIVILEGED) + *attributes |= MT_USER; + } + + uint64_t ns_bit = (desc >> NS_SHIFT) & 1U; + + if (ns_bit == 1U) + *attributes |= MT_NS; + + uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime); + + if ((desc & xn_mask) == xn_mask) { + *attributes |= MT_EXECUTE_NEVER; + } else { + assert((desc & xn_mask) == 0U); + } + + return 0; +} + + +int xlat_get_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va, + uint32_t *attr) +{ + return xlat_get_mem_attributes_internal(ctx, base_va, attr, + NULL, NULL, NULL); +} + + +int xlat_change_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va, + size_t size, uint32_t attr) +{ + /* Note: This implementation isn't optimized. */ + + assert(ctx != NULL); + assert(ctx->initialized); + + unsigned long long virt_addr_space_size = + (unsigned long long)ctx->va_max_address + 1U; + assert(virt_addr_space_size > 0U); + + if (!IS_PAGE_ALIGNED(base_va)) { + WARN("%s: Address 0x%lx is not aligned on a page boundary.\n", + __func__, base_va); + return -EINVAL; + } + + if (size == 0U) { + WARN("%s: Size is 0.\n", __func__); + return -EINVAL; + } + + if ((size % PAGE_SIZE) != 0U) { + WARN("%s: Size 0x%zx is not a multiple of a page size.\n", + __func__, size); + return -EINVAL; + } + + if (((attr & MT_EXECUTE_NEVER) == 0U) && ((attr & MT_RW) != 0U)) { + WARN("%s: Mapping memory as read-write and executable not allowed.\n", + __func__); + return -EINVAL; + } + + size_t pages_count = size / PAGE_SIZE; + + VERBOSE("Changing memory attributes of %zu pages starting from address 0x%lx...\n", + pages_count, base_va); + + uintptr_t base_va_original = base_va; + + /* + * Sanity checks. + */ + for (unsigned int i = 0U; i < pages_count; ++i) { + const uint64_t *entry; + uint64_t desc, attr_index; + unsigned int level; + + entry = find_xlat_table_entry(base_va, + ctx->base_table, + ctx->base_table_entries, + virt_addr_space_size, + &level); + if (entry == NULL) { + WARN("Address 0x%lx is not mapped.\n", base_va); + return -EINVAL; + } + + desc = *entry; + + /* + * Check that all the required pages are mapped at page + * granularity. + */ + if (((desc & DESC_MASK) != PAGE_DESC) || + (level != XLAT_TABLE_LEVEL_MAX)) { + WARN("Address 0x%lx is not mapped at the right granularity.\n", + base_va); + WARN("Granularity is 0x%lx, should be 0x%lx.\n", + XLAT_BLOCK_SIZE(level), PAGE_SIZE); + return -EINVAL; + } + + /* + * If the region type is device, it shouldn't be executable. + */ + attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; + if (attr_index == ATTR_DEVICE_INDEX) { + if ((attr & MT_EXECUTE_NEVER) == 0U) { + WARN("Setting device memory as executable at address 0x%lx.", + base_va); + return -EINVAL; + } + } + + base_va += PAGE_SIZE; + } + + /* Restore original value. */ + base_va = base_va_original; + + for (unsigned int i = 0U; i < pages_count; ++i) { + + uint32_t old_attr = 0U, new_attr; + uint64_t *entry = NULL; + unsigned int level = 0U; + unsigned long long addr_pa = 0ULL; + + (void) xlat_get_mem_attributes_internal(ctx, base_va, &old_attr, + &entry, &addr_pa, &level); + + /* + * From attr, only MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and + * MT_USER/MT_PRIVILEGED are taken into account. Any other + * information is ignored. + */ + + /* Clean the old attributes so that they can be rebuilt. */ + new_attr = old_attr & ~(MT_RW | MT_EXECUTE_NEVER | MT_USER); + + /* + * Update attributes, but filter out the ones this function + * isn't allowed to change. + */ + new_attr |= attr & (MT_RW | MT_EXECUTE_NEVER | MT_USER); + + /* + * The break-before-make sequence requires writing an invalid + * descriptor and making sure that the system sees the change + * before writing the new descriptor. + */ + *entry = INVALID_DESC; +#if !HW_ASSISTED_COHERENCY + dccvac((uintptr_t)entry); +#endif + /* Invalidate any cached copy of this mapping in the TLBs. */ + xlat_arch_tlbi_va(base_va, ctx->xlat_regime); + + /* Ensure completion of the invalidation. */ + xlat_arch_tlbi_va_sync(); + + /* Write new descriptor */ + *entry = xlat_desc(ctx, new_attr, addr_pa, level); +#if !HW_ASSISTED_COHERENCY + dccvac((uintptr_t)entry); +#endif + base_va += PAGE_SIZE; + } + + /* Ensure that the last descriptor written is seen by the system. */ + dsbish(); + + return 0; +} diff --git a/lib/zlib/adler32.c b/lib/zlib/adler32.c new file mode 100644 index 0000000..04b81d2 --- /dev/null +++ b/lib/zlib/adler32.c @@ -0,0 +1,164 @@ +/* adler32.c -- compute the Adler-32 checksum of a data stream + * Copyright (C) 1995-2011, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#include "zutil.h" + +#define BASE 65521U /* largest prime smaller than 65536 */ +#define NMAX 5552 +/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ + +#define DO1(buf,i) {adler += (buf)[i]; sum2 += adler;} +#define DO2(buf,i) DO1(buf,i); DO1(buf,i+1); +#define DO4(buf,i) DO2(buf,i); DO2(buf,i+2); +#define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); +#define DO16(buf) DO8(buf,0); DO8(buf,8); + +/* use NO_DIVIDE if your processor does not do division in hardware -- + try it both ways to see which is faster */ +#ifdef NO_DIVIDE +/* note that this assumes BASE is 65521, where 65536 % 65521 == 15 + (thank you to John Reiser for pointing this out) */ +# define CHOP(a) \ + do { \ + unsigned long tmp = a >> 16; \ + a &= 0xffffUL; \ + a += (tmp << 4) - tmp; \ + } while (0) +# define MOD28(a) \ + do { \ + CHOP(a); \ + if (a >= BASE) a -= BASE; \ + } while (0) +# define MOD(a) \ + do { \ + CHOP(a); \ + MOD28(a); \ + } while (0) +# define MOD63(a) \ + do { /* this assumes a is not negative */ \ + z_off64_t tmp = a >> 32; \ + a &= 0xffffffffL; \ + a += (tmp << 8) - (tmp << 5) + tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + if (a >= BASE) a -= BASE; \ + } while (0) +#else +# define MOD(a) a %= BASE +# define MOD28(a) a %= BASE +# define MOD63(a) a %= BASE +#endif + +/* ========================================================================= */ +uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, z_size_t len) { + unsigned long sum2; + unsigned n; + + /* split Adler-32 into component sums */ + sum2 = (adler >> 16) & 0xffff; + adler &= 0xffff; + + /* in case user likes doing a byte at a time, keep it fast */ + if (len == 1) { + adler += buf[0]; + if (adler >= BASE) + adler -= BASE; + sum2 += adler; + if (sum2 >= BASE) + sum2 -= BASE; + return adler | (sum2 << 16); + } + + /* initial Adler-32 value (deferred check for len == 1 speed) */ + if (buf == Z_NULL) + return 1L; + + /* in case short lengths are provided, keep it somewhat fast */ + if (len < 16) { + while (len--) { + adler += *buf++; + sum2 += adler; + } + if (adler >= BASE) + adler -= BASE; + MOD28(sum2); /* only added so many BASE's */ + return adler | (sum2 << 16); + } + + /* do length NMAX blocks -- requires just one modulo operation */ + while (len >= NMAX) { + len -= NMAX; + n = NMAX / 16; /* NMAX is divisible by 16 */ + do { + DO16(buf); /* 16 sums unrolled */ + buf += 16; + } while (--n); + MOD(adler); + MOD(sum2); + } + + /* do remaining bytes (less than NMAX, still just one modulo) */ + if (len) { /* avoid modulos if none remaining */ + while (len >= 16) { + len -= 16; + DO16(buf); + buf += 16; + } + while (len--) { + adler += *buf++; + sum2 += adler; + } + MOD(adler); + MOD(sum2); + } + + /* return recombined sums */ + return adler | (sum2 << 16); +} + +/* ========================================================================= */ +uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len) { + return adler32_z(adler, buf, len); +} + +/* ========================================================================= */ +local uLong adler32_combine_(uLong adler1, uLong adler2, z_off64_t len2) { + unsigned long sum1; + unsigned long sum2; + unsigned rem; + + /* for negative len, return invalid adler32 as a clue for debugging */ + if (len2 < 0) + return 0xffffffffUL; + + /* the derivation of this formula is left as an exercise for the reader */ + MOD63(len2); /* assumes len2 >= 0 */ + rem = (unsigned)len2; + sum1 = adler1 & 0xffff; + sum2 = rem * sum1; + MOD(sum2); + sum1 += (adler2 & 0xffff) + BASE - 1; + sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; + if (sum1 >= BASE) sum1 -= BASE; + if (sum1 >= BASE) sum1 -= BASE; + if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); + if (sum2 >= BASE) sum2 -= BASE; + return sum1 | (sum2 << 16); +} + +/* ========================================================================= */ +uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2, z_off_t len2) { + return adler32_combine_(adler1, adler2, len2); +} + +uLong ZEXPORT adler32_combine64(uLong adler1, uLong adler2, z_off64_t len2) { + return adler32_combine_(adler1, adler2, len2); +} diff --git a/lib/zlib/crc32.c b/lib/zlib/crc32.c new file mode 100644 index 0000000..6c38f5c --- /dev/null +++ b/lib/zlib/crc32.c @@ -0,0 +1,1049 @@ +/* crc32.c -- compute the CRC-32 of a data stream + * Copyright (C) 1995-2022 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + * + * This interleaved implementation of a CRC makes use of pipelined multiple + * arithmetic-logic units, commonly found in modern CPU cores. It is due to + * Kadatch and Jenkins (2010). See doc/crc-doc.1.0.pdf in this distribution. + */ + +/* @(#) $Id$ */ + +/* + Note on the use of DYNAMIC_CRC_TABLE: there is no mutex or semaphore + protection on the static variables used to control the first-use generation + of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should + first call get_crc_table() to initialize the tables before allowing more than + one thread to use crc32(). + + MAKECRCH can be #defined to write out crc32.h. A main() routine is also + produced, so that this one source file can be compiled to an executable. + */ + +#ifdef MAKECRCH +# include <stdio.h> +# ifndef DYNAMIC_CRC_TABLE +# define DYNAMIC_CRC_TABLE +# endif /* !DYNAMIC_CRC_TABLE */ +#endif /* MAKECRCH */ + +#include "zutil.h" /* for Z_U4, Z_U8, z_crc_t, and FAR definitions */ + + /* + A CRC of a message is computed on N braids of words in the message, where + each word consists of W bytes (4 or 8). If N is 3, for example, then three + running sparse CRCs are calculated respectively on each braid, at these + indices in the array of words: 0, 3, 6, ..., 1, 4, 7, ..., and 2, 5, 8, ... + This is done starting at a word boundary, and continues until as many blocks + of N * W bytes as are available have been processed. The results are combined + into a single CRC at the end. For this code, N must be in the range 1..6 and + W must be 4 or 8. The upper limit on N can be increased if desired by adding + more #if blocks, extending the patterns apparent in the code. In addition, + crc32.h would need to be regenerated, if the maximum N value is increased. + + N and W are chosen empirically by benchmarking the execution time on a given + processor. The choices for N and W below were based on testing on Intel Kaby + Lake i7, AMD Ryzen 7, ARM Cortex-A57, Sparc64-VII, PowerPC POWER9, and MIPS64 + Octeon II processors. The Intel, AMD, and ARM processors were all fastest + with N=5, W=8. The Sparc, PowerPC, and MIPS64 were all fastest at N=5, W=4. + They were all tested with either gcc or clang, all using the -O3 optimization + level. Your mileage may vary. + */ + +/* Define N */ +#ifdef Z_TESTN +# define N Z_TESTN +#else +# define N 5 +#endif +#if N < 1 || N > 6 +# error N must be in 1..6 +#endif + +/* + z_crc_t must be at least 32 bits. z_word_t must be at least as long as + z_crc_t. It is assumed here that z_word_t is either 32 bits or 64 bits, and + that bytes are eight bits. + */ + +/* + Define W and the associated z_word_t type. If W is not defined, then a + braided calculation is not used, and the associated tables and code are not + compiled. + */ +#ifdef Z_TESTW +# if Z_TESTW-1 != -1 +# define W Z_TESTW +# endif +#else +# ifdef MAKECRCH +# define W 8 /* required for MAKECRCH */ +# else +# if defined(__x86_64__) || defined(__aarch64__) +# define W 8 +# else +# define W 4 +# endif +# endif +#endif +#ifdef W +# if W == 8 && defined(Z_U8) + typedef Z_U8 z_word_t; +# elif defined(Z_U4) +# undef W +# define W 4 + typedef Z_U4 z_word_t; +# else +# undef W +# endif +#endif + +/* If available, use the ARM processor CRC32 instruction. */ +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) && W == 8 +# define ARMCRC32 +#endif + +#if defined(W) && (!defined(ARMCRC32) || defined(DYNAMIC_CRC_TABLE)) +/* + Swap the bytes in a z_word_t to convert between little and big endian. Any + self-respecting compiler will optimize this to a single machine byte-swap + instruction, if one is available. This assumes that word_t is either 32 bits + or 64 bits. + */ +local z_word_t byte_swap(z_word_t word) { +# if W == 8 + return + (word & 0xff00000000000000) >> 56 | + (word & 0xff000000000000) >> 40 | + (word & 0xff0000000000) >> 24 | + (word & 0xff00000000) >> 8 | + (word & 0xff000000) << 8 | + (word & 0xff0000) << 24 | + (word & 0xff00) << 40 | + (word & 0xff) << 56; +# else /* W == 4 */ + return + (word & 0xff000000) >> 24 | + (word & 0xff0000) >> 8 | + (word & 0xff00) << 8 | + (word & 0xff) << 24; +# endif +} +#endif + +#ifdef DYNAMIC_CRC_TABLE +/* ========================================================================= + * Table of powers of x for combining CRC-32s, filled in by make_crc_table() + * below. + */ + local z_crc_t FAR x2n_table[32]; +#else +/* ========================================================================= + * Tables for byte-wise and braided CRC-32 calculations, and a table of powers + * of x for combining CRC-32s, all made by make_crc_table(). + */ +# include "crc32.h" +#endif + +/* CRC polynomial. */ +#define POLY 0xedb88320 /* p(x) reflected, with x^32 implied */ + +/* + Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC polynomial, + reflected. For speed, this requires that a not be zero. + */ +local z_crc_t multmodp(z_crc_t a, z_crc_t b) { + z_crc_t m, p; + + m = (z_crc_t)1 << 31; + p = 0; + for (;;) { + if (a & m) { + p ^= b; + if ((a & (m - 1)) == 0) + break; + } + m >>= 1; + b = b & 1 ? (b >> 1) ^ POLY : b >> 1; + } + return p; +} + +/* + Return x^(n * 2^k) modulo p(x). Requires that x2n_table[] has been + initialized. + */ +local z_crc_t x2nmodp(z_off64_t n, unsigned k) { + z_crc_t p; + + p = (z_crc_t)1 << 31; /* x^0 == 1 */ + while (n) { + if (n & 1) + p = multmodp(x2n_table[k & 31], p); + n >>= 1; + k++; + } + return p; +} + +#ifdef DYNAMIC_CRC_TABLE +/* ========================================================================= + * Build the tables for byte-wise and braided CRC-32 calculations, and a table + * of powers of x for combining CRC-32s. + */ +local z_crc_t FAR crc_table[256]; +#ifdef W + local z_word_t FAR crc_big_table[256]; + local z_crc_t FAR crc_braid_table[W][256]; + local z_word_t FAR crc_braid_big_table[W][256]; + local void braid(z_crc_t [][256], z_word_t [][256], int, int); +#endif +#ifdef MAKECRCH + local void write_table(FILE *, const z_crc_t FAR *, int); + local void write_table32hi(FILE *, const z_word_t FAR *, int); + local void write_table64(FILE *, const z_word_t FAR *, int); +#endif /* MAKECRCH */ + +/* + Define a once() function depending on the availability of atomics. If this is + compiled with DYNAMIC_CRC_TABLE defined, and if CRCs will be computed in + multiple threads, and if atomics are not available, then get_crc_table() must + be called to initialize the tables and must return before any threads are + allowed to compute or combine CRCs. + */ + +/* Definition of once functionality. */ +typedef struct once_s once_t; + +/* Check for the availability of atomics. */ +#if defined(__STDC__) && __STDC_VERSION__ >= 201112L && \ + !defined(__STDC_NO_ATOMICS__) + +#include <stdatomic.h> + +/* Structure for once(), which must be initialized with ONCE_INIT. */ +struct once_s { + atomic_flag begun; + atomic_int done; +}; +#define ONCE_INIT {ATOMIC_FLAG_INIT, 0} + +/* + Run the provided init() function exactly once, even if multiple threads + invoke once() at the same time. The state must be a once_t initialized with + ONCE_INIT. + */ +local void once(once_t *state, void (*init)(void)) { + if (!atomic_load(&state->done)) { + if (atomic_flag_test_and_set(&state->begun)) + while (!atomic_load(&state->done)) + ; + else { + init(); + atomic_store(&state->done, 1); + } + } +} + +#else /* no atomics */ + +/* Structure for once(), which must be initialized with ONCE_INIT. */ +struct once_s { + volatile int begun; + volatile int done; +}; +#define ONCE_INIT {0, 0} + +/* Test and set. Alas, not atomic, but tries to minimize the period of + vulnerability. */ +local int test_and_set(int volatile *flag) { + int was; + + was = *flag; + *flag = 1; + return was; +} + +/* Run the provided init() function once. This is not thread-safe. */ +local void once(once_t *state, void (*init)(void)) { + if (!state->done) { + if (test_and_set(&state->begun)) + while (!state->done) + ; + else { + init(); + state->done = 1; + } + } +} + +#endif + +/* State for once(). */ +local once_t made = ONCE_INIT; + +/* + Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: + x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. + + Polynomials over GF(2) are represented in binary, one bit per coefficient, + with the lowest powers in the most significant bit. Then adding polynomials + is just exclusive-or, and multiplying a polynomial by x is a right shift by + one. If we call the above polynomial p, and represent a byte as the + polynomial q, also with the lowest power in the most significant bit (so the + byte 0xb1 is the polynomial x^7+x^3+x^2+1), then the CRC is (q*x^32) mod p, + where a mod b means the remainder after dividing a by b. + + This calculation is done using the shift-register method of multiplying and + taking the remainder. The register is initialized to zero, and for each + incoming bit, x^32 is added mod p to the register if the bit is a one (where + x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by x + (which is shifting right by one and adding x^32 mod p if the bit shifted out + is a one). We start with the highest power (least significant bit) of q and + repeat for all eight bits of q. + + The table is simply the CRC of all possible eight bit values. This is all the + information needed to generate CRCs on data a byte at a time for all + combinations of CRC register values and incoming bytes. + */ + +local void make_crc_table(void) { + unsigned i, j, n; + z_crc_t p; + + /* initialize the CRC of bytes tables */ + for (i = 0; i < 256; i++) { + p = i; + for (j = 0; j < 8; j++) + p = p & 1 ? (p >> 1) ^ POLY : p >> 1; + crc_table[i] = p; +#ifdef W + crc_big_table[i] = byte_swap(p); +#endif + } + + /* initialize the x^2^n mod p(x) table */ + p = (z_crc_t)1 << 30; /* x^1 */ + x2n_table[0] = p; + for (n = 1; n < 32; n++) + x2n_table[n] = p = multmodp(p, p); + +#ifdef W + /* initialize the braiding tables -- needs x2n_table[] */ + braid(crc_braid_table, crc_braid_big_table, N, W); +#endif + +#ifdef MAKECRCH + { + /* + The crc32.h header file contains tables for both 32-bit and 64-bit + z_word_t's, and so requires a 64-bit type be available. In that case, + z_word_t must be defined to be 64-bits. This code then also generates + and writes out the tables for the case that z_word_t is 32 bits. + */ +#if !defined(W) || W != 8 +# error Need a 64-bit integer type in order to generate crc32.h. +#endif + FILE *out; + int k, n; + z_crc_t ltl[8][256]; + z_word_t big[8][256]; + + out = fopen("crc32.h", "w"); + if (out == NULL) return; + + /* write out little-endian CRC table to crc32.h */ + fprintf(out, + "/* crc32.h -- tables for rapid CRC calculation\n" + " * Generated automatically by crc32.c\n */\n" + "\n" + "local const z_crc_t FAR crc_table[] = {\n" + " "); + write_table(out, crc_table, 256); + fprintf(out, + "};\n"); + + /* write out big-endian CRC table for 64-bit z_word_t to crc32.h */ + fprintf(out, + "\n" + "#ifdef W\n" + "\n" + "#if W == 8\n" + "\n" + "local const z_word_t FAR crc_big_table[] = {\n" + " "); + write_table64(out, crc_big_table, 256); + fprintf(out, + "};\n"); + + /* write out big-endian CRC table for 32-bit z_word_t to crc32.h */ + fprintf(out, + "\n" + "#else /* W == 4 */\n" + "\n" + "local const z_word_t FAR crc_big_table[] = {\n" + " "); + write_table32hi(out, crc_big_table, 256); + fprintf(out, + "};\n" + "\n" + "#endif\n"); + + /* write out braid tables for each value of N */ + for (n = 1; n <= 6; n++) { + fprintf(out, + "\n" + "#if N == %d\n", n); + + /* compute braid tables for this N and 64-bit word_t */ + braid(ltl, big, n, 8); + + /* write out braid tables for 64-bit z_word_t to crc32.h */ + fprintf(out, + "\n" + "#if W == 8\n" + "\n" + "local const z_crc_t FAR crc_braid_table[][256] = {\n"); + for (k = 0; k < 8; k++) { + fprintf(out, " {"); + write_table(out, ltl[k], 256); + fprintf(out, "}%s", k < 7 ? ",\n" : ""); + } + fprintf(out, + "};\n" + "\n" + "local const z_word_t FAR crc_braid_big_table[][256] = {\n"); + for (k = 0; k < 8; k++) { + fprintf(out, " {"); + write_table64(out, big[k], 256); + fprintf(out, "}%s", k < 7 ? ",\n" : ""); + } + fprintf(out, + "};\n"); + + /* compute braid tables for this N and 32-bit word_t */ + braid(ltl, big, n, 4); + + /* write out braid tables for 32-bit z_word_t to crc32.h */ + fprintf(out, + "\n" + "#else /* W == 4 */\n" + "\n" + "local const z_crc_t FAR crc_braid_table[][256] = {\n"); + for (k = 0; k < 4; k++) { + fprintf(out, " {"); + write_table(out, ltl[k], 256); + fprintf(out, "}%s", k < 3 ? ",\n" : ""); + } + fprintf(out, + "};\n" + "\n" + "local const z_word_t FAR crc_braid_big_table[][256] = {\n"); + for (k = 0; k < 4; k++) { + fprintf(out, " {"); + write_table32hi(out, big[k], 256); + fprintf(out, "}%s", k < 3 ? ",\n" : ""); + } + fprintf(out, + "};\n" + "\n" + "#endif\n" + "\n" + "#endif\n"); + } + fprintf(out, + "\n" + "#endif\n"); + + /* write out zeros operator table to crc32.h */ + fprintf(out, + "\n" + "local const z_crc_t FAR x2n_table[] = {\n" + " "); + write_table(out, x2n_table, 32); + fprintf(out, + "};\n"); + fclose(out); + } +#endif /* MAKECRCH */ +} + +#ifdef MAKECRCH + +/* + Write the 32-bit values in table[0..k-1] to out, five per line in + hexadecimal separated by commas. + */ +local void write_table(FILE *out, const z_crc_t FAR *table, int k) { + int n; + + for (n = 0; n < k; n++) + fprintf(out, "%s0x%08lx%s", n == 0 || n % 5 ? "" : " ", + (unsigned long)(table[n]), + n == k - 1 ? "" : (n % 5 == 4 ? ",\n" : ", ")); +} + +/* + Write the high 32-bits of each value in table[0..k-1] to out, five per line + in hexadecimal separated by commas. + */ +local void write_table32hi(FILE *out, const z_word_t FAR *table, int k) { + int n; + + for (n = 0; n < k; n++) + fprintf(out, "%s0x%08lx%s", n == 0 || n % 5 ? "" : " ", + (unsigned long)(table[n] >> 32), + n == k - 1 ? "" : (n % 5 == 4 ? ",\n" : ", ")); +} + +/* + Write the 64-bit values in table[0..k-1] to out, three per line in + hexadecimal separated by commas. This assumes that if there is a 64-bit + type, then there is also a long long integer type, and it is at least 64 + bits. If not, then the type cast and format string can be adjusted + accordingly. + */ +local void write_table64(FILE *out, const z_word_t FAR *table, int k) { + int n; + + for (n = 0; n < k; n++) + fprintf(out, "%s0x%016llx%s", n == 0 || n % 3 ? "" : " ", + (unsigned long long)(table[n]), + n == k - 1 ? "" : (n % 3 == 2 ? ",\n" : ", ")); +} + +/* Actually do the deed. */ +int main(void) { + make_crc_table(); + return 0; +} + +#endif /* MAKECRCH */ + +#ifdef W +/* + Generate the little and big-endian braid tables for the given n and z_word_t + size w. Each array must have room for w blocks of 256 elements. + */ +local void braid(z_crc_t ltl[][256], z_word_t big[][256], int n, int w) { + int k; + z_crc_t i, p, q; + for (k = 0; k < w; k++) { + p = x2nmodp((n * w + 3 - k) << 3, 0); + ltl[k][0] = 0; + big[w - 1 - k][0] = 0; + for (i = 1; i < 256; i++) { + ltl[k][i] = q = multmodp(i << 24, p); + big[w - 1 - k][i] = byte_swap(q); + } + } +} +#endif + +#endif /* DYNAMIC_CRC_TABLE */ + +/* ========================================================================= + * This function can be used by asm versions of crc32(), and to force the + * generation of the CRC tables in a threaded application. + */ +const z_crc_t FAR * ZEXPORT get_crc_table(void) { +#ifdef DYNAMIC_CRC_TABLE + once(&made, make_crc_table); +#endif /* DYNAMIC_CRC_TABLE */ + return (const z_crc_t FAR *)crc_table; +} + +/* ========================================================================= + * Use ARM machine instructions if available. This will compute the CRC about + * ten times faster than the braided calculation. This code does not check for + * the presence of the CRC instruction at run time. __ARM_FEATURE_CRC32 will + * only be defined if the compilation specifies an ARM processor architecture + * that has the instructions. For example, compiling with -march=armv8.1-a or + * -march=armv8-a+crc, or -march=native if the compile machine has the crc32 + * instructions. + */ +#ifdef ARMCRC32 + +/* + Constants empirically determined to maximize speed. These values are from + measurements on a Cortex-A57. Your mileage may vary. + */ +#define Z_BATCH 3990 /* number of words in a batch */ +#define Z_BATCH_ZEROS 0xa10d3d0c /* computed from Z_BATCH = 3990 */ +#define Z_BATCH_MIN 800 /* fewest words in a final batch */ + +unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, + z_size_t len) { + z_crc_t val; + z_word_t crc1, crc2; + const z_word_t *word; + z_word_t val0, val1, val2; + z_size_t last, last2, i; + z_size_t num; + + /* Return initial CRC, if requested. */ + if (buf == Z_NULL) return 0; + +#ifdef DYNAMIC_CRC_TABLE + once(&made, make_crc_table); +#endif /* DYNAMIC_CRC_TABLE */ + + /* Pre-condition the CRC */ + crc = (~crc) & 0xffffffff; + + /* Compute the CRC up to a word boundary. */ + while (len && ((z_size_t)buf & 7) != 0) { + len--; + val = *buf++; + __asm__ volatile("crc32b %w0, %w0, %w1" : "+r"(crc) : "r"(val)); + } + + /* Prepare to compute the CRC on full 64-bit words word[0..num-1]. */ + word = (z_word_t const *)buf; + num = len >> 3; + len &= 7; + + /* Do three interleaved CRCs to realize the throughput of one crc32x + instruction per cycle. Each CRC is calculated on Z_BATCH words. The + three CRCs are combined into a single CRC after each set of batches. */ + while (num >= 3 * Z_BATCH) { + crc1 = 0; + crc2 = 0; + for (i = 0; i < Z_BATCH; i++) { + val0 = word[i]; + val1 = word[i + Z_BATCH]; + val2 = word[i + 2 * Z_BATCH]; + __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc) : "r"(val0)); + __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc1) : "r"(val1)); + __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc2) : "r"(val2)); + } + word += 3 * Z_BATCH; + num -= 3 * Z_BATCH; + crc = multmodp(Z_BATCH_ZEROS, crc) ^ crc1; + crc = multmodp(Z_BATCH_ZEROS, crc) ^ crc2; + } + + /* Do one last smaller batch with the remaining words, if there are enough + to pay for the combination of CRCs. */ + last = num / 3; + if (last >= Z_BATCH_MIN) { + last2 = last << 1; + crc1 = 0; + crc2 = 0; + for (i = 0; i < last; i++) { + val0 = word[i]; + val1 = word[i + last]; + val2 = word[i + last2]; + __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc) : "r"(val0)); + __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc1) : "r"(val1)); + __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc2) : "r"(val2)); + } + word += 3 * last; + num -= 3 * last; + val = x2nmodp(last, 6); + crc = multmodp(val, crc) ^ crc1; + crc = multmodp(val, crc) ^ crc2; + } + + /* Compute the CRC on any remaining words. */ + for (i = 0; i < num; i++) { + val0 = word[i]; + __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc) : "r"(val0)); + } + word += num; + + /* Complete the CRC on any remaining bytes. */ + buf = (const unsigned char FAR *)word; + while (len) { + len--; + val = *buf++; + __asm__ volatile("crc32b %w0, %w0, %w1" : "+r"(crc) : "r"(val)); + } + + /* Return the CRC, post-conditioned. */ + return crc ^ 0xffffffff; +} + +#else + +#ifdef W + +/* + Return the CRC of the W bytes in the word_t data, taking the + least-significant byte of the word as the first byte of data, without any pre + or post conditioning. This is used to combine the CRCs of each braid. + */ +local z_crc_t crc_word(z_word_t data) { + int k; + for (k = 0; k < W; k++) + data = (data >> 8) ^ crc_table[data & 0xff]; + return (z_crc_t)data; +} + +local z_word_t crc_word_big(z_word_t data) { + int k; + for (k = 0; k < W; k++) + data = (data << 8) ^ + crc_big_table[(data >> ((W - 1) << 3)) & 0xff]; + return data; +} + +#endif + +/* ========================================================================= */ +unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, + z_size_t len) { + /* Return initial CRC, if requested. */ + if (buf == Z_NULL) return 0; + +#ifdef DYNAMIC_CRC_TABLE + once(&made, make_crc_table); +#endif /* DYNAMIC_CRC_TABLE */ + + /* Pre-condition the CRC */ + crc = (~crc) & 0xffffffff; + +#ifdef W + + /* If provided enough bytes, do a braided CRC calculation. */ + if (len >= N * W + W - 1) { + z_size_t blks; + z_word_t const *words; + unsigned endian; + int k; + + /* Compute the CRC up to a z_word_t boundary. */ + while (len && ((z_size_t)buf & (W - 1)) != 0) { + len--; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + } + + /* Compute the CRC on as many N z_word_t blocks as are available. */ + blks = len / (N * W); + len -= blks * N * W; + words = (z_word_t const *)buf; + + /* Do endian check at execution time instead of compile time, since ARM + processors can change the endianness at execution time. If the + compiler knows what the endianness will be, it can optimize out the + check and the unused branch. */ + endian = 1; + if (*(unsigned char *)&endian) { + /* Little endian. */ + + z_crc_t crc0; + z_word_t word0; +#if N > 1 + z_crc_t crc1; + z_word_t word1; +#if N > 2 + z_crc_t crc2; + z_word_t word2; +#if N > 3 + z_crc_t crc3; + z_word_t word3; +#if N > 4 + z_crc_t crc4; + z_word_t word4; +#if N > 5 + z_crc_t crc5; + z_word_t word5; +#endif +#endif +#endif +#endif +#endif + + /* Initialize the CRC for each braid. */ + crc0 = crc; +#if N > 1 + crc1 = 0; +#if N > 2 + crc2 = 0; +#if N > 3 + crc3 = 0; +#if N > 4 + crc4 = 0; +#if N > 5 + crc5 = 0; +#endif +#endif +#endif +#endif +#endif + + /* + Process the first blks-1 blocks, computing the CRCs on each braid + independently. + */ + while (--blks) { + /* Load the word for each braid into registers. */ + word0 = crc0 ^ words[0]; +#if N > 1 + word1 = crc1 ^ words[1]; +#if N > 2 + word2 = crc2 ^ words[2]; +#if N > 3 + word3 = crc3 ^ words[3]; +#if N > 4 + word4 = crc4 ^ words[4]; +#if N > 5 + word5 = crc5 ^ words[5]; +#endif +#endif +#endif +#endif +#endif + words += N; + + /* Compute and update the CRC for each word. The loop should + get unrolled. */ + crc0 = crc_braid_table[0][word0 & 0xff]; +#if N > 1 + crc1 = crc_braid_table[0][word1 & 0xff]; +#if N > 2 + crc2 = crc_braid_table[0][word2 & 0xff]; +#if N > 3 + crc3 = crc_braid_table[0][word3 & 0xff]; +#if N > 4 + crc4 = crc_braid_table[0][word4 & 0xff]; +#if N > 5 + crc5 = crc_braid_table[0][word5 & 0xff]; +#endif +#endif +#endif +#endif +#endif + for (k = 1; k < W; k++) { + crc0 ^= crc_braid_table[k][(word0 >> (k << 3)) & 0xff]; +#if N > 1 + crc1 ^= crc_braid_table[k][(word1 >> (k << 3)) & 0xff]; +#if N > 2 + crc2 ^= crc_braid_table[k][(word2 >> (k << 3)) & 0xff]; +#if N > 3 + crc3 ^= crc_braid_table[k][(word3 >> (k << 3)) & 0xff]; +#if N > 4 + crc4 ^= crc_braid_table[k][(word4 >> (k << 3)) & 0xff]; +#if N > 5 + crc5 ^= crc_braid_table[k][(word5 >> (k << 3)) & 0xff]; +#endif +#endif +#endif +#endif +#endif + } + } + + /* + Process the last block, combining the CRCs of the N braids at the + same time. + */ + crc = crc_word(crc0 ^ words[0]); +#if N > 1 + crc = crc_word(crc1 ^ words[1] ^ crc); +#if N > 2 + crc = crc_word(crc2 ^ words[2] ^ crc); +#if N > 3 + crc = crc_word(crc3 ^ words[3] ^ crc); +#if N > 4 + crc = crc_word(crc4 ^ words[4] ^ crc); +#if N > 5 + crc = crc_word(crc5 ^ words[5] ^ crc); +#endif +#endif +#endif +#endif +#endif + words += N; + } + else { + /* Big endian. */ + + z_word_t crc0, word0, comb; +#if N > 1 + z_word_t crc1, word1; +#if N > 2 + z_word_t crc2, word2; +#if N > 3 + z_word_t crc3, word3; +#if N > 4 + z_word_t crc4, word4; +#if N > 5 + z_word_t crc5, word5; +#endif +#endif +#endif +#endif +#endif + + /* Initialize the CRC for each braid. */ + crc0 = byte_swap(crc); +#if N > 1 + crc1 = 0; +#if N > 2 + crc2 = 0; +#if N > 3 + crc3 = 0; +#if N > 4 + crc4 = 0; +#if N > 5 + crc5 = 0; +#endif +#endif +#endif +#endif +#endif + + /* + Process the first blks-1 blocks, computing the CRCs on each braid + independently. + */ + while (--blks) { + /* Load the word for each braid into registers. */ + word0 = crc0 ^ words[0]; +#if N > 1 + word1 = crc1 ^ words[1]; +#if N > 2 + word2 = crc2 ^ words[2]; +#if N > 3 + word3 = crc3 ^ words[3]; +#if N > 4 + word4 = crc4 ^ words[4]; +#if N > 5 + word5 = crc5 ^ words[5]; +#endif +#endif +#endif +#endif +#endif + words += N; + + /* Compute and update the CRC for each word. The loop should + get unrolled. */ + crc0 = crc_braid_big_table[0][word0 & 0xff]; +#if N > 1 + crc1 = crc_braid_big_table[0][word1 & 0xff]; +#if N > 2 + crc2 = crc_braid_big_table[0][word2 & 0xff]; +#if N > 3 + crc3 = crc_braid_big_table[0][word3 & 0xff]; +#if N > 4 + crc4 = crc_braid_big_table[0][word4 & 0xff]; +#if N > 5 + crc5 = crc_braid_big_table[0][word5 & 0xff]; +#endif +#endif +#endif +#endif +#endif + for (k = 1; k < W; k++) { + crc0 ^= crc_braid_big_table[k][(word0 >> (k << 3)) & 0xff]; +#if N > 1 + crc1 ^= crc_braid_big_table[k][(word1 >> (k << 3)) & 0xff]; +#if N > 2 + crc2 ^= crc_braid_big_table[k][(word2 >> (k << 3)) & 0xff]; +#if N > 3 + crc3 ^= crc_braid_big_table[k][(word3 >> (k << 3)) & 0xff]; +#if N > 4 + crc4 ^= crc_braid_big_table[k][(word4 >> (k << 3)) & 0xff]; +#if N > 5 + crc5 ^= crc_braid_big_table[k][(word5 >> (k << 3)) & 0xff]; +#endif +#endif +#endif +#endif +#endif + } + } + + /* + Process the last block, combining the CRCs of the N braids at the + same time. + */ + comb = crc_word_big(crc0 ^ words[0]); +#if N > 1 + comb = crc_word_big(crc1 ^ words[1] ^ comb); +#if N > 2 + comb = crc_word_big(crc2 ^ words[2] ^ comb); +#if N > 3 + comb = crc_word_big(crc3 ^ words[3] ^ comb); +#if N > 4 + comb = crc_word_big(crc4 ^ words[4] ^ comb); +#if N > 5 + comb = crc_word_big(crc5 ^ words[5] ^ comb); +#endif +#endif +#endif +#endif +#endif + words += N; + crc = byte_swap(comb); + } + + /* + Update the pointer to the remaining bytes to process. + */ + buf = (unsigned char const *)words; + } + +#endif /* W */ + + /* Complete the computation of the CRC on any remaining bytes. */ + while (len >= 8) { + len -= 8; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + } + while (len) { + len--; + crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff]; + } + + /* Return the CRC, post-conditioned. */ + return crc ^ 0xffffffff; +} + +#endif + +/* ========================================================================= */ +unsigned long ZEXPORT crc32(unsigned long crc, const unsigned char FAR *buf, + uInt len) { + return crc32_z(crc, buf, len); +} + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine64(uLong crc1, uLong crc2, z_off64_t len2) { +#ifdef DYNAMIC_CRC_TABLE + once(&made, make_crc_table); +#endif /* DYNAMIC_CRC_TABLE */ + return multmodp(x2nmodp(len2, 3), crc1) ^ (crc2 & 0xffffffff); +} + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2) { + return crc32_combine64(crc1, crc2, (z_off64_t)len2); +} + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine_gen64(z_off64_t len2) { +#ifdef DYNAMIC_CRC_TABLE + once(&made, make_crc_table); +#endif /* DYNAMIC_CRC_TABLE */ + return x2nmodp(len2, 3); +} + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine_gen(z_off_t len2) { + return crc32_combine_gen64((z_off64_t)len2); +} + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op) { + return multmodp(op, crc1) ^ (crc2 & 0xffffffff); +} diff --git a/lib/zlib/crc32.h b/lib/zlib/crc32.h new file mode 100644 index 0000000..137df68 --- /dev/null +++ b/lib/zlib/crc32.h @@ -0,0 +1,9446 @@ +/* crc32.h -- tables for rapid CRC calculation + * Generated automatically by crc32.c + */ + +local const z_crc_t FAR crc_table[] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, + 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, + 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, + 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, + 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, + 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, + 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, + 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, + 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, + 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, + 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, + 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, + 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, + 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, + 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, + 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, + 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, + 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, + 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, + 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, + 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, + 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, + 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, + 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, + 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, + 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, + 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, + 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, + 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, + 0x2d02ef8d}; + +#ifdef W + +#if W == 8 + +local const z_word_t FAR crc_big_table[] = { + 0x0000000000000000, 0x9630077700000000, 0x2c610eee00000000, + 0xba51099900000000, 0x19c46d0700000000, 0x8ff46a7000000000, + 0x35a563e900000000, 0xa395649e00000000, 0x3288db0e00000000, + 0xa4b8dc7900000000, 0x1ee9d5e000000000, 0x88d9d29700000000, + 0x2b4cb60900000000, 0xbd7cb17e00000000, 0x072db8e700000000, + 0x911dbf9000000000, 0x6410b71d00000000, 0xf220b06a00000000, + 0x4871b9f300000000, 0xde41be8400000000, 0x7dd4da1a00000000, + 0xebe4dd6d00000000, 0x51b5d4f400000000, 0xc785d38300000000, + 0x56986c1300000000, 0xc0a86b6400000000, 0x7af962fd00000000, + 0xecc9658a00000000, 0x4f5c011400000000, 0xd96c066300000000, + 0x633d0ffa00000000, 0xf50d088d00000000, 0xc8206e3b00000000, + 0x5e10694c00000000, 0xe44160d500000000, 0x727167a200000000, + 0xd1e4033c00000000, 0x47d4044b00000000, 0xfd850dd200000000, + 0x6bb50aa500000000, 0xfaa8b53500000000, 0x6c98b24200000000, + 0xd6c9bbdb00000000, 0x40f9bcac00000000, 0xe36cd83200000000, + 0x755cdf4500000000, 0xcf0dd6dc00000000, 0x593dd1ab00000000, + 0xac30d92600000000, 0x3a00de5100000000, 0x8051d7c800000000, + 0x1661d0bf00000000, 0xb5f4b42100000000, 0x23c4b35600000000, + 0x9995bacf00000000, 0x0fa5bdb800000000, 0x9eb8022800000000, + 0x0888055f00000000, 0xb2d90cc600000000, 0x24e90bb100000000, + 0x877c6f2f00000000, 0x114c685800000000, 0xab1d61c100000000, + 0x3d2d66b600000000, 0x9041dc7600000000, 0x0671db0100000000, + 0xbc20d29800000000, 0x2a10d5ef00000000, 0x8985b17100000000, + 0x1fb5b60600000000, 0xa5e4bf9f00000000, 0x33d4b8e800000000, + 0xa2c9077800000000, 0x34f9000f00000000, 0x8ea8099600000000, + 0x18980ee100000000, 0xbb0d6a7f00000000, 0x2d3d6d0800000000, + 0x976c649100000000, 0x015c63e600000000, 0xf4516b6b00000000, + 0x62616c1c00000000, 0xd830658500000000, 0x4e0062f200000000, + 0xed95066c00000000, 0x7ba5011b00000000, 0xc1f4088200000000, + 0x57c40ff500000000, 0xc6d9b06500000000, 0x50e9b71200000000, + 0xeab8be8b00000000, 0x7c88b9fc00000000, 0xdf1ddd6200000000, + 0x492dda1500000000, 0xf37cd38c00000000, 0x654cd4fb00000000, + 0x5861b24d00000000, 0xce51b53a00000000, 0x7400bca300000000, + 0xe230bbd400000000, 0x41a5df4a00000000, 0xd795d83d00000000, + 0x6dc4d1a400000000, 0xfbf4d6d300000000, 0x6ae9694300000000, + 0xfcd96e3400000000, 0x468867ad00000000, 0xd0b860da00000000, + 0x732d044400000000, 0xe51d033300000000, 0x5f4c0aaa00000000, + 0xc97c0ddd00000000, 0x3c71055000000000, 0xaa41022700000000, + 0x10100bbe00000000, 0x86200cc900000000, 0x25b5685700000000, + 0xb3856f2000000000, 0x09d466b900000000, 0x9fe461ce00000000, + 0x0ef9de5e00000000, 0x98c9d92900000000, 0x2298d0b000000000, + 0xb4a8d7c700000000, 0x173db35900000000, 0x810db42e00000000, + 0x3b5cbdb700000000, 0xad6cbac000000000, 0x2083b8ed00000000, + 0xb6b3bf9a00000000, 0x0ce2b60300000000, 0x9ad2b17400000000, + 0x3947d5ea00000000, 0xaf77d29d00000000, 0x1526db0400000000, + 0x8316dc7300000000, 0x120b63e300000000, 0x843b649400000000, + 0x3e6a6d0d00000000, 0xa85a6a7a00000000, 0x0bcf0ee400000000, + 0x9dff099300000000, 0x27ae000a00000000, 0xb19e077d00000000, + 0x44930ff000000000, 0xd2a3088700000000, 0x68f2011e00000000, + 0xfec2066900000000, 0x5d5762f700000000, 0xcb67658000000000, + 0x71366c1900000000, 0xe7066b6e00000000, 0x761bd4fe00000000, + 0xe02bd38900000000, 0x5a7ada1000000000, 0xcc4add6700000000, + 0x6fdfb9f900000000, 0xf9efbe8e00000000, 0x43beb71700000000, + 0xd58eb06000000000, 0xe8a3d6d600000000, 0x7e93d1a100000000, + 0xc4c2d83800000000, 0x52f2df4f00000000, 0xf167bbd100000000, + 0x6757bca600000000, 0xdd06b53f00000000, 0x4b36b24800000000, + 0xda2b0dd800000000, 0x4c1b0aaf00000000, 0xf64a033600000000, + 0x607a044100000000, 0xc3ef60df00000000, 0x55df67a800000000, + 0xef8e6e3100000000, 0x79be694600000000, 0x8cb361cb00000000, + 0x1a8366bc00000000, 0xa0d26f2500000000, 0x36e2685200000000, + 0x95770ccc00000000, 0x03470bbb00000000, 0xb916022200000000, + 0x2f26055500000000, 0xbe3bbac500000000, 0x280bbdb200000000, + 0x925ab42b00000000, 0x046ab35c00000000, 0xa7ffd7c200000000, + 0x31cfd0b500000000, 0x8b9ed92c00000000, 0x1daede5b00000000, + 0xb0c2649b00000000, 0x26f263ec00000000, 0x9ca36a7500000000, + 0x0a936d0200000000, 0xa906099c00000000, 0x3f360eeb00000000, + 0x8567077200000000, 0x1357000500000000, 0x824abf9500000000, + 0x147ab8e200000000, 0xae2bb17b00000000, 0x381bb60c00000000, + 0x9b8ed29200000000, 0x0dbed5e500000000, 0xb7efdc7c00000000, + 0x21dfdb0b00000000, 0xd4d2d38600000000, 0x42e2d4f100000000, + 0xf8b3dd6800000000, 0x6e83da1f00000000, 0xcd16be8100000000, + 0x5b26b9f600000000, 0xe177b06f00000000, 0x7747b71800000000, + 0xe65a088800000000, 0x706a0fff00000000, 0xca3b066600000000, + 0x5c0b011100000000, 0xff9e658f00000000, 0x69ae62f800000000, + 0xd3ff6b6100000000, 0x45cf6c1600000000, 0x78e20aa000000000, + 0xeed20dd700000000, 0x5483044e00000000, 0xc2b3033900000000, + 0x612667a700000000, 0xf71660d000000000, 0x4d47694900000000, + 0xdb776e3e00000000, 0x4a6ad1ae00000000, 0xdc5ad6d900000000, + 0x660bdf4000000000, 0xf03bd83700000000, 0x53aebca900000000, + 0xc59ebbde00000000, 0x7fcfb24700000000, 0xe9ffb53000000000, + 0x1cf2bdbd00000000, 0x8ac2baca00000000, 0x3093b35300000000, + 0xa6a3b42400000000, 0x0536d0ba00000000, 0x9306d7cd00000000, + 0x2957de5400000000, 0xbf67d92300000000, 0x2e7a66b300000000, + 0xb84a61c400000000, 0x021b685d00000000, 0x942b6f2a00000000, + 0x37be0bb400000000, 0xa18e0cc300000000, 0x1bdf055a00000000, + 0x8def022d00000000}; + +#else /* W == 4 */ + +local const z_word_t FAR crc_big_table[] = { + 0x00000000, 0x96300777, 0x2c610eee, 0xba510999, 0x19c46d07, + 0x8ff46a70, 0x35a563e9, 0xa395649e, 0x3288db0e, 0xa4b8dc79, + 0x1ee9d5e0, 0x88d9d297, 0x2b4cb609, 0xbd7cb17e, 0x072db8e7, + 0x911dbf90, 0x6410b71d, 0xf220b06a, 0x4871b9f3, 0xde41be84, + 0x7dd4da1a, 0xebe4dd6d, 0x51b5d4f4, 0xc785d383, 0x56986c13, + 0xc0a86b64, 0x7af962fd, 0xecc9658a, 0x4f5c0114, 0xd96c0663, + 0x633d0ffa, 0xf50d088d, 0xc8206e3b, 0x5e10694c, 0xe44160d5, + 0x727167a2, 0xd1e4033c, 0x47d4044b, 0xfd850dd2, 0x6bb50aa5, + 0xfaa8b535, 0x6c98b242, 0xd6c9bbdb, 0x40f9bcac, 0xe36cd832, + 0x755cdf45, 0xcf0dd6dc, 0x593dd1ab, 0xac30d926, 0x3a00de51, + 0x8051d7c8, 0x1661d0bf, 0xb5f4b421, 0x23c4b356, 0x9995bacf, + 0x0fa5bdb8, 0x9eb80228, 0x0888055f, 0xb2d90cc6, 0x24e90bb1, + 0x877c6f2f, 0x114c6858, 0xab1d61c1, 0x3d2d66b6, 0x9041dc76, + 0x0671db01, 0xbc20d298, 0x2a10d5ef, 0x8985b171, 0x1fb5b606, + 0xa5e4bf9f, 0x33d4b8e8, 0xa2c90778, 0x34f9000f, 0x8ea80996, + 0x18980ee1, 0xbb0d6a7f, 0x2d3d6d08, 0x976c6491, 0x015c63e6, + 0xf4516b6b, 0x62616c1c, 0xd8306585, 0x4e0062f2, 0xed95066c, + 0x7ba5011b, 0xc1f40882, 0x57c40ff5, 0xc6d9b065, 0x50e9b712, + 0xeab8be8b, 0x7c88b9fc, 0xdf1ddd62, 0x492dda15, 0xf37cd38c, + 0x654cd4fb, 0x5861b24d, 0xce51b53a, 0x7400bca3, 0xe230bbd4, + 0x41a5df4a, 0xd795d83d, 0x6dc4d1a4, 0xfbf4d6d3, 0x6ae96943, + 0xfcd96e34, 0x468867ad, 0xd0b860da, 0x732d0444, 0xe51d0333, + 0x5f4c0aaa, 0xc97c0ddd, 0x3c710550, 0xaa410227, 0x10100bbe, + 0x86200cc9, 0x25b56857, 0xb3856f20, 0x09d466b9, 0x9fe461ce, + 0x0ef9de5e, 0x98c9d929, 0x2298d0b0, 0xb4a8d7c7, 0x173db359, + 0x810db42e, 0x3b5cbdb7, 0xad6cbac0, 0x2083b8ed, 0xb6b3bf9a, + 0x0ce2b603, 0x9ad2b174, 0x3947d5ea, 0xaf77d29d, 0x1526db04, + 0x8316dc73, 0x120b63e3, 0x843b6494, 0x3e6a6d0d, 0xa85a6a7a, + 0x0bcf0ee4, 0x9dff0993, 0x27ae000a, 0xb19e077d, 0x44930ff0, + 0xd2a30887, 0x68f2011e, 0xfec20669, 0x5d5762f7, 0xcb676580, + 0x71366c19, 0xe7066b6e, 0x761bd4fe, 0xe02bd389, 0x5a7ada10, + 0xcc4add67, 0x6fdfb9f9, 0xf9efbe8e, 0x43beb717, 0xd58eb060, + 0xe8a3d6d6, 0x7e93d1a1, 0xc4c2d838, 0x52f2df4f, 0xf167bbd1, + 0x6757bca6, 0xdd06b53f, 0x4b36b248, 0xda2b0dd8, 0x4c1b0aaf, + 0xf64a0336, 0x607a0441, 0xc3ef60df, 0x55df67a8, 0xef8e6e31, + 0x79be6946, 0x8cb361cb, 0x1a8366bc, 0xa0d26f25, 0x36e26852, + 0x95770ccc, 0x03470bbb, 0xb9160222, 0x2f260555, 0xbe3bbac5, + 0x280bbdb2, 0x925ab42b, 0x046ab35c, 0xa7ffd7c2, 0x31cfd0b5, + 0x8b9ed92c, 0x1daede5b, 0xb0c2649b, 0x26f263ec, 0x9ca36a75, + 0x0a936d02, 0xa906099c, 0x3f360eeb, 0x85670772, 0x13570005, + 0x824abf95, 0x147ab8e2, 0xae2bb17b, 0x381bb60c, 0x9b8ed292, + 0x0dbed5e5, 0xb7efdc7c, 0x21dfdb0b, 0xd4d2d386, 0x42e2d4f1, + 0xf8b3dd68, 0x6e83da1f, 0xcd16be81, 0x5b26b9f6, 0xe177b06f, + 0x7747b718, 0xe65a0888, 0x706a0fff, 0xca3b0666, 0x5c0b0111, + 0xff9e658f, 0x69ae62f8, 0xd3ff6b61, 0x45cf6c16, 0x78e20aa0, + 0xeed20dd7, 0x5483044e, 0xc2b30339, 0x612667a7, 0xf71660d0, + 0x4d476949, 0xdb776e3e, 0x4a6ad1ae, 0xdc5ad6d9, 0x660bdf40, + 0xf03bd837, 0x53aebca9, 0xc59ebbde, 0x7fcfb247, 0xe9ffb530, + 0x1cf2bdbd, 0x8ac2baca, 0x3093b353, 0xa6a3b424, 0x0536d0ba, + 0x9306d7cd, 0x2957de54, 0xbf67d923, 0x2e7a66b3, 0xb84a61c4, + 0x021b685d, 0x942b6f2a, 0x37be0bb4, 0xa18e0cc3, 0x1bdf055a, + 0x8def022d}; + +#endif + +#if N == 1 + +#if W == 8 + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0xccaa009e, 0x4225077d, 0x8e8f07e3, 0x844a0efa, + 0x48e00e64, 0xc66f0987, 0x0ac50919, 0xd3e51bb5, 0x1f4f1b2b, + 0x91c01cc8, 0x5d6a1c56, 0x57af154f, 0x9b0515d1, 0x158a1232, + 0xd92012ac, 0x7cbb312b, 0xb01131b5, 0x3e9e3656, 0xf23436c8, + 0xf8f13fd1, 0x345b3f4f, 0xbad438ac, 0x767e3832, 0xaf5e2a9e, + 0x63f42a00, 0xed7b2de3, 0x21d12d7d, 0x2b142464, 0xe7be24fa, + 0x69312319, 0xa59b2387, 0xf9766256, 0x35dc62c8, 0xbb53652b, + 0x77f965b5, 0x7d3c6cac, 0xb1966c32, 0x3f196bd1, 0xf3b36b4f, + 0x2a9379e3, 0xe639797d, 0x68b67e9e, 0xa41c7e00, 0xaed97719, + 0x62737787, 0xecfc7064, 0x205670fa, 0x85cd537d, 0x496753e3, + 0xc7e85400, 0x0b42549e, 0x01875d87, 0xcd2d5d19, 0x43a25afa, + 0x8f085a64, 0x562848c8, 0x9a824856, 0x140d4fb5, 0xd8a74f2b, + 0xd2624632, 0x1ec846ac, 0x9047414f, 0x5ced41d1, 0x299dc2ed, + 0xe537c273, 0x6bb8c590, 0xa712c50e, 0xadd7cc17, 0x617dcc89, + 0xeff2cb6a, 0x2358cbf4, 0xfa78d958, 0x36d2d9c6, 0xb85dde25, + 0x74f7debb, 0x7e32d7a2, 0xb298d73c, 0x3c17d0df, 0xf0bdd041, + 0x5526f3c6, 0x998cf358, 0x1703f4bb, 0xdba9f425, 0xd16cfd3c, + 0x1dc6fda2, 0x9349fa41, 0x5fe3fadf, 0x86c3e873, 0x4a69e8ed, + 0xc4e6ef0e, 0x084cef90, 0x0289e689, 0xce23e617, 0x40ace1f4, + 0x8c06e16a, 0xd0eba0bb, 0x1c41a025, 0x92cea7c6, 0x5e64a758, + 0x54a1ae41, 0x980baedf, 0x1684a93c, 0xda2ea9a2, 0x030ebb0e, + 0xcfa4bb90, 0x412bbc73, 0x8d81bced, 0x8744b5f4, 0x4beeb56a, + 0xc561b289, 0x09cbb217, 0xac509190, 0x60fa910e, 0xee7596ed, + 0x22df9673, 0x281a9f6a, 0xe4b09ff4, 0x6a3f9817, 0xa6959889, + 0x7fb58a25, 0xb31f8abb, 0x3d908d58, 0xf13a8dc6, 0xfbff84df, + 0x37558441, 0xb9da83a2, 0x7570833c, 0x533b85da, 0x9f918544, + 0x111e82a7, 0xddb48239, 0xd7718b20, 0x1bdb8bbe, 0x95548c5d, + 0x59fe8cc3, 0x80de9e6f, 0x4c749ef1, 0xc2fb9912, 0x0e51998c, + 0x04949095, 0xc83e900b, 0x46b197e8, 0x8a1b9776, 0x2f80b4f1, + 0xe32ab46f, 0x6da5b38c, 0xa10fb312, 0xabcaba0b, 0x6760ba95, + 0xe9efbd76, 0x2545bde8, 0xfc65af44, 0x30cfafda, 0xbe40a839, + 0x72eaa8a7, 0x782fa1be, 0xb485a120, 0x3a0aa6c3, 0xf6a0a65d, + 0xaa4de78c, 0x66e7e712, 0xe868e0f1, 0x24c2e06f, 0x2e07e976, + 0xe2ade9e8, 0x6c22ee0b, 0xa088ee95, 0x79a8fc39, 0xb502fca7, + 0x3b8dfb44, 0xf727fbda, 0xfde2f2c3, 0x3148f25d, 0xbfc7f5be, + 0x736df520, 0xd6f6d6a7, 0x1a5cd639, 0x94d3d1da, 0x5879d144, + 0x52bcd85d, 0x9e16d8c3, 0x1099df20, 0xdc33dfbe, 0x0513cd12, + 0xc9b9cd8c, 0x4736ca6f, 0x8b9ccaf1, 0x8159c3e8, 0x4df3c376, + 0xc37cc495, 0x0fd6c40b, 0x7aa64737, 0xb60c47a9, 0x3883404a, + 0xf42940d4, 0xfeec49cd, 0x32464953, 0xbcc94eb0, 0x70634e2e, + 0xa9435c82, 0x65e95c1c, 0xeb665bff, 0x27cc5b61, 0x2d095278, + 0xe1a352e6, 0x6f2c5505, 0xa386559b, 0x061d761c, 0xcab77682, + 0x44387161, 0x889271ff, 0x825778e6, 0x4efd7878, 0xc0727f9b, + 0x0cd87f05, 0xd5f86da9, 0x19526d37, 0x97dd6ad4, 0x5b776a4a, + 0x51b26353, 0x9d1863cd, 0x1397642e, 0xdf3d64b0, 0x83d02561, + 0x4f7a25ff, 0xc1f5221c, 0x0d5f2282, 0x079a2b9b, 0xcb302b05, + 0x45bf2ce6, 0x89152c78, 0x50353ed4, 0x9c9f3e4a, 0x121039a9, + 0xdeba3937, 0xd47f302e, 0x18d530b0, 0x965a3753, 0x5af037cd, + 0xff6b144a, 0x33c114d4, 0xbd4e1337, 0x71e413a9, 0x7b211ab0, + 0xb78b1a2e, 0x39041dcd, 0xf5ae1d53, 0x2c8e0fff, 0xe0240f61, + 0x6eab0882, 0xa201081c, 0xa8c40105, 0x646e019b, 0xeae10678, + 0x264b06e6}, + {0x00000000, 0xa6770bb4, 0x979f1129, 0x31e81a9d, 0xf44f2413, + 0x52382fa7, 0x63d0353a, 0xc5a73e8e, 0x33ef4e67, 0x959845d3, + 0xa4705f4e, 0x020754fa, 0xc7a06a74, 0x61d761c0, 0x503f7b5d, + 0xf64870e9, 0x67de9cce, 0xc1a9977a, 0xf0418de7, 0x56368653, + 0x9391b8dd, 0x35e6b369, 0x040ea9f4, 0xa279a240, 0x5431d2a9, + 0xf246d91d, 0xc3aec380, 0x65d9c834, 0xa07ef6ba, 0x0609fd0e, + 0x37e1e793, 0x9196ec27, 0xcfbd399c, 0x69ca3228, 0x582228b5, + 0xfe552301, 0x3bf21d8f, 0x9d85163b, 0xac6d0ca6, 0x0a1a0712, + 0xfc5277fb, 0x5a257c4f, 0x6bcd66d2, 0xcdba6d66, 0x081d53e8, + 0xae6a585c, 0x9f8242c1, 0x39f54975, 0xa863a552, 0x0e14aee6, + 0x3ffcb47b, 0x998bbfcf, 0x5c2c8141, 0xfa5b8af5, 0xcbb39068, + 0x6dc49bdc, 0x9b8ceb35, 0x3dfbe081, 0x0c13fa1c, 0xaa64f1a8, + 0x6fc3cf26, 0xc9b4c492, 0xf85cde0f, 0x5e2bd5bb, 0x440b7579, + 0xe27c7ecd, 0xd3946450, 0x75e36fe4, 0xb044516a, 0x16335ade, + 0x27db4043, 0x81ac4bf7, 0x77e43b1e, 0xd19330aa, 0xe07b2a37, + 0x460c2183, 0x83ab1f0d, 0x25dc14b9, 0x14340e24, 0xb2430590, + 0x23d5e9b7, 0x85a2e203, 0xb44af89e, 0x123df32a, 0xd79acda4, + 0x71edc610, 0x4005dc8d, 0xe672d739, 0x103aa7d0, 0xb64dac64, + 0x87a5b6f9, 0x21d2bd4d, 0xe47583c3, 0x42028877, 0x73ea92ea, + 0xd59d995e, 0x8bb64ce5, 0x2dc14751, 0x1c295dcc, 0xba5e5678, + 0x7ff968f6, 0xd98e6342, 0xe86679df, 0x4e11726b, 0xb8590282, + 0x1e2e0936, 0x2fc613ab, 0x89b1181f, 0x4c162691, 0xea612d25, + 0xdb8937b8, 0x7dfe3c0c, 0xec68d02b, 0x4a1fdb9f, 0x7bf7c102, + 0xdd80cab6, 0x1827f438, 0xbe50ff8c, 0x8fb8e511, 0x29cfeea5, + 0xdf879e4c, 0x79f095f8, 0x48188f65, 0xee6f84d1, 0x2bc8ba5f, + 0x8dbfb1eb, 0xbc57ab76, 0x1a20a0c2, 0x8816eaf2, 0x2e61e146, + 0x1f89fbdb, 0xb9fef06f, 0x7c59cee1, 0xda2ec555, 0xebc6dfc8, + 0x4db1d47c, 0xbbf9a495, 0x1d8eaf21, 0x2c66b5bc, 0x8a11be08, + 0x4fb68086, 0xe9c18b32, 0xd82991af, 0x7e5e9a1b, 0xefc8763c, + 0x49bf7d88, 0x78576715, 0xde206ca1, 0x1b87522f, 0xbdf0599b, + 0x8c184306, 0x2a6f48b2, 0xdc27385b, 0x7a5033ef, 0x4bb82972, + 0xedcf22c6, 0x28681c48, 0x8e1f17fc, 0xbff70d61, 0x198006d5, + 0x47abd36e, 0xe1dcd8da, 0xd034c247, 0x7643c9f3, 0xb3e4f77d, + 0x1593fcc9, 0x247be654, 0x820cede0, 0x74449d09, 0xd23396bd, + 0xe3db8c20, 0x45ac8794, 0x800bb91a, 0x267cb2ae, 0x1794a833, + 0xb1e3a387, 0x20754fa0, 0x86024414, 0xb7ea5e89, 0x119d553d, + 0xd43a6bb3, 0x724d6007, 0x43a57a9a, 0xe5d2712e, 0x139a01c7, + 0xb5ed0a73, 0x840510ee, 0x22721b5a, 0xe7d525d4, 0x41a22e60, + 0x704a34fd, 0xd63d3f49, 0xcc1d9f8b, 0x6a6a943f, 0x5b828ea2, + 0xfdf58516, 0x3852bb98, 0x9e25b02c, 0xafcdaab1, 0x09baa105, + 0xfff2d1ec, 0x5985da58, 0x686dc0c5, 0xce1acb71, 0x0bbdf5ff, + 0xadcafe4b, 0x9c22e4d6, 0x3a55ef62, 0xabc30345, 0x0db408f1, + 0x3c5c126c, 0x9a2b19d8, 0x5f8c2756, 0xf9fb2ce2, 0xc813367f, + 0x6e643dcb, 0x982c4d22, 0x3e5b4696, 0x0fb35c0b, 0xa9c457bf, + 0x6c636931, 0xca146285, 0xfbfc7818, 0x5d8b73ac, 0x03a0a617, + 0xa5d7ada3, 0x943fb73e, 0x3248bc8a, 0xf7ef8204, 0x519889b0, + 0x6070932d, 0xc6079899, 0x304fe870, 0x9638e3c4, 0xa7d0f959, + 0x01a7f2ed, 0xc400cc63, 0x6277c7d7, 0x539fdd4a, 0xf5e8d6fe, + 0x647e3ad9, 0xc209316d, 0xf3e12bf0, 0x55962044, 0x90311eca, + 0x3646157e, 0x07ae0fe3, 0xa1d90457, 0x579174be, 0xf1e67f0a, + 0xc00e6597, 0x66796e23, 0xa3de50ad, 0x05a95b19, 0x34414184, + 0x92364a30}, + {0x00000000, 0xcb5cd3a5, 0x4dc8a10b, 0x869472ae, 0x9b914216, + 0x50cd91b3, 0xd659e31d, 0x1d0530b8, 0xec53826d, 0x270f51c8, + 0xa19b2366, 0x6ac7f0c3, 0x77c2c07b, 0xbc9e13de, 0x3a0a6170, + 0xf156b2d5, 0x03d6029b, 0xc88ad13e, 0x4e1ea390, 0x85427035, + 0x9847408d, 0x531b9328, 0xd58fe186, 0x1ed33223, 0xef8580f6, + 0x24d95353, 0xa24d21fd, 0x6911f258, 0x7414c2e0, 0xbf481145, + 0x39dc63eb, 0xf280b04e, 0x07ac0536, 0xccf0d693, 0x4a64a43d, + 0x81387798, 0x9c3d4720, 0x57619485, 0xd1f5e62b, 0x1aa9358e, + 0xebff875b, 0x20a354fe, 0xa6372650, 0x6d6bf5f5, 0x706ec54d, + 0xbb3216e8, 0x3da66446, 0xf6fab7e3, 0x047a07ad, 0xcf26d408, + 0x49b2a6a6, 0x82ee7503, 0x9feb45bb, 0x54b7961e, 0xd223e4b0, + 0x197f3715, 0xe82985c0, 0x23755665, 0xa5e124cb, 0x6ebdf76e, + 0x73b8c7d6, 0xb8e41473, 0x3e7066dd, 0xf52cb578, 0x0f580a6c, + 0xc404d9c9, 0x4290ab67, 0x89cc78c2, 0x94c9487a, 0x5f959bdf, + 0xd901e971, 0x125d3ad4, 0xe30b8801, 0x28575ba4, 0xaec3290a, + 0x659ffaaf, 0x789aca17, 0xb3c619b2, 0x35526b1c, 0xfe0eb8b9, + 0x0c8e08f7, 0xc7d2db52, 0x4146a9fc, 0x8a1a7a59, 0x971f4ae1, + 0x5c439944, 0xdad7ebea, 0x118b384f, 0xe0dd8a9a, 0x2b81593f, + 0xad152b91, 0x6649f834, 0x7b4cc88c, 0xb0101b29, 0x36846987, + 0xfdd8ba22, 0x08f40f5a, 0xc3a8dcff, 0x453cae51, 0x8e607df4, + 0x93654d4c, 0x58399ee9, 0xdeadec47, 0x15f13fe2, 0xe4a78d37, + 0x2ffb5e92, 0xa96f2c3c, 0x6233ff99, 0x7f36cf21, 0xb46a1c84, + 0x32fe6e2a, 0xf9a2bd8f, 0x0b220dc1, 0xc07ede64, 0x46eaacca, + 0x8db67f6f, 0x90b34fd7, 0x5bef9c72, 0xdd7beedc, 0x16273d79, + 0xe7718fac, 0x2c2d5c09, 0xaab92ea7, 0x61e5fd02, 0x7ce0cdba, + 0xb7bc1e1f, 0x31286cb1, 0xfa74bf14, 0x1eb014d8, 0xd5ecc77d, + 0x5378b5d3, 0x98246676, 0x852156ce, 0x4e7d856b, 0xc8e9f7c5, + 0x03b52460, 0xf2e396b5, 0x39bf4510, 0xbf2b37be, 0x7477e41b, + 0x6972d4a3, 0xa22e0706, 0x24ba75a8, 0xefe6a60d, 0x1d661643, + 0xd63ac5e6, 0x50aeb748, 0x9bf264ed, 0x86f75455, 0x4dab87f0, + 0xcb3ff55e, 0x006326fb, 0xf135942e, 0x3a69478b, 0xbcfd3525, + 0x77a1e680, 0x6aa4d638, 0xa1f8059d, 0x276c7733, 0xec30a496, + 0x191c11ee, 0xd240c24b, 0x54d4b0e5, 0x9f886340, 0x828d53f8, + 0x49d1805d, 0xcf45f2f3, 0x04192156, 0xf54f9383, 0x3e134026, + 0xb8873288, 0x73dbe12d, 0x6eded195, 0xa5820230, 0x2316709e, + 0xe84aa33b, 0x1aca1375, 0xd196c0d0, 0x5702b27e, 0x9c5e61db, + 0x815b5163, 0x4a0782c6, 0xcc93f068, 0x07cf23cd, 0xf6999118, + 0x3dc542bd, 0xbb513013, 0x700de3b6, 0x6d08d30e, 0xa65400ab, + 0x20c07205, 0xeb9ca1a0, 0x11e81eb4, 0xdab4cd11, 0x5c20bfbf, + 0x977c6c1a, 0x8a795ca2, 0x41258f07, 0xc7b1fda9, 0x0ced2e0c, + 0xfdbb9cd9, 0x36e74f7c, 0xb0733dd2, 0x7b2fee77, 0x662adecf, + 0xad760d6a, 0x2be27fc4, 0xe0beac61, 0x123e1c2f, 0xd962cf8a, + 0x5ff6bd24, 0x94aa6e81, 0x89af5e39, 0x42f38d9c, 0xc467ff32, + 0x0f3b2c97, 0xfe6d9e42, 0x35314de7, 0xb3a53f49, 0x78f9ecec, + 0x65fcdc54, 0xaea00ff1, 0x28347d5f, 0xe368aefa, 0x16441b82, + 0xdd18c827, 0x5b8cba89, 0x90d0692c, 0x8dd55994, 0x46898a31, + 0xc01df89f, 0x0b412b3a, 0xfa1799ef, 0x314b4a4a, 0xb7df38e4, + 0x7c83eb41, 0x6186dbf9, 0xaada085c, 0x2c4e7af2, 0xe712a957, + 0x15921919, 0xdececabc, 0x585ab812, 0x93066bb7, 0x8e035b0f, + 0x455f88aa, 0xc3cbfa04, 0x089729a1, 0xf9c19b74, 0x329d48d1, + 0xb4093a7f, 0x7f55e9da, 0x6250d962, 0xa90c0ac7, 0x2f987869, + 0xe4c4abcc}, + {0x00000000, 0x3d6029b0, 0x7ac05360, 0x47a07ad0, 0xf580a6c0, + 0xc8e08f70, 0x8f40f5a0, 0xb220dc10, 0x30704bc1, 0x0d106271, + 0x4ab018a1, 0x77d03111, 0xc5f0ed01, 0xf890c4b1, 0xbf30be61, + 0x825097d1, 0x60e09782, 0x5d80be32, 0x1a20c4e2, 0x2740ed52, + 0x95603142, 0xa80018f2, 0xefa06222, 0xd2c04b92, 0x5090dc43, + 0x6df0f5f3, 0x2a508f23, 0x1730a693, 0xa5107a83, 0x98705333, + 0xdfd029e3, 0xe2b00053, 0xc1c12f04, 0xfca106b4, 0xbb017c64, + 0x866155d4, 0x344189c4, 0x0921a074, 0x4e81daa4, 0x73e1f314, + 0xf1b164c5, 0xccd14d75, 0x8b7137a5, 0xb6111e15, 0x0431c205, + 0x3951ebb5, 0x7ef19165, 0x4391b8d5, 0xa121b886, 0x9c419136, + 0xdbe1ebe6, 0xe681c256, 0x54a11e46, 0x69c137f6, 0x2e614d26, + 0x13016496, 0x9151f347, 0xac31daf7, 0xeb91a027, 0xd6f18997, + 0x64d15587, 0x59b17c37, 0x1e1106e7, 0x23712f57, 0x58f35849, + 0x659371f9, 0x22330b29, 0x1f532299, 0xad73fe89, 0x9013d739, + 0xd7b3ade9, 0xead38459, 0x68831388, 0x55e33a38, 0x124340e8, + 0x2f236958, 0x9d03b548, 0xa0639cf8, 0xe7c3e628, 0xdaa3cf98, + 0x3813cfcb, 0x0573e67b, 0x42d39cab, 0x7fb3b51b, 0xcd93690b, + 0xf0f340bb, 0xb7533a6b, 0x8a3313db, 0x0863840a, 0x3503adba, + 0x72a3d76a, 0x4fc3feda, 0xfde322ca, 0xc0830b7a, 0x872371aa, + 0xba43581a, 0x9932774d, 0xa4525efd, 0xe3f2242d, 0xde920d9d, + 0x6cb2d18d, 0x51d2f83d, 0x167282ed, 0x2b12ab5d, 0xa9423c8c, + 0x9422153c, 0xd3826fec, 0xeee2465c, 0x5cc29a4c, 0x61a2b3fc, + 0x2602c92c, 0x1b62e09c, 0xf9d2e0cf, 0xc4b2c97f, 0x8312b3af, + 0xbe729a1f, 0x0c52460f, 0x31326fbf, 0x7692156f, 0x4bf23cdf, + 0xc9a2ab0e, 0xf4c282be, 0xb362f86e, 0x8e02d1de, 0x3c220dce, + 0x0142247e, 0x46e25eae, 0x7b82771e, 0xb1e6b092, 0x8c869922, + 0xcb26e3f2, 0xf646ca42, 0x44661652, 0x79063fe2, 0x3ea64532, + 0x03c66c82, 0x8196fb53, 0xbcf6d2e3, 0xfb56a833, 0xc6368183, + 0x74165d93, 0x49767423, 0x0ed60ef3, 0x33b62743, 0xd1062710, + 0xec660ea0, 0xabc67470, 0x96a65dc0, 0x248681d0, 0x19e6a860, + 0x5e46d2b0, 0x6326fb00, 0xe1766cd1, 0xdc164561, 0x9bb63fb1, + 0xa6d61601, 0x14f6ca11, 0x2996e3a1, 0x6e369971, 0x5356b0c1, + 0x70279f96, 0x4d47b626, 0x0ae7ccf6, 0x3787e546, 0x85a73956, + 0xb8c710e6, 0xff676a36, 0xc2074386, 0x4057d457, 0x7d37fde7, + 0x3a978737, 0x07f7ae87, 0xb5d77297, 0x88b75b27, 0xcf1721f7, + 0xf2770847, 0x10c70814, 0x2da721a4, 0x6a075b74, 0x576772c4, + 0xe547aed4, 0xd8278764, 0x9f87fdb4, 0xa2e7d404, 0x20b743d5, + 0x1dd76a65, 0x5a7710b5, 0x67173905, 0xd537e515, 0xe857cca5, + 0xaff7b675, 0x92979fc5, 0xe915e8db, 0xd475c16b, 0x93d5bbbb, + 0xaeb5920b, 0x1c954e1b, 0x21f567ab, 0x66551d7b, 0x5b3534cb, + 0xd965a31a, 0xe4058aaa, 0xa3a5f07a, 0x9ec5d9ca, 0x2ce505da, + 0x11852c6a, 0x562556ba, 0x6b457f0a, 0x89f57f59, 0xb49556e9, + 0xf3352c39, 0xce550589, 0x7c75d999, 0x4115f029, 0x06b58af9, + 0x3bd5a349, 0xb9853498, 0x84e51d28, 0xc34567f8, 0xfe254e48, + 0x4c059258, 0x7165bbe8, 0x36c5c138, 0x0ba5e888, 0x28d4c7df, + 0x15b4ee6f, 0x521494bf, 0x6f74bd0f, 0xdd54611f, 0xe03448af, + 0xa794327f, 0x9af41bcf, 0x18a48c1e, 0x25c4a5ae, 0x6264df7e, + 0x5f04f6ce, 0xed242ade, 0xd044036e, 0x97e479be, 0xaa84500e, + 0x4834505d, 0x755479ed, 0x32f4033d, 0x0f942a8d, 0xbdb4f69d, + 0x80d4df2d, 0xc774a5fd, 0xfa148c4d, 0x78441b9c, 0x4524322c, + 0x028448fc, 0x3fe4614c, 0x8dc4bd5c, 0xb0a494ec, 0xf704ee3c, + 0xca64c78c}, + {0x00000000, 0xb8bc6765, 0xaa09c88b, 0x12b5afee, 0x8f629757, + 0x37def032, 0x256b5fdc, 0x9dd738b9, 0xc5b428ef, 0x7d084f8a, + 0x6fbde064, 0xd7018701, 0x4ad6bfb8, 0xf26ad8dd, 0xe0df7733, + 0x58631056, 0x5019579f, 0xe8a530fa, 0xfa109f14, 0x42acf871, + 0xdf7bc0c8, 0x67c7a7ad, 0x75720843, 0xcdce6f26, 0x95ad7f70, + 0x2d111815, 0x3fa4b7fb, 0x8718d09e, 0x1acfe827, 0xa2738f42, + 0xb0c620ac, 0x087a47c9, 0xa032af3e, 0x188ec85b, 0x0a3b67b5, + 0xb28700d0, 0x2f503869, 0x97ec5f0c, 0x8559f0e2, 0x3de59787, + 0x658687d1, 0xdd3ae0b4, 0xcf8f4f5a, 0x7733283f, 0xeae41086, + 0x525877e3, 0x40edd80d, 0xf851bf68, 0xf02bf8a1, 0x48979fc4, + 0x5a22302a, 0xe29e574f, 0x7f496ff6, 0xc7f50893, 0xd540a77d, + 0x6dfcc018, 0x359fd04e, 0x8d23b72b, 0x9f9618c5, 0x272a7fa0, + 0xbafd4719, 0x0241207c, 0x10f48f92, 0xa848e8f7, 0x9b14583d, + 0x23a83f58, 0x311d90b6, 0x89a1f7d3, 0x1476cf6a, 0xaccaa80f, + 0xbe7f07e1, 0x06c36084, 0x5ea070d2, 0xe61c17b7, 0xf4a9b859, + 0x4c15df3c, 0xd1c2e785, 0x697e80e0, 0x7bcb2f0e, 0xc377486b, + 0xcb0d0fa2, 0x73b168c7, 0x6104c729, 0xd9b8a04c, 0x446f98f5, + 0xfcd3ff90, 0xee66507e, 0x56da371b, 0x0eb9274d, 0xb6054028, + 0xa4b0efc6, 0x1c0c88a3, 0x81dbb01a, 0x3967d77f, 0x2bd27891, + 0x936e1ff4, 0x3b26f703, 0x839a9066, 0x912f3f88, 0x299358ed, + 0xb4446054, 0x0cf80731, 0x1e4da8df, 0xa6f1cfba, 0xfe92dfec, + 0x462eb889, 0x549b1767, 0xec277002, 0x71f048bb, 0xc94c2fde, + 0xdbf98030, 0x6345e755, 0x6b3fa09c, 0xd383c7f9, 0xc1366817, + 0x798a0f72, 0xe45d37cb, 0x5ce150ae, 0x4e54ff40, 0xf6e89825, + 0xae8b8873, 0x1637ef16, 0x048240f8, 0xbc3e279d, 0x21e91f24, + 0x99557841, 0x8be0d7af, 0x335cb0ca, 0xed59b63b, 0x55e5d15e, + 0x47507eb0, 0xffec19d5, 0x623b216c, 0xda874609, 0xc832e9e7, + 0x708e8e82, 0x28ed9ed4, 0x9051f9b1, 0x82e4565f, 0x3a58313a, + 0xa78f0983, 0x1f336ee6, 0x0d86c108, 0xb53aa66d, 0xbd40e1a4, + 0x05fc86c1, 0x1749292f, 0xaff54e4a, 0x322276f3, 0x8a9e1196, + 0x982bbe78, 0x2097d91d, 0x78f4c94b, 0xc048ae2e, 0xd2fd01c0, + 0x6a4166a5, 0xf7965e1c, 0x4f2a3979, 0x5d9f9697, 0xe523f1f2, + 0x4d6b1905, 0xf5d77e60, 0xe762d18e, 0x5fdeb6eb, 0xc2098e52, + 0x7ab5e937, 0x680046d9, 0xd0bc21bc, 0x88df31ea, 0x3063568f, + 0x22d6f961, 0x9a6a9e04, 0x07bda6bd, 0xbf01c1d8, 0xadb46e36, + 0x15080953, 0x1d724e9a, 0xa5ce29ff, 0xb77b8611, 0x0fc7e174, + 0x9210d9cd, 0x2aacbea8, 0x38191146, 0x80a57623, 0xd8c66675, + 0x607a0110, 0x72cfaefe, 0xca73c99b, 0x57a4f122, 0xef189647, + 0xfdad39a9, 0x45115ecc, 0x764dee06, 0xcef18963, 0xdc44268d, + 0x64f841e8, 0xf92f7951, 0x41931e34, 0x5326b1da, 0xeb9ad6bf, + 0xb3f9c6e9, 0x0b45a18c, 0x19f00e62, 0xa14c6907, 0x3c9b51be, + 0x842736db, 0x96929935, 0x2e2efe50, 0x2654b999, 0x9ee8defc, + 0x8c5d7112, 0x34e11677, 0xa9362ece, 0x118a49ab, 0x033fe645, + 0xbb838120, 0xe3e09176, 0x5b5cf613, 0x49e959fd, 0xf1553e98, + 0x6c820621, 0xd43e6144, 0xc68bceaa, 0x7e37a9cf, 0xd67f4138, + 0x6ec3265d, 0x7c7689b3, 0xc4caeed6, 0x591dd66f, 0xe1a1b10a, + 0xf3141ee4, 0x4ba87981, 0x13cb69d7, 0xab770eb2, 0xb9c2a15c, + 0x017ec639, 0x9ca9fe80, 0x241599e5, 0x36a0360b, 0x8e1c516e, + 0x866616a7, 0x3eda71c2, 0x2c6fde2c, 0x94d3b949, 0x090481f0, + 0xb1b8e695, 0xa30d497b, 0x1bb12e1e, 0x43d23e48, 0xfb6e592d, + 0xe9dbf6c3, 0x516791a6, 0xccb0a91f, 0x740cce7a, 0x66b96194, + 0xde0506f1}, + {0x00000000, 0x01c26a37, 0x0384d46e, 0x0246be59, 0x0709a8dc, + 0x06cbc2eb, 0x048d7cb2, 0x054f1685, 0x0e1351b8, 0x0fd13b8f, + 0x0d9785d6, 0x0c55efe1, 0x091af964, 0x08d89353, 0x0a9e2d0a, + 0x0b5c473d, 0x1c26a370, 0x1de4c947, 0x1fa2771e, 0x1e601d29, + 0x1b2f0bac, 0x1aed619b, 0x18abdfc2, 0x1969b5f5, 0x1235f2c8, + 0x13f798ff, 0x11b126a6, 0x10734c91, 0x153c5a14, 0x14fe3023, + 0x16b88e7a, 0x177ae44d, 0x384d46e0, 0x398f2cd7, 0x3bc9928e, + 0x3a0bf8b9, 0x3f44ee3c, 0x3e86840b, 0x3cc03a52, 0x3d025065, + 0x365e1758, 0x379c7d6f, 0x35dac336, 0x3418a901, 0x3157bf84, + 0x3095d5b3, 0x32d36bea, 0x331101dd, 0x246be590, 0x25a98fa7, + 0x27ef31fe, 0x262d5bc9, 0x23624d4c, 0x22a0277b, 0x20e69922, + 0x2124f315, 0x2a78b428, 0x2bbade1f, 0x29fc6046, 0x283e0a71, + 0x2d711cf4, 0x2cb376c3, 0x2ef5c89a, 0x2f37a2ad, 0x709a8dc0, + 0x7158e7f7, 0x731e59ae, 0x72dc3399, 0x7793251c, 0x76514f2b, + 0x7417f172, 0x75d59b45, 0x7e89dc78, 0x7f4bb64f, 0x7d0d0816, + 0x7ccf6221, 0x798074a4, 0x78421e93, 0x7a04a0ca, 0x7bc6cafd, + 0x6cbc2eb0, 0x6d7e4487, 0x6f38fade, 0x6efa90e9, 0x6bb5866c, + 0x6a77ec5b, 0x68315202, 0x69f33835, 0x62af7f08, 0x636d153f, + 0x612bab66, 0x60e9c151, 0x65a6d7d4, 0x6464bde3, 0x662203ba, + 0x67e0698d, 0x48d7cb20, 0x4915a117, 0x4b531f4e, 0x4a917579, + 0x4fde63fc, 0x4e1c09cb, 0x4c5ab792, 0x4d98dda5, 0x46c49a98, + 0x4706f0af, 0x45404ef6, 0x448224c1, 0x41cd3244, 0x400f5873, + 0x4249e62a, 0x438b8c1d, 0x54f16850, 0x55330267, 0x5775bc3e, + 0x56b7d609, 0x53f8c08c, 0x523aaabb, 0x507c14e2, 0x51be7ed5, + 0x5ae239e8, 0x5b2053df, 0x5966ed86, 0x58a487b1, 0x5deb9134, + 0x5c29fb03, 0x5e6f455a, 0x5fad2f6d, 0xe1351b80, 0xe0f771b7, + 0xe2b1cfee, 0xe373a5d9, 0xe63cb35c, 0xe7fed96b, 0xe5b86732, + 0xe47a0d05, 0xef264a38, 0xeee4200f, 0xeca29e56, 0xed60f461, + 0xe82fe2e4, 0xe9ed88d3, 0xebab368a, 0xea695cbd, 0xfd13b8f0, + 0xfcd1d2c7, 0xfe976c9e, 0xff5506a9, 0xfa1a102c, 0xfbd87a1b, + 0xf99ec442, 0xf85cae75, 0xf300e948, 0xf2c2837f, 0xf0843d26, + 0xf1465711, 0xf4094194, 0xf5cb2ba3, 0xf78d95fa, 0xf64fffcd, + 0xd9785d60, 0xd8ba3757, 0xdafc890e, 0xdb3ee339, 0xde71f5bc, + 0xdfb39f8b, 0xddf521d2, 0xdc374be5, 0xd76b0cd8, 0xd6a966ef, + 0xd4efd8b6, 0xd52db281, 0xd062a404, 0xd1a0ce33, 0xd3e6706a, + 0xd2241a5d, 0xc55efe10, 0xc49c9427, 0xc6da2a7e, 0xc7184049, + 0xc25756cc, 0xc3953cfb, 0xc1d382a2, 0xc011e895, 0xcb4dafa8, + 0xca8fc59f, 0xc8c97bc6, 0xc90b11f1, 0xcc440774, 0xcd866d43, + 0xcfc0d31a, 0xce02b92d, 0x91af9640, 0x906dfc77, 0x922b422e, + 0x93e92819, 0x96a63e9c, 0x976454ab, 0x9522eaf2, 0x94e080c5, + 0x9fbcc7f8, 0x9e7eadcf, 0x9c381396, 0x9dfa79a1, 0x98b56f24, + 0x99770513, 0x9b31bb4a, 0x9af3d17d, 0x8d893530, 0x8c4b5f07, + 0x8e0de15e, 0x8fcf8b69, 0x8a809dec, 0x8b42f7db, 0x89044982, + 0x88c623b5, 0x839a6488, 0x82580ebf, 0x801eb0e6, 0x81dcdad1, + 0x8493cc54, 0x8551a663, 0x8717183a, 0x86d5720d, 0xa9e2d0a0, + 0xa820ba97, 0xaa6604ce, 0xaba46ef9, 0xaeeb787c, 0xaf29124b, + 0xad6fac12, 0xacadc625, 0xa7f18118, 0xa633eb2f, 0xa4755576, + 0xa5b73f41, 0xa0f829c4, 0xa13a43f3, 0xa37cfdaa, 0xa2be979d, + 0xb5c473d0, 0xb40619e7, 0xb640a7be, 0xb782cd89, 0xb2cddb0c, + 0xb30fb13b, 0xb1490f62, 0xb08b6555, 0xbbd72268, 0xba15485f, + 0xb853f606, 0xb9919c31, 0xbcde8ab4, 0xbd1ce083, 0xbf5a5eda, + 0xbe9834ed}, + {0x00000000, 0x191b3141, 0x32366282, 0x2b2d53c3, 0x646cc504, + 0x7d77f445, 0x565aa786, 0x4f4196c7, 0xc8d98a08, 0xd1c2bb49, + 0xfaefe88a, 0xe3f4d9cb, 0xacb54f0c, 0xb5ae7e4d, 0x9e832d8e, + 0x87981ccf, 0x4ac21251, 0x53d92310, 0x78f470d3, 0x61ef4192, + 0x2eaed755, 0x37b5e614, 0x1c98b5d7, 0x05838496, 0x821b9859, + 0x9b00a918, 0xb02dfadb, 0xa936cb9a, 0xe6775d5d, 0xff6c6c1c, + 0xd4413fdf, 0xcd5a0e9e, 0x958424a2, 0x8c9f15e3, 0xa7b24620, + 0xbea97761, 0xf1e8e1a6, 0xe8f3d0e7, 0xc3de8324, 0xdac5b265, + 0x5d5daeaa, 0x44469feb, 0x6f6bcc28, 0x7670fd69, 0x39316bae, + 0x202a5aef, 0x0b07092c, 0x121c386d, 0xdf4636f3, 0xc65d07b2, + 0xed705471, 0xf46b6530, 0xbb2af3f7, 0xa231c2b6, 0x891c9175, + 0x9007a034, 0x179fbcfb, 0x0e848dba, 0x25a9de79, 0x3cb2ef38, + 0x73f379ff, 0x6ae848be, 0x41c51b7d, 0x58de2a3c, 0xf0794f05, + 0xe9627e44, 0xc24f2d87, 0xdb541cc6, 0x94158a01, 0x8d0ebb40, + 0xa623e883, 0xbf38d9c2, 0x38a0c50d, 0x21bbf44c, 0x0a96a78f, + 0x138d96ce, 0x5ccc0009, 0x45d73148, 0x6efa628b, 0x77e153ca, + 0xbabb5d54, 0xa3a06c15, 0x888d3fd6, 0x91960e97, 0xded79850, + 0xc7cca911, 0xece1fad2, 0xf5facb93, 0x7262d75c, 0x6b79e61d, + 0x4054b5de, 0x594f849f, 0x160e1258, 0x0f152319, 0x243870da, + 0x3d23419b, 0x65fd6ba7, 0x7ce65ae6, 0x57cb0925, 0x4ed03864, + 0x0191aea3, 0x188a9fe2, 0x33a7cc21, 0x2abcfd60, 0xad24e1af, + 0xb43fd0ee, 0x9f12832d, 0x8609b26c, 0xc94824ab, 0xd05315ea, + 0xfb7e4629, 0xe2657768, 0x2f3f79f6, 0x362448b7, 0x1d091b74, + 0x04122a35, 0x4b53bcf2, 0x52488db3, 0x7965de70, 0x607eef31, + 0xe7e6f3fe, 0xfefdc2bf, 0xd5d0917c, 0xcccba03d, 0x838a36fa, + 0x9a9107bb, 0xb1bc5478, 0xa8a76539, 0x3b83984b, 0x2298a90a, + 0x09b5fac9, 0x10aecb88, 0x5fef5d4f, 0x46f46c0e, 0x6dd93fcd, + 0x74c20e8c, 0xf35a1243, 0xea412302, 0xc16c70c1, 0xd8774180, + 0x9736d747, 0x8e2de606, 0xa500b5c5, 0xbc1b8484, 0x71418a1a, + 0x685abb5b, 0x4377e898, 0x5a6cd9d9, 0x152d4f1e, 0x0c367e5f, + 0x271b2d9c, 0x3e001cdd, 0xb9980012, 0xa0833153, 0x8bae6290, + 0x92b553d1, 0xddf4c516, 0xc4eff457, 0xefc2a794, 0xf6d996d5, + 0xae07bce9, 0xb71c8da8, 0x9c31de6b, 0x852aef2a, 0xca6b79ed, + 0xd37048ac, 0xf85d1b6f, 0xe1462a2e, 0x66de36e1, 0x7fc507a0, + 0x54e85463, 0x4df36522, 0x02b2f3e5, 0x1ba9c2a4, 0x30849167, + 0x299fa026, 0xe4c5aeb8, 0xfdde9ff9, 0xd6f3cc3a, 0xcfe8fd7b, + 0x80a96bbc, 0x99b25afd, 0xb29f093e, 0xab84387f, 0x2c1c24b0, + 0x350715f1, 0x1e2a4632, 0x07317773, 0x4870e1b4, 0x516bd0f5, + 0x7a468336, 0x635db277, 0xcbfad74e, 0xd2e1e60f, 0xf9ccb5cc, + 0xe0d7848d, 0xaf96124a, 0xb68d230b, 0x9da070c8, 0x84bb4189, + 0x03235d46, 0x1a386c07, 0x31153fc4, 0x280e0e85, 0x674f9842, + 0x7e54a903, 0x5579fac0, 0x4c62cb81, 0x8138c51f, 0x9823f45e, + 0xb30ea79d, 0xaa1596dc, 0xe554001b, 0xfc4f315a, 0xd7626299, + 0xce7953d8, 0x49e14f17, 0x50fa7e56, 0x7bd72d95, 0x62cc1cd4, + 0x2d8d8a13, 0x3496bb52, 0x1fbbe891, 0x06a0d9d0, 0x5e7ef3ec, + 0x4765c2ad, 0x6c48916e, 0x7553a02f, 0x3a1236e8, 0x230907a9, + 0x0824546a, 0x113f652b, 0x96a779e4, 0x8fbc48a5, 0xa4911b66, + 0xbd8a2a27, 0xf2cbbce0, 0xebd08da1, 0xc0fdde62, 0xd9e6ef23, + 0x14bce1bd, 0x0da7d0fc, 0x268a833f, 0x3f91b27e, 0x70d024b9, + 0x69cb15f8, 0x42e6463b, 0x5bfd777a, 0xdc656bb5, 0xc57e5af4, + 0xee530937, 0xf7483876, 0xb809aeb1, 0xa1129ff0, 0x8a3fcc33, + 0x9324fd72}, + {0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, + 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, + 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, + 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, + 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, + 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, + 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, + 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, + 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, + 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, + 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, + 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, + 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, + 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, + 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, + 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, + 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, + 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, + 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, + 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, + 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, + 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, + 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, + 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, + 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, + 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, + 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, + 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, + 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, + 0x2d02ef8d}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x0000000000000000, 0x9630077700000000, 0x2c610eee00000000, + 0xba51099900000000, 0x19c46d0700000000, 0x8ff46a7000000000, + 0x35a563e900000000, 0xa395649e00000000, 0x3288db0e00000000, + 0xa4b8dc7900000000, 0x1ee9d5e000000000, 0x88d9d29700000000, + 0x2b4cb60900000000, 0xbd7cb17e00000000, 0x072db8e700000000, + 0x911dbf9000000000, 0x6410b71d00000000, 0xf220b06a00000000, + 0x4871b9f300000000, 0xde41be8400000000, 0x7dd4da1a00000000, + 0xebe4dd6d00000000, 0x51b5d4f400000000, 0xc785d38300000000, + 0x56986c1300000000, 0xc0a86b6400000000, 0x7af962fd00000000, + 0xecc9658a00000000, 0x4f5c011400000000, 0xd96c066300000000, + 0x633d0ffa00000000, 0xf50d088d00000000, 0xc8206e3b00000000, + 0x5e10694c00000000, 0xe44160d500000000, 0x727167a200000000, + 0xd1e4033c00000000, 0x47d4044b00000000, 0xfd850dd200000000, + 0x6bb50aa500000000, 0xfaa8b53500000000, 0x6c98b24200000000, + 0xd6c9bbdb00000000, 0x40f9bcac00000000, 0xe36cd83200000000, + 0x755cdf4500000000, 0xcf0dd6dc00000000, 0x593dd1ab00000000, + 0xac30d92600000000, 0x3a00de5100000000, 0x8051d7c800000000, + 0x1661d0bf00000000, 0xb5f4b42100000000, 0x23c4b35600000000, + 0x9995bacf00000000, 0x0fa5bdb800000000, 0x9eb8022800000000, + 0x0888055f00000000, 0xb2d90cc600000000, 0x24e90bb100000000, + 0x877c6f2f00000000, 0x114c685800000000, 0xab1d61c100000000, + 0x3d2d66b600000000, 0x9041dc7600000000, 0x0671db0100000000, + 0xbc20d29800000000, 0x2a10d5ef00000000, 0x8985b17100000000, + 0x1fb5b60600000000, 0xa5e4bf9f00000000, 0x33d4b8e800000000, + 0xa2c9077800000000, 0x34f9000f00000000, 0x8ea8099600000000, + 0x18980ee100000000, 0xbb0d6a7f00000000, 0x2d3d6d0800000000, + 0x976c649100000000, 0x015c63e600000000, 0xf4516b6b00000000, + 0x62616c1c00000000, 0xd830658500000000, 0x4e0062f200000000, + 0xed95066c00000000, 0x7ba5011b00000000, 0xc1f4088200000000, + 0x57c40ff500000000, 0xc6d9b06500000000, 0x50e9b71200000000, + 0xeab8be8b00000000, 0x7c88b9fc00000000, 0xdf1ddd6200000000, + 0x492dda1500000000, 0xf37cd38c00000000, 0x654cd4fb00000000, + 0x5861b24d00000000, 0xce51b53a00000000, 0x7400bca300000000, + 0xe230bbd400000000, 0x41a5df4a00000000, 0xd795d83d00000000, + 0x6dc4d1a400000000, 0xfbf4d6d300000000, 0x6ae9694300000000, + 0xfcd96e3400000000, 0x468867ad00000000, 0xd0b860da00000000, + 0x732d044400000000, 0xe51d033300000000, 0x5f4c0aaa00000000, + 0xc97c0ddd00000000, 0x3c71055000000000, 0xaa41022700000000, + 0x10100bbe00000000, 0x86200cc900000000, 0x25b5685700000000, + 0xb3856f2000000000, 0x09d466b900000000, 0x9fe461ce00000000, + 0x0ef9de5e00000000, 0x98c9d92900000000, 0x2298d0b000000000, + 0xb4a8d7c700000000, 0x173db35900000000, 0x810db42e00000000, + 0x3b5cbdb700000000, 0xad6cbac000000000, 0x2083b8ed00000000, + 0xb6b3bf9a00000000, 0x0ce2b60300000000, 0x9ad2b17400000000, + 0x3947d5ea00000000, 0xaf77d29d00000000, 0x1526db0400000000, + 0x8316dc7300000000, 0x120b63e300000000, 0x843b649400000000, + 0x3e6a6d0d00000000, 0xa85a6a7a00000000, 0x0bcf0ee400000000, + 0x9dff099300000000, 0x27ae000a00000000, 0xb19e077d00000000, + 0x44930ff000000000, 0xd2a3088700000000, 0x68f2011e00000000, + 0xfec2066900000000, 0x5d5762f700000000, 0xcb67658000000000, + 0x71366c1900000000, 0xe7066b6e00000000, 0x761bd4fe00000000, + 0xe02bd38900000000, 0x5a7ada1000000000, 0xcc4add6700000000, + 0x6fdfb9f900000000, 0xf9efbe8e00000000, 0x43beb71700000000, + 0xd58eb06000000000, 0xe8a3d6d600000000, 0x7e93d1a100000000, + 0xc4c2d83800000000, 0x52f2df4f00000000, 0xf167bbd100000000, + 0x6757bca600000000, 0xdd06b53f00000000, 0x4b36b24800000000, + 0xda2b0dd800000000, 0x4c1b0aaf00000000, 0xf64a033600000000, + 0x607a044100000000, 0xc3ef60df00000000, 0x55df67a800000000, + 0xef8e6e3100000000, 0x79be694600000000, 0x8cb361cb00000000, + 0x1a8366bc00000000, 0xa0d26f2500000000, 0x36e2685200000000, + 0x95770ccc00000000, 0x03470bbb00000000, 0xb916022200000000, + 0x2f26055500000000, 0xbe3bbac500000000, 0x280bbdb200000000, + 0x925ab42b00000000, 0x046ab35c00000000, 0xa7ffd7c200000000, + 0x31cfd0b500000000, 0x8b9ed92c00000000, 0x1daede5b00000000, + 0xb0c2649b00000000, 0x26f263ec00000000, 0x9ca36a7500000000, + 0x0a936d0200000000, 0xa906099c00000000, 0x3f360eeb00000000, + 0x8567077200000000, 0x1357000500000000, 0x824abf9500000000, + 0x147ab8e200000000, 0xae2bb17b00000000, 0x381bb60c00000000, + 0x9b8ed29200000000, 0x0dbed5e500000000, 0xb7efdc7c00000000, + 0x21dfdb0b00000000, 0xd4d2d38600000000, 0x42e2d4f100000000, + 0xf8b3dd6800000000, 0x6e83da1f00000000, 0xcd16be8100000000, + 0x5b26b9f600000000, 0xe177b06f00000000, 0x7747b71800000000, + 0xe65a088800000000, 0x706a0fff00000000, 0xca3b066600000000, + 0x5c0b011100000000, 0xff9e658f00000000, 0x69ae62f800000000, + 0xd3ff6b6100000000, 0x45cf6c1600000000, 0x78e20aa000000000, + 0xeed20dd700000000, 0x5483044e00000000, 0xc2b3033900000000, + 0x612667a700000000, 0xf71660d000000000, 0x4d47694900000000, + 0xdb776e3e00000000, 0x4a6ad1ae00000000, 0xdc5ad6d900000000, + 0x660bdf4000000000, 0xf03bd83700000000, 0x53aebca900000000, + 0xc59ebbde00000000, 0x7fcfb24700000000, 0xe9ffb53000000000, + 0x1cf2bdbd00000000, 0x8ac2baca00000000, 0x3093b35300000000, + 0xa6a3b42400000000, 0x0536d0ba00000000, 0x9306d7cd00000000, + 0x2957de5400000000, 0xbf67d92300000000, 0x2e7a66b300000000, + 0xb84a61c400000000, 0x021b685d00000000, 0x942b6f2a00000000, + 0x37be0bb400000000, 0xa18e0cc300000000, 0x1bdf055a00000000, + 0x8def022d00000000}, + {0x0000000000000000, 0x41311b1900000000, 0x8262363200000000, + 0xc3532d2b00000000, 0x04c56c6400000000, 0x45f4777d00000000, + 0x86a75a5600000000, 0xc796414f00000000, 0x088ad9c800000000, + 0x49bbc2d100000000, 0x8ae8effa00000000, 0xcbd9f4e300000000, + 0x0c4fb5ac00000000, 0x4d7eaeb500000000, 0x8e2d839e00000000, + 0xcf1c988700000000, 0x5112c24a00000000, 0x1023d95300000000, + 0xd370f47800000000, 0x9241ef6100000000, 0x55d7ae2e00000000, + 0x14e6b53700000000, 0xd7b5981c00000000, 0x9684830500000000, + 0x59981b8200000000, 0x18a9009b00000000, 0xdbfa2db000000000, + 0x9acb36a900000000, 0x5d5d77e600000000, 0x1c6c6cff00000000, + 0xdf3f41d400000000, 0x9e0e5acd00000000, 0xa224849500000000, + 0xe3159f8c00000000, 0x2046b2a700000000, 0x6177a9be00000000, + 0xa6e1e8f100000000, 0xe7d0f3e800000000, 0x2483dec300000000, + 0x65b2c5da00000000, 0xaaae5d5d00000000, 0xeb9f464400000000, + 0x28cc6b6f00000000, 0x69fd707600000000, 0xae6b313900000000, + 0xef5a2a2000000000, 0x2c09070b00000000, 0x6d381c1200000000, + 0xf33646df00000000, 0xb2075dc600000000, 0x715470ed00000000, + 0x30656bf400000000, 0xf7f32abb00000000, 0xb6c231a200000000, + 0x75911c8900000000, 0x34a0079000000000, 0xfbbc9f1700000000, + 0xba8d840e00000000, 0x79dea92500000000, 0x38efb23c00000000, + 0xff79f37300000000, 0xbe48e86a00000000, 0x7d1bc54100000000, + 0x3c2ade5800000000, 0x054f79f000000000, 0x447e62e900000000, + 0x872d4fc200000000, 0xc61c54db00000000, 0x018a159400000000, + 0x40bb0e8d00000000, 0x83e823a600000000, 0xc2d938bf00000000, + 0x0dc5a03800000000, 0x4cf4bb2100000000, 0x8fa7960a00000000, + 0xce968d1300000000, 0x0900cc5c00000000, 0x4831d74500000000, + 0x8b62fa6e00000000, 0xca53e17700000000, 0x545dbbba00000000, + 0x156ca0a300000000, 0xd63f8d8800000000, 0x970e969100000000, + 0x5098d7de00000000, 0x11a9ccc700000000, 0xd2fae1ec00000000, + 0x93cbfaf500000000, 0x5cd7627200000000, 0x1de6796b00000000, + 0xdeb5544000000000, 0x9f844f5900000000, 0x58120e1600000000, + 0x1923150f00000000, 0xda70382400000000, 0x9b41233d00000000, + 0xa76bfd6500000000, 0xe65ae67c00000000, 0x2509cb5700000000, + 0x6438d04e00000000, 0xa3ae910100000000, 0xe29f8a1800000000, + 0x21cca73300000000, 0x60fdbc2a00000000, 0xafe124ad00000000, + 0xeed03fb400000000, 0x2d83129f00000000, 0x6cb2098600000000, + 0xab2448c900000000, 0xea1553d000000000, 0x29467efb00000000, + 0x687765e200000000, 0xf6793f2f00000000, 0xb748243600000000, + 0x741b091d00000000, 0x352a120400000000, 0xf2bc534b00000000, + 0xb38d485200000000, 0x70de657900000000, 0x31ef7e6000000000, + 0xfef3e6e700000000, 0xbfc2fdfe00000000, 0x7c91d0d500000000, + 0x3da0cbcc00000000, 0xfa368a8300000000, 0xbb07919a00000000, + 0x7854bcb100000000, 0x3965a7a800000000, 0x4b98833b00000000, + 0x0aa9982200000000, 0xc9fab50900000000, 0x88cbae1000000000, + 0x4f5def5f00000000, 0x0e6cf44600000000, 0xcd3fd96d00000000, + 0x8c0ec27400000000, 0x43125af300000000, 0x022341ea00000000, + 0xc1706cc100000000, 0x804177d800000000, 0x47d7369700000000, + 0x06e62d8e00000000, 0xc5b500a500000000, 0x84841bbc00000000, + 0x1a8a417100000000, 0x5bbb5a6800000000, 0x98e8774300000000, + 0xd9d96c5a00000000, 0x1e4f2d1500000000, 0x5f7e360c00000000, + 0x9c2d1b2700000000, 0xdd1c003e00000000, 0x120098b900000000, + 0x533183a000000000, 0x9062ae8b00000000, 0xd153b59200000000, + 0x16c5f4dd00000000, 0x57f4efc400000000, 0x94a7c2ef00000000, + 0xd596d9f600000000, 0xe9bc07ae00000000, 0xa88d1cb700000000, + 0x6bde319c00000000, 0x2aef2a8500000000, 0xed796bca00000000, + 0xac4870d300000000, 0x6f1b5df800000000, 0x2e2a46e100000000, + 0xe136de6600000000, 0xa007c57f00000000, 0x6354e85400000000, + 0x2265f34d00000000, 0xe5f3b20200000000, 0xa4c2a91b00000000, + 0x6791843000000000, 0x26a09f2900000000, 0xb8aec5e400000000, + 0xf99fdefd00000000, 0x3accf3d600000000, 0x7bfde8cf00000000, + 0xbc6ba98000000000, 0xfd5ab29900000000, 0x3e099fb200000000, + 0x7f3884ab00000000, 0xb0241c2c00000000, 0xf115073500000000, + 0x32462a1e00000000, 0x7377310700000000, 0xb4e1704800000000, + 0xf5d06b5100000000, 0x3683467a00000000, 0x77b25d6300000000, + 0x4ed7facb00000000, 0x0fe6e1d200000000, 0xccb5ccf900000000, + 0x8d84d7e000000000, 0x4a1296af00000000, 0x0b238db600000000, + 0xc870a09d00000000, 0x8941bb8400000000, 0x465d230300000000, + 0x076c381a00000000, 0xc43f153100000000, 0x850e0e2800000000, + 0x42984f6700000000, 0x03a9547e00000000, 0xc0fa795500000000, + 0x81cb624c00000000, 0x1fc5388100000000, 0x5ef4239800000000, + 0x9da70eb300000000, 0xdc9615aa00000000, 0x1b0054e500000000, + 0x5a314ffc00000000, 0x996262d700000000, 0xd85379ce00000000, + 0x174fe14900000000, 0x567efa5000000000, 0x952dd77b00000000, + 0xd41ccc6200000000, 0x138a8d2d00000000, 0x52bb963400000000, + 0x91e8bb1f00000000, 0xd0d9a00600000000, 0xecf37e5e00000000, + 0xadc2654700000000, 0x6e91486c00000000, 0x2fa0537500000000, + 0xe836123a00000000, 0xa907092300000000, 0x6a54240800000000, + 0x2b653f1100000000, 0xe479a79600000000, 0xa548bc8f00000000, + 0x661b91a400000000, 0x272a8abd00000000, 0xe0bccbf200000000, + 0xa18dd0eb00000000, 0x62defdc000000000, 0x23efe6d900000000, + 0xbde1bc1400000000, 0xfcd0a70d00000000, 0x3f838a2600000000, + 0x7eb2913f00000000, 0xb924d07000000000, 0xf815cb6900000000, + 0x3b46e64200000000, 0x7a77fd5b00000000, 0xb56b65dc00000000, + 0xf45a7ec500000000, 0x370953ee00000000, 0x763848f700000000, + 0xb1ae09b800000000, 0xf09f12a100000000, 0x33cc3f8a00000000, + 0x72fd249300000000}, + {0x0000000000000000, 0x376ac20100000000, 0x6ed4840300000000, + 0x59be460200000000, 0xdca8090700000000, 0xebc2cb0600000000, + 0xb27c8d0400000000, 0x85164f0500000000, 0xb851130e00000000, + 0x8f3bd10f00000000, 0xd685970d00000000, 0xe1ef550c00000000, + 0x64f91a0900000000, 0x5393d80800000000, 0x0a2d9e0a00000000, + 0x3d475c0b00000000, 0x70a3261c00000000, 0x47c9e41d00000000, + 0x1e77a21f00000000, 0x291d601e00000000, 0xac0b2f1b00000000, + 0x9b61ed1a00000000, 0xc2dfab1800000000, 0xf5b5691900000000, + 0xc8f2351200000000, 0xff98f71300000000, 0xa626b11100000000, + 0x914c731000000000, 0x145a3c1500000000, 0x2330fe1400000000, + 0x7a8eb81600000000, 0x4de47a1700000000, 0xe0464d3800000000, + 0xd72c8f3900000000, 0x8e92c93b00000000, 0xb9f80b3a00000000, + 0x3cee443f00000000, 0x0b84863e00000000, 0x523ac03c00000000, + 0x6550023d00000000, 0x58175e3600000000, 0x6f7d9c3700000000, + 0x36c3da3500000000, 0x01a9183400000000, 0x84bf573100000000, + 0xb3d5953000000000, 0xea6bd33200000000, 0xdd01113300000000, + 0x90e56b2400000000, 0xa78fa92500000000, 0xfe31ef2700000000, + 0xc95b2d2600000000, 0x4c4d622300000000, 0x7b27a02200000000, + 0x2299e62000000000, 0x15f3242100000000, 0x28b4782a00000000, + 0x1fdeba2b00000000, 0x4660fc2900000000, 0x710a3e2800000000, + 0xf41c712d00000000, 0xc376b32c00000000, 0x9ac8f52e00000000, + 0xada2372f00000000, 0xc08d9a7000000000, 0xf7e7587100000000, + 0xae591e7300000000, 0x9933dc7200000000, 0x1c25937700000000, + 0x2b4f517600000000, 0x72f1177400000000, 0x459bd57500000000, + 0x78dc897e00000000, 0x4fb64b7f00000000, 0x16080d7d00000000, + 0x2162cf7c00000000, 0xa474807900000000, 0x931e427800000000, + 0xcaa0047a00000000, 0xfdcac67b00000000, 0xb02ebc6c00000000, + 0x87447e6d00000000, 0xdefa386f00000000, 0xe990fa6e00000000, + 0x6c86b56b00000000, 0x5bec776a00000000, 0x0252316800000000, + 0x3538f36900000000, 0x087faf6200000000, 0x3f156d6300000000, + 0x66ab2b6100000000, 0x51c1e96000000000, 0xd4d7a66500000000, + 0xe3bd646400000000, 0xba03226600000000, 0x8d69e06700000000, + 0x20cbd74800000000, 0x17a1154900000000, 0x4e1f534b00000000, + 0x7975914a00000000, 0xfc63de4f00000000, 0xcb091c4e00000000, + 0x92b75a4c00000000, 0xa5dd984d00000000, 0x989ac44600000000, + 0xaff0064700000000, 0xf64e404500000000, 0xc124824400000000, + 0x4432cd4100000000, 0x73580f4000000000, 0x2ae6494200000000, + 0x1d8c8b4300000000, 0x5068f15400000000, 0x6702335500000000, + 0x3ebc755700000000, 0x09d6b75600000000, 0x8cc0f85300000000, + 0xbbaa3a5200000000, 0xe2147c5000000000, 0xd57ebe5100000000, + 0xe839e25a00000000, 0xdf53205b00000000, 0x86ed665900000000, + 0xb187a45800000000, 0x3491eb5d00000000, 0x03fb295c00000000, + 0x5a456f5e00000000, 0x6d2fad5f00000000, 0x801b35e100000000, + 0xb771f7e000000000, 0xeecfb1e200000000, 0xd9a573e300000000, + 0x5cb33ce600000000, 0x6bd9fee700000000, 0x3267b8e500000000, + 0x050d7ae400000000, 0x384a26ef00000000, 0x0f20e4ee00000000, + 0x569ea2ec00000000, 0x61f460ed00000000, 0xe4e22fe800000000, + 0xd388ede900000000, 0x8a36abeb00000000, 0xbd5c69ea00000000, + 0xf0b813fd00000000, 0xc7d2d1fc00000000, 0x9e6c97fe00000000, + 0xa90655ff00000000, 0x2c101afa00000000, 0x1b7ad8fb00000000, + 0x42c49ef900000000, 0x75ae5cf800000000, 0x48e900f300000000, + 0x7f83c2f200000000, 0x263d84f000000000, 0x115746f100000000, + 0x944109f400000000, 0xa32bcbf500000000, 0xfa958df700000000, + 0xcdff4ff600000000, 0x605d78d900000000, 0x5737bad800000000, + 0x0e89fcda00000000, 0x39e33edb00000000, 0xbcf571de00000000, + 0x8b9fb3df00000000, 0xd221f5dd00000000, 0xe54b37dc00000000, + 0xd80c6bd700000000, 0xef66a9d600000000, 0xb6d8efd400000000, + 0x81b22dd500000000, 0x04a462d000000000, 0x33cea0d100000000, + 0x6a70e6d300000000, 0x5d1a24d200000000, 0x10fe5ec500000000, + 0x27949cc400000000, 0x7e2adac600000000, 0x494018c700000000, + 0xcc5657c200000000, 0xfb3c95c300000000, 0xa282d3c100000000, + 0x95e811c000000000, 0xa8af4dcb00000000, 0x9fc58fca00000000, + 0xc67bc9c800000000, 0xf1110bc900000000, 0x740744cc00000000, + 0x436d86cd00000000, 0x1ad3c0cf00000000, 0x2db902ce00000000, + 0x4096af9100000000, 0x77fc6d9000000000, 0x2e422b9200000000, + 0x1928e99300000000, 0x9c3ea69600000000, 0xab54649700000000, + 0xf2ea229500000000, 0xc580e09400000000, 0xf8c7bc9f00000000, + 0xcfad7e9e00000000, 0x9613389c00000000, 0xa179fa9d00000000, + 0x246fb59800000000, 0x1305779900000000, 0x4abb319b00000000, + 0x7dd1f39a00000000, 0x3035898d00000000, 0x075f4b8c00000000, + 0x5ee10d8e00000000, 0x698bcf8f00000000, 0xec9d808a00000000, + 0xdbf7428b00000000, 0x8249048900000000, 0xb523c68800000000, + 0x88649a8300000000, 0xbf0e588200000000, 0xe6b01e8000000000, + 0xd1dadc8100000000, 0x54cc938400000000, 0x63a6518500000000, + 0x3a18178700000000, 0x0d72d58600000000, 0xa0d0e2a900000000, + 0x97ba20a800000000, 0xce0466aa00000000, 0xf96ea4ab00000000, + 0x7c78ebae00000000, 0x4b1229af00000000, 0x12ac6fad00000000, + 0x25c6adac00000000, 0x1881f1a700000000, 0x2feb33a600000000, + 0x765575a400000000, 0x413fb7a500000000, 0xc429f8a000000000, + 0xf3433aa100000000, 0xaafd7ca300000000, 0x9d97bea200000000, + 0xd073c4b500000000, 0xe71906b400000000, 0xbea740b600000000, + 0x89cd82b700000000, 0x0cdbcdb200000000, 0x3bb10fb300000000, + 0x620f49b100000000, 0x55658bb000000000, 0x6822d7bb00000000, + 0x5f4815ba00000000, 0x06f653b800000000, 0x319c91b900000000, + 0xb48adebc00000000, 0x83e01cbd00000000, 0xda5e5abf00000000, + 0xed3498be00000000}, + {0x0000000000000000, 0x6567bcb800000000, 0x8bc809aa00000000, + 0xeeafb51200000000, 0x5797628f00000000, 0x32f0de3700000000, + 0xdc5f6b2500000000, 0xb938d79d00000000, 0xef28b4c500000000, + 0x8a4f087d00000000, 0x64e0bd6f00000000, 0x018701d700000000, + 0xb8bfd64a00000000, 0xddd86af200000000, 0x3377dfe000000000, + 0x5610635800000000, 0x9f57195000000000, 0xfa30a5e800000000, + 0x149f10fa00000000, 0x71f8ac4200000000, 0xc8c07bdf00000000, + 0xada7c76700000000, 0x4308727500000000, 0x266fcecd00000000, + 0x707fad9500000000, 0x1518112d00000000, 0xfbb7a43f00000000, + 0x9ed0188700000000, 0x27e8cf1a00000000, 0x428f73a200000000, + 0xac20c6b000000000, 0xc9477a0800000000, 0x3eaf32a000000000, + 0x5bc88e1800000000, 0xb5673b0a00000000, 0xd00087b200000000, + 0x6938502f00000000, 0x0c5fec9700000000, 0xe2f0598500000000, + 0x8797e53d00000000, 0xd187866500000000, 0xb4e03add00000000, + 0x5a4f8fcf00000000, 0x3f28337700000000, 0x8610e4ea00000000, + 0xe377585200000000, 0x0dd8ed4000000000, 0x68bf51f800000000, + 0xa1f82bf000000000, 0xc49f974800000000, 0x2a30225a00000000, + 0x4f579ee200000000, 0xf66f497f00000000, 0x9308f5c700000000, + 0x7da740d500000000, 0x18c0fc6d00000000, 0x4ed09f3500000000, + 0x2bb7238d00000000, 0xc518969f00000000, 0xa07f2a2700000000, + 0x1947fdba00000000, 0x7c20410200000000, 0x928ff41000000000, + 0xf7e848a800000000, 0x3d58149b00000000, 0x583fa82300000000, + 0xb6901d3100000000, 0xd3f7a18900000000, 0x6acf761400000000, + 0x0fa8caac00000000, 0xe1077fbe00000000, 0x8460c30600000000, + 0xd270a05e00000000, 0xb7171ce600000000, 0x59b8a9f400000000, + 0x3cdf154c00000000, 0x85e7c2d100000000, 0xe0807e6900000000, + 0x0e2fcb7b00000000, 0x6b4877c300000000, 0xa20f0dcb00000000, + 0xc768b17300000000, 0x29c7046100000000, 0x4ca0b8d900000000, + 0xf5986f4400000000, 0x90ffd3fc00000000, 0x7e5066ee00000000, + 0x1b37da5600000000, 0x4d27b90e00000000, 0x284005b600000000, + 0xc6efb0a400000000, 0xa3880c1c00000000, 0x1ab0db8100000000, + 0x7fd7673900000000, 0x9178d22b00000000, 0xf41f6e9300000000, + 0x03f7263b00000000, 0x66909a8300000000, 0x883f2f9100000000, + 0xed58932900000000, 0x546044b400000000, 0x3107f80c00000000, + 0xdfa84d1e00000000, 0xbacff1a600000000, 0xecdf92fe00000000, + 0x89b82e4600000000, 0x67179b5400000000, 0x027027ec00000000, + 0xbb48f07100000000, 0xde2f4cc900000000, 0x3080f9db00000000, + 0x55e7456300000000, 0x9ca03f6b00000000, 0xf9c783d300000000, + 0x176836c100000000, 0x720f8a7900000000, 0xcb375de400000000, + 0xae50e15c00000000, 0x40ff544e00000000, 0x2598e8f600000000, + 0x73888bae00000000, 0x16ef371600000000, 0xf840820400000000, + 0x9d273ebc00000000, 0x241fe92100000000, 0x4178559900000000, + 0xafd7e08b00000000, 0xcab05c3300000000, 0x3bb659ed00000000, + 0x5ed1e55500000000, 0xb07e504700000000, 0xd519ecff00000000, + 0x6c213b6200000000, 0x094687da00000000, 0xe7e932c800000000, + 0x828e8e7000000000, 0xd49eed2800000000, 0xb1f9519000000000, + 0x5f56e48200000000, 0x3a31583a00000000, 0x83098fa700000000, + 0xe66e331f00000000, 0x08c1860d00000000, 0x6da63ab500000000, + 0xa4e140bd00000000, 0xc186fc0500000000, 0x2f29491700000000, + 0x4a4ef5af00000000, 0xf376223200000000, 0x96119e8a00000000, + 0x78be2b9800000000, 0x1dd9972000000000, 0x4bc9f47800000000, + 0x2eae48c000000000, 0xc001fdd200000000, 0xa566416a00000000, + 0x1c5e96f700000000, 0x79392a4f00000000, 0x97969f5d00000000, + 0xf2f123e500000000, 0x05196b4d00000000, 0x607ed7f500000000, + 0x8ed162e700000000, 0xebb6de5f00000000, 0x528e09c200000000, + 0x37e9b57a00000000, 0xd946006800000000, 0xbc21bcd000000000, + 0xea31df8800000000, 0x8f56633000000000, 0x61f9d62200000000, + 0x049e6a9a00000000, 0xbda6bd0700000000, 0xd8c101bf00000000, + 0x366eb4ad00000000, 0x5309081500000000, 0x9a4e721d00000000, + 0xff29cea500000000, 0x11867bb700000000, 0x74e1c70f00000000, + 0xcdd9109200000000, 0xa8beac2a00000000, 0x4611193800000000, + 0x2376a58000000000, 0x7566c6d800000000, 0x10017a6000000000, + 0xfeaecf7200000000, 0x9bc973ca00000000, 0x22f1a45700000000, + 0x479618ef00000000, 0xa939adfd00000000, 0xcc5e114500000000, + 0x06ee4d7600000000, 0x6389f1ce00000000, 0x8d2644dc00000000, + 0xe841f86400000000, 0x51792ff900000000, 0x341e934100000000, + 0xdab1265300000000, 0xbfd69aeb00000000, 0xe9c6f9b300000000, + 0x8ca1450b00000000, 0x620ef01900000000, 0x07694ca100000000, + 0xbe519b3c00000000, 0xdb36278400000000, 0x3599929600000000, + 0x50fe2e2e00000000, 0x99b9542600000000, 0xfcdee89e00000000, + 0x12715d8c00000000, 0x7716e13400000000, 0xce2e36a900000000, + 0xab498a1100000000, 0x45e63f0300000000, 0x208183bb00000000, + 0x7691e0e300000000, 0x13f65c5b00000000, 0xfd59e94900000000, + 0x983e55f100000000, 0x2106826c00000000, 0x44613ed400000000, + 0xaace8bc600000000, 0xcfa9377e00000000, 0x38417fd600000000, + 0x5d26c36e00000000, 0xb389767c00000000, 0xd6eecac400000000, + 0x6fd61d5900000000, 0x0ab1a1e100000000, 0xe41e14f300000000, + 0x8179a84b00000000, 0xd769cb1300000000, 0xb20e77ab00000000, + 0x5ca1c2b900000000, 0x39c67e0100000000, 0x80fea99c00000000, + 0xe599152400000000, 0x0b36a03600000000, 0x6e511c8e00000000, + 0xa716668600000000, 0xc271da3e00000000, 0x2cde6f2c00000000, + 0x49b9d39400000000, 0xf081040900000000, 0x95e6b8b100000000, + 0x7b490da300000000, 0x1e2eb11b00000000, 0x483ed24300000000, + 0x2d596efb00000000, 0xc3f6dbe900000000, 0xa691675100000000, + 0x1fa9b0cc00000000, 0x7ace0c7400000000, 0x9461b96600000000, + 0xf10605de00000000}, + {0x0000000000000000, 0xb029603d00000000, 0x6053c07a00000000, + 0xd07aa04700000000, 0xc0a680f500000000, 0x708fe0c800000000, + 0xa0f5408f00000000, 0x10dc20b200000000, 0xc14b703000000000, + 0x7162100d00000000, 0xa118b04a00000000, 0x1131d07700000000, + 0x01edf0c500000000, 0xb1c490f800000000, 0x61be30bf00000000, + 0xd197508200000000, 0x8297e06000000000, 0x32be805d00000000, + 0xe2c4201a00000000, 0x52ed402700000000, 0x4231609500000000, + 0xf21800a800000000, 0x2262a0ef00000000, 0x924bc0d200000000, + 0x43dc905000000000, 0xf3f5f06d00000000, 0x238f502a00000000, + 0x93a6301700000000, 0x837a10a500000000, 0x3353709800000000, + 0xe329d0df00000000, 0x5300b0e200000000, 0x042fc1c100000000, + 0xb406a1fc00000000, 0x647c01bb00000000, 0xd455618600000000, + 0xc489413400000000, 0x74a0210900000000, 0xa4da814e00000000, + 0x14f3e17300000000, 0xc564b1f100000000, 0x754dd1cc00000000, + 0xa537718b00000000, 0x151e11b600000000, 0x05c2310400000000, + 0xb5eb513900000000, 0x6591f17e00000000, 0xd5b8914300000000, + 0x86b821a100000000, 0x3691419c00000000, 0xe6ebe1db00000000, + 0x56c281e600000000, 0x461ea15400000000, 0xf637c16900000000, + 0x264d612e00000000, 0x9664011300000000, 0x47f3519100000000, + 0xf7da31ac00000000, 0x27a091eb00000000, 0x9789f1d600000000, + 0x8755d16400000000, 0x377cb15900000000, 0xe706111e00000000, + 0x572f712300000000, 0x4958f35800000000, 0xf971936500000000, + 0x290b332200000000, 0x9922531f00000000, 0x89fe73ad00000000, + 0x39d7139000000000, 0xe9adb3d700000000, 0x5984d3ea00000000, + 0x8813836800000000, 0x383ae35500000000, 0xe840431200000000, + 0x5869232f00000000, 0x48b5039d00000000, 0xf89c63a000000000, + 0x28e6c3e700000000, 0x98cfa3da00000000, 0xcbcf133800000000, + 0x7be6730500000000, 0xab9cd34200000000, 0x1bb5b37f00000000, + 0x0b6993cd00000000, 0xbb40f3f000000000, 0x6b3a53b700000000, + 0xdb13338a00000000, 0x0a84630800000000, 0xbaad033500000000, + 0x6ad7a37200000000, 0xdafec34f00000000, 0xca22e3fd00000000, + 0x7a0b83c000000000, 0xaa71238700000000, 0x1a5843ba00000000, + 0x4d77329900000000, 0xfd5e52a400000000, 0x2d24f2e300000000, + 0x9d0d92de00000000, 0x8dd1b26c00000000, 0x3df8d25100000000, + 0xed82721600000000, 0x5dab122b00000000, 0x8c3c42a900000000, + 0x3c15229400000000, 0xec6f82d300000000, 0x5c46e2ee00000000, + 0x4c9ac25c00000000, 0xfcb3a26100000000, 0x2cc9022600000000, + 0x9ce0621b00000000, 0xcfe0d2f900000000, 0x7fc9b2c400000000, + 0xafb3128300000000, 0x1f9a72be00000000, 0x0f46520c00000000, + 0xbf6f323100000000, 0x6f15927600000000, 0xdf3cf24b00000000, + 0x0eaba2c900000000, 0xbe82c2f400000000, 0x6ef862b300000000, + 0xded1028e00000000, 0xce0d223c00000000, 0x7e24420100000000, + 0xae5ee24600000000, 0x1e77827b00000000, 0x92b0e6b100000000, + 0x2299868c00000000, 0xf2e326cb00000000, 0x42ca46f600000000, + 0x5216664400000000, 0xe23f067900000000, 0x3245a63e00000000, + 0x826cc60300000000, 0x53fb968100000000, 0xe3d2f6bc00000000, + 0x33a856fb00000000, 0x838136c600000000, 0x935d167400000000, + 0x2374764900000000, 0xf30ed60e00000000, 0x4327b63300000000, + 0x102706d100000000, 0xa00e66ec00000000, 0x7074c6ab00000000, + 0xc05da69600000000, 0xd081862400000000, 0x60a8e61900000000, + 0xb0d2465e00000000, 0x00fb266300000000, 0xd16c76e100000000, + 0x614516dc00000000, 0xb13fb69b00000000, 0x0116d6a600000000, + 0x11caf61400000000, 0xa1e3962900000000, 0x7199366e00000000, + 0xc1b0565300000000, 0x969f277000000000, 0x26b6474d00000000, + 0xf6cce70a00000000, 0x46e5873700000000, 0x5639a78500000000, + 0xe610c7b800000000, 0x366a67ff00000000, 0x864307c200000000, + 0x57d4574000000000, 0xe7fd377d00000000, 0x3787973a00000000, + 0x87aef70700000000, 0x9772d7b500000000, 0x275bb78800000000, + 0xf72117cf00000000, 0x470877f200000000, 0x1408c71000000000, + 0xa421a72d00000000, 0x745b076a00000000, 0xc472675700000000, + 0xd4ae47e500000000, 0x648727d800000000, 0xb4fd879f00000000, + 0x04d4e7a200000000, 0xd543b72000000000, 0x656ad71d00000000, + 0xb510775a00000000, 0x0539176700000000, 0x15e537d500000000, + 0xa5cc57e800000000, 0x75b6f7af00000000, 0xc59f979200000000, + 0xdbe815e900000000, 0x6bc175d400000000, 0xbbbbd59300000000, + 0x0b92b5ae00000000, 0x1b4e951c00000000, 0xab67f52100000000, + 0x7b1d556600000000, 0xcb34355b00000000, 0x1aa365d900000000, + 0xaa8a05e400000000, 0x7af0a5a300000000, 0xcad9c59e00000000, + 0xda05e52c00000000, 0x6a2c851100000000, 0xba56255600000000, + 0x0a7f456b00000000, 0x597ff58900000000, 0xe95695b400000000, + 0x392c35f300000000, 0x890555ce00000000, 0x99d9757c00000000, + 0x29f0154100000000, 0xf98ab50600000000, 0x49a3d53b00000000, + 0x983485b900000000, 0x281de58400000000, 0xf86745c300000000, + 0x484e25fe00000000, 0x5892054c00000000, 0xe8bb657100000000, + 0x38c1c53600000000, 0x88e8a50b00000000, 0xdfc7d42800000000, + 0x6feeb41500000000, 0xbf94145200000000, 0x0fbd746f00000000, + 0x1f6154dd00000000, 0xaf4834e000000000, 0x7f3294a700000000, + 0xcf1bf49a00000000, 0x1e8ca41800000000, 0xaea5c42500000000, + 0x7edf646200000000, 0xcef6045f00000000, 0xde2a24ed00000000, + 0x6e0344d000000000, 0xbe79e49700000000, 0x0e5084aa00000000, + 0x5d50344800000000, 0xed79547500000000, 0x3d03f43200000000, + 0x8d2a940f00000000, 0x9df6b4bd00000000, 0x2ddfd48000000000, + 0xfda574c700000000, 0x4d8c14fa00000000, 0x9c1b447800000000, + 0x2c32244500000000, 0xfc48840200000000, 0x4c61e43f00000000, + 0x5cbdc48d00000000, 0xec94a4b000000000, 0x3cee04f700000000, + 0x8cc764ca00000000}, + {0x0000000000000000, 0xa5d35ccb00000000, 0x0ba1c84d00000000, + 0xae72948600000000, 0x1642919b00000000, 0xb391cd5000000000, + 0x1de359d600000000, 0xb830051d00000000, 0x6d8253ec00000000, + 0xc8510f2700000000, 0x66239ba100000000, 0xc3f0c76a00000000, + 0x7bc0c27700000000, 0xde139ebc00000000, 0x70610a3a00000000, + 0xd5b256f100000000, 0x9b02d60300000000, 0x3ed18ac800000000, + 0x90a31e4e00000000, 0x3570428500000000, 0x8d40479800000000, + 0x28931b5300000000, 0x86e18fd500000000, 0x2332d31e00000000, + 0xf68085ef00000000, 0x5353d92400000000, 0xfd214da200000000, + 0x58f2116900000000, 0xe0c2147400000000, 0x451148bf00000000, + 0xeb63dc3900000000, 0x4eb080f200000000, 0x3605ac0700000000, + 0x93d6f0cc00000000, 0x3da4644a00000000, 0x9877388100000000, + 0x20473d9c00000000, 0x8594615700000000, 0x2be6f5d100000000, + 0x8e35a91a00000000, 0x5b87ffeb00000000, 0xfe54a32000000000, + 0x502637a600000000, 0xf5f56b6d00000000, 0x4dc56e7000000000, + 0xe81632bb00000000, 0x4664a63d00000000, 0xe3b7faf600000000, + 0xad077a0400000000, 0x08d426cf00000000, 0xa6a6b24900000000, + 0x0375ee8200000000, 0xbb45eb9f00000000, 0x1e96b75400000000, + 0xb0e423d200000000, 0x15377f1900000000, 0xc08529e800000000, + 0x6556752300000000, 0xcb24e1a500000000, 0x6ef7bd6e00000000, + 0xd6c7b87300000000, 0x7314e4b800000000, 0xdd66703e00000000, + 0x78b52cf500000000, 0x6c0a580f00000000, 0xc9d904c400000000, + 0x67ab904200000000, 0xc278cc8900000000, 0x7a48c99400000000, + 0xdf9b955f00000000, 0x71e901d900000000, 0xd43a5d1200000000, + 0x01880be300000000, 0xa45b572800000000, 0x0a29c3ae00000000, + 0xaffa9f6500000000, 0x17ca9a7800000000, 0xb219c6b300000000, + 0x1c6b523500000000, 0xb9b80efe00000000, 0xf7088e0c00000000, + 0x52dbd2c700000000, 0xfca9464100000000, 0x597a1a8a00000000, + 0xe14a1f9700000000, 0x4499435c00000000, 0xeaebd7da00000000, + 0x4f388b1100000000, 0x9a8adde000000000, 0x3f59812b00000000, + 0x912b15ad00000000, 0x34f8496600000000, 0x8cc84c7b00000000, + 0x291b10b000000000, 0x8769843600000000, 0x22bad8fd00000000, + 0x5a0ff40800000000, 0xffdca8c300000000, 0x51ae3c4500000000, + 0xf47d608e00000000, 0x4c4d659300000000, 0xe99e395800000000, + 0x47ecadde00000000, 0xe23ff11500000000, 0x378da7e400000000, + 0x925efb2f00000000, 0x3c2c6fa900000000, 0x99ff336200000000, + 0x21cf367f00000000, 0x841c6ab400000000, 0x2a6efe3200000000, + 0x8fbda2f900000000, 0xc10d220b00000000, 0x64de7ec000000000, + 0xcaacea4600000000, 0x6f7fb68d00000000, 0xd74fb39000000000, + 0x729cef5b00000000, 0xdcee7bdd00000000, 0x793d271600000000, + 0xac8f71e700000000, 0x095c2d2c00000000, 0xa72eb9aa00000000, + 0x02fde56100000000, 0xbacde07c00000000, 0x1f1ebcb700000000, + 0xb16c283100000000, 0x14bf74fa00000000, 0xd814b01e00000000, + 0x7dc7ecd500000000, 0xd3b5785300000000, 0x7666249800000000, + 0xce56218500000000, 0x6b857d4e00000000, 0xc5f7e9c800000000, + 0x6024b50300000000, 0xb596e3f200000000, 0x1045bf3900000000, + 0xbe372bbf00000000, 0x1be4777400000000, 0xa3d4726900000000, + 0x06072ea200000000, 0xa875ba2400000000, 0x0da6e6ef00000000, + 0x4316661d00000000, 0xe6c53ad600000000, 0x48b7ae5000000000, + 0xed64f29b00000000, 0x5554f78600000000, 0xf087ab4d00000000, + 0x5ef53fcb00000000, 0xfb26630000000000, 0x2e9435f100000000, + 0x8b47693a00000000, 0x2535fdbc00000000, 0x80e6a17700000000, + 0x38d6a46a00000000, 0x9d05f8a100000000, 0x33776c2700000000, + 0x96a430ec00000000, 0xee111c1900000000, 0x4bc240d200000000, + 0xe5b0d45400000000, 0x4063889f00000000, 0xf8538d8200000000, + 0x5d80d14900000000, 0xf3f245cf00000000, 0x5621190400000000, + 0x83934ff500000000, 0x2640133e00000000, 0x883287b800000000, + 0x2de1db7300000000, 0x95d1de6e00000000, 0x300282a500000000, + 0x9e70162300000000, 0x3ba34ae800000000, 0x7513ca1a00000000, + 0xd0c096d100000000, 0x7eb2025700000000, 0xdb615e9c00000000, + 0x63515b8100000000, 0xc682074a00000000, 0x68f093cc00000000, + 0xcd23cf0700000000, 0x189199f600000000, 0xbd42c53d00000000, + 0x133051bb00000000, 0xb6e30d7000000000, 0x0ed3086d00000000, + 0xab0054a600000000, 0x0572c02000000000, 0xa0a19ceb00000000, + 0xb41ee81100000000, 0x11cdb4da00000000, 0xbfbf205c00000000, + 0x1a6c7c9700000000, 0xa25c798a00000000, 0x078f254100000000, + 0xa9fdb1c700000000, 0x0c2eed0c00000000, 0xd99cbbfd00000000, + 0x7c4fe73600000000, 0xd23d73b000000000, 0x77ee2f7b00000000, + 0xcfde2a6600000000, 0x6a0d76ad00000000, 0xc47fe22b00000000, + 0x61acbee000000000, 0x2f1c3e1200000000, 0x8acf62d900000000, + 0x24bdf65f00000000, 0x816eaa9400000000, 0x395eaf8900000000, + 0x9c8df34200000000, 0x32ff67c400000000, 0x972c3b0f00000000, + 0x429e6dfe00000000, 0xe74d313500000000, 0x493fa5b300000000, + 0xececf97800000000, 0x54dcfc6500000000, 0xf10fa0ae00000000, + 0x5f7d342800000000, 0xfaae68e300000000, 0x821b441600000000, + 0x27c818dd00000000, 0x89ba8c5b00000000, 0x2c69d09000000000, + 0x9459d58d00000000, 0x318a894600000000, 0x9ff81dc000000000, + 0x3a2b410b00000000, 0xef9917fa00000000, 0x4a4a4b3100000000, + 0xe438dfb700000000, 0x41eb837c00000000, 0xf9db866100000000, + 0x5c08daaa00000000, 0xf27a4e2c00000000, 0x57a912e700000000, + 0x1919921500000000, 0xbccacede00000000, 0x12b85a5800000000, + 0xb76b069300000000, 0x0f5b038e00000000, 0xaa885f4500000000, + 0x04facbc300000000, 0xa129970800000000, 0x749bc1f900000000, + 0xd1489d3200000000, 0x7f3a09b400000000, 0xdae9557f00000000, + 0x62d9506200000000, 0xc70a0ca900000000, 0x6978982f00000000, + 0xccabc4e400000000}, + {0x0000000000000000, 0xb40b77a600000000, 0x29119f9700000000, + 0x9d1ae83100000000, 0x13244ff400000000, 0xa72f385200000000, + 0x3a35d06300000000, 0x8e3ea7c500000000, 0x674eef3300000000, + 0xd345989500000000, 0x4e5f70a400000000, 0xfa54070200000000, + 0x746aa0c700000000, 0xc061d76100000000, 0x5d7b3f5000000000, + 0xe97048f600000000, 0xce9cde6700000000, 0x7a97a9c100000000, + 0xe78d41f000000000, 0x5386365600000000, 0xddb8919300000000, + 0x69b3e63500000000, 0xf4a90e0400000000, 0x40a279a200000000, + 0xa9d2315400000000, 0x1dd946f200000000, 0x80c3aec300000000, + 0x34c8d96500000000, 0xbaf67ea000000000, 0x0efd090600000000, + 0x93e7e13700000000, 0x27ec969100000000, 0x9c39bdcf00000000, + 0x2832ca6900000000, 0xb528225800000000, 0x012355fe00000000, + 0x8f1df23b00000000, 0x3b16859d00000000, 0xa60c6dac00000000, + 0x12071a0a00000000, 0xfb7752fc00000000, 0x4f7c255a00000000, + 0xd266cd6b00000000, 0x666dbacd00000000, 0xe8531d0800000000, + 0x5c586aae00000000, 0xc142829f00000000, 0x7549f53900000000, + 0x52a563a800000000, 0xe6ae140e00000000, 0x7bb4fc3f00000000, + 0xcfbf8b9900000000, 0x41812c5c00000000, 0xf58a5bfa00000000, + 0x6890b3cb00000000, 0xdc9bc46d00000000, 0x35eb8c9b00000000, + 0x81e0fb3d00000000, 0x1cfa130c00000000, 0xa8f164aa00000000, + 0x26cfc36f00000000, 0x92c4b4c900000000, 0x0fde5cf800000000, + 0xbbd52b5e00000000, 0x79750b4400000000, 0xcd7e7ce200000000, + 0x506494d300000000, 0xe46fe37500000000, 0x6a5144b000000000, + 0xde5a331600000000, 0x4340db2700000000, 0xf74bac8100000000, + 0x1e3be47700000000, 0xaa3093d100000000, 0x372a7be000000000, + 0x83210c4600000000, 0x0d1fab8300000000, 0xb914dc2500000000, + 0x240e341400000000, 0x900543b200000000, 0xb7e9d52300000000, + 0x03e2a28500000000, 0x9ef84ab400000000, 0x2af33d1200000000, + 0xa4cd9ad700000000, 0x10c6ed7100000000, 0x8ddc054000000000, + 0x39d772e600000000, 0xd0a73a1000000000, 0x64ac4db600000000, + 0xf9b6a58700000000, 0x4dbdd22100000000, 0xc38375e400000000, + 0x7788024200000000, 0xea92ea7300000000, 0x5e999dd500000000, + 0xe54cb68b00000000, 0x5147c12d00000000, 0xcc5d291c00000000, + 0x78565eba00000000, 0xf668f97f00000000, 0x42638ed900000000, + 0xdf7966e800000000, 0x6b72114e00000000, 0x820259b800000000, + 0x36092e1e00000000, 0xab13c62f00000000, 0x1f18b18900000000, + 0x9126164c00000000, 0x252d61ea00000000, 0xb83789db00000000, + 0x0c3cfe7d00000000, 0x2bd068ec00000000, 0x9fdb1f4a00000000, + 0x02c1f77b00000000, 0xb6ca80dd00000000, 0x38f4271800000000, + 0x8cff50be00000000, 0x11e5b88f00000000, 0xa5eecf2900000000, + 0x4c9e87df00000000, 0xf895f07900000000, 0x658f184800000000, + 0xd1846fee00000000, 0x5fbac82b00000000, 0xebb1bf8d00000000, + 0x76ab57bc00000000, 0xc2a0201a00000000, 0xf2ea168800000000, + 0x46e1612e00000000, 0xdbfb891f00000000, 0x6ff0feb900000000, + 0xe1ce597c00000000, 0x55c52eda00000000, 0xc8dfc6eb00000000, + 0x7cd4b14d00000000, 0x95a4f9bb00000000, 0x21af8e1d00000000, + 0xbcb5662c00000000, 0x08be118a00000000, 0x8680b64f00000000, + 0x328bc1e900000000, 0xaf9129d800000000, 0x1b9a5e7e00000000, + 0x3c76c8ef00000000, 0x887dbf4900000000, 0x1567577800000000, + 0xa16c20de00000000, 0x2f52871b00000000, 0x9b59f0bd00000000, + 0x0643188c00000000, 0xb2486f2a00000000, 0x5b3827dc00000000, + 0xef33507a00000000, 0x7229b84b00000000, 0xc622cfed00000000, + 0x481c682800000000, 0xfc171f8e00000000, 0x610df7bf00000000, + 0xd506801900000000, 0x6ed3ab4700000000, 0xdad8dce100000000, + 0x47c234d000000000, 0xf3c9437600000000, 0x7df7e4b300000000, + 0xc9fc931500000000, 0x54e67b2400000000, 0xe0ed0c8200000000, + 0x099d447400000000, 0xbd9633d200000000, 0x208cdbe300000000, + 0x9487ac4500000000, 0x1ab90b8000000000, 0xaeb27c2600000000, + 0x33a8941700000000, 0x87a3e3b100000000, 0xa04f752000000000, + 0x1444028600000000, 0x895eeab700000000, 0x3d559d1100000000, + 0xb36b3ad400000000, 0x07604d7200000000, 0x9a7aa54300000000, + 0x2e71d2e500000000, 0xc7019a1300000000, 0x730aedb500000000, + 0xee10058400000000, 0x5a1b722200000000, 0xd425d5e700000000, + 0x602ea24100000000, 0xfd344a7000000000, 0x493f3dd600000000, + 0x8b9f1dcc00000000, 0x3f946a6a00000000, 0xa28e825b00000000, + 0x1685f5fd00000000, 0x98bb523800000000, 0x2cb0259e00000000, + 0xb1aacdaf00000000, 0x05a1ba0900000000, 0xecd1f2ff00000000, + 0x58da855900000000, 0xc5c06d6800000000, 0x71cb1ace00000000, + 0xfff5bd0b00000000, 0x4bfecaad00000000, 0xd6e4229c00000000, + 0x62ef553a00000000, 0x4503c3ab00000000, 0xf108b40d00000000, + 0x6c125c3c00000000, 0xd8192b9a00000000, 0x56278c5f00000000, + 0xe22cfbf900000000, 0x7f3613c800000000, 0xcb3d646e00000000, + 0x224d2c9800000000, 0x96465b3e00000000, 0x0b5cb30f00000000, + 0xbf57c4a900000000, 0x3169636c00000000, 0x856214ca00000000, + 0x1878fcfb00000000, 0xac738b5d00000000, 0x17a6a00300000000, + 0xa3add7a500000000, 0x3eb73f9400000000, 0x8abc483200000000, + 0x0482eff700000000, 0xb089985100000000, 0x2d93706000000000, + 0x999807c600000000, 0x70e84f3000000000, 0xc4e3389600000000, + 0x59f9d0a700000000, 0xedf2a70100000000, 0x63cc00c400000000, + 0xd7c7776200000000, 0x4add9f5300000000, 0xfed6e8f500000000, + 0xd93a7e6400000000, 0x6d3109c200000000, 0xf02be1f300000000, + 0x4420965500000000, 0xca1e319000000000, 0x7e15463600000000, + 0xe30fae0700000000, 0x5704d9a100000000, 0xbe74915700000000, + 0x0a7fe6f100000000, 0x97650ec000000000, 0x236e796600000000, + 0xad50dea300000000, 0x195ba90500000000, 0x8441413400000000, + 0x304a369200000000}, + {0x0000000000000000, 0x9e00aacc00000000, 0x7d07254200000000, + 0xe3078f8e00000000, 0xfa0e4a8400000000, 0x640ee04800000000, + 0x87096fc600000000, 0x1909c50a00000000, 0xb51be5d300000000, + 0x2b1b4f1f00000000, 0xc81cc09100000000, 0x561c6a5d00000000, + 0x4f15af5700000000, 0xd115059b00000000, 0x32128a1500000000, + 0xac1220d900000000, 0x2b31bb7c00000000, 0xb53111b000000000, + 0x56369e3e00000000, 0xc83634f200000000, 0xd13ff1f800000000, + 0x4f3f5b3400000000, 0xac38d4ba00000000, 0x32387e7600000000, + 0x9e2a5eaf00000000, 0x002af46300000000, 0xe32d7bed00000000, + 0x7d2dd12100000000, 0x6424142b00000000, 0xfa24bee700000000, + 0x1923316900000000, 0x87239ba500000000, 0x566276f900000000, + 0xc862dc3500000000, 0x2b6553bb00000000, 0xb565f97700000000, + 0xac6c3c7d00000000, 0x326c96b100000000, 0xd16b193f00000000, + 0x4f6bb3f300000000, 0xe379932a00000000, 0x7d7939e600000000, + 0x9e7eb66800000000, 0x007e1ca400000000, 0x1977d9ae00000000, + 0x8777736200000000, 0x6470fcec00000000, 0xfa70562000000000, + 0x7d53cd8500000000, 0xe353674900000000, 0x0054e8c700000000, + 0x9e54420b00000000, 0x875d870100000000, 0x195d2dcd00000000, + 0xfa5aa24300000000, 0x645a088f00000000, 0xc848285600000000, + 0x5648829a00000000, 0xb54f0d1400000000, 0x2b4fa7d800000000, + 0x324662d200000000, 0xac46c81e00000000, 0x4f41479000000000, + 0xd141ed5c00000000, 0xedc29d2900000000, 0x73c237e500000000, + 0x90c5b86b00000000, 0x0ec512a700000000, 0x17ccd7ad00000000, + 0x89cc7d6100000000, 0x6acbf2ef00000000, 0xf4cb582300000000, + 0x58d978fa00000000, 0xc6d9d23600000000, 0x25de5db800000000, + 0xbbdef77400000000, 0xa2d7327e00000000, 0x3cd798b200000000, + 0xdfd0173c00000000, 0x41d0bdf000000000, 0xc6f3265500000000, + 0x58f38c9900000000, 0xbbf4031700000000, 0x25f4a9db00000000, + 0x3cfd6cd100000000, 0xa2fdc61d00000000, 0x41fa499300000000, + 0xdffae35f00000000, 0x73e8c38600000000, 0xede8694a00000000, + 0x0eefe6c400000000, 0x90ef4c0800000000, 0x89e6890200000000, + 0x17e623ce00000000, 0xf4e1ac4000000000, 0x6ae1068c00000000, + 0xbba0ebd000000000, 0x25a0411c00000000, 0xc6a7ce9200000000, + 0x58a7645e00000000, 0x41aea15400000000, 0xdfae0b9800000000, + 0x3ca9841600000000, 0xa2a92eda00000000, 0x0ebb0e0300000000, + 0x90bba4cf00000000, 0x73bc2b4100000000, 0xedbc818d00000000, + 0xf4b5448700000000, 0x6ab5ee4b00000000, 0x89b261c500000000, + 0x17b2cb0900000000, 0x909150ac00000000, 0x0e91fa6000000000, + 0xed9675ee00000000, 0x7396df2200000000, 0x6a9f1a2800000000, + 0xf49fb0e400000000, 0x17983f6a00000000, 0x899895a600000000, + 0x258ab57f00000000, 0xbb8a1fb300000000, 0x588d903d00000000, + 0xc68d3af100000000, 0xdf84fffb00000000, 0x4184553700000000, + 0xa283dab900000000, 0x3c83707500000000, 0xda853b5300000000, + 0x4485919f00000000, 0xa7821e1100000000, 0x3982b4dd00000000, + 0x208b71d700000000, 0xbe8bdb1b00000000, 0x5d8c549500000000, + 0xc38cfe5900000000, 0x6f9ede8000000000, 0xf19e744c00000000, + 0x1299fbc200000000, 0x8c99510e00000000, 0x9590940400000000, + 0x0b903ec800000000, 0xe897b14600000000, 0x76971b8a00000000, + 0xf1b4802f00000000, 0x6fb42ae300000000, 0x8cb3a56d00000000, + 0x12b30fa100000000, 0x0bbacaab00000000, 0x95ba606700000000, + 0x76bdefe900000000, 0xe8bd452500000000, 0x44af65fc00000000, + 0xdaafcf3000000000, 0x39a840be00000000, 0xa7a8ea7200000000, + 0xbea12f7800000000, 0x20a185b400000000, 0xc3a60a3a00000000, + 0x5da6a0f600000000, 0x8ce74daa00000000, 0x12e7e76600000000, + 0xf1e068e800000000, 0x6fe0c22400000000, 0x76e9072e00000000, + 0xe8e9ade200000000, 0x0bee226c00000000, 0x95ee88a000000000, + 0x39fca87900000000, 0xa7fc02b500000000, 0x44fb8d3b00000000, + 0xdafb27f700000000, 0xc3f2e2fd00000000, 0x5df2483100000000, + 0xbef5c7bf00000000, 0x20f56d7300000000, 0xa7d6f6d600000000, + 0x39d65c1a00000000, 0xdad1d39400000000, 0x44d1795800000000, + 0x5dd8bc5200000000, 0xc3d8169e00000000, 0x20df991000000000, + 0xbedf33dc00000000, 0x12cd130500000000, 0x8ccdb9c900000000, + 0x6fca364700000000, 0xf1ca9c8b00000000, 0xe8c3598100000000, + 0x76c3f34d00000000, 0x95c47cc300000000, 0x0bc4d60f00000000, + 0x3747a67a00000000, 0xa9470cb600000000, 0x4a40833800000000, + 0xd44029f400000000, 0xcd49ecfe00000000, 0x5349463200000000, + 0xb04ec9bc00000000, 0x2e4e637000000000, 0x825c43a900000000, + 0x1c5ce96500000000, 0xff5b66eb00000000, 0x615bcc2700000000, + 0x7852092d00000000, 0xe652a3e100000000, 0x05552c6f00000000, + 0x9b5586a300000000, 0x1c761d0600000000, 0x8276b7ca00000000, + 0x6171384400000000, 0xff71928800000000, 0xe678578200000000, + 0x7878fd4e00000000, 0x9b7f72c000000000, 0x057fd80c00000000, + 0xa96df8d500000000, 0x376d521900000000, 0xd46add9700000000, + 0x4a6a775b00000000, 0x5363b25100000000, 0xcd63189d00000000, + 0x2e64971300000000, 0xb0643ddf00000000, 0x6125d08300000000, + 0xff257a4f00000000, 0x1c22f5c100000000, 0x82225f0d00000000, + 0x9b2b9a0700000000, 0x052b30cb00000000, 0xe62cbf4500000000, + 0x782c158900000000, 0xd43e355000000000, 0x4a3e9f9c00000000, + 0xa939101200000000, 0x3739bade00000000, 0x2e307fd400000000, + 0xb030d51800000000, 0x53375a9600000000, 0xcd37f05a00000000, + 0x4a146bff00000000, 0xd414c13300000000, 0x37134ebd00000000, + 0xa913e47100000000, 0xb01a217b00000000, 0x2e1a8bb700000000, + 0xcd1d043900000000, 0x531daef500000000, 0xff0f8e2c00000000, + 0x610f24e000000000, 0x8208ab6e00000000, 0x1c0801a200000000, + 0x0501c4a800000000, 0x9b016e6400000000, 0x7806e1ea00000000, + 0xe6064b2600000000}}; + +#else /* W == 4 */ + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0xb8bc6765, 0xaa09c88b, 0x12b5afee, 0x8f629757, + 0x37def032, 0x256b5fdc, 0x9dd738b9, 0xc5b428ef, 0x7d084f8a, + 0x6fbde064, 0xd7018701, 0x4ad6bfb8, 0xf26ad8dd, 0xe0df7733, + 0x58631056, 0x5019579f, 0xe8a530fa, 0xfa109f14, 0x42acf871, + 0xdf7bc0c8, 0x67c7a7ad, 0x75720843, 0xcdce6f26, 0x95ad7f70, + 0x2d111815, 0x3fa4b7fb, 0x8718d09e, 0x1acfe827, 0xa2738f42, + 0xb0c620ac, 0x087a47c9, 0xa032af3e, 0x188ec85b, 0x0a3b67b5, + 0xb28700d0, 0x2f503869, 0x97ec5f0c, 0x8559f0e2, 0x3de59787, + 0x658687d1, 0xdd3ae0b4, 0xcf8f4f5a, 0x7733283f, 0xeae41086, + 0x525877e3, 0x40edd80d, 0xf851bf68, 0xf02bf8a1, 0x48979fc4, + 0x5a22302a, 0xe29e574f, 0x7f496ff6, 0xc7f50893, 0xd540a77d, + 0x6dfcc018, 0x359fd04e, 0x8d23b72b, 0x9f9618c5, 0x272a7fa0, + 0xbafd4719, 0x0241207c, 0x10f48f92, 0xa848e8f7, 0x9b14583d, + 0x23a83f58, 0x311d90b6, 0x89a1f7d3, 0x1476cf6a, 0xaccaa80f, + 0xbe7f07e1, 0x06c36084, 0x5ea070d2, 0xe61c17b7, 0xf4a9b859, + 0x4c15df3c, 0xd1c2e785, 0x697e80e0, 0x7bcb2f0e, 0xc377486b, + 0xcb0d0fa2, 0x73b168c7, 0x6104c729, 0xd9b8a04c, 0x446f98f5, + 0xfcd3ff90, 0xee66507e, 0x56da371b, 0x0eb9274d, 0xb6054028, + 0xa4b0efc6, 0x1c0c88a3, 0x81dbb01a, 0x3967d77f, 0x2bd27891, + 0x936e1ff4, 0x3b26f703, 0x839a9066, 0x912f3f88, 0x299358ed, + 0xb4446054, 0x0cf80731, 0x1e4da8df, 0xa6f1cfba, 0xfe92dfec, + 0x462eb889, 0x549b1767, 0xec277002, 0x71f048bb, 0xc94c2fde, + 0xdbf98030, 0x6345e755, 0x6b3fa09c, 0xd383c7f9, 0xc1366817, + 0x798a0f72, 0xe45d37cb, 0x5ce150ae, 0x4e54ff40, 0xf6e89825, + 0xae8b8873, 0x1637ef16, 0x048240f8, 0xbc3e279d, 0x21e91f24, + 0x99557841, 0x8be0d7af, 0x335cb0ca, 0xed59b63b, 0x55e5d15e, + 0x47507eb0, 0xffec19d5, 0x623b216c, 0xda874609, 0xc832e9e7, + 0x708e8e82, 0x28ed9ed4, 0x9051f9b1, 0x82e4565f, 0x3a58313a, + 0xa78f0983, 0x1f336ee6, 0x0d86c108, 0xb53aa66d, 0xbd40e1a4, + 0x05fc86c1, 0x1749292f, 0xaff54e4a, 0x322276f3, 0x8a9e1196, + 0x982bbe78, 0x2097d91d, 0x78f4c94b, 0xc048ae2e, 0xd2fd01c0, + 0x6a4166a5, 0xf7965e1c, 0x4f2a3979, 0x5d9f9697, 0xe523f1f2, + 0x4d6b1905, 0xf5d77e60, 0xe762d18e, 0x5fdeb6eb, 0xc2098e52, + 0x7ab5e937, 0x680046d9, 0xd0bc21bc, 0x88df31ea, 0x3063568f, + 0x22d6f961, 0x9a6a9e04, 0x07bda6bd, 0xbf01c1d8, 0xadb46e36, + 0x15080953, 0x1d724e9a, 0xa5ce29ff, 0xb77b8611, 0x0fc7e174, + 0x9210d9cd, 0x2aacbea8, 0x38191146, 0x80a57623, 0xd8c66675, + 0x607a0110, 0x72cfaefe, 0xca73c99b, 0x57a4f122, 0xef189647, + 0xfdad39a9, 0x45115ecc, 0x764dee06, 0xcef18963, 0xdc44268d, + 0x64f841e8, 0xf92f7951, 0x41931e34, 0x5326b1da, 0xeb9ad6bf, + 0xb3f9c6e9, 0x0b45a18c, 0x19f00e62, 0xa14c6907, 0x3c9b51be, + 0x842736db, 0x96929935, 0x2e2efe50, 0x2654b999, 0x9ee8defc, + 0x8c5d7112, 0x34e11677, 0xa9362ece, 0x118a49ab, 0x033fe645, + 0xbb838120, 0xe3e09176, 0x5b5cf613, 0x49e959fd, 0xf1553e98, + 0x6c820621, 0xd43e6144, 0xc68bceaa, 0x7e37a9cf, 0xd67f4138, + 0x6ec3265d, 0x7c7689b3, 0xc4caeed6, 0x591dd66f, 0xe1a1b10a, + 0xf3141ee4, 0x4ba87981, 0x13cb69d7, 0xab770eb2, 0xb9c2a15c, + 0x017ec639, 0x9ca9fe80, 0x241599e5, 0x36a0360b, 0x8e1c516e, + 0x866616a7, 0x3eda71c2, 0x2c6fde2c, 0x94d3b949, 0x090481f0, + 0xb1b8e695, 0xa30d497b, 0x1bb12e1e, 0x43d23e48, 0xfb6e592d, + 0xe9dbf6c3, 0x516791a6, 0xccb0a91f, 0x740cce7a, 0x66b96194, + 0xde0506f1}, + {0x00000000, 0x01c26a37, 0x0384d46e, 0x0246be59, 0x0709a8dc, + 0x06cbc2eb, 0x048d7cb2, 0x054f1685, 0x0e1351b8, 0x0fd13b8f, + 0x0d9785d6, 0x0c55efe1, 0x091af964, 0x08d89353, 0x0a9e2d0a, + 0x0b5c473d, 0x1c26a370, 0x1de4c947, 0x1fa2771e, 0x1e601d29, + 0x1b2f0bac, 0x1aed619b, 0x18abdfc2, 0x1969b5f5, 0x1235f2c8, + 0x13f798ff, 0x11b126a6, 0x10734c91, 0x153c5a14, 0x14fe3023, + 0x16b88e7a, 0x177ae44d, 0x384d46e0, 0x398f2cd7, 0x3bc9928e, + 0x3a0bf8b9, 0x3f44ee3c, 0x3e86840b, 0x3cc03a52, 0x3d025065, + 0x365e1758, 0x379c7d6f, 0x35dac336, 0x3418a901, 0x3157bf84, + 0x3095d5b3, 0x32d36bea, 0x331101dd, 0x246be590, 0x25a98fa7, + 0x27ef31fe, 0x262d5bc9, 0x23624d4c, 0x22a0277b, 0x20e69922, + 0x2124f315, 0x2a78b428, 0x2bbade1f, 0x29fc6046, 0x283e0a71, + 0x2d711cf4, 0x2cb376c3, 0x2ef5c89a, 0x2f37a2ad, 0x709a8dc0, + 0x7158e7f7, 0x731e59ae, 0x72dc3399, 0x7793251c, 0x76514f2b, + 0x7417f172, 0x75d59b45, 0x7e89dc78, 0x7f4bb64f, 0x7d0d0816, + 0x7ccf6221, 0x798074a4, 0x78421e93, 0x7a04a0ca, 0x7bc6cafd, + 0x6cbc2eb0, 0x6d7e4487, 0x6f38fade, 0x6efa90e9, 0x6bb5866c, + 0x6a77ec5b, 0x68315202, 0x69f33835, 0x62af7f08, 0x636d153f, + 0x612bab66, 0x60e9c151, 0x65a6d7d4, 0x6464bde3, 0x662203ba, + 0x67e0698d, 0x48d7cb20, 0x4915a117, 0x4b531f4e, 0x4a917579, + 0x4fde63fc, 0x4e1c09cb, 0x4c5ab792, 0x4d98dda5, 0x46c49a98, + 0x4706f0af, 0x45404ef6, 0x448224c1, 0x41cd3244, 0x400f5873, + 0x4249e62a, 0x438b8c1d, 0x54f16850, 0x55330267, 0x5775bc3e, + 0x56b7d609, 0x53f8c08c, 0x523aaabb, 0x507c14e2, 0x51be7ed5, + 0x5ae239e8, 0x5b2053df, 0x5966ed86, 0x58a487b1, 0x5deb9134, + 0x5c29fb03, 0x5e6f455a, 0x5fad2f6d, 0xe1351b80, 0xe0f771b7, + 0xe2b1cfee, 0xe373a5d9, 0xe63cb35c, 0xe7fed96b, 0xe5b86732, + 0xe47a0d05, 0xef264a38, 0xeee4200f, 0xeca29e56, 0xed60f461, + 0xe82fe2e4, 0xe9ed88d3, 0xebab368a, 0xea695cbd, 0xfd13b8f0, + 0xfcd1d2c7, 0xfe976c9e, 0xff5506a9, 0xfa1a102c, 0xfbd87a1b, + 0xf99ec442, 0xf85cae75, 0xf300e948, 0xf2c2837f, 0xf0843d26, + 0xf1465711, 0xf4094194, 0xf5cb2ba3, 0xf78d95fa, 0xf64fffcd, + 0xd9785d60, 0xd8ba3757, 0xdafc890e, 0xdb3ee339, 0xde71f5bc, + 0xdfb39f8b, 0xddf521d2, 0xdc374be5, 0xd76b0cd8, 0xd6a966ef, + 0xd4efd8b6, 0xd52db281, 0xd062a404, 0xd1a0ce33, 0xd3e6706a, + 0xd2241a5d, 0xc55efe10, 0xc49c9427, 0xc6da2a7e, 0xc7184049, + 0xc25756cc, 0xc3953cfb, 0xc1d382a2, 0xc011e895, 0xcb4dafa8, + 0xca8fc59f, 0xc8c97bc6, 0xc90b11f1, 0xcc440774, 0xcd866d43, + 0xcfc0d31a, 0xce02b92d, 0x91af9640, 0x906dfc77, 0x922b422e, + 0x93e92819, 0x96a63e9c, 0x976454ab, 0x9522eaf2, 0x94e080c5, + 0x9fbcc7f8, 0x9e7eadcf, 0x9c381396, 0x9dfa79a1, 0x98b56f24, + 0x99770513, 0x9b31bb4a, 0x9af3d17d, 0x8d893530, 0x8c4b5f07, + 0x8e0de15e, 0x8fcf8b69, 0x8a809dec, 0x8b42f7db, 0x89044982, + 0x88c623b5, 0x839a6488, 0x82580ebf, 0x801eb0e6, 0x81dcdad1, + 0x8493cc54, 0x8551a663, 0x8717183a, 0x86d5720d, 0xa9e2d0a0, + 0xa820ba97, 0xaa6604ce, 0xaba46ef9, 0xaeeb787c, 0xaf29124b, + 0xad6fac12, 0xacadc625, 0xa7f18118, 0xa633eb2f, 0xa4755576, + 0xa5b73f41, 0xa0f829c4, 0xa13a43f3, 0xa37cfdaa, 0xa2be979d, + 0xb5c473d0, 0xb40619e7, 0xb640a7be, 0xb782cd89, 0xb2cddb0c, + 0xb30fb13b, 0xb1490f62, 0xb08b6555, 0xbbd72268, 0xba15485f, + 0xb853f606, 0xb9919c31, 0xbcde8ab4, 0xbd1ce083, 0xbf5a5eda, + 0xbe9834ed}, + {0x00000000, 0x191b3141, 0x32366282, 0x2b2d53c3, 0x646cc504, + 0x7d77f445, 0x565aa786, 0x4f4196c7, 0xc8d98a08, 0xd1c2bb49, + 0xfaefe88a, 0xe3f4d9cb, 0xacb54f0c, 0xb5ae7e4d, 0x9e832d8e, + 0x87981ccf, 0x4ac21251, 0x53d92310, 0x78f470d3, 0x61ef4192, + 0x2eaed755, 0x37b5e614, 0x1c98b5d7, 0x05838496, 0x821b9859, + 0x9b00a918, 0xb02dfadb, 0xa936cb9a, 0xe6775d5d, 0xff6c6c1c, + 0xd4413fdf, 0xcd5a0e9e, 0x958424a2, 0x8c9f15e3, 0xa7b24620, + 0xbea97761, 0xf1e8e1a6, 0xe8f3d0e7, 0xc3de8324, 0xdac5b265, + 0x5d5daeaa, 0x44469feb, 0x6f6bcc28, 0x7670fd69, 0x39316bae, + 0x202a5aef, 0x0b07092c, 0x121c386d, 0xdf4636f3, 0xc65d07b2, + 0xed705471, 0xf46b6530, 0xbb2af3f7, 0xa231c2b6, 0x891c9175, + 0x9007a034, 0x179fbcfb, 0x0e848dba, 0x25a9de79, 0x3cb2ef38, + 0x73f379ff, 0x6ae848be, 0x41c51b7d, 0x58de2a3c, 0xf0794f05, + 0xe9627e44, 0xc24f2d87, 0xdb541cc6, 0x94158a01, 0x8d0ebb40, + 0xa623e883, 0xbf38d9c2, 0x38a0c50d, 0x21bbf44c, 0x0a96a78f, + 0x138d96ce, 0x5ccc0009, 0x45d73148, 0x6efa628b, 0x77e153ca, + 0xbabb5d54, 0xa3a06c15, 0x888d3fd6, 0x91960e97, 0xded79850, + 0xc7cca911, 0xece1fad2, 0xf5facb93, 0x7262d75c, 0x6b79e61d, + 0x4054b5de, 0x594f849f, 0x160e1258, 0x0f152319, 0x243870da, + 0x3d23419b, 0x65fd6ba7, 0x7ce65ae6, 0x57cb0925, 0x4ed03864, + 0x0191aea3, 0x188a9fe2, 0x33a7cc21, 0x2abcfd60, 0xad24e1af, + 0xb43fd0ee, 0x9f12832d, 0x8609b26c, 0xc94824ab, 0xd05315ea, + 0xfb7e4629, 0xe2657768, 0x2f3f79f6, 0x362448b7, 0x1d091b74, + 0x04122a35, 0x4b53bcf2, 0x52488db3, 0x7965de70, 0x607eef31, + 0xe7e6f3fe, 0xfefdc2bf, 0xd5d0917c, 0xcccba03d, 0x838a36fa, + 0x9a9107bb, 0xb1bc5478, 0xa8a76539, 0x3b83984b, 0x2298a90a, + 0x09b5fac9, 0x10aecb88, 0x5fef5d4f, 0x46f46c0e, 0x6dd93fcd, + 0x74c20e8c, 0xf35a1243, 0xea412302, 0xc16c70c1, 0xd8774180, + 0x9736d747, 0x8e2de606, 0xa500b5c5, 0xbc1b8484, 0x71418a1a, + 0x685abb5b, 0x4377e898, 0x5a6cd9d9, 0x152d4f1e, 0x0c367e5f, + 0x271b2d9c, 0x3e001cdd, 0xb9980012, 0xa0833153, 0x8bae6290, + 0x92b553d1, 0xddf4c516, 0xc4eff457, 0xefc2a794, 0xf6d996d5, + 0xae07bce9, 0xb71c8da8, 0x9c31de6b, 0x852aef2a, 0xca6b79ed, + 0xd37048ac, 0xf85d1b6f, 0xe1462a2e, 0x66de36e1, 0x7fc507a0, + 0x54e85463, 0x4df36522, 0x02b2f3e5, 0x1ba9c2a4, 0x30849167, + 0x299fa026, 0xe4c5aeb8, 0xfdde9ff9, 0xd6f3cc3a, 0xcfe8fd7b, + 0x80a96bbc, 0x99b25afd, 0xb29f093e, 0xab84387f, 0x2c1c24b0, + 0x350715f1, 0x1e2a4632, 0x07317773, 0x4870e1b4, 0x516bd0f5, + 0x7a468336, 0x635db277, 0xcbfad74e, 0xd2e1e60f, 0xf9ccb5cc, + 0xe0d7848d, 0xaf96124a, 0xb68d230b, 0x9da070c8, 0x84bb4189, + 0x03235d46, 0x1a386c07, 0x31153fc4, 0x280e0e85, 0x674f9842, + 0x7e54a903, 0x5579fac0, 0x4c62cb81, 0x8138c51f, 0x9823f45e, + 0xb30ea79d, 0xaa1596dc, 0xe554001b, 0xfc4f315a, 0xd7626299, + 0xce7953d8, 0x49e14f17, 0x50fa7e56, 0x7bd72d95, 0x62cc1cd4, + 0x2d8d8a13, 0x3496bb52, 0x1fbbe891, 0x06a0d9d0, 0x5e7ef3ec, + 0x4765c2ad, 0x6c48916e, 0x7553a02f, 0x3a1236e8, 0x230907a9, + 0x0824546a, 0x113f652b, 0x96a779e4, 0x8fbc48a5, 0xa4911b66, + 0xbd8a2a27, 0xf2cbbce0, 0xebd08da1, 0xc0fdde62, 0xd9e6ef23, + 0x14bce1bd, 0x0da7d0fc, 0x268a833f, 0x3f91b27e, 0x70d024b9, + 0x69cb15f8, 0x42e6463b, 0x5bfd777a, 0xdc656bb5, 0xc57e5af4, + 0xee530937, 0xf7483876, 0xb809aeb1, 0xa1129ff0, 0x8a3fcc33, + 0x9324fd72}, + {0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, + 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, + 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, + 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, + 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, + 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, + 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, + 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, + 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, + 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, + 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, + 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, + 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, + 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, + 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, + 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, + 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, + 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, + 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, + 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, + 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, + 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, + 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, + 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, + 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, + 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, + 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, + 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, + 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, + 0x2d02ef8d}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x00000000, 0x96300777, 0x2c610eee, 0xba510999, 0x19c46d07, + 0x8ff46a70, 0x35a563e9, 0xa395649e, 0x3288db0e, 0xa4b8dc79, + 0x1ee9d5e0, 0x88d9d297, 0x2b4cb609, 0xbd7cb17e, 0x072db8e7, + 0x911dbf90, 0x6410b71d, 0xf220b06a, 0x4871b9f3, 0xde41be84, + 0x7dd4da1a, 0xebe4dd6d, 0x51b5d4f4, 0xc785d383, 0x56986c13, + 0xc0a86b64, 0x7af962fd, 0xecc9658a, 0x4f5c0114, 0xd96c0663, + 0x633d0ffa, 0xf50d088d, 0xc8206e3b, 0x5e10694c, 0xe44160d5, + 0x727167a2, 0xd1e4033c, 0x47d4044b, 0xfd850dd2, 0x6bb50aa5, + 0xfaa8b535, 0x6c98b242, 0xd6c9bbdb, 0x40f9bcac, 0xe36cd832, + 0x755cdf45, 0xcf0dd6dc, 0x593dd1ab, 0xac30d926, 0x3a00de51, + 0x8051d7c8, 0x1661d0bf, 0xb5f4b421, 0x23c4b356, 0x9995bacf, + 0x0fa5bdb8, 0x9eb80228, 0x0888055f, 0xb2d90cc6, 0x24e90bb1, + 0x877c6f2f, 0x114c6858, 0xab1d61c1, 0x3d2d66b6, 0x9041dc76, + 0x0671db01, 0xbc20d298, 0x2a10d5ef, 0x8985b171, 0x1fb5b606, + 0xa5e4bf9f, 0x33d4b8e8, 0xa2c90778, 0x34f9000f, 0x8ea80996, + 0x18980ee1, 0xbb0d6a7f, 0x2d3d6d08, 0x976c6491, 0x015c63e6, + 0xf4516b6b, 0x62616c1c, 0xd8306585, 0x4e0062f2, 0xed95066c, + 0x7ba5011b, 0xc1f40882, 0x57c40ff5, 0xc6d9b065, 0x50e9b712, + 0xeab8be8b, 0x7c88b9fc, 0xdf1ddd62, 0x492dda15, 0xf37cd38c, + 0x654cd4fb, 0x5861b24d, 0xce51b53a, 0x7400bca3, 0xe230bbd4, + 0x41a5df4a, 0xd795d83d, 0x6dc4d1a4, 0xfbf4d6d3, 0x6ae96943, + 0xfcd96e34, 0x468867ad, 0xd0b860da, 0x732d0444, 0xe51d0333, + 0x5f4c0aaa, 0xc97c0ddd, 0x3c710550, 0xaa410227, 0x10100bbe, + 0x86200cc9, 0x25b56857, 0xb3856f20, 0x09d466b9, 0x9fe461ce, + 0x0ef9de5e, 0x98c9d929, 0x2298d0b0, 0xb4a8d7c7, 0x173db359, + 0x810db42e, 0x3b5cbdb7, 0xad6cbac0, 0x2083b8ed, 0xb6b3bf9a, + 0x0ce2b603, 0x9ad2b174, 0x3947d5ea, 0xaf77d29d, 0x1526db04, + 0x8316dc73, 0x120b63e3, 0x843b6494, 0x3e6a6d0d, 0xa85a6a7a, + 0x0bcf0ee4, 0x9dff0993, 0x27ae000a, 0xb19e077d, 0x44930ff0, + 0xd2a30887, 0x68f2011e, 0xfec20669, 0x5d5762f7, 0xcb676580, + 0x71366c19, 0xe7066b6e, 0x761bd4fe, 0xe02bd389, 0x5a7ada10, + 0xcc4add67, 0x6fdfb9f9, 0xf9efbe8e, 0x43beb717, 0xd58eb060, + 0xe8a3d6d6, 0x7e93d1a1, 0xc4c2d838, 0x52f2df4f, 0xf167bbd1, + 0x6757bca6, 0xdd06b53f, 0x4b36b248, 0xda2b0dd8, 0x4c1b0aaf, + 0xf64a0336, 0x607a0441, 0xc3ef60df, 0x55df67a8, 0xef8e6e31, + 0x79be6946, 0x8cb361cb, 0x1a8366bc, 0xa0d26f25, 0x36e26852, + 0x95770ccc, 0x03470bbb, 0xb9160222, 0x2f260555, 0xbe3bbac5, + 0x280bbdb2, 0x925ab42b, 0x046ab35c, 0xa7ffd7c2, 0x31cfd0b5, + 0x8b9ed92c, 0x1daede5b, 0xb0c2649b, 0x26f263ec, 0x9ca36a75, + 0x0a936d02, 0xa906099c, 0x3f360eeb, 0x85670772, 0x13570005, + 0x824abf95, 0x147ab8e2, 0xae2bb17b, 0x381bb60c, 0x9b8ed292, + 0x0dbed5e5, 0xb7efdc7c, 0x21dfdb0b, 0xd4d2d386, 0x42e2d4f1, + 0xf8b3dd68, 0x6e83da1f, 0xcd16be81, 0x5b26b9f6, 0xe177b06f, + 0x7747b718, 0xe65a0888, 0x706a0fff, 0xca3b0666, 0x5c0b0111, + 0xff9e658f, 0x69ae62f8, 0xd3ff6b61, 0x45cf6c16, 0x78e20aa0, + 0xeed20dd7, 0x5483044e, 0xc2b30339, 0x612667a7, 0xf71660d0, + 0x4d476949, 0xdb776e3e, 0x4a6ad1ae, 0xdc5ad6d9, 0x660bdf40, + 0xf03bd837, 0x53aebca9, 0xc59ebbde, 0x7fcfb247, 0xe9ffb530, + 0x1cf2bdbd, 0x8ac2baca, 0x3093b353, 0xa6a3b424, 0x0536d0ba, + 0x9306d7cd, 0x2957de54, 0xbf67d923, 0x2e7a66b3, 0xb84a61c4, + 0x021b685d, 0x942b6f2a, 0x37be0bb4, 0xa18e0cc3, 0x1bdf055a, + 0x8def022d}, + {0x00000000, 0x41311b19, 0x82623632, 0xc3532d2b, 0x04c56c64, + 0x45f4777d, 0x86a75a56, 0xc796414f, 0x088ad9c8, 0x49bbc2d1, + 0x8ae8effa, 0xcbd9f4e3, 0x0c4fb5ac, 0x4d7eaeb5, 0x8e2d839e, + 0xcf1c9887, 0x5112c24a, 0x1023d953, 0xd370f478, 0x9241ef61, + 0x55d7ae2e, 0x14e6b537, 0xd7b5981c, 0x96848305, 0x59981b82, + 0x18a9009b, 0xdbfa2db0, 0x9acb36a9, 0x5d5d77e6, 0x1c6c6cff, + 0xdf3f41d4, 0x9e0e5acd, 0xa2248495, 0xe3159f8c, 0x2046b2a7, + 0x6177a9be, 0xa6e1e8f1, 0xe7d0f3e8, 0x2483dec3, 0x65b2c5da, + 0xaaae5d5d, 0xeb9f4644, 0x28cc6b6f, 0x69fd7076, 0xae6b3139, + 0xef5a2a20, 0x2c09070b, 0x6d381c12, 0xf33646df, 0xb2075dc6, + 0x715470ed, 0x30656bf4, 0xf7f32abb, 0xb6c231a2, 0x75911c89, + 0x34a00790, 0xfbbc9f17, 0xba8d840e, 0x79dea925, 0x38efb23c, + 0xff79f373, 0xbe48e86a, 0x7d1bc541, 0x3c2ade58, 0x054f79f0, + 0x447e62e9, 0x872d4fc2, 0xc61c54db, 0x018a1594, 0x40bb0e8d, + 0x83e823a6, 0xc2d938bf, 0x0dc5a038, 0x4cf4bb21, 0x8fa7960a, + 0xce968d13, 0x0900cc5c, 0x4831d745, 0x8b62fa6e, 0xca53e177, + 0x545dbbba, 0x156ca0a3, 0xd63f8d88, 0x970e9691, 0x5098d7de, + 0x11a9ccc7, 0xd2fae1ec, 0x93cbfaf5, 0x5cd76272, 0x1de6796b, + 0xdeb55440, 0x9f844f59, 0x58120e16, 0x1923150f, 0xda703824, + 0x9b41233d, 0xa76bfd65, 0xe65ae67c, 0x2509cb57, 0x6438d04e, + 0xa3ae9101, 0xe29f8a18, 0x21cca733, 0x60fdbc2a, 0xafe124ad, + 0xeed03fb4, 0x2d83129f, 0x6cb20986, 0xab2448c9, 0xea1553d0, + 0x29467efb, 0x687765e2, 0xf6793f2f, 0xb7482436, 0x741b091d, + 0x352a1204, 0xf2bc534b, 0xb38d4852, 0x70de6579, 0x31ef7e60, + 0xfef3e6e7, 0xbfc2fdfe, 0x7c91d0d5, 0x3da0cbcc, 0xfa368a83, + 0xbb07919a, 0x7854bcb1, 0x3965a7a8, 0x4b98833b, 0x0aa99822, + 0xc9fab509, 0x88cbae10, 0x4f5def5f, 0x0e6cf446, 0xcd3fd96d, + 0x8c0ec274, 0x43125af3, 0x022341ea, 0xc1706cc1, 0x804177d8, + 0x47d73697, 0x06e62d8e, 0xc5b500a5, 0x84841bbc, 0x1a8a4171, + 0x5bbb5a68, 0x98e87743, 0xd9d96c5a, 0x1e4f2d15, 0x5f7e360c, + 0x9c2d1b27, 0xdd1c003e, 0x120098b9, 0x533183a0, 0x9062ae8b, + 0xd153b592, 0x16c5f4dd, 0x57f4efc4, 0x94a7c2ef, 0xd596d9f6, + 0xe9bc07ae, 0xa88d1cb7, 0x6bde319c, 0x2aef2a85, 0xed796bca, + 0xac4870d3, 0x6f1b5df8, 0x2e2a46e1, 0xe136de66, 0xa007c57f, + 0x6354e854, 0x2265f34d, 0xe5f3b202, 0xa4c2a91b, 0x67918430, + 0x26a09f29, 0xb8aec5e4, 0xf99fdefd, 0x3accf3d6, 0x7bfde8cf, + 0xbc6ba980, 0xfd5ab299, 0x3e099fb2, 0x7f3884ab, 0xb0241c2c, + 0xf1150735, 0x32462a1e, 0x73773107, 0xb4e17048, 0xf5d06b51, + 0x3683467a, 0x77b25d63, 0x4ed7facb, 0x0fe6e1d2, 0xccb5ccf9, + 0x8d84d7e0, 0x4a1296af, 0x0b238db6, 0xc870a09d, 0x8941bb84, + 0x465d2303, 0x076c381a, 0xc43f1531, 0x850e0e28, 0x42984f67, + 0x03a9547e, 0xc0fa7955, 0x81cb624c, 0x1fc53881, 0x5ef42398, + 0x9da70eb3, 0xdc9615aa, 0x1b0054e5, 0x5a314ffc, 0x996262d7, + 0xd85379ce, 0x174fe149, 0x567efa50, 0x952dd77b, 0xd41ccc62, + 0x138a8d2d, 0x52bb9634, 0x91e8bb1f, 0xd0d9a006, 0xecf37e5e, + 0xadc26547, 0x6e91486c, 0x2fa05375, 0xe836123a, 0xa9070923, + 0x6a542408, 0x2b653f11, 0xe479a796, 0xa548bc8f, 0x661b91a4, + 0x272a8abd, 0xe0bccbf2, 0xa18dd0eb, 0x62defdc0, 0x23efe6d9, + 0xbde1bc14, 0xfcd0a70d, 0x3f838a26, 0x7eb2913f, 0xb924d070, + 0xf815cb69, 0x3b46e642, 0x7a77fd5b, 0xb56b65dc, 0xf45a7ec5, + 0x370953ee, 0x763848f7, 0xb1ae09b8, 0xf09f12a1, 0x33cc3f8a, + 0x72fd2493}, + {0x00000000, 0x376ac201, 0x6ed48403, 0x59be4602, 0xdca80907, + 0xebc2cb06, 0xb27c8d04, 0x85164f05, 0xb851130e, 0x8f3bd10f, + 0xd685970d, 0xe1ef550c, 0x64f91a09, 0x5393d808, 0x0a2d9e0a, + 0x3d475c0b, 0x70a3261c, 0x47c9e41d, 0x1e77a21f, 0x291d601e, + 0xac0b2f1b, 0x9b61ed1a, 0xc2dfab18, 0xf5b56919, 0xc8f23512, + 0xff98f713, 0xa626b111, 0x914c7310, 0x145a3c15, 0x2330fe14, + 0x7a8eb816, 0x4de47a17, 0xe0464d38, 0xd72c8f39, 0x8e92c93b, + 0xb9f80b3a, 0x3cee443f, 0x0b84863e, 0x523ac03c, 0x6550023d, + 0x58175e36, 0x6f7d9c37, 0x36c3da35, 0x01a91834, 0x84bf5731, + 0xb3d59530, 0xea6bd332, 0xdd011133, 0x90e56b24, 0xa78fa925, + 0xfe31ef27, 0xc95b2d26, 0x4c4d6223, 0x7b27a022, 0x2299e620, + 0x15f32421, 0x28b4782a, 0x1fdeba2b, 0x4660fc29, 0x710a3e28, + 0xf41c712d, 0xc376b32c, 0x9ac8f52e, 0xada2372f, 0xc08d9a70, + 0xf7e75871, 0xae591e73, 0x9933dc72, 0x1c259377, 0x2b4f5176, + 0x72f11774, 0x459bd575, 0x78dc897e, 0x4fb64b7f, 0x16080d7d, + 0x2162cf7c, 0xa4748079, 0x931e4278, 0xcaa0047a, 0xfdcac67b, + 0xb02ebc6c, 0x87447e6d, 0xdefa386f, 0xe990fa6e, 0x6c86b56b, + 0x5bec776a, 0x02523168, 0x3538f369, 0x087faf62, 0x3f156d63, + 0x66ab2b61, 0x51c1e960, 0xd4d7a665, 0xe3bd6464, 0xba032266, + 0x8d69e067, 0x20cbd748, 0x17a11549, 0x4e1f534b, 0x7975914a, + 0xfc63de4f, 0xcb091c4e, 0x92b75a4c, 0xa5dd984d, 0x989ac446, + 0xaff00647, 0xf64e4045, 0xc1248244, 0x4432cd41, 0x73580f40, + 0x2ae64942, 0x1d8c8b43, 0x5068f154, 0x67023355, 0x3ebc7557, + 0x09d6b756, 0x8cc0f853, 0xbbaa3a52, 0xe2147c50, 0xd57ebe51, + 0xe839e25a, 0xdf53205b, 0x86ed6659, 0xb187a458, 0x3491eb5d, + 0x03fb295c, 0x5a456f5e, 0x6d2fad5f, 0x801b35e1, 0xb771f7e0, + 0xeecfb1e2, 0xd9a573e3, 0x5cb33ce6, 0x6bd9fee7, 0x3267b8e5, + 0x050d7ae4, 0x384a26ef, 0x0f20e4ee, 0x569ea2ec, 0x61f460ed, + 0xe4e22fe8, 0xd388ede9, 0x8a36abeb, 0xbd5c69ea, 0xf0b813fd, + 0xc7d2d1fc, 0x9e6c97fe, 0xa90655ff, 0x2c101afa, 0x1b7ad8fb, + 0x42c49ef9, 0x75ae5cf8, 0x48e900f3, 0x7f83c2f2, 0x263d84f0, + 0x115746f1, 0x944109f4, 0xa32bcbf5, 0xfa958df7, 0xcdff4ff6, + 0x605d78d9, 0x5737bad8, 0x0e89fcda, 0x39e33edb, 0xbcf571de, + 0x8b9fb3df, 0xd221f5dd, 0xe54b37dc, 0xd80c6bd7, 0xef66a9d6, + 0xb6d8efd4, 0x81b22dd5, 0x04a462d0, 0x33cea0d1, 0x6a70e6d3, + 0x5d1a24d2, 0x10fe5ec5, 0x27949cc4, 0x7e2adac6, 0x494018c7, + 0xcc5657c2, 0xfb3c95c3, 0xa282d3c1, 0x95e811c0, 0xa8af4dcb, + 0x9fc58fca, 0xc67bc9c8, 0xf1110bc9, 0x740744cc, 0x436d86cd, + 0x1ad3c0cf, 0x2db902ce, 0x4096af91, 0x77fc6d90, 0x2e422b92, + 0x1928e993, 0x9c3ea696, 0xab546497, 0xf2ea2295, 0xc580e094, + 0xf8c7bc9f, 0xcfad7e9e, 0x9613389c, 0xa179fa9d, 0x246fb598, + 0x13057799, 0x4abb319b, 0x7dd1f39a, 0x3035898d, 0x075f4b8c, + 0x5ee10d8e, 0x698bcf8f, 0xec9d808a, 0xdbf7428b, 0x82490489, + 0xb523c688, 0x88649a83, 0xbf0e5882, 0xe6b01e80, 0xd1dadc81, + 0x54cc9384, 0x63a65185, 0x3a181787, 0x0d72d586, 0xa0d0e2a9, + 0x97ba20a8, 0xce0466aa, 0xf96ea4ab, 0x7c78ebae, 0x4b1229af, + 0x12ac6fad, 0x25c6adac, 0x1881f1a7, 0x2feb33a6, 0x765575a4, + 0x413fb7a5, 0xc429f8a0, 0xf3433aa1, 0xaafd7ca3, 0x9d97bea2, + 0xd073c4b5, 0xe71906b4, 0xbea740b6, 0x89cd82b7, 0x0cdbcdb2, + 0x3bb10fb3, 0x620f49b1, 0x55658bb0, 0x6822d7bb, 0x5f4815ba, + 0x06f653b8, 0x319c91b9, 0xb48adebc, 0x83e01cbd, 0xda5e5abf, + 0xed3498be}, + {0x00000000, 0x6567bcb8, 0x8bc809aa, 0xeeafb512, 0x5797628f, + 0x32f0de37, 0xdc5f6b25, 0xb938d79d, 0xef28b4c5, 0x8a4f087d, + 0x64e0bd6f, 0x018701d7, 0xb8bfd64a, 0xddd86af2, 0x3377dfe0, + 0x56106358, 0x9f571950, 0xfa30a5e8, 0x149f10fa, 0x71f8ac42, + 0xc8c07bdf, 0xada7c767, 0x43087275, 0x266fcecd, 0x707fad95, + 0x1518112d, 0xfbb7a43f, 0x9ed01887, 0x27e8cf1a, 0x428f73a2, + 0xac20c6b0, 0xc9477a08, 0x3eaf32a0, 0x5bc88e18, 0xb5673b0a, + 0xd00087b2, 0x6938502f, 0x0c5fec97, 0xe2f05985, 0x8797e53d, + 0xd1878665, 0xb4e03add, 0x5a4f8fcf, 0x3f283377, 0x8610e4ea, + 0xe3775852, 0x0dd8ed40, 0x68bf51f8, 0xa1f82bf0, 0xc49f9748, + 0x2a30225a, 0x4f579ee2, 0xf66f497f, 0x9308f5c7, 0x7da740d5, + 0x18c0fc6d, 0x4ed09f35, 0x2bb7238d, 0xc518969f, 0xa07f2a27, + 0x1947fdba, 0x7c204102, 0x928ff410, 0xf7e848a8, 0x3d58149b, + 0x583fa823, 0xb6901d31, 0xd3f7a189, 0x6acf7614, 0x0fa8caac, + 0xe1077fbe, 0x8460c306, 0xd270a05e, 0xb7171ce6, 0x59b8a9f4, + 0x3cdf154c, 0x85e7c2d1, 0xe0807e69, 0x0e2fcb7b, 0x6b4877c3, + 0xa20f0dcb, 0xc768b173, 0x29c70461, 0x4ca0b8d9, 0xf5986f44, + 0x90ffd3fc, 0x7e5066ee, 0x1b37da56, 0x4d27b90e, 0x284005b6, + 0xc6efb0a4, 0xa3880c1c, 0x1ab0db81, 0x7fd76739, 0x9178d22b, + 0xf41f6e93, 0x03f7263b, 0x66909a83, 0x883f2f91, 0xed589329, + 0x546044b4, 0x3107f80c, 0xdfa84d1e, 0xbacff1a6, 0xecdf92fe, + 0x89b82e46, 0x67179b54, 0x027027ec, 0xbb48f071, 0xde2f4cc9, + 0x3080f9db, 0x55e74563, 0x9ca03f6b, 0xf9c783d3, 0x176836c1, + 0x720f8a79, 0xcb375de4, 0xae50e15c, 0x40ff544e, 0x2598e8f6, + 0x73888bae, 0x16ef3716, 0xf8408204, 0x9d273ebc, 0x241fe921, + 0x41785599, 0xafd7e08b, 0xcab05c33, 0x3bb659ed, 0x5ed1e555, + 0xb07e5047, 0xd519ecff, 0x6c213b62, 0x094687da, 0xe7e932c8, + 0x828e8e70, 0xd49eed28, 0xb1f95190, 0x5f56e482, 0x3a31583a, + 0x83098fa7, 0xe66e331f, 0x08c1860d, 0x6da63ab5, 0xa4e140bd, + 0xc186fc05, 0x2f294917, 0x4a4ef5af, 0xf3762232, 0x96119e8a, + 0x78be2b98, 0x1dd99720, 0x4bc9f478, 0x2eae48c0, 0xc001fdd2, + 0xa566416a, 0x1c5e96f7, 0x79392a4f, 0x97969f5d, 0xf2f123e5, + 0x05196b4d, 0x607ed7f5, 0x8ed162e7, 0xebb6de5f, 0x528e09c2, + 0x37e9b57a, 0xd9460068, 0xbc21bcd0, 0xea31df88, 0x8f566330, + 0x61f9d622, 0x049e6a9a, 0xbda6bd07, 0xd8c101bf, 0x366eb4ad, + 0x53090815, 0x9a4e721d, 0xff29cea5, 0x11867bb7, 0x74e1c70f, + 0xcdd91092, 0xa8beac2a, 0x46111938, 0x2376a580, 0x7566c6d8, + 0x10017a60, 0xfeaecf72, 0x9bc973ca, 0x22f1a457, 0x479618ef, + 0xa939adfd, 0xcc5e1145, 0x06ee4d76, 0x6389f1ce, 0x8d2644dc, + 0xe841f864, 0x51792ff9, 0x341e9341, 0xdab12653, 0xbfd69aeb, + 0xe9c6f9b3, 0x8ca1450b, 0x620ef019, 0x07694ca1, 0xbe519b3c, + 0xdb362784, 0x35999296, 0x50fe2e2e, 0x99b95426, 0xfcdee89e, + 0x12715d8c, 0x7716e134, 0xce2e36a9, 0xab498a11, 0x45e63f03, + 0x208183bb, 0x7691e0e3, 0x13f65c5b, 0xfd59e949, 0x983e55f1, + 0x2106826c, 0x44613ed4, 0xaace8bc6, 0xcfa9377e, 0x38417fd6, + 0x5d26c36e, 0xb389767c, 0xd6eecac4, 0x6fd61d59, 0x0ab1a1e1, + 0xe41e14f3, 0x8179a84b, 0xd769cb13, 0xb20e77ab, 0x5ca1c2b9, + 0x39c67e01, 0x80fea99c, 0xe5991524, 0x0b36a036, 0x6e511c8e, + 0xa7166686, 0xc271da3e, 0x2cde6f2c, 0x49b9d394, 0xf0810409, + 0x95e6b8b1, 0x7b490da3, 0x1e2eb11b, 0x483ed243, 0x2d596efb, + 0xc3f6dbe9, 0xa6916751, 0x1fa9b0cc, 0x7ace0c74, 0x9461b966, + 0xf10605de}}; + +#endif + +#endif + +#if N == 2 + +#if W == 8 + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0xae689191, 0x87a02563, 0x29c8b4f2, 0xd4314c87, + 0x7a59dd16, 0x539169e4, 0xfdf9f875, 0x73139f4f, 0xdd7b0ede, + 0xf4b3ba2c, 0x5adb2bbd, 0xa722d3c8, 0x094a4259, 0x2082f6ab, + 0x8eea673a, 0xe6273e9e, 0x484faf0f, 0x61871bfd, 0xcfef8a6c, + 0x32167219, 0x9c7ee388, 0xb5b6577a, 0x1bdec6eb, 0x9534a1d1, + 0x3b5c3040, 0x129484b2, 0xbcfc1523, 0x4105ed56, 0xef6d7cc7, + 0xc6a5c835, 0x68cd59a4, 0x173f7b7d, 0xb957eaec, 0x909f5e1e, + 0x3ef7cf8f, 0xc30e37fa, 0x6d66a66b, 0x44ae1299, 0xeac68308, + 0x642ce432, 0xca4475a3, 0xe38cc151, 0x4de450c0, 0xb01da8b5, + 0x1e753924, 0x37bd8dd6, 0x99d51c47, 0xf11845e3, 0x5f70d472, + 0x76b86080, 0xd8d0f111, 0x25290964, 0x8b4198f5, 0xa2892c07, + 0x0ce1bd96, 0x820bdaac, 0x2c634b3d, 0x05abffcf, 0xabc36e5e, + 0x563a962b, 0xf85207ba, 0xd19ab348, 0x7ff222d9, 0x2e7ef6fa, + 0x8016676b, 0xa9ded399, 0x07b64208, 0xfa4fba7d, 0x54272bec, + 0x7def9f1e, 0xd3870e8f, 0x5d6d69b5, 0xf305f824, 0xdacd4cd6, + 0x74a5dd47, 0x895c2532, 0x2734b4a3, 0x0efc0051, 0xa09491c0, + 0xc859c864, 0x663159f5, 0x4ff9ed07, 0xe1917c96, 0x1c6884e3, + 0xb2001572, 0x9bc8a180, 0x35a03011, 0xbb4a572b, 0x1522c6ba, + 0x3cea7248, 0x9282e3d9, 0x6f7b1bac, 0xc1138a3d, 0xe8db3ecf, + 0x46b3af5e, 0x39418d87, 0x97291c16, 0xbee1a8e4, 0x10893975, + 0xed70c100, 0x43185091, 0x6ad0e463, 0xc4b875f2, 0x4a5212c8, + 0xe43a8359, 0xcdf237ab, 0x639aa63a, 0x9e635e4f, 0x300bcfde, + 0x19c37b2c, 0xb7abeabd, 0xdf66b319, 0x710e2288, 0x58c6967a, + 0xf6ae07eb, 0x0b57ff9e, 0xa53f6e0f, 0x8cf7dafd, 0x229f4b6c, + 0xac752c56, 0x021dbdc7, 0x2bd50935, 0x85bd98a4, 0x784460d1, + 0xd62cf140, 0xffe445b2, 0x518cd423, 0x5cfdedf4, 0xf2957c65, + 0xdb5dc897, 0x75355906, 0x88cca173, 0x26a430e2, 0x0f6c8410, + 0xa1041581, 0x2fee72bb, 0x8186e32a, 0xa84e57d8, 0x0626c649, + 0xfbdf3e3c, 0x55b7afad, 0x7c7f1b5f, 0xd2178ace, 0xbadad36a, + 0x14b242fb, 0x3d7af609, 0x93126798, 0x6eeb9fed, 0xc0830e7c, + 0xe94bba8e, 0x47232b1f, 0xc9c94c25, 0x67a1ddb4, 0x4e696946, + 0xe001f8d7, 0x1df800a2, 0xb3909133, 0x9a5825c1, 0x3430b450, + 0x4bc29689, 0xe5aa0718, 0xcc62b3ea, 0x620a227b, 0x9ff3da0e, + 0x319b4b9f, 0x1853ff6d, 0xb63b6efc, 0x38d109c6, 0x96b99857, + 0xbf712ca5, 0x1119bd34, 0xece04541, 0x4288d4d0, 0x6b406022, + 0xc528f1b3, 0xade5a817, 0x038d3986, 0x2a458d74, 0x842d1ce5, + 0x79d4e490, 0xd7bc7501, 0xfe74c1f3, 0x501c5062, 0xdef63758, + 0x709ea6c9, 0x5956123b, 0xf73e83aa, 0x0ac77bdf, 0xa4afea4e, + 0x8d675ebc, 0x230fcf2d, 0x72831b0e, 0xdceb8a9f, 0xf5233e6d, + 0x5b4baffc, 0xa6b25789, 0x08dac618, 0x211272ea, 0x8f7ae37b, + 0x01908441, 0xaff815d0, 0x8630a122, 0x285830b3, 0xd5a1c8c6, + 0x7bc95957, 0x5201eda5, 0xfc697c34, 0x94a42590, 0x3accb401, + 0x130400f3, 0xbd6c9162, 0x40956917, 0xeefdf886, 0xc7354c74, + 0x695ddde5, 0xe7b7badf, 0x49df2b4e, 0x60179fbc, 0xce7f0e2d, + 0x3386f658, 0x9dee67c9, 0xb426d33b, 0x1a4e42aa, 0x65bc6073, + 0xcbd4f1e2, 0xe21c4510, 0x4c74d481, 0xb18d2cf4, 0x1fe5bd65, + 0x362d0997, 0x98459806, 0x16afff3c, 0xb8c76ead, 0x910fda5f, + 0x3f674bce, 0xc29eb3bb, 0x6cf6222a, 0x453e96d8, 0xeb560749, + 0x839b5eed, 0x2df3cf7c, 0x043b7b8e, 0xaa53ea1f, 0x57aa126a, + 0xf9c283fb, 0xd00a3709, 0x7e62a698, 0xf088c1a2, 0x5ee05033, + 0x7728e4c1, 0xd9407550, 0x24b98d25, 0x8ad11cb4, 0xa319a846, + 0x0d7139d7}, + {0x00000000, 0xb9fbdbe8, 0xa886b191, 0x117d6a79, 0x8a7c6563, + 0x3387be8b, 0x22fad4f2, 0x9b010f1a, 0xcf89cc87, 0x7672176f, + 0x670f7d16, 0xdef4a6fe, 0x45f5a9e4, 0xfc0e720c, 0xed731875, + 0x5488c39d, 0x44629f4f, 0xfd9944a7, 0xece42ede, 0x551ff536, + 0xce1efa2c, 0x77e521c4, 0x66984bbd, 0xdf639055, 0x8beb53c8, + 0x32108820, 0x236de259, 0x9a9639b1, 0x019736ab, 0xb86ced43, + 0xa911873a, 0x10ea5cd2, 0x88c53e9e, 0x313ee576, 0x20438f0f, + 0x99b854e7, 0x02b95bfd, 0xbb428015, 0xaa3fea6c, 0x13c43184, + 0x474cf219, 0xfeb729f1, 0xefca4388, 0x56319860, 0xcd30977a, + 0x74cb4c92, 0x65b626eb, 0xdc4dfd03, 0xcca7a1d1, 0x755c7a39, + 0x64211040, 0xdddacba8, 0x46dbc4b2, 0xff201f5a, 0xee5d7523, + 0x57a6aecb, 0x032e6d56, 0xbad5b6be, 0xaba8dcc7, 0x1253072f, + 0x89520835, 0x30a9d3dd, 0x21d4b9a4, 0x982f624c, 0xcafb7b7d, + 0x7300a095, 0x627dcaec, 0xdb861104, 0x40871e1e, 0xf97cc5f6, + 0xe801af8f, 0x51fa7467, 0x0572b7fa, 0xbc896c12, 0xadf4066b, + 0x140fdd83, 0x8f0ed299, 0x36f50971, 0x27886308, 0x9e73b8e0, + 0x8e99e432, 0x37623fda, 0x261f55a3, 0x9fe48e4b, 0x04e58151, + 0xbd1e5ab9, 0xac6330c0, 0x1598eb28, 0x411028b5, 0xf8ebf35d, + 0xe9969924, 0x506d42cc, 0xcb6c4dd6, 0x7297963e, 0x63eafc47, + 0xda1127af, 0x423e45e3, 0xfbc59e0b, 0xeab8f472, 0x53432f9a, + 0xc8422080, 0x71b9fb68, 0x60c49111, 0xd93f4af9, 0x8db78964, + 0x344c528c, 0x253138f5, 0x9ccae31d, 0x07cbec07, 0xbe3037ef, + 0xaf4d5d96, 0x16b6867e, 0x065cdaac, 0xbfa70144, 0xaeda6b3d, + 0x1721b0d5, 0x8c20bfcf, 0x35db6427, 0x24a60e5e, 0x9d5dd5b6, + 0xc9d5162b, 0x702ecdc3, 0x6153a7ba, 0xd8a87c52, 0x43a97348, + 0xfa52a8a0, 0xeb2fc2d9, 0x52d41931, 0x4e87f0bb, 0xf77c2b53, + 0xe601412a, 0x5ffa9ac2, 0xc4fb95d8, 0x7d004e30, 0x6c7d2449, + 0xd586ffa1, 0x810e3c3c, 0x38f5e7d4, 0x29888dad, 0x90735645, + 0x0b72595f, 0xb28982b7, 0xa3f4e8ce, 0x1a0f3326, 0x0ae56ff4, + 0xb31eb41c, 0xa263de65, 0x1b98058d, 0x80990a97, 0x3962d17f, + 0x281fbb06, 0x91e460ee, 0xc56ca373, 0x7c97789b, 0x6dea12e2, + 0xd411c90a, 0x4f10c610, 0xf6eb1df8, 0xe7967781, 0x5e6dac69, + 0xc642ce25, 0x7fb915cd, 0x6ec47fb4, 0xd73fa45c, 0x4c3eab46, + 0xf5c570ae, 0xe4b81ad7, 0x5d43c13f, 0x09cb02a2, 0xb030d94a, + 0xa14db333, 0x18b668db, 0x83b767c1, 0x3a4cbc29, 0x2b31d650, + 0x92ca0db8, 0x8220516a, 0x3bdb8a82, 0x2aa6e0fb, 0x935d3b13, + 0x085c3409, 0xb1a7efe1, 0xa0da8598, 0x19215e70, 0x4da99ded, + 0xf4524605, 0xe52f2c7c, 0x5cd4f794, 0xc7d5f88e, 0x7e2e2366, + 0x6f53491f, 0xd6a892f7, 0x847c8bc6, 0x3d87502e, 0x2cfa3a57, + 0x9501e1bf, 0x0e00eea5, 0xb7fb354d, 0xa6865f34, 0x1f7d84dc, + 0x4bf54741, 0xf20e9ca9, 0xe373f6d0, 0x5a882d38, 0xc1892222, + 0x7872f9ca, 0x690f93b3, 0xd0f4485b, 0xc01e1489, 0x79e5cf61, + 0x6898a518, 0xd1637ef0, 0x4a6271ea, 0xf399aa02, 0xe2e4c07b, + 0x5b1f1b93, 0x0f97d80e, 0xb66c03e6, 0xa711699f, 0x1eeab277, + 0x85ebbd6d, 0x3c106685, 0x2d6d0cfc, 0x9496d714, 0x0cb9b558, + 0xb5426eb0, 0xa43f04c9, 0x1dc4df21, 0x86c5d03b, 0x3f3e0bd3, + 0x2e4361aa, 0x97b8ba42, 0xc33079df, 0x7acba237, 0x6bb6c84e, + 0xd24d13a6, 0x494c1cbc, 0xf0b7c754, 0xe1caad2d, 0x583176c5, + 0x48db2a17, 0xf120f1ff, 0xe05d9b86, 0x59a6406e, 0xc2a74f74, + 0x7b5c949c, 0x6a21fee5, 0xd3da250d, 0x8752e690, 0x3ea93d78, + 0x2fd45701, 0x962f8ce9, 0x0d2e83f3, 0xb4d5581b, 0xa5a83262, + 0x1c53e98a}, + {0x00000000, 0x9d0fe176, 0xe16ec4ad, 0x7c6125db, 0x19ac8f1b, + 0x84a36e6d, 0xf8c24bb6, 0x65cdaac0, 0x33591e36, 0xae56ff40, + 0xd237da9b, 0x4f383bed, 0x2af5912d, 0xb7fa705b, 0xcb9b5580, + 0x5694b4f6, 0x66b23c6c, 0xfbbddd1a, 0x87dcf8c1, 0x1ad319b7, + 0x7f1eb377, 0xe2115201, 0x9e7077da, 0x037f96ac, 0x55eb225a, + 0xc8e4c32c, 0xb485e6f7, 0x298a0781, 0x4c47ad41, 0xd1484c37, + 0xad2969ec, 0x3026889a, 0xcd6478d8, 0x506b99ae, 0x2c0abc75, + 0xb1055d03, 0xd4c8f7c3, 0x49c716b5, 0x35a6336e, 0xa8a9d218, + 0xfe3d66ee, 0x63328798, 0x1f53a243, 0x825c4335, 0xe791e9f5, + 0x7a9e0883, 0x06ff2d58, 0x9bf0cc2e, 0xabd644b4, 0x36d9a5c2, + 0x4ab88019, 0xd7b7616f, 0xb27acbaf, 0x2f752ad9, 0x53140f02, + 0xce1bee74, 0x988f5a82, 0x0580bbf4, 0x79e19e2f, 0xe4ee7f59, + 0x8123d599, 0x1c2c34ef, 0x604d1134, 0xfd42f042, 0x41b9f7f1, + 0xdcb61687, 0xa0d7335c, 0x3dd8d22a, 0x581578ea, 0xc51a999c, + 0xb97bbc47, 0x24745d31, 0x72e0e9c7, 0xefef08b1, 0x938e2d6a, + 0x0e81cc1c, 0x6b4c66dc, 0xf64387aa, 0x8a22a271, 0x172d4307, + 0x270bcb9d, 0xba042aeb, 0xc6650f30, 0x5b6aee46, 0x3ea74486, + 0xa3a8a5f0, 0xdfc9802b, 0x42c6615d, 0x1452d5ab, 0x895d34dd, + 0xf53c1106, 0x6833f070, 0x0dfe5ab0, 0x90f1bbc6, 0xec909e1d, + 0x719f7f6b, 0x8cdd8f29, 0x11d26e5f, 0x6db34b84, 0xf0bcaaf2, + 0x95710032, 0x087ee144, 0x741fc49f, 0xe91025e9, 0xbf84911f, + 0x228b7069, 0x5eea55b2, 0xc3e5b4c4, 0xa6281e04, 0x3b27ff72, + 0x4746daa9, 0xda493bdf, 0xea6fb345, 0x77605233, 0x0b0177e8, + 0x960e969e, 0xf3c33c5e, 0x6eccdd28, 0x12adf8f3, 0x8fa21985, + 0xd936ad73, 0x44394c05, 0x385869de, 0xa55788a8, 0xc09a2268, + 0x5d95c31e, 0x21f4e6c5, 0xbcfb07b3, 0x8373efe2, 0x1e7c0e94, + 0x621d2b4f, 0xff12ca39, 0x9adf60f9, 0x07d0818f, 0x7bb1a454, + 0xe6be4522, 0xb02af1d4, 0x2d2510a2, 0x51443579, 0xcc4bd40f, + 0xa9867ecf, 0x34899fb9, 0x48e8ba62, 0xd5e75b14, 0xe5c1d38e, + 0x78ce32f8, 0x04af1723, 0x99a0f655, 0xfc6d5c95, 0x6162bde3, + 0x1d039838, 0x800c794e, 0xd698cdb8, 0x4b972cce, 0x37f60915, + 0xaaf9e863, 0xcf3442a3, 0x523ba3d5, 0x2e5a860e, 0xb3556778, + 0x4e17973a, 0xd318764c, 0xaf795397, 0x3276b2e1, 0x57bb1821, + 0xcab4f957, 0xb6d5dc8c, 0x2bda3dfa, 0x7d4e890c, 0xe041687a, + 0x9c204da1, 0x012facd7, 0x64e20617, 0xf9ede761, 0x858cc2ba, + 0x188323cc, 0x28a5ab56, 0xb5aa4a20, 0xc9cb6ffb, 0x54c48e8d, + 0x3109244d, 0xac06c53b, 0xd067e0e0, 0x4d680196, 0x1bfcb560, + 0x86f35416, 0xfa9271cd, 0x679d90bb, 0x02503a7b, 0x9f5fdb0d, + 0xe33efed6, 0x7e311fa0, 0xc2ca1813, 0x5fc5f965, 0x23a4dcbe, + 0xbeab3dc8, 0xdb669708, 0x4669767e, 0x3a0853a5, 0xa707b2d3, + 0xf1930625, 0x6c9ce753, 0x10fdc288, 0x8df223fe, 0xe83f893e, + 0x75306848, 0x09514d93, 0x945eace5, 0xa478247f, 0x3977c509, + 0x4516e0d2, 0xd81901a4, 0xbdd4ab64, 0x20db4a12, 0x5cba6fc9, + 0xc1b58ebf, 0x97213a49, 0x0a2edb3f, 0x764ffee4, 0xeb401f92, + 0x8e8db552, 0x13825424, 0x6fe371ff, 0xf2ec9089, 0x0fae60cb, + 0x92a181bd, 0xeec0a466, 0x73cf4510, 0x1602efd0, 0x8b0d0ea6, + 0xf76c2b7d, 0x6a63ca0b, 0x3cf77efd, 0xa1f89f8b, 0xdd99ba50, + 0x40965b26, 0x255bf1e6, 0xb8541090, 0xc435354b, 0x593ad43d, + 0x691c5ca7, 0xf413bdd1, 0x8872980a, 0x157d797c, 0x70b0d3bc, + 0xedbf32ca, 0x91de1711, 0x0cd1f667, 0x5a454291, 0xc74aa3e7, + 0xbb2b863c, 0x2624674a, 0x43e9cd8a, 0xdee62cfc, 0xa2870927, + 0x3f88e851}, + {0x00000000, 0xdd96d985, 0x605cb54b, 0xbdca6cce, 0xc0b96a96, + 0x1d2fb313, 0xa0e5dfdd, 0x7d730658, 0x5a03d36d, 0x87950ae8, + 0x3a5f6626, 0xe7c9bfa3, 0x9abab9fb, 0x472c607e, 0xfae60cb0, + 0x2770d535, 0xb407a6da, 0x69917f5f, 0xd45b1391, 0x09cdca14, + 0x74becc4c, 0xa92815c9, 0x14e27907, 0xc974a082, 0xee0475b7, + 0x3392ac32, 0x8e58c0fc, 0x53ce1979, 0x2ebd1f21, 0xf32bc6a4, + 0x4ee1aa6a, 0x937773ef, 0xb37e4bf5, 0x6ee89270, 0xd322febe, + 0x0eb4273b, 0x73c72163, 0xae51f8e6, 0x139b9428, 0xce0d4dad, + 0xe97d9898, 0x34eb411d, 0x89212dd3, 0x54b7f456, 0x29c4f20e, + 0xf4522b8b, 0x49984745, 0x940e9ec0, 0x0779ed2f, 0xdaef34aa, + 0x67255864, 0xbab381e1, 0xc7c087b9, 0x1a565e3c, 0xa79c32f2, + 0x7a0aeb77, 0x5d7a3e42, 0x80ece7c7, 0x3d268b09, 0xe0b0528c, + 0x9dc354d4, 0x40558d51, 0xfd9fe19f, 0x2009381a, 0xbd8d91ab, + 0x601b482e, 0xddd124e0, 0x0047fd65, 0x7d34fb3d, 0xa0a222b8, + 0x1d684e76, 0xc0fe97f3, 0xe78e42c6, 0x3a189b43, 0x87d2f78d, + 0x5a442e08, 0x27372850, 0xfaa1f1d5, 0x476b9d1b, 0x9afd449e, + 0x098a3771, 0xd41ceef4, 0x69d6823a, 0xb4405bbf, 0xc9335de7, + 0x14a58462, 0xa96fe8ac, 0x74f93129, 0x5389e41c, 0x8e1f3d99, + 0x33d55157, 0xee4388d2, 0x93308e8a, 0x4ea6570f, 0xf36c3bc1, + 0x2efae244, 0x0ef3da5e, 0xd36503db, 0x6eaf6f15, 0xb339b690, + 0xce4ab0c8, 0x13dc694d, 0xae160583, 0x7380dc06, 0x54f00933, + 0x8966d0b6, 0x34acbc78, 0xe93a65fd, 0x944963a5, 0x49dfba20, + 0xf415d6ee, 0x29830f6b, 0xbaf47c84, 0x6762a501, 0xdaa8c9cf, + 0x073e104a, 0x7a4d1612, 0xa7dbcf97, 0x1a11a359, 0xc7877adc, + 0xe0f7afe9, 0x3d61766c, 0x80ab1aa2, 0x5d3dc327, 0x204ec57f, + 0xfdd81cfa, 0x40127034, 0x9d84a9b1, 0xa06a2517, 0x7dfcfc92, + 0xc036905c, 0x1da049d9, 0x60d34f81, 0xbd459604, 0x008ffaca, + 0xdd19234f, 0xfa69f67a, 0x27ff2fff, 0x9a354331, 0x47a39ab4, + 0x3ad09cec, 0xe7464569, 0x5a8c29a7, 0x871af022, 0x146d83cd, + 0xc9fb5a48, 0x74313686, 0xa9a7ef03, 0xd4d4e95b, 0x094230de, + 0xb4885c10, 0x691e8595, 0x4e6e50a0, 0x93f88925, 0x2e32e5eb, + 0xf3a43c6e, 0x8ed73a36, 0x5341e3b3, 0xee8b8f7d, 0x331d56f8, + 0x13146ee2, 0xce82b767, 0x7348dba9, 0xaede022c, 0xd3ad0474, + 0x0e3bddf1, 0xb3f1b13f, 0x6e6768ba, 0x4917bd8f, 0x9481640a, + 0x294b08c4, 0xf4ddd141, 0x89aed719, 0x54380e9c, 0xe9f26252, + 0x3464bbd7, 0xa713c838, 0x7a8511bd, 0xc74f7d73, 0x1ad9a4f6, + 0x67aaa2ae, 0xba3c7b2b, 0x07f617e5, 0xda60ce60, 0xfd101b55, + 0x2086c2d0, 0x9d4cae1e, 0x40da779b, 0x3da971c3, 0xe03fa846, + 0x5df5c488, 0x80631d0d, 0x1de7b4bc, 0xc0716d39, 0x7dbb01f7, + 0xa02dd872, 0xdd5ede2a, 0x00c807af, 0xbd026b61, 0x6094b2e4, + 0x47e467d1, 0x9a72be54, 0x27b8d29a, 0xfa2e0b1f, 0x875d0d47, + 0x5acbd4c2, 0xe701b80c, 0x3a976189, 0xa9e01266, 0x7476cbe3, + 0xc9bca72d, 0x142a7ea8, 0x695978f0, 0xb4cfa175, 0x0905cdbb, + 0xd493143e, 0xf3e3c10b, 0x2e75188e, 0x93bf7440, 0x4e29adc5, + 0x335aab9d, 0xeecc7218, 0x53061ed6, 0x8e90c753, 0xae99ff49, + 0x730f26cc, 0xcec54a02, 0x13539387, 0x6e2095df, 0xb3b64c5a, + 0x0e7c2094, 0xd3eaf911, 0xf49a2c24, 0x290cf5a1, 0x94c6996f, + 0x495040ea, 0x342346b2, 0xe9b59f37, 0x547ff3f9, 0x89e92a7c, + 0x1a9e5993, 0xc7088016, 0x7ac2ecd8, 0xa754355d, 0xda273305, + 0x07b1ea80, 0xba7b864e, 0x67ed5fcb, 0x409d8afe, 0x9d0b537b, + 0x20c13fb5, 0xfd57e630, 0x8024e068, 0x5db239ed, 0xe0785523, + 0x3dee8ca6}, + {0x00000000, 0x9ba54c6f, 0xec3b9e9f, 0x779ed2f0, 0x03063b7f, + 0x98a37710, 0xef3da5e0, 0x7498e98f, 0x060c76fe, 0x9da93a91, + 0xea37e861, 0x7192a40e, 0x050a4d81, 0x9eaf01ee, 0xe931d31e, + 0x72949f71, 0x0c18edfc, 0x97bda193, 0xe0237363, 0x7b863f0c, + 0x0f1ed683, 0x94bb9aec, 0xe325481c, 0x78800473, 0x0a149b02, + 0x91b1d76d, 0xe62f059d, 0x7d8a49f2, 0x0912a07d, 0x92b7ec12, + 0xe5293ee2, 0x7e8c728d, 0x1831dbf8, 0x83949797, 0xf40a4567, + 0x6faf0908, 0x1b37e087, 0x8092ace8, 0xf70c7e18, 0x6ca93277, + 0x1e3dad06, 0x8598e169, 0xf2063399, 0x69a37ff6, 0x1d3b9679, + 0x869eda16, 0xf10008e6, 0x6aa54489, 0x14293604, 0x8f8c7a6b, + 0xf812a89b, 0x63b7e4f4, 0x172f0d7b, 0x8c8a4114, 0xfb1493e4, + 0x60b1df8b, 0x122540fa, 0x89800c95, 0xfe1ede65, 0x65bb920a, + 0x11237b85, 0x8a8637ea, 0xfd18e51a, 0x66bda975, 0x3063b7f0, + 0xabc6fb9f, 0xdc58296f, 0x47fd6500, 0x33658c8f, 0xa8c0c0e0, + 0xdf5e1210, 0x44fb5e7f, 0x366fc10e, 0xadca8d61, 0xda545f91, + 0x41f113fe, 0x3569fa71, 0xaeccb61e, 0xd95264ee, 0x42f72881, + 0x3c7b5a0c, 0xa7de1663, 0xd040c493, 0x4be588fc, 0x3f7d6173, + 0xa4d82d1c, 0xd346ffec, 0x48e3b383, 0x3a772cf2, 0xa1d2609d, + 0xd64cb26d, 0x4de9fe02, 0x3971178d, 0xa2d45be2, 0xd54a8912, + 0x4eefc57d, 0x28526c08, 0xb3f72067, 0xc469f297, 0x5fccbef8, + 0x2b545777, 0xb0f11b18, 0xc76fc9e8, 0x5cca8587, 0x2e5e1af6, + 0xb5fb5699, 0xc2658469, 0x59c0c806, 0x2d582189, 0xb6fd6de6, + 0xc163bf16, 0x5ac6f379, 0x244a81f4, 0xbfefcd9b, 0xc8711f6b, + 0x53d45304, 0x274cba8b, 0xbce9f6e4, 0xcb772414, 0x50d2687b, + 0x2246f70a, 0xb9e3bb65, 0xce7d6995, 0x55d825fa, 0x2140cc75, + 0xbae5801a, 0xcd7b52ea, 0x56de1e85, 0x60c76fe0, 0xfb62238f, + 0x8cfcf17f, 0x1759bd10, 0x63c1549f, 0xf86418f0, 0x8ffaca00, + 0x145f866f, 0x66cb191e, 0xfd6e5571, 0x8af08781, 0x1155cbee, + 0x65cd2261, 0xfe686e0e, 0x89f6bcfe, 0x1253f091, 0x6cdf821c, + 0xf77ace73, 0x80e41c83, 0x1b4150ec, 0x6fd9b963, 0xf47cf50c, + 0x83e227fc, 0x18476b93, 0x6ad3f4e2, 0xf176b88d, 0x86e86a7d, + 0x1d4d2612, 0x69d5cf9d, 0xf27083f2, 0x85ee5102, 0x1e4b1d6d, + 0x78f6b418, 0xe353f877, 0x94cd2a87, 0x0f6866e8, 0x7bf08f67, + 0xe055c308, 0x97cb11f8, 0x0c6e5d97, 0x7efac2e6, 0xe55f8e89, + 0x92c15c79, 0x09641016, 0x7dfcf999, 0xe659b5f6, 0x91c76706, + 0x0a622b69, 0x74ee59e4, 0xef4b158b, 0x98d5c77b, 0x03708b14, + 0x77e8629b, 0xec4d2ef4, 0x9bd3fc04, 0x0076b06b, 0x72e22f1a, + 0xe9476375, 0x9ed9b185, 0x057cfdea, 0x71e41465, 0xea41580a, + 0x9ddf8afa, 0x067ac695, 0x50a4d810, 0xcb01947f, 0xbc9f468f, + 0x273a0ae0, 0x53a2e36f, 0xc807af00, 0xbf997df0, 0x243c319f, + 0x56a8aeee, 0xcd0de281, 0xba933071, 0x21367c1e, 0x55ae9591, + 0xce0bd9fe, 0xb9950b0e, 0x22304761, 0x5cbc35ec, 0xc7197983, + 0xb087ab73, 0x2b22e71c, 0x5fba0e93, 0xc41f42fc, 0xb381900c, + 0x2824dc63, 0x5ab04312, 0xc1150f7d, 0xb68bdd8d, 0x2d2e91e2, + 0x59b6786d, 0xc2133402, 0xb58de6f2, 0x2e28aa9d, 0x489503e8, + 0xd3304f87, 0xa4ae9d77, 0x3f0bd118, 0x4b933897, 0xd03674f8, + 0xa7a8a608, 0x3c0dea67, 0x4e997516, 0xd53c3979, 0xa2a2eb89, + 0x3907a7e6, 0x4d9f4e69, 0xd63a0206, 0xa1a4d0f6, 0x3a019c99, + 0x448dee14, 0xdf28a27b, 0xa8b6708b, 0x33133ce4, 0x478bd56b, + 0xdc2e9904, 0xabb04bf4, 0x3015079b, 0x428198ea, 0xd924d485, + 0xaeba0675, 0x351f4a1a, 0x4187a395, 0xda22effa, 0xadbc3d0a, + 0x36197165}, + {0x00000000, 0xc18edfc0, 0x586cb9c1, 0x99e26601, 0xb0d97382, + 0x7157ac42, 0xe8b5ca43, 0x293b1583, 0xbac3e145, 0x7b4d3e85, + 0xe2af5884, 0x23218744, 0x0a1a92c7, 0xcb944d07, 0x52762b06, + 0x93f8f4c6, 0xaef6c4cb, 0x6f781b0b, 0xf69a7d0a, 0x3714a2ca, + 0x1e2fb749, 0xdfa16889, 0x46430e88, 0x87cdd148, 0x1435258e, + 0xd5bbfa4e, 0x4c599c4f, 0x8dd7438f, 0xa4ec560c, 0x656289cc, + 0xfc80efcd, 0x3d0e300d, 0x869c8fd7, 0x47125017, 0xdef03616, + 0x1f7ee9d6, 0x3645fc55, 0xf7cb2395, 0x6e294594, 0xafa79a54, + 0x3c5f6e92, 0xfdd1b152, 0x6433d753, 0xa5bd0893, 0x8c861d10, + 0x4d08c2d0, 0xd4eaa4d1, 0x15647b11, 0x286a4b1c, 0xe9e494dc, + 0x7006f2dd, 0xb1882d1d, 0x98b3389e, 0x593de75e, 0xc0df815f, + 0x01515e9f, 0x92a9aa59, 0x53277599, 0xcac51398, 0x0b4bcc58, + 0x2270d9db, 0xe3fe061b, 0x7a1c601a, 0xbb92bfda, 0xd64819ef, + 0x17c6c62f, 0x8e24a02e, 0x4faa7fee, 0x66916a6d, 0xa71fb5ad, + 0x3efdd3ac, 0xff730c6c, 0x6c8bf8aa, 0xad05276a, 0x34e7416b, + 0xf5699eab, 0xdc528b28, 0x1ddc54e8, 0x843e32e9, 0x45b0ed29, + 0x78bedd24, 0xb93002e4, 0x20d264e5, 0xe15cbb25, 0xc867aea6, + 0x09e97166, 0x900b1767, 0x5185c8a7, 0xc27d3c61, 0x03f3e3a1, + 0x9a1185a0, 0x5b9f5a60, 0x72a44fe3, 0xb32a9023, 0x2ac8f622, + 0xeb4629e2, 0x50d49638, 0x915a49f8, 0x08b82ff9, 0xc936f039, + 0xe00de5ba, 0x21833a7a, 0xb8615c7b, 0x79ef83bb, 0xea17777d, + 0x2b99a8bd, 0xb27bcebc, 0x73f5117c, 0x5ace04ff, 0x9b40db3f, + 0x02a2bd3e, 0xc32c62fe, 0xfe2252f3, 0x3fac8d33, 0xa64eeb32, + 0x67c034f2, 0x4efb2171, 0x8f75feb1, 0x169798b0, 0xd7194770, + 0x44e1b3b6, 0x856f6c76, 0x1c8d0a77, 0xdd03d5b7, 0xf438c034, + 0x35b61ff4, 0xac5479f5, 0x6ddaa635, 0x77e1359f, 0xb66fea5f, + 0x2f8d8c5e, 0xee03539e, 0xc738461d, 0x06b699dd, 0x9f54ffdc, + 0x5eda201c, 0xcd22d4da, 0x0cac0b1a, 0x954e6d1b, 0x54c0b2db, + 0x7dfba758, 0xbc757898, 0x25971e99, 0xe419c159, 0xd917f154, + 0x18992e94, 0x817b4895, 0x40f59755, 0x69ce82d6, 0xa8405d16, + 0x31a23b17, 0xf02ce4d7, 0x63d41011, 0xa25acfd1, 0x3bb8a9d0, + 0xfa367610, 0xd30d6393, 0x1283bc53, 0x8b61da52, 0x4aef0592, + 0xf17dba48, 0x30f36588, 0xa9110389, 0x689fdc49, 0x41a4c9ca, + 0x802a160a, 0x19c8700b, 0xd846afcb, 0x4bbe5b0d, 0x8a3084cd, + 0x13d2e2cc, 0xd25c3d0c, 0xfb67288f, 0x3ae9f74f, 0xa30b914e, + 0x62854e8e, 0x5f8b7e83, 0x9e05a143, 0x07e7c742, 0xc6691882, + 0xef520d01, 0x2edcd2c1, 0xb73eb4c0, 0x76b06b00, 0xe5489fc6, + 0x24c64006, 0xbd242607, 0x7caaf9c7, 0x5591ec44, 0x941f3384, + 0x0dfd5585, 0xcc738a45, 0xa1a92c70, 0x6027f3b0, 0xf9c595b1, + 0x384b4a71, 0x11705ff2, 0xd0fe8032, 0x491ce633, 0x889239f3, + 0x1b6acd35, 0xdae412f5, 0x430674f4, 0x8288ab34, 0xabb3beb7, + 0x6a3d6177, 0xf3df0776, 0x3251d8b6, 0x0f5fe8bb, 0xced1377b, + 0x5733517a, 0x96bd8eba, 0xbf869b39, 0x7e0844f9, 0xe7ea22f8, + 0x2664fd38, 0xb59c09fe, 0x7412d63e, 0xedf0b03f, 0x2c7e6fff, + 0x05457a7c, 0xc4cba5bc, 0x5d29c3bd, 0x9ca71c7d, 0x2735a3a7, + 0xe6bb7c67, 0x7f591a66, 0xbed7c5a6, 0x97ecd025, 0x56620fe5, + 0xcf8069e4, 0x0e0eb624, 0x9df642e2, 0x5c789d22, 0xc59afb23, + 0x041424e3, 0x2d2f3160, 0xeca1eea0, 0x754388a1, 0xb4cd5761, + 0x89c3676c, 0x484db8ac, 0xd1afdead, 0x1021016d, 0x391a14ee, + 0xf894cb2e, 0x6176ad2f, 0xa0f872ef, 0x33008629, 0xf28e59e9, + 0x6b6c3fe8, 0xaae2e028, 0x83d9f5ab, 0x42572a6b, 0xdbb54c6a, + 0x1a3b93aa}, + {0x00000000, 0xefc26b3e, 0x04f5d03d, 0xeb37bb03, 0x09eba07a, + 0xe629cb44, 0x0d1e7047, 0xe2dc1b79, 0x13d740f4, 0xfc152bca, + 0x172290c9, 0xf8e0fbf7, 0x1a3ce08e, 0xf5fe8bb0, 0x1ec930b3, + 0xf10b5b8d, 0x27ae81e8, 0xc86cead6, 0x235b51d5, 0xcc993aeb, + 0x2e452192, 0xc1874aac, 0x2ab0f1af, 0xc5729a91, 0x3479c11c, + 0xdbbbaa22, 0x308c1121, 0xdf4e7a1f, 0x3d926166, 0xd2500a58, + 0x3967b15b, 0xd6a5da65, 0x4f5d03d0, 0xa09f68ee, 0x4ba8d3ed, + 0xa46ab8d3, 0x46b6a3aa, 0xa974c894, 0x42437397, 0xad8118a9, + 0x5c8a4324, 0xb348281a, 0x587f9319, 0xb7bdf827, 0x5561e35e, + 0xbaa38860, 0x51943363, 0xbe56585d, 0x68f38238, 0x8731e906, + 0x6c065205, 0x83c4393b, 0x61182242, 0x8eda497c, 0x65edf27f, + 0x8a2f9941, 0x7b24c2cc, 0x94e6a9f2, 0x7fd112f1, 0x901379cf, + 0x72cf62b6, 0x9d0d0988, 0x763ab28b, 0x99f8d9b5, 0x9eba07a0, + 0x71786c9e, 0x9a4fd79d, 0x758dbca3, 0x9751a7da, 0x7893cce4, + 0x93a477e7, 0x7c661cd9, 0x8d6d4754, 0x62af2c6a, 0x89989769, + 0x665afc57, 0x8486e72e, 0x6b448c10, 0x80733713, 0x6fb15c2d, + 0xb9148648, 0x56d6ed76, 0xbde15675, 0x52233d4b, 0xb0ff2632, + 0x5f3d4d0c, 0xb40af60f, 0x5bc89d31, 0xaac3c6bc, 0x4501ad82, + 0xae361681, 0x41f47dbf, 0xa32866c6, 0x4cea0df8, 0xa7ddb6fb, + 0x481fddc5, 0xd1e70470, 0x3e256f4e, 0xd512d44d, 0x3ad0bf73, + 0xd80ca40a, 0x37cecf34, 0xdcf97437, 0x333b1f09, 0xc2304484, + 0x2df22fba, 0xc6c594b9, 0x2907ff87, 0xcbdbe4fe, 0x24198fc0, + 0xcf2e34c3, 0x20ec5ffd, 0xf6498598, 0x198beea6, 0xf2bc55a5, + 0x1d7e3e9b, 0xffa225e2, 0x10604edc, 0xfb57f5df, 0x14959ee1, + 0xe59ec56c, 0x0a5cae52, 0xe16b1551, 0x0ea97e6f, 0xec756516, + 0x03b70e28, 0xe880b52b, 0x0742de15, 0xe6050901, 0x09c7623f, + 0xe2f0d93c, 0x0d32b202, 0xefeea97b, 0x002cc245, 0xeb1b7946, + 0x04d91278, 0xf5d249f5, 0x1a1022cb, 0xf12799c8, 0x1ee5f2f6, + 0xfc39e98f, 0x13fb82b1, 0xf8cc39b2, 0x170e528c, 0xc1ab88e9, + 0x2e69e3d7, 0xc55e58d4, 0x2a9c33ea, 0xc8402893, 0x278243ad, + 0xccb5f8ae, 0x23779390, 0xd27cc81d, 0x3dbea323, 0xd6891820, + 0x394b731e, 0xdb976867, 0x34550359, 0xdf62b85a, 0x30a0d364, + 0xa9580ad1, 0x469a61ef, 0xadaddaec, 0x426fb1d2, 0xa0b3aaab, + 0x4f71c195, 0xa4467a96, 0x4b8411a8, 0xba8f4a25, 0x554d211b, + 0xbe7a9a18, 0x51b8f126, 0xb364ea5f, 0x5ca68161, 0xb7913a62, + 0x5853515c, 0x8ef68b39, 0x6134e007, 0x8a035b04, 0x65c1303a, + 0x871d2b43, 0x68df407d, 0x83e8fb7e, 0x6c2a9040, 0x9d21cbcd, + 0x72e3a0f3, 0x99d41bf0, 0x761670ce, 0x94ca6bb7, 0x7b080089, + 0x903fbb8a, 0x7ffdd0b4, 0x78bf0ea1, 0x977d659f, 0x7c4ade9c, + 0x9388b5a2, 0x7154aedb, 0x9e96c5e5, 0x75a17ee6, 0x9a6315d8, + 0x6b684e55, 0x84aa256b, 0x6f9d9e68, 0x805ff556, 0x6283ee2f, + 0x8d418511, 0x66763e12, 0x89b4552c, 0x5f118f49, 0xb0d3e477, + 0x5be45f74, 0xb426344a, 0x56fa2f33, 0xb938440d, 0x520fff0e, + 0xbdcd9430, 0x4cc6cfbd, 0xa304a483, 0x48331f80, 0xa7f174be, + 0x452d6fc7, 0xaaef04f9, 0x41d8bffa, 0xae1ad4c4, 0x37e20d71, + 0xd820664f, 0x3317dd4c, 0xdcd5b672, 0x3e09ad0b, 0xd1cbc635, + 0x3afc7d36, 0xd53e1608, 0x24354d85, 0xcbf726bb, 0x20c09db8, + 0xcf02f686, 0x2ddeedff, 0xc21c86c1, 0x292b3dc2, 0xc6e956fc, + 0x104c8c99, 0xff8ee7a7, 0x14b95ca4, 0xfb7b379a, 0x19a72ce3, + 0xf66547dd, 0x1d52fcde, 0xf29097e0, 0x039bcc6d, 0xec59a753, + 0x076e1c50, 0xe8ac776e, 0x0a706c17, 0xe5b20729, 0x0e85bc2a, + 0xe147d714}, + {0x00000000, 0x177b1443, 0x2ef62886, 0x398d3cc5, 0x5dec510c, + 0x4a97454f, 0x731a798a, 0x64616dc9, 0xbbd8a218, 0xaca3b65b, + 0x952e8a9e, 0x82559edd, 0xe634f314, 0xf14fe757, 0xc8c2db92, + 0xdfb9cfd1, 0xacc04271, 0xbbbb5632, 0x82366af7, 0x954d7eb4, + 0xf12c137d, 0xe657073e, 0xdfda3bfb, 0xc8a12fb8, 0x1718e069, + 0x0063f42a, 0x39eec8ef, 0x2e95dcac, 0x4af4b165, 0x5d8fa526, + 0x640299e3, 0x73798da0, 0x82f182a3, 0x958a96e0, 0xac07aa25, + 0xbb7cbe66, 0xdf1dd3af, 0xc866c7ec, 0xf1ebfb29, 0xe690ef6a, + 0x392920bb, 0x2e5234f8, 0x17df083d, 0x00a41c7e, 0x64c571b7, + 0x73be65f4, 0x4a335931, 0x5d484d72, 0x2e31c0d2, 0x394ad491, + 0x00c7e854, 0x17bcfc17, 0x73dd91de, 0x64a6859d, 0x5d2bb958, + 0x4a50ad1b, 0x95e962ca, 0x82927689, 0xbb1f4a4c, 0xac645e0f, + 0xc80533c6, 0xdf7e2785, 0xe6f31b40, 0xf1880f03, 0xde920307, + 0xc9e91744, 0xf0642b81, 0xe71f3fc2, 0x837e520b, 0x94054648, + 0xad887a8d, 0xbaf36ece, 0x654aa11f, 0x7231b55c, 0x4bbc8999, + 0x5cc79dda, 0x38a6f013, 0x2fdde450, 0x1650d895, 0x012bccd6, + 0x72524176, 0x65295535, 0x5ca469f0, 0x4bdf7db3, 0x2fbe107a, + 0x38c50439, 0x014838fc, 0x16332cbf, 0xc98ae36e, 0xdef1f72d, + 0xe77ccbe8, 0xf007dfab, 0x9466b262, 0x831da621, 0xba909ae4, + 0xadeb8ea7, 0x5c6381a4, 0x4b1895e7, 0x7295a922, 0x65eebd61, + 0x018fd0a8, 0x16f4c4eb, 0x2f79f82e, 0x3802ec6d, 0xe7bb23bc, + 0xf0c037ff, 0xc94d0b3a, 0xde361f79, 0xba5772b0, 0xad2c66f3, + 0x94a15a36, 0x83da4e75, 0xf0a3c3d5, 0xe7d8d796, 0xde55eb53, + 0xc92eff10, 0xad4f92d9, 0xba34869a, 0x83b9ba5f, 0x94c2ae1c, + 0x4b7b61cd, 0x5c00758e, 0x658d494b, 0x72f65d08, 0x169730c1, + 0x01ec2482, 0x38611847, 0x2f1a0c04, 0x6655004f, 0x712e140c, + 0x48a328c9, 0x5fd83c8a, 0x3bb95143, 0x2cc24500, 0x154f79c5, + 0x02346d86, 0xdd8da257, 0xcaf6b614, 0xf37b8ad1, 0xe4009e92, + 0x8061f35b, 0x971ae718, 0xae97dbdd, 0xb9eccf9e, 0xca95423e, + 0xddee567d, 0xe4636ab8, 0xf3187efb, 0x97791332, 0x80020771, + 0xb98f3bb4, 0xaef42ff7, 0x714de026, 0x6636f465, 0x5fbbc8a0, + 0x48c0dce3, 0x2ca1b12a, 0x3bdaa569, 0x025799ac, 0x152c8def, + 0xe4a482ec, 0xf3df96af, 0xca52aa6a, 0xdd29be29, 0xb948d3e0, + 0xae33c7a3, 0x97befb66, 0x80c5ef25, 0x5f7c20f4, 0x480734b7, + 0x718a0872, 0x66f11c31, 0x029071f8, 0x15eb65bb, 0x2c66597e, + 0x3b1d4d3d, 0x4864c09d, 0x5f1fd4de, 0x6692e81b, 0x71e9fc58, + 0x15889191, 0x02f385d2, 0x3b7eb917, 0x2c05ad54, 0xf3bc6285, + 0xe4c776c6, 0xdd4a4a03, 0xca315e40, 0xae503389, 0xb92b27ca, + 0x80a61b0f, 0x97dd0f4c, 0xb8c70348, 0xafbc170b, 0x96312bce, + 0x814a3f8d, 0xe52b5244, 0xf2504607, 0xcbdd7ac2, 0xdca66e81, + 0x031fa150, 0x1464b513, 0x2de989d6, 0x3a929d95, 0x5ef3f05c, + 0x4988e41f, 0x7005d8da, 0x677ecc99, 0x14074139, 0x037c557a, + 0x3af169bf, 0x2d8a7dfc, 0x49eb1035, 0x5e900476, 0x671d38b3, + 0x70662cf0, 0xafdfe321, 0xb8a4f762, 0x8129cba7, 0x9652dfe4, + 0xf233b22d, 0xe548a66e, 0xdcc59aab, 0xcbbe8ee8, 0x3a3681eb, + 0x2d4d95a8, 0x14c0a96d, 0x03bbbd2e, 0x67dad0e7, 0x70a1c4a4, + 0x492cf861, 0x5e57ec22, 0x81ee23f3, 0x969537b0, 0xaf180b75, + 0xb8631f36, 0xdc0272ff, 0xcb7966bc, 0xf2f45a79, 0xe58f4e3a, + 0x96f6c39a, 0x818dd7d9, 0xb800eb1c, 0xaf7bff5f, 0xcb1a9296, + 0xdc6186d5, 0xe5ecba10, 0xf297ae53, 0x2d2e6182, 0x3a5575c1, + 0x03d84904, 0x14a35d47, 0x70c2308e, 0x67b924cd, 0x5e341808, + 0x494f0c4b}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x0000000000000000, 0x43147b1700000000, 0x8628f62e00000000, + 0xc53c8d3900000000, 0x0c51ec5d00000000, 0x4f45974a00000000, + 0x8a791a7300000000, 0xc96d616400000000, 0x18a2d8bb00000000, + 0x5bb6a3ac00000000, 0x9e8a2e9500000000, 0xdd9e558200000000, + 0x14f334e600000000, 0x57e74ff100000000, 0x92dbc2c800000000, + 0xd1cfb9df00000000, 0x7142c0ac00000000, 0x3256bbbb00000000, + 0xf76a368200000000, 0xb47e4d9500000000, 0x7d132cf100000000, + 0x3e0757e600000000, 0xfb3bdadf00000000, 0xb82fa1c800000000, + 0x69e0181700000000, 0x2af4630000000000, 0xefc8ee3900000000, + 0xacdc952e00000000, 0x65b1f44a00000000, 0x26a58f5d00000000, + 0xe399026400000000, 0xa08d797300000000, 0xa382f18200000000, + 0xe0968a9500000000, 0x25aa07ac00000000, 0x66be7cbb00000000, + 0xafd31ddf00000000, 0xecc766c800000000, 0x29fbebf100000000, + 0x6aef90e600000000, 0xbb20293900000000, 0xf834522e00000000, + 0x3d08df1700000000, 0x7e1ca40000000000, 0xb771c56400000000, + 0xf465be7300000000, 0x3159334a00000000, 0x724d485d00000000, + 0xd2c0312e00000000, 0x91d44a3900000000, 0x54e8c70000000000, + 0x17fcbc1700000000, 0xde91dd7300000000, 0x9d85a66400000000, + 0x58b92b5d00000000, 0x1bad504a00000000, 0xca62e99500000000, + 0x8976928200000000, 0x4c4a1fbb00000000, 0x0f5e64ac00000000, + 0xc63305c800000000, 0x85277edf00000000, 0x401bf3e600000000, + 0x030f88f100000000, 0x070392de00000000, 0x4417e9c900000000, + 0x812b64f000000000, 0xc23f1fe700000000, 0x0b527e8300000000, + 0x4846059400000000, 0x8d7a88ad00000000, 0xce6ef3ba00000000, + 0x1fa14a6500000000, 0x5cb5317200000000, 0x9989bc4b00000000, + 0xda9dc75c00000000, 0x13f0a63800000000, 0x50e4dd2f00000000, + 0x95d8501600000000, 0xd6cc2b0100000000, 0x7641527200000000, + 0x3555296500000000, 0xf069a45c00000000, 0xb37ddf4b00000000, + 0x7a10be2f00000000, 0x3904c53800000000, 0xfc38480100000000, + 0xbf2c331600000000, 0x6ee38ac900000000, 0x2df7f1de00000000, + 0xe8cb7ce700000000, 0xabdf07f000000000, 0x62b2669400000000, + 0x21a61d8300000000, 0xe49a90ba00000000, 0xa78eebad00000000, + 0xa481635c00000000, 0xe795184b00000000, 0x22a9957200000000, + 0x61bdee6500000000, 0xa8d08f0100000000, 0xebc4f41600000000, + 0x2ef8792f00000000, 0x6dec023800000000, 0xbc23bbe700000000, + 0xff37c0f000000000, 0x3a0b4dc900000000, 0x791f36de00000000, + 0xb07257ba00000000, 0xf3662cad00000000, 0x365aa19400000000, + 0x754eda8300000000, 0xd5c3a3f000000000, 0x96d7d8e700000000, + 0x53eb55de00000000, 0x10ff2ec900000000, 0xd9924fad00000000, + 0x9a8634ba00000000, 0x5fbab98300000000, 0x1caec29400000000, + 0xcd617b4b00000000, 0x8e75005c00000000, 0x4b498d6500000000, + 0x085df67200000000, 0xc130971600000000, 0x8224ec0100000000, + 0x4718613800000000, 0x040c1a2f00000000, 0x4f00556600000000, + 0x0c142e7100000000, 0xc928a34800000000, 0x8a3cd85f00000000, + 0x4351b93b00000000, 0x0045c22c00000000, 0xc5794f1500000000, + 0x866d340200000000, 0x57a28ddd00000000, 0x14b6f6ca00000000, + 0xd18a7bf300000000, 0x929e00e400000000, 0x5bf3618000000000, + 0x18e71a9700000000, 0xdddb97ae00000000, 0x9ecfecb900000000, + 0x3e4295ca00000000, 0x7d56eedd00000000, 0xb86a63e400000000, + 0xfb7e18f300000000, 0x3213799700000000, 0x7107028000000000, + 0xb43b8fb900000000, 0xf72ff4ae00000000, 0x26e04d7100000000, + 0x65f4366600000000, 0xa0c8bb5f00000000, 0xe3dcc04800000000, + 0x2ab1a12c00000000, 0x69a5da3b00000000, 0xac99570200000000, + 0xef8d2c1500000000, 0xec82a4e400000000, 0xaf96dff300000000, + 0x6aaa52ca00000000, 0x29be29dd00000000, 0xe0d348b900000000, + 0xa3c733ae00000000, 0x66fbbe9700000000, 0x25efc58000000000, + 0xf4207c5f00000000, 0xb734074800000000, 0x72088a7100000000, + 0x311cf16600000000, 0xf871900200000000, 0xbb65eb1500000000, + 0x7e59662c00000000, 0x3d4d1d3b00000000, 0x9dc0644800000000, + 0xded41f5f00000000, 0x1be8926600000000, 0x58fce97100000000, + 0x9191881500000000, 0xd285f30200000000, 0x17b97e3b00000000, + 0x54ad052c00000000, 0x8562bcf300000000, 0xc676c7e400000000, + 0x034a4add00000000, 0x405e31ca00000000, 0x893350ae00000000, + 0xca272bb900000000, 0x0f1ba68000000000, 0x4c0fdd9700000000, + 0x4803c7b800000000, 0x0b17bcaf00000000, 0xce2b319600000000, + 0x8d3f4a8100000000, 0x44522be500000000, 0x074650f200000000, + 0xc27addcb00000000, 0x816ea6dc00000000, 0x50a11f0300000000, + 0x13b5641400000000, 0xd689e92d00000000, 0x959d923a00000000, + 0x5cf0f35e00000000, 0x1fe4884900000000, 0xdad8057000000000, + 0x99cc7e6700000000, 0x3941071400000000, 0x7a557c0300000000, + 0xbf69f13a00000000, 0xfc7d8a2d00000000, 0x3510eb4900000000, + 0x7604905e00000000, 0xb3381d6700000000, 0xf02c667000000000, + 0x21e3dfaf00000000, 0x62f7a4b800000000, 0xa7cb298100000000, + 0xe4df529600000000, 0x2db233f200000000, 0x6ea648e500000000, + 0xab9ac5dc00000000, 0xe88ebecb00000000, 0xeb81363a00000000, + 0xa8954d2d00000000, 0x6da9c01400000000, 0x2ebdbb0300000000, + 0xe7d0da6700000000, 0xa4c4a17000000000, 0x61f82c4900000000, + 0x22ec575e00000000, 0xf323ee8100000000, 0xb037959600000000, + 0x750b18af00000000, 0x361f63b800000000, 0xff7202dc00000000, + 0xbc6679cb00000000, 0x795af4f200000000, 0x3a4e8fe500000000, + 0x9ac3f69600000000, 0xd9d78d8100000000, 0x1ceb00b800000000, + 0x5fff7baf00000000, 0x96921acb00000000, 0xd58661dc00000000, + 0x10baece500000000, 0x53ae97f200000000, 0x82612e2d00000000, + 0xc175553a00000000, 0x0449d80300000000, 0x475da31400000000, + 0x8e30c27000000000, 0xcd24b96700000000, 0x0818345e00000000, + 0x4b0c4f4900000000}, + {0x0000000000000000, 0x3e6bc2ef00000000, 0x3dd0f50400000000, + 0x03bb37eb00000000, 0x7aa0eb0900000000, 0x44cb29e600000000, + 0x47701e0d00000000, 0x791bdce200000000, 0xf440d71300000000, + 0xca2b15fc00000000, 0xc990221700000000, 0xf7fbe0f800000000, + 0x8ee03c1a00000000, 0xb08bfef500000000, 0xb330c91e00000000, + 0x8d5b0bf100000000, 0xe881ae2700000000, 0xd6ea6cc800000000, + 0xd5515b2300000000, 0xeb3a99cc00000000, 0x9221452e00000000, + 0xac4a87c100000000, 0xaff1b02a00000000, 0x919a72c500000000, + 0x1cc1793400000000, 0x22aabbdb00000000, 0x21118c3000000000, + 0x1f7a4edf00000000, 0x6661923d00000000, 0x580a50d200000000, + 0x5bb1673900000000, 0x65daa5d600000000, 0xd0035d4f00000000, + 0xee689fa000000000, 0xedd3a84b00000000, 0xd3b86aa400000000, + 0xaaa3b64600000000, 0x94c874a900000000, 0x9773434200000000, + 0xa91881ad00000000, 0x24438a5c00000000, 0x1a2848b300000000, + 0x19937f5800000000, 0x27f8bdb700000000, 0x5ee3615500000000, + 0x6088a3ba00000000, 0x6333945100000000, 0x5d5856be00000000, + 0x3882f36800000000, 0x06e9318700000000, 0x0552066c00000000, + 0x3b39c48300000000, 0x4222186100000000, 0x7c49da8e00000000, + 0x7ff2ed6500000000, 0x41992f8a00000000, 0xccc2247b00000000, + 0xf2a9e69400000000, 0xf112d17f00000000, 0xcf79139000000000, + 0xb662cf7200000000, 0x88090d9d00000000, 0x8bb23a7600000000, + 0xb5d9f89900000000, 0xa007ba9e00000000, 0x9e6c787100000000, + 0x9dd74f9a00000000, 0xa3bc8d7500000000, 0xdaa7519700000000, + 0xe4cc937800000000, 0xe777a49300000000, 0xd91c667c00000000, + 0x54476d8d00000000, 0x6a2caf6200000000, 0x6997988900000000, + 0x57fc5a6600000000, 0x2ee7868400000000, 0x108c446b00000000, + 0x1337738000000000, 0x2d5cb16f00000000, 0x488614b900000000, + 0x76edd65600000000, 0x7556e1bd00000000, 0x4b3d235200000000, + 0x3226ffb000000000, 0x0c4d3d5f00000000, 0x0ff60ab400000000, + 0x319dc85b00000000, 0xbcc6c3aa00000000, 0x82ad014500000000, + 0x811636ae00000000, 0xbf7df44100000000, 0xc66628a300000000, + 0xf80dea4c00000000, 0xfbb6dda700000000, 0xc5dd1f4800000000, + 0x7004e7d100000000, 0x4e6f253e00000000, 0x4dd412d500000000, + 0x73bfd03a00000000, 0x0aa40cd800000000, 0x34cfce3700000000, + 0x3774f9dc00000000, 0x091f3b3300000000, 0x844430c200000000, + 0xba2ff22d00000000, 0xb994c5c600000000, 0x87ff072900000000, + 0xfee4dbcb00000000, 0xc08f192400000000, 0xc3342ecf00000000, + 0xfd5fec2000000000, 0x988549f600000000, 0xa6ee8b1900000000, + 0xa555bcf200000000, 0x9b3e7e1d00000000, 0xe225a2ff00000000, + 0xdc4e601000000000, 0xdff557fb00000000, 0xe19e951400000000, + 0x6cc59ee500000000, 0x52ae5c0a00000000, 0x51156be100000000, + 0x6f7ea90e00000000, 0x166575ec00000000, 0x280eb70300000000, + 0x2bb580e800000000, 0x15de420700000000, 0x010905e600000000, + 0x3f62c70900000000, 0x3cd9f0e200000000, 0x02b2320d00000000, + 0x7ba9eeef00000000, 0x45c22c0000000000, 0x46791beb00000000, + 0x7812d90400000000, 0xf549d2f500000000, 0xcb22101a00000000, + 0xc89927f100000000, 0xf6f2e51e00000000, 0x8fe939fc00000000, + 0xb182fb1300000000, 0xb239ccf800000000, 0x8c520e1700000000, + 0xe988abc100000000, 0xd7e3692e00000000, 0xd4585ec500000000, + 0xea339c2a00000000, 0x932840c800000000, 0xad43822700000000, + 0xaef8b5cc00000000, 0x9093772300000000, 0x1dc87cd200000000, + 0x23a3be3d00000000, 0x201889d600000000, 0x1e734b3900000000, + 0x676897db00000000, 0x5903553400000000, 0x5ab862df00000000, + 0x64d3a03000000000, 0xd10a58a900000000, 0xef619a4600000000, + 0xecdaadad00000000, 0xd2b16f4200000000, 0xabaab3a000000000, + 0x95c1714f00000000, 0x967a46a400000000, 0xa811844b00000000, + 0x254a8fba00000000, 0x1b214d5500000000, 0x189a7abe00000000, + 0x26f1b85100000000, 0x5fea64b300000000, 0x6181a65c00000000, + 0x623a91b700000000, 0x5c51535800000000, 0x398bf68e00000000, + 0x07e0346100000000, 0x045b038a00000000, 0x3a30c16500000000, + 0x432b1d8700000000, 0x7d40df6800000000, 0x7efbe88300000000, + 0x40902a6c00000000, 0xcdcb219d00000000, 0xf3a0e37200000000, + 0xf01bd49900000000, 0xce70167600000000, 0xb76bca9400000000, + 0x8900087b00000000, 0x8abb3f9000000000, 0xb4d0fd7f00000000, + 0xa10ebf7800000000, 0x9f657d9700000000, 0x9cde4a7c00000000, + 0xa2b5889300000000, 0xdbae547100000000, 0xe5c5969e00000000, + 0xe67ea17500000000, 0xd815639a00000000, 0x554e686b00000000, + 0x6b25aa8400000000, 0x689e9d6f00000000, 0x56f55f8000000000, + 0x2fee836200000000, 0x1185418d00000000, 0x123e766600000000, + 0x2c55b48900000000, 0x498f115f00000000, 0x77e4d3b000000000, + 0x745fe45b00000000, 0x4a3426b400000000, 0x332ffa5600000000, + 0x0d4438b900000000, 0x0eff0f5200000000, 0x3094cdbd00000000, + 0xbdcfc64c00000000, 0x83a404a300000000, 0x801f334800000000, + 0xbe74f1a700000000, 0xc76f2d4500000000, 0xf904efaa00000000, + 0xfabfd84100000000, 0xc4d41aae00000000, 0x710de23700000000, + 0x4f6620d800000000, 0x4cdd173300000000, 0x72b6d5dc00000000, + 0x0bad093e00000000, 0x35c6cbd100000000, 0x367dfc3a00000000, + 0x08163ed500000000, 0x854d352400000000, 0xbb26f7cb00000000, + 0xb89dc02000000000, 0x86f602cf00000000, 0xffedde2d00000000, + 0xc1861cc200000000, 0xc23d2b2900000000, 0xfc56e9c600000000, + 0x998c4c1000000000, 0xa7e78eff00000000, 0xa45cb91400000000, + 0x9a377bfb00000000, 0xe32ca71900000000, 0xdd4765f600000000, + 0xdefc521d00000000, 0xe09790f200000000, 0x6dcc9b0300000000, + 0x53a759ec00000000, 0x501c6e0700000000, 0x6e77ace800000000, + 0x176c700a00000000, 0x2907b2e500000000, 0x2abc850e00000000, + 0x14d747e100000000}, + {0x0000000000000000, 0xc0df8ec100000000, 0xc1b96c5800000000, + 0x0166e29900000000, 0x8273d9b000000000, 0x42ac577100000000, + 0x43cab5e800000000, 0x83153b2900000000, 0x45e1c3ba00000000, + 0x853e4d7b00000000, 0x8458afe200000000, 0x4487212300000000, + 0xc7921a0a00000000, 0x074d94cb00000000, 0x062b765200000000, + 0xc6f4f89300000000, 0xcbc4f6ae00000000, 0x0b1b786f00000000, + 0x0a7d9af600000000, 0xcaa2143700000000, 0x49b72f1e00000000, + 0x8968a1df00000000, 0x880e434600000000, 0x48d1cd8700000000, + 0x8e25351400000000, 0x4efabbd500000000, 0x4f9c594c00000000, + 0x8f43d78d00000000, 0x0c56eca400000000, 0xcc89626500000000, + 0xcdef80fc00000000, 0x0d300e3d00000000, 0xd78f9c8600000000, + 0x1750124700000000, 0x1636f0de00000000, 0xd6e97e1f00000000, + 0x55fc453600000000, 0x9523cbf700000000, 0x9445296e00000000, + 0x549aa7af00000000, 0x926e5f3c00000000, 0x52b1d1fd00000000, + 0x53d7336400000000, 0x9308bda500000000, 0x101d868c00000000, + 0xd0c2084d00000000, 0xd1a4ead400000000, 0x117b641500000000, + 0x1c4b6a2800000000, 0xdc94e4e900000000, 0xddf2067000000000, + 0x1d2d88b100000000, 0x9e38b39800000000, 0x5ee73d5900000000, + 0x5f81dfc000000000, 0x9f5e510100000000, 0x59aaa99200000000, + 0x9975275300000000, 0x9813c5ca00000000, 0x58cc4b0b00000000, + 0xdbd9702200000000, 0x1b06fee300000000, 0x1a601c7a00000000, + 0xdabf92bb00000000, 0xef1948d600000000, 0x2fc6c61700000000, + 0x2ea0248e00000000, 0xee7faa4f00000000, 0x6d6a916600000000, + 0xadb51fa700000000, 0xacd3fd3e00000000, 0x6c0c73ff00000000, + 0xaaf88b6c00000000, 0x6a2705ad00000000, 0x6b41e73400000000, + 0xab9e69f500000000, 0x288b52dc00000000, 0xe854dc1d00000000, + 0xe9323e8400000000, 0x29edb04500000000, 0x24ddbe7800000000, + 0xe40230b900000000, 0xe564d22000000000, 0x25bb5ce100000000, + 0xa6ae67c800000000, 0x6671e90900000000, 0x67170b9000000000, + 0xa7c8855100000000, 0x613c7dc200000000, 0xa1e3f30300000000, + 0xa085119a00000000, 0x605a9f5b00000000, 0xe34fa47200000000, + 0x23902ab300000000, 0x22f6c82a00000000, 0xe22946eb00000000, + 0x3896d45000000000, 0xf8495a9100000000, 0xf92fb80800000000, + 0x39f036c900000000, 0xbae50de000000000, 0x7a3a832100000000, + 0x7b5c61b800000000, 0xbb83ef7900000000, 0x7d7717ea00000000, + 0xbda8992b00000000, 0xbcce7bb200000000, 0x7c11f57300000000, + 0xff04ce5a00000000, 0x3fdb409b00000000, 0x3ebda20200000000, + 0xfe622cc300000000, 0xf35222fe00000000, 0x338dac3f00000000, + 0x32eb4ea600000000, 0xf234c06700000000, 0x7121fb4e00000000, + 0xb1fe758f00000000, 0xb098971600000000, 0x704719d700000000, + 0xb6b3e14400000000, 0x766c6f8500000000, 0x770a8d1c00000000, + 0xb7d503dd00000000, 0x34c038f400000000, 0xf41fb63500000000, + 0xf57954ac00000000, 0x35a6da6d00000000, 0x9f35e17700000000, + 0x5fea6fb600000000, 0x5e8c8d2f00000000, 0x9e5303ee00000000, + 0x1d4638c700000000, 0xdd99b60600000000, 0xdcff549f00000000, + 0x1c20da5e00000000, 0xdad422cd00000000, 0x1a0bac0c00000000, + 0x1b6d4e9500000000, 0xdbb2c05400000000, 0x58a7fb7d00000000, + 0x987875bc00000000, 0x991e972500000000, 0x59c119e400000000, + 0x54f117d900000000, 0x942e991800000000, 0x95487b8100000000, + 0x5597f54000000000, 0xd682ce6900000000, 0x165d40a800000000, + 0x173ba23100000000, 0xd7e42cf000000000, 0x1110d46300000000, + 0xd1cf5aa200000000, 0xd0a9b83b00000000, 0x107636fa00000000, + 0x93630dd300000000, 0x53bc831200000000, 0x52da618b00000000, + 0x9205ef4a00000000, 0x48ba7df100000000, 0x8865f33000000000, + 0x890311a900000000, 0x49dc9f6800000000, 0xcac9a44100000000, + 0x0a162a8000000000, 0x0b70c81900000000, 0xcbaf46d800000000, + 0x0d5bbe4b00000000, 0xcd84308a00000000, 0xcce2d21300000000, + 0x0c3d5cd200000000, 0x8f2867fb00000000, 0x4ff7e93a00000000, + 0x4e910ba300000000, 0x8e4e856200000000, 0x837e8b5f00000000, + 0x43a1059e00000000, 0x42c7e70700000000, 0x821869c600000000, + 0x010d52ef00000000, 0xc1d2dc2e00000000, 0xc0b43eb700000000, + 0x006bb07600000000, 0xc69f48e500000000, 0x0640c62400000000, + 0x072624bd00000000, 0xc7f9aa7c00000000, 0x44ec915500000000, + 0x84331f9400000000, 0x8555fd0d00000000, 0x458a73cc00000000, + 0x702ca9a100000000, 0xb0f3276000000000, 0xb195c5f900000000, + 0x714a4b3800000000, 0xf25f701100000000, 0x3280fed000000000, + 0x33e61c4900000000, 0xf339928800000000, 0x35cd6a1b00000000, + 0xf512e4da00000000, 0xf474064300000000, 0x34ab888200000000, + 0xb7beb3ab00000000, 0x77613d6a00000000, 0x7607dff300000000, + 0xb6d8513200000000, 0xbbe85f0f00000000, 0x7b37d1ce00000000, + 0x7a51335700000000, 0xba8ebd9600000000, 0x399b86bf00000000, + 0xf944087e00000000, 0xf822eae700000000, 0x38fd642600000000, + 0xfe099cb500000000, 0x3ed6127400000000, 0x3fb0f0ed00000000, + 0xff6f7e2c00000000, 0x7c7a450500000000, 0xbca5cbc400000000, + 0xbdc3295d00000000, 0x7d1ca79c00000000, 0xa7a3352700000000, + 0x677cbbe600000000, 0x661a597f00000000, 0xa6c5d7be00000000, + 0x25d0ec9700000000, 0xe50f625600000000, 0xe46980cf00000000, + 0x24b60e0e00000000, 0xe242f69d00000000, 0x229d785c00000000, + 0x23fb9ac500000000, 0xe324140400000000, 0x60312f2d00000000, + 0xa0eea1ec00000000, 0xa188437500000000, 0x6157cdb400000000, + 0x6c67c38900000000, 0xacb84d4800000000, 0xaddeafd100000000, + 0x6d01211000000000, 0xee141a3900000000, 0x2ecb94f800000000, + 0x2fad766100000000, 0xef72f8a000000000, 0x2986003300000000, + 0xe9598ef200000000, 0xe83f6c6b00000000, 0x28e0e2aa00000000, + 0xabf5d98300000000, 0x6b2a574200000000, 0x6a4cb5db00000000, + 0xaa933b1a00000000}, + {0x0000000000000000, 0x6f4ca59b00000000, 0x9f9e3bec00000000, + 0xf0d29e7700000000, 0x7f3b060300000000, 0x1077a39800000000, + 0xe0a53def00000000, 0x8fe9987400000000, 0xfe760c0600000000, + 0x913aa99d00000000, 0x61e837ea00000000, 0x0ea4927100000000, + 0x814d0a0500000000, 0xee01af9e00000000, 0x1ed331e900000000, + 0x719f947200000000, 0xfced180c00000000, 0x93a1bd9700000000, + 0x637323e000000000, 0x0c3f867b00000000, 0x83d61e0f00000000, + 0xec9abb9400000000, 0x1c4825e300000000, 0x7304807800000000, + 0x029b140a00000000, 0x6dd7b19100000000, 0x9d052fe600000000, + 0xf2498a7d00000000, 0x7da0120900000000, 0x12ecb79200000000, + 0xe23e29e500000000, 0x8d728c7e00000000, 0xf8db311800000000, + 0x9797948300000000, 0x67450af400000000, 0x0809af6f00000000, + 0x87e0371b00000000, 0xe8ac928000000000, 0x187e0cf700000000, + 0x7732a96c00000000, 0x06ad3d1e00000000, 0x69e1988500000000, + 0x993306f200000000, 0xf67fa36900000000, 0x79963b1d00000000, + 0x16da9e8600000000, 0xe60800f100000000, 0x8944a56a00000000, + 0x0436291400000000, 0x6b7a8c8f00000000, 0x9ba812f800000000, + 0xf4e4b76300000000, 0x7b0d2f1700000000, 0x14418a8c00000000, + 0xe49314fb00000000, 0x8bdfb16000000000, 0xfa40251200000000, + 0x950c808900000000, 0x65de1efe00000000, 0x0a92bb6500000000, + 0x857b231100000000, 0xea37868a00000000, 0x1ae518fd00000000, + 0x75a9bd6600000000, 0xf0b7633000000000, 0x9ffbc6ab00000000, + 0x6f2958dc00000000, 0x0065fd4700000000, 0x8f8c653300000000, + 0xe0c0c0a800000000, 0x10125edf00000000, 0x7f5efb4400000000, + 0x0ec16f3600000000, 0x618dcaad00000000, 0x915f54da00000000, + 0xfe13f14100000000, 0x71fa693500000000, 0x1eb6ccae00000000, + 0xee6452d900000000, 0x8128f74200000000, 0x0c5a7b3c00000000, + 0x6316dea700000000, 0x93c440d000000000, 0xfc88e54b00000000, + 0x73617d3f00000000, 0x1c2dd8a400000000, 0xecff46d300000000, + 0x83b3e34800000000, 0xf22c773a00000000, 0x9d60d2a100000000, + 0x6db24cd600000000, 0x02fee94d00000000, 0x8d17713900000000, + 0xe25bd4a200000000, 0x12894ad500000000, 0x7dc5ef4e00000000, + 0x086c522800000000, 0x6720f7b300000000, 0x97f269c400000000, + 0xf8becc5f00000000, 0x7757542b00000000, 0x181bf1b000000000, + 0xe8c96fc700000000, 0x8785ca5c00000000, 0xf61a5e2e00000000, + 0x9956fbb500000000, 0x698465c200000000, 0x06c8c05900000000, + 0x8921582d00000000, 0xe66dfdb600000000, 0x16bf63c100000000, + 0x79f3c65a00000000, 0xf4814a2400000000, 0x9bcdefbf00000000, + 0x6b1f71c800000000, 0x0453d45300000000, 0x8bba4c2700000000, + 0xe4f6e9bc00000000, 0x142477cb00000000, 0x7b68d25000000000, + 0x0af7462200000000, 0x65bbe3b900000000, 0x95697dce00000000, + 0xfa25d85500000000, 0x75cc402100000000, 0x1a80e5ba00000000, + 0xea527bcd00000000, 0x851ede5600000000, 0xe06fc76000000000, + 0x8f2362fb00000000, 0x7ff1fc8c00000000, 0x10bd591700000000, + 0x9f54c16300000000, 0xf01864f800000000, 0x00cafa8f00000000, + 0x6f865f1400000000, 0x1e19cb6600000000, 0x71556efd00000000, + 0x8187f08a00000000, 0xeecb551100000000, 0x6122cd6500000000, + 0x0e6e68fe00000000, 0xfebcf68900000000, 0x91f0531200000000, + 0x1c82df6c00000000, 0x73ce7af700000000, 0x831ce48000000000, + 0xec50411b00000000, 0x63b9d96f00000000, 0x0cf57cf400000000, + 0xfc27e28300000000, 0x936b471800000000, 0xe2f4d36a00000000, + 0x8db876f100000000, 0x7d6ae88600000000, 0x12264d1d00000000, + 0x9dcfd56900000000, 0xf28370f200000000, 0x0251ee8500000000, + 0x6d1d4b1e00000000, 0x18b4f67800000000, 0x77f853e300000000, + 0x872acd9400000000, 0xe866680f00000000, 0x678ff07b00000000, + 0x08c355e000000000, 0xf811cb9700000000, 0x975d6e0c00000000, + 0xe6c2fa7e00000000, 0x898e5fe500000000, 0x795cc19200000000, + 0x1610640900000000, 0x99f9fc7d00000000, 0xf6b559e600000000, + 0x0667c79100000000, 0x692b620a00000000, 0xe459ee7400000000, + 0x8b154bef00000000, 0x7bc7d59800000000, 0x148b700300000000, + 0x9b62e87700000000, 0xf42e4dec00000000, 0x04fcd39b00000000, + 0x6bb0760000000000, 0x1a2fe27200000000, 0x756347e900000000, + 0x85b1d99e00000000, 0xeafd7c0500000000, 0x6514e47100000000, + 0x0a5841ea00000000, 0xfa8adf9d00000000, 0x95c67a0600000000, + 0x10d8a45000000000, 0x7f9401cb00000000, 0x8f469fbc00000000, + 0xe00a3a2700000000, 0x6fe3a25300000000, 0x00af07c800000000, + 0xf07d99bf00000000, 0x9f313c2400000000, 0xeeaea85600000000, + 0x81e20dcd00000000, 0x713093ba00000000, 0x1e7c362100000000, + 0x9195ae5500000000, 0xfed90bce00000000, 0x0e0b95b900000000, + 0x6147302200000000, 0xec35bc5c00000000, 0x837919c700000000, + 0x73ab87b000000000, 0x1ce7222b00000000, 0x930eba5f00000000, + 0xfc421fc400000000, 0x0c9081b300000000, 0x63dc242800000000, + 0x1243b05a00000000, 0x7d0f15c100000000, 0x8ddd8bb600000000, + 0xe2912e2d00000000, 0x6d78b65900000000, 0x023413c200000000, + 0xf2e68db500000000, 0x9daa282e00000000, 0xe803954800000000, + 0x874f30d300000000, 0x779daea400000000, 0x18d10b3f00000000, + 0x9738934b00000000, 0xf87436d000000000, 0x08a6a8a700000000, + 0x67ea0d3c00000000, 0x1675994e00000000, 0x79393cd500000000, + 0x89eba2a200000000, 0xe6a7073900000000, 0x694e9f4d00000000, + 0x06023ad600000000, 0xf6d0a4a100000000, 0x999c013a00000000, + 0x14ee8d4400000000, 0x7ba228df00000000, 0x8b70b6a800000000, + 0xe43c133300000000, 0x6bd58b4700000000, 0x04992edc00000000, + 0xf44bb0ab00000000, 0x9b07153000000000, 0xea98814200000000, + 0x85d424d900000000, 0x7506baae00000000, 0x1a4a1f3500000000, + 0x95a3874100000000, 0xfaef22da00000000, 0x0a3dbcad00000000, + 0x6571193600000000}, + {0x0000000000000000, 0x85d996dd00000000, 0x4bb55c6000000000, + 0xce6ccabd00000000, 0x966ab9c000000000, 0x13b32f1d00000000, + 0xdddfe5a000000000, 0x5806737d00000000, 0x6dd3035a00000000, + 0xe80a958700000000, 0x26665f3a00000000, 0xa3bfc9e700000000, + 0xfbb9ba9a00000000, 0x7e602c4700000000, 0xb00ce6fa00000000, + 0x35d5702700000000, 0xdaa607b400000000, 0x5f7f916900000000, + 0x91135bd400000000, 0x14cacd0900000000, 0x4cccbe7400000000, + 0xc91528a900000000, 0x0779e21400000000, 0x82a074c900000000, + 0xb77504ee00000000, 0x32ac923300000000, 0xfcc0588e00000000, + 0x7919ce5300000000, 0x211fbd2e00000000, 0xa4c62bf300000000, + 0x6aaae14e00000000, 0xef73779300000000, 0xf54b7eb300000000, + 0x7092e86e00000000, 0xbefe22d300000000, 0x3b27b40e00000000, + 0x6321c77300000000, 0xe6f851ae00000000, 0x28949b1300000000, + 0xad4d0dce00000000, 0x98987de900000000, 0x1d41eb3400000000, + 0xd32d218900000000, 0x56f4b75400000000, 0x0ef2c42900000000, + 0x8b2b52f400000000, 0x4547984900000000, 0xc09e0e9400000000, + 0x2fed790700000000, 0xaa34efda00000000, 0x6458256700000000, + 0xe181b3ba00000000, 0xb987c0c700000000, 0x3c5e561a00000000, + 0xf2329ca700000000, 0x77eb0a7a00000000, 0x423e7a5d00000000, + 0xc7e7ec8000000000, 0x098b263d00000000, 0x8c52b0e000000000, + 0xd454c39d00000000, 0x518d554000000000, 0x9fe19ffd00000000, + 0x1a38092000000000, 0xab918dbd00000000, 0x2e481b6000000000, + 0xe024d1dd00000000, 0x65fd470000000000, 0x3dfb347d00000000, + 0xb822a2a000000000, 0x764e681d00000000, 0xf397fec000000000, + 0xc6428ee700000000, 0x439b183a00000000, 0x8df7d28700000000, + 0x082e445a00000000, 0x5028372700000000, 0xd5f1a1fa00000000, + 0x1b9d6b4700000000, 0x9e44fd9a00000000, 0x71378a0900000000, + 0xf4ee1cd400000000, 0x3a82d66900000000, 0xbf5b40b400000000, + 0xe75d33c900000000, 0x6284a51400000000, 0xace86fa900000000, + 0x2931f97400000000, 0x1ce4895300000000, 0x993d1f8e00000000, + 0x5751d53300000000, 0xd28843ee00000000, 0x8a8e309300000000, + 0x0f57a64e00000000, 0xc13b6cf300000000, 0x44e2fa2e00000000, + 0x5edaf30e00000000, 0xdb0365d300000000, 0x156faf6e00000000, + 0x90b639b300000000, 0xc8b04ace00000000, 0x4d69dc1300000000, + 0x830516ae00000000, 0x06dc807300000000, 0x3309f05400000000, + 0xb6d0668900000000, 0x78bcac3400000000, 0xfd653ae900000000, + 0xa563499400000000, 0x20badf4900000000, 0xeed615f400000000, + 0x6b0f832900000000, 0x847cf4ba00000000, 0x01a5626700000000, + 0xcfc9a8da00000000, 0x4a103e0700000000, 0x12164d7a00000000, + 0x97cfdba700000000, 0x59a3111a00000000, 0xdc7a87c700000000, + 0xe9aff7e000000000, 0x6c76613d00000000, 0xa21aab8000000000, + 0x27c33d5d00000000, 0x7fc54e2000000000, 0xfa1cd8fd00000000, + 0x3470124000000000, 0xb1a9849d00000000, 0x17256aa000000000, + 0x92fcfc7d00000000, 0x5c9036c000000000, 0xd949a01d00000000, + 0x814fd36000000000, 0x049645bd00000000, 0xcafa8f0000000000, + 0x4f2319dd00000000, 0x7af669fa00000000, 0xff2fff2700000000, + 0x3143359a00000000, 0xb49aa34700000000, 0xec9cd03a00000000, + 0x694546e700000000, 0xa7298c5a00000000, 0x22f01a8700000000, + 0xcd836d1400000000, 0x485afbc900000000, 0x8636317400000000, + 0x03efa7a900000000, 0x5be9d4d400000000, 0xde30420900000000, + 0x105c88b400000000, 0x95851e6900000000, 0xa0506e4e00000000, + 0x2589f89300000000, 0xebe5322e00000000, 0x6e3ca4f300000000, + 0x363ad78e00000000, 0xb3e3415300000000, 0x7d8f8bee00000000, + 0xf8561d3300000000, 0xe26e141300000000, 0x67b782ce00000000, + 0xa9db487300000000, 0x2c02deae00000000, 0x7404add300000000, + 0xf1dd3b0e00000000, 0x3fb1f1b300000000, 0xba68676e00000000, + 0x8fbd174900000000, 0x0a64819400000000, 0xc4084b2900000000, + 0x41d1ddf400000000, 0x19d7ae8900000000, 0x9c0e385400000000, + 0x5262f2e900000000, 0xd7bb643400000000, 0x38c813a700000000, + 0xbd11857a00000000, 0x737d4fc700000000, 0xf6a4d91a00000000, + 0xaea2aa6700000000, 0x2b7b3cba00000000, 0xe517f60700000000, + 0x60ce60da00000000, 0x551b10fd00000000, 0xd0c2862000000000, + 0x1eae4c9d00000000, 0x9b77da4000000000, 0xc371a93d00000000, + 0x46a83fe000000000, 0x88c4f55d00000000, 0x0d1d638000000000, + 0xbcb4e71d00000000, 0x396d71c000000000, 0xf701bb7d00000000, + 0x72d82da000000000, 0x2ade5edd00000000, 0xaf07c80000000000, + 0x616b02bd00000000, 0xe4b2946000000000, 0xd167e44700000000, + 0x54be729a00000000, 0x9ad2b82700000000, 0x1f0b2efa00000000, + 0x470d5d8700000000, 0xc2d4cb5a00000000, 0x0cb801e700000000, + 0x8961973a00000000, 0x6612e0a900000000, 0xe3cb767400000000, + 0x2da7bcc900000000, 0xa87e2a1400000000, 0xf078596900000000, + 0x75a1cfb400000000, 0xbbcd050900000000, 0x3e1493d400000000, + 0x0bc1e3f300000000, 0x8e18752e00000000, 0x4074bf9300000000, + 0xc5ad294e00000000, 0x9dab5a3300000000, 0x1872ccee00000000, + 0xd61e065300000000, 0x53c7908e00000000, 0x49ff99ae00000000, + 0xcc260f7300000000, 0x024ac5ce00000000, 0x8793531300000000, + 0xdf95206e00000000, 0x5a4cb6b300000000, 0x94207c0e00000000, + 0x11f9ead300000000, 0x242c9af400000000, 0xa1f50c2900000000, + 0x6f99c69400000000, 0xea40504900000000, 0xb246233400000000, + 0x379fb5e900000000, 0xf9f37f5400000000, 0x7c2ae98900000000, + 0x93599e1a00000000, 0x168008c700000000, 0xd8ecc27a00000000, + 0x5d3554a700000000, 0x053327da00000000, 0x80eab10700000000, + 0x4e867bba00000000, 0xcb5fed6700000000, 0xfe8a9d4000000000, + 0x7b530b9d00000000, 0xb53fc12000000000, 0x30e657fd00000000, + 0x68e0248000000000, 0xed39b25d00000000, 0x235578e000000000, + 0xa68cee3d00000000}, + {0x0000000000000000, 0x76e10f9d00000000, 0xadc46ee100000000, + 0xdb25617c00000000, 0x1b8fac1900000000, 0x6d6ea38400000000, + 0xb64bc2f800000000, 0xc0aacd6500000000, 0x361e593300000000, + 0x40ff56ae00000000, 0x9bda37d200000000, 0xed3b384f00000000, + 0x2d91f52a00000000, 0x5b70fab700000000, 0x80559bcb00000000, + 0xf6b4945600000000, 0x6c3cb26600000000, 0x1addbdfb00000000, + 0xc1f8dc8700000000, 0xb719d31a00000000, 0x77b31e7f00000000, + 0x015211e200000000, 0xda77709e00000000, 0xac967f0300000000, + 0x5a22eb5500000000, 0x2cc3e4c800000000, 0xf7e685b400000000, + 0x81078a2900000000, 0x41ad474c00000000, 0x374c48d100000000, + 0xec6929ad00000000, 0x9a88263000000000, 0xd87864cd00000000, + 0xae996b5000000000, 0x75bc0a2c00000000, 0x035d05b100000000, + 0xc3f7c8d400000000, 0xb516c74900000000, 0x6e33a63500000000, + 0x18d2a9a800000000, 0xee663dfe00000000, 0x9887326300000000, + 0x43a2531f00000000, 0x35435c8200000000, 0xf5e991e700000000, + 0x83089e7a00000000, 0x582dff0600000000, 0x2eccf09b00000000, + 0xb444d6ab00000000, 0xc2a5d93600000000, 0x1980b84a00000000, + 0x6f61b7d700000000, 0xafcb7ab200000000, 0xd92a752f00000000, + 0x020f145300000000, 0x74ee1bce00000000, 0x825a8f9800000000, + 0xf4bb800500000000, 0x2f9ee17900000000, 0x597feee400000000, + 0x99d5238100000000, 0xef342c1c00000000, 0x34114d6000000000, + 0x42f042fd00000000, 0xf1f7b94100000000, 0x8716b6dc00000000, + 0x5c33d7a000000000, 0x2ad2d83d00000000, 0xea78155800000000, + 0x9c991ac500000000, 0x47bc7bb900000000, 0x315d742400000000, + 0xc7e9e07200000000, 0xb108efef00000000, 0x6a2d8e9300000000, + 0x1ccc810e00000000, 0xdc664c6b00000000, 0xaa8743f600000000, + 0x71a2228a00000000, 0x07432d1700000000, 0x9dcb0b2700000000, + 0xeb2a04ba00000000, 0x300f65c600000000, 0x46ee6a5b00000000, + 0x8644a73e00000000, 0xf0a5a8a300000000, 0x2b80c9df00000000, + 0x5d61c64200000000, 0xabd5521400000000, 0xdd345d8900000000, + 0x06113cf500000000, 0x70f0336800000000, 0xb05afe0d00000000, + 0xc6bbf19000000000, 0x1d9e90ec00000000, 0x6b7f9f7100000000, + 0x298fdd8c00000000, 0x5f6ed21100000000, 0x844bb36d00000000, + 0xf2aabcf000000000, 0x3200719500000000, 0x44e17e0800000000, + 0x9fc41f7400000000, 0xe92510e900000000, 0x1f9184bf00000000, + 0x69708b2200000000, 0xb255ea5e00000000, 0xc4b4e5c300000000, + 0x041e28a600000000, 0x72ff273b00000000, 0xa9da464700000000, + 0xdf3b49da00000000, 0x45b36fea00000000, 0x3352607700000000, + 0xe877010b00000000, 0x9e960e9600000000, 0x5e3cc3f300000000, + 0x28ddcc6e00000000, 0xf3f8ad1200000000, 0x8519a28f00000000, + 0x73ad36d900000000, 0x054c394400000000, 0xde69583800000000, + 0xa88857a500000000, 0x68229ac000000000, 0x1ec3955d00000000, + 0xc5e6f42100000000, 0xb307fbbc00000000, 0xe2ef738300000000, + 0x940e7c1e00000000, 0x4f2b1d6200000000, 0x39ca12ff00000000, + 0xf960df9a00000000, 0x8f81d00700000000, 0x54a4b17b00000000, + 0x2245bee600000000, 0xd4f12ab000000000, 0xa210252d00000000, + 0x7935445100000000, 0x0fd44bcc00000000, 0xcf7e86a900000000, + 0xb99f893400000000, 0x62bae84800000000, 0x145be7d500000000, + 0x8ed3c1e500000000, 0xf832ce7800000000, 0x2317af0400000000, + 0x55f6a09900000000, 0x955c6dfc00000000, 0xe3bd626100000000, + 0x3898031d00000000, 0x4e790c8000000000, 0xb8cd98d600000000, + 0xce2c974b00000000, 0x1509f63700000000, 0x63e8f9aa00000000, + 0xa34234cf00000000, 0xd5a33b5200000000, 0x0e865a2e00000000, + 0x786755b300000000, 0x3a97174e00000000, 0x4c7618d300000000, + 0x975379af00000000, 0xe1b2763200000000, 0x2118bb5700000000, + 0x57f9b4ca00000000, 0x8cdcd5b600000000, 0xfa3dda2b00000000, + 0x0c894e7d00000000, 0x7a6841e000000000, 0xa14d209c00000000, + 0xd7ac2f0100000000, 0x1706e26400000000, 0x61e7edf900000000, + 0xbac28c8500000000, 0xcc23831800000000, 0x56aba52800000000, + 0x204aaab500000000, 0xfb6fcbc900000000, 0x8d8ec45400000000, + 0x4d24093100000000, 0x3bc506ac00000000, 0xe0e067d000000000, + 0x9601684d00000000, 0x60b5fc1b00000000, 0x1654f38600000000, + 0xcd7192fa00000000, 0xbb909d6700000000, 0x7b3a500200000000, + 0x0ddb5f9f00000000, 0xd6fe3ee300000000, 0xa01f317e00000000, + 0x1318cac200000000, 0x65f9c55f00000000, 0xbedca42300000000, + 0xc83dabbe00000000, 0x089766db00000000, 0x7e76694600000000, + 0xa553083a00000000, 0xd3b207a700000000, 0x250693f100000000, + 0x53e79c6c00000000, 0x88c2fd1000000000, 0xfe23f28d00000000, + 0x3e893fe800000000, 0x4868307500000000, 0x934d510900000000, + 0xe5ac5e9400000000, 0x7f2478a400000000, 0x09c5773900000000, + 0xd2e0164500000000, 0xa40119d800000000, 0x64abd4bd00000000, + 0x124adb2000000000, 0xc96fba5c00000000, 0xbf8eb5c100000000, + 0x493a219700000000, 0x3fdb2e0a00000000, 0xe4fe4f7600000000, + 0x921f40eb00000000, 0x52b58d8e00000000, 0x2454821300000000, + 0xff71e36f00000000, 0x8990ecf200000000, 0xcb60ae0f00000000, + 0xbd81a19200000000, 0x66a4c0ee00000000, 0x1045cf7300000000, + 0xd0ef021600000000, 0xa60e0d8b00000000, 0x7d2b6cf700000000, + 0x0bca636a00000000, 0xfd7ef73c00000000, 0x8b9ff8a100000000, + 0x50ba99dd00000000, 0x265b964000000000, 0xe6f15b2500000000, + 0x901054b800000000, 0x4b3535c400000000, 0x3dd43a5900000000, + 0xa75c1c6900000000, 0xd1bd13f400000000, 0x0a98728800000000, + 0x7c797d1500000000, 0xbcd3b07000000000, 0xca32bfed00000000, + 0x1117de9100000000, 0x67f6d10c00000000, 0x9142455a00000000, + 0xe7a34ac700000000, 0x3c862bbb00000000, 0x4a67242600000000, + 0x8acde94300000000, 0xfc2ce6de00000000, 0x270987a200000000, + 0x51e8883f00000000}, + {0x0000000000000000, 0xe8dbfbb900000000, 0x91b186a800000000, + 0x796a7d1100000000, 0x63657c8a00000000, 0x8bbe873300000000, + 0xf2d4fa2200000000, 0x1a0f019b00000000, 0x87cc89cf00000000, + 0x6f17727600000000, 0x167d0f6700000000, 0xfea6f4de00000000, + 0xe4a9f54500000000, 0x0c720efc00000000, 0x751873ed00000000, + 0x9dc3885400000000, 0x4f9f624400000000, 0xa74499fd00000000, + 0xde2ee4ec00000000, 0x36f51f5500000000, 0x2cfa1ece00000000, + 0xc421e57700000000, 0xbd4b986600000000, 0x559063df00000000, + 0xc853eb8b00000000, 0x2088103200000000, 0x59e26d2300000000, + 0xb139969a00000000, 0xab36970100000000, 0x43ed6cb800000000, + 0x3a8711a900000000, 0xd25cea1000000000, 0x9e3ec58800000000, + 0x76e53e3100000000, 0x0f8f432000000000, 0xe754b89900000000, + 0xfd5bb90200000000, 0x158042bb00000000, 0x6cea3faa00000000, + 0x8431c41300000000, 0x19f24c4700000000, 0xf129b7fe00000000, + 0x8843caef00000000, 0x6098315600000000, 0x7a9730cd00000000, + 0x924ccb7400000000, 0xeb26b66500000000, 0x03fd4ddc00000000, + 0xd1a1a7cc00000000, 0x397a5c7500000000, 0x4010216400000000, + 0xa8cbdadd00000000, 0xb2c4db4600000000, 0x5a1f20ff00000000, + 0x23755dee00000000, 0xcbaea65700000000, 0x566d2e0300000000, + 0xbeb6d5ba00000000, 0xc7dca8ab00000000, 0x2f07531200000000, + 0x3508528900000000, 0xddd3a93000000000, 0xa4b9d42100000000, + 0x4c622f9800000000, 0x7d7bfbca00000000, 0x95a0007300000000, + 0xecca7d6200000000, 0x041186db00000000, 0x1e1e874000000000, + 0xf6c57cf900000000, 0x8faf01e800000000, 0x6774fa5100000000, + 0xfab7720500000000, 0x126c89bc00000000, 0x6b06f4ad00000000, + 0x83dd0f1400000000, 0x99d20e8f00000000, 0x7109f53600000000, + 0x0863882700000000, 0xe0b8739e00000000, 0x32e4998e00000000, + 0xda3f623700000000, 0xa3551f2600000000, 0x4b8ee49f00000000, + 0x5181e50400000000, 0xb95a1ebd00000000, 0xc03063ac00000000, + 0x28eb981500000000, 0xb528104100000000, 0x5df3ebf800000000, + 0x249996e900000000, 0xcc426d5000000000, 0xd64d6ccb00000000, + 0x3e96977200000000, 0x47fcea6300000000, 0xaf2711da00000000, + 0xe3453e4200000000, 0x0b9ec5fb00000000, 0x72f4b8ea00000000, + 0x9a2f435300000000, 0x802042c800000000, 0x68fbb97100000000, + 0x1191c46000000000, 0xf94a3fd900000000, 0x6489b78d00000000, + 0x8c524c3400000000, 0xf538312500000000, 0x1de3ca9c00000000, + 0x07eccb0700000000, 0xef3730be00000000, 0x965d4daf00000000, + 0x7e86b61600000000, 0xacda5c0600000000, 0x4401a7bf00000000, + 0x3d6bdaae00000000, 0xd5b0211700000000, 0xcfbf208c00000000, + 0x2764db3500000000, 0x5e0ea62400000000, 0xb6d55d9d00000000, + 0x2b16d5c900000000, 0xc3cd2e7000000000, 0xbaa7536100000000, + 0x527ca8d800000000, 0x4873a94300000000, 0xa0a852fa00000000, + 0xd9c22feb00000000, 0x3119d45200000000, 0xbbf0874e00000000, + 0x532b7cf700000000, 0x2a4101e600000000, 0xc29afa5f00000000, + 0xd895fbc400000000, 0x304e007d00000000, 0x49247d6c00000000, + 0xa1ff86d500000000, 0x3c3c0e8100000000, 0xd4e7f53800000000, + 0xad8d882900000000, 0x4556739000000000, 0x5f59720b00000000, + 0xb78289b200000000, 0xcee8f4a300000000, 0x26330f1a00000000, + 0xf46fe50a00000000, 0x1cb41eb300000000, 0x65de63a200000000, + 0x8d05981b00000000, 0x970a998000000000, 0x7fd1623900000000, + 0x06bb1f2800000000, 0xee60e49100000000, 0x73a36cc500000000, + 0x9b78977c00000000, 0xe212ea6d00000000, 0x0ac911d400000000, + 0x10c6104f00000000, 0xf81debf600000000, 0x817796e700000000, + 0x69ac6d5e00000000, 0x25ce42c600000000, 0xcd15b97f00000000, + 0xb47fc46e00000000, 0x5ca43fd700000000, 0x46ab3e4c00000000, + 0xae70c5f500000000, 0xd71ab8e400000000, 0x3fc1435d00000000, + 0xa202cb0900000000, 0x4ad930b000000000, 0x33b34da100000000, + 0xdb68b61800000000, 0xc167b78300000000, 0x29bc4c3a00000000, + 0x50d6312b00000000, 0xb80dca9200000000, 0x6a51208200000000, + 0x828adb3b00000000, 0xfbe0a62a00000000, 0x133b5d9300000000, + 0x09345c0800000000, 0xe1efa7b100000000, 0x9885daa000000000, + 0x705e211900000000, 0xed9da94d00000000, 0x054652f400000000, + 0x7c2c2fe500000000, 0x94f7d45c00000000, 0x8ef8d5c700000000, + 0x66232e7e00000000, 0x1f49536f00000000, 0xf792a8d600000000, + 0xc68b7c8400000000, 0x2e50873d00000000, 0x573afa2c00000000, + 0xbfe1019500000000, 0xa5ee000e00000000, 0x4d35fbb700000000, + 0x345f86a600000000, 0xdc847d1f00000000, 0x4147f54b00000000, + 0xa99c0ef200000000, 0xd0f673e300000000, 0x382d885a00000000, + 0x222289c100000000, 0xcaf9727800000000, 0xb3930f6900000000, + 0x5b48f4d000000000, 0x89141ec000000000, 0x61cfe57900000000, + 0x18a5986800000000, 0xf07e63d100000000, 0xea71624a00000000, + 0x02aa99f300000000, 0x7bc0e4e200000000, 0x931b1f5b00000000, + 0x0ed8970f00000000, 0xe6036cb600000000, 0x9f6911a700000000, + 0x77b2ea1e00000000, 0x6dbdeb8500000000, 0x8566103c00000000, + 0xfc0c6d2d00000000, 0x14d7969400000000, 0x58b5b90c00000000, + 0xb06e42b500000000, 0xc9043fa400000000, 0x21dfc41d00000000, + 0x3bd0c58600000000, 0xd30b3e3f00000000, 0xaa61432e00000000, + 0x42bab89700000000, 0xdf7930c300000000, 0x37a2cb7a00000000, + 0x4ec8b66b00000000, 0xa6134dd200000000, 0xbc1c4c4900000000, + 0x54c7b7f000000000, 0x2dadcae100000000, 0xc576315800000000, + 0x172adb4800000000, 0xfff120f100000000, 0x869b5de000000000, + 0x6e40a65900000000, 0x744fa7c200000000, 0x9c945c7b00000000, + 0xe5fe216a00000000, 0x0d25dad300000000, 0x90e6528700000000, + 0x783da93e00000000, 0x0157d42f00000000, 0xe98c2f9600000000, + 0xf3832e0d00000000, 0x1b58d5b400000000, 0x6232a8a500000000, + 0x8ae9531c00000000}, + {0x0000000000000000, 0x919168ae00000000, 0x6325a08700000000, + 0xf2b4c82900000000, 0x874c31d400000000, 0x16dd597a00000000, + 0xe469915300000000, 0x75f8f9fd00000000, 0x4f9f137300000000, + 0xde0e7bdd00000000, 0x2cbab3f400000000, 0xbd2bdb5a00000000, + 0xc8d322a700000000, 0x59424a0900000000, 0xabf6822000000000, + 0x3a67ea8e00000000, 0x9e3e27e600000000, 0x0faf4f4800000000, + 0xfd1b876100000000, 0x6c8aefcf00000000, 0x1972163200000000, + 0x88e37e9c00000000, 0x7a57b6b500000000, 0xebc6de1b00000000, + 0xd1a1349500000000, 0x40305c3b00000000, 0xb284941200000000, + 0x2315fcbc00000000, 0x56ed054100000000, 0xc77c6def00000000, + 0x35c8a5c600000000, 0xa459cd6800000000, 0x7d7b3f1700000000, + 0xecea57b900000000, 0x1e5e9f9000000000, 0x8fcff73e00000000, + 0xfa370ec300000000, 0x6ba6666d00000000, 0x9912ae4400000000, + 0x0883c6ea00000000, 0x32e42c6400000000, 0xa37544ca00000000, + 0x51c18ce300000000, 0xc050e44d00000000, 0xb5a81db000000000, + 0x2439751e00000000, 0xd68dbd3700000000, 0x471cd59900000000, + 0xe34518f100000000, 0x72d4705f00000000, 0x8060b87600000000, + 0x11f1d0d800000000, 0x6409292500000000, 0xf598418b00000000, + 0x072c89a200000000, 0x96bde10c00000000, 0xacda0b8200000000, + 0x3d4b632c00000000, 0xcfffab0500000000, 0x5e6ec3ab00000000, + 0x2b963a5600000000, 0xba0752f800000000, 0x48b39ad100000000, + 0xd922f27f00000000, 0xfaf67e2e00000000, 0x6b67168000000000, + 0x99d3dea900000000, 0x0842b60700000000, 0x7dba4ffa00000000, + 0xec2b275400000000, 0x1e9fef7d00000000, 0x8f0e87d300000000, + 0xb5696d5d00000000, 0x24f805f300000000, 0xd64ccdda00000000, + 0x47dda57400000000, 0x32255c8900000000, 0xa3b4342700000000, + 0x5100fc0e00000000, 0xc09194a000000000, 0x64c859c800000000, + 0xf559316600000000, 0x07edf94f00000000, 0x967c91e100000000, + 0xe384681c00000000, 0x721500b200000000, 0x80a1c89b00000000, + 0x1130a03500000000, 0x2b574abb00000000, 0xbac6221500000000, + 0x4872ea3c00000000, 0xd9e3829200000000, 0xac1b7b6f00000000, + 0x3d8a13c100000000, 0xcf3edbe800000000, 0x5eafb34600000000, + 0x878d413900000000, 0x161c299700000000, 0xe4a8e1be00000000, + 0x7539891000000000, 0x00c170ed00000000, 0x9150184300000000, + 0x63e4d06a00000000, 0xf275b8c400000000, 0xc812524a00000000, + 0x59833ae400000000, 0xab37f2cd00000000, 0x3aa69a6300000000, + 0x4f5e639e00000000, 0xdecf0b3000000000, 0x2c7bc31900000000, + 0xbdeaabb700000000, 0x19b366df00000000, 0x88220e7100000000, + 0x7a96c65800000000, 0xeb07aef600000000, 0x9eff570b00000000, + 0x0f6e3fa500000000, 0xfddaf78c00000000, 0x6c4b9f2200000000, + 0x562c75ac00000000, 0xc7bd1d0200000000, 0x3509d52b00000000, + 0xa498bd8500000000, 0xd160447800000000, 0x40f12cd600000000, + 0xb245e4ff00000000, 0x23d48c5100000000, 0xf4edfd5c00000000, + 0x657c95f200000000, 0x97c85ddb00000000, 0x0659357500000000, + 0x73a1cc8800000000, 0xe230a42600000000, 0x10846c0f00000000, + 0x811504a100000000, 0xbb72ee2f00000000, 0x2ae3868100000000, + 0xd8574ea800000000, 0x49c6260600000000, 0x3c3edffb00000000, + 0xadafb75500000000, 0x5f1b7f7c00000000, 0xce8a17d200000000, + 0x6ad3daba00000000, 0xfb42b21400000000, 0x09f67a3d00000000, + 0x9867129300000000, 0xed9feb6e00000000, 0x7c0e83c000000000, + 0x8eba4be900000000, 0x1f2b234700000000, 0x254cc9c900000000, + 0xb4dda16700000000, 0x4669694e00000000, 0xd7f801e000000000, + 0xa200f81d00000000, 0x339190b300000000, 0xc125589a00000000, + 0x50b4303400000000, 0x8996c24b00000000, 0x1807aae500000000, + 0xeab362cc00000000, 0x7b220a6200000000, 0x0edaf39f00000000, + 0x9f4b9b3100000000, 0x6dff531800000000, 0xfc6e3bb600000000, + 0xc609d13800000000, 0x5798b99600000000, 0xa52c71bf00000000, + 0x34bd191100000000, 0x4145e0ec00000000, 0xd0d4884200000000, + 0x2260406b00000000, 0xb3f128c500000000, 0x17a8e5ad00000000, + 0x86398d0300000000, 0x748d452a00000000, 0xe51c2d8400000000, + 0x90e4d47900000000, 0x0175bcd700000000, 0xf3c174fe00000000, + 0x62501c5000000000, 0x5837f6de00000000, 0xc9a69e7000000000, + 0x3b12565900000000, 0xaa833ef700000000, 0xdf7bc70a00000000, + 0x4eeaafa400000000, 0xbc5e678d00000000, 0x2dcf0f2300000000, + 0x0e1b837200000000, 0x9f8aebdc00000000, 0x6d3e23f500000000, + 0xfcaf4b5b00000000, 0x8957b2a600000000, 0x18c6da0800000000, + 0xea72122100000000, 0x7be37a8f00000000, 0x4184900100000000, + 0xd015f8af00000000, 0x22a1308600000000, 0xb330582800000000, + 0xc6c8a1d500000000, 0x5759c97b00000000, 0xa5ed015200000000, + 0x347c69fc00000000, 0x9025a49400000000, 0x01b4cc3a00000000, + 0xf300041300000000, 0x62916cbd00000000, 0x1769954000000000, + 0x86f8fdee00000000, 0x744c35c700000000, 0xe5dd5d6900000000, + 0xdfbab7e700000000, 0x4e2bdf4900000000, 0xbc9f176000000000, + 0x2d0e7fce00000000, 0x58f6863300000000, 0xc967ee9d00000000, + 0x3bd326b400000000, 0xaa424e1a00000000, 0x7360bc6500000000, + 0xe2f1d4cb00000000, 0x10451ce200000000, 0x81d4744c00000000, + 0xf42c8db100000000, 0x65bde51f00000000, 0x97092d3600000000, + 0x0698459800000000, 0x3cffaf1600000000, 0xad6ec7b800000000, + 0x5fda0f9100000000, 0xce4b673f00000000, 0xbbb39ec200000000, + 0x2a22f66c00000000, 0xd8963e4500000000, 0x490756eb00000000, + 0xed5e9b8300000000, 0x7ccff32d00000000, 0x8e7b3b0400000000, + 0x1fea53aa00000000, 0x6a12aa5700000000, 0xfb83c2f900000000, + 0x09370ad000000000, 0x98a6627e00000000, 0xa2c188f000000000, + 0x3350e05e00000000, 0xc1e4287700000000, 0x507540d900000000, + 0x258db92400000000, 0xb41cd18a00000000, 0x46a819a300000000, + 0xd739710d00000000}}; + +#else /* W == 4 */ + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0xccaa009e, 0x4225077d, 0x8e8f07e3, 0x844a0efa, + 0x48e00e64, 0xc66f0987, 0x0ac50919, 0xd3e51bb5, 0x1f4f1b2b, + 0x91c01cc8, 0x5d6a1c56, 0x57af154f, 0x9b0515d1, 0x158a1232, + 0xd92012ac, 0x7cbb312b, 0xb01131b5, 0x3e9e3656, 0xf23436c8, + 0xf8f13fd1, 0x345b3f4f, 0xbad438ac, 0x767e3832, 0xaf5e2a9e, + 0x63f42a00, 0xed7b2de3, 0x21d12d7d, 0x2b142464, 0xe7be24fa, + 0x69312319, 0xa59b2387, 0xf9766256, 0x35dc62c8, 0xbb53652b, + 0x77f965b5, 0x7d3c6cac, 0xb1966c32, 0x3f196bd1, 0xf3b36b4f, + 0x2a9379e3, 0xe639797d, 0x68b67e9e, 0xa41c7e00, 0xaed97719, + 0x62737787, 0xecfc7064, 0x205670fa, 0x85cd537d, 0x496753e3, + 0xc7e85400, 0x0b42549e, 0x01875d87, 0xcd2d5d19, 0x43a25afa, + 0x8f085a64, 0x562848c8, 0x9a824856, 0x140d4fb5, 0xd8a74f2b, + 0xd2624632, 0x1ec846ac, 0x9047414f, 0x5ced41d1, 0x299dc2ed, + 0xe537c273, 0x6bb8c590, 0xa712c50e, 0xadd7cc17, 0x617dcc89, + 0xeff2cb6a, 0x2358cbf4, 0xfa78d958, 0x36d2d9c6, 0xb85dde25, + 0x74f7debb, 0x7e32d7a2, 0xb298d73c, 0x3c17d0df, 0xf0bdd041, + 0x5526f3c6, 0x998cf358, 0x1703f4bb, 0xdba9f425, 0xd16cfd3c, + 0x1dc6fda2, 0x9349fa41, 0x5fe3fadf, 0x86c3e873, 0x4a69e8ed, + 0xc4e6ef0e, 0x084cef90, 0x0289e689, 0xce23e617, 0x40ace1f4, + 0x8c06e16a, 0xd0eba0bb, 0x1c41a025, 0x92cea7c6, 0x5e64a758, + 0x54a1ae41, 0x980baedf, 0x1684a93c, 0xda2ea9a2, 0x030ebb0e, + 0xcfa4bb90, 0x412bbc73, 0x8d81bced, 0x8744b5f4, 0x4beeb56a, + 0xc561b289, 0x09cbb217, 0xac509190, 0x60fa910e, 0xee7596ed, + 0x22df9673, 0x281a9f6a, 0xe4b09ff4, 0x6a3f9817, 0xa6959889, + 0x7fb58a25, 0xb31f8abb, 0x3d908d58, 0xf13a8dc6, 0xfbff84df, + 0x37558441, 0xb9da83a2, 0x7570833c, 0x533b85da, 0x9f918544, + 0x111e82a7, 0xddb48239, 0xd7718b20, 0x1bdb8bbe, 0x95548c5d, + 0x59fe8cc3, 0x80de9e6f, 0x4c749ef1, 0xc2fb9912, 0x0e51998c, + 0x04949095, 0xc83e900b, 0x46b197e8, 0x8a1b9776, 0x2f80b4f1, + 0xe32ab46f, 0x6da5b38c, 0xa10fb312, 0xabcaba0b, 0x6760ba95, + 0xe9efbd76, 0x2545bde8, 0xfc65af44, 0x30cfafda, 0xbe40a839, + 0x72eaa8a7, 0x782fa1be, 0xb485a120, 0x3a0aa6c3, 0xf6a0a65d, + 0xaa4de78c, 0x66e7e712, 0xe868e0f1, 0x24c2e06f, 0x2e07e976, + 0xe2ade9e8, 0x6c22ee0b, 0xa088ee95, 0x79a8fc39, 0xb502fca7, + 0x3b8dfb44, 0xf727fbda, 0xfde2f2c3, 0x3148f25d, 0xbfc7f5be, + 0x736df520, 0xd6f6d6a7, 0x1a5cd639, 0x94d3d1da, 0x5879d144, + 0x52bcd85d, 0x9e16d8c3, 0x1099df20, 0xdc33dfbe, 0x0513cd12, + 0xc9b9cd8c, 0x4736ca6f, 0x8b9ccaf1, 0x8159c3e8, 0x4df3c376, + 0xc37cc495, 0x0fd6c40b, 0x7aa64737, 0xb60c47a9, 0x3883404a, + 0xf42940d4, 0xfeec49cd, 0x32464953, 0xbcc94eb0, 0x70634e2e, + 0xa9435c82, 0x65e95c1c, 0xeb665bff, 0x27cc5b61, 0x2d095278, + 0xe1a352e6, 0x6f2c5505, 0xa386559b, 0x061d761c, 0xcab77682, + 0x44387161, 0x889271ff, 0x825778e6, 0x4efd7878, 0xc0727f9b, + 0x0cd87f05, 0xd5f86da9, 0x19526d37, 0x97dd6ad4, 0x5b776a4a, + 0x51b26353, 0x9d1863cd, 0x1397642e, 0xdf3d64b0, 0x83d02561, + 0x4f7a25ff, 0xc1f5221c, 0x0d5f2282, 0x079a2b9b, 0xcb302b05, + 0x45bf2ce6, 0x89152c78, 0x50353ed4, 0x9c9f3e4a, 0x121039a9, + 0xdeba3937, 0xd47f302e, 0x18d530b0, 0x965a3753, 0x5af037cd, + 0xff6b144a, 0x33c114d4, 0xbd4e1337, 0x71e413a9, 0x7b211ab0, + 0xb78b1a2e, 0x39041dcd, 0xf5ae1d53, 0x2c8e0fff, 0xe0240f61, + 0x6eab0882, 0xa201081c, 0xa8c40105, 0x646e019b, 0xeae10678, + 0x264b06e6}, + {0x00000000, 0xa6770bb4, 0x979f1129, 0x31e81a9d, 0xf44f2413, + 0x52382fa7, 0x63d0353a, 0xc5a73e8e, 0x33ef4e67, 0x959845d3, + 0xa4705f4e, 0x020754fa, 0xc7a06a74, 0x61d761c0, 0x503f7b5d, + 0xf64870e9, 0x67de9cce, 0xc1a9977a, 0xf0418de7, 0x56368653, + 0x9391b8dd, 0x35e6b369, 0x040ea9f4, 0xa279a240, 0x5431d2a9, + 0xf246d91d, 0xc3aec380, 0x65d9c834, 0xa07ef6ba, 0x0609fd0e, + 0x37e1e793, 0x9196ec27, 0xcfbd399c, 0x69ca3228, 0x582228b5, + 0xfe552301, 0x3bf21d8f, 0x9d85163b, 0xac6d0ca6, 0x0a1a0712, + 0xfc5277fb, 0x5a257c4f, 0x6bcd66d2, 0xcdba6d66, 0x081d53e8, + 0xae6a585c, 0x9f8242c1, 0x39f54975, 0xa863a552, 0x0e14aee6, + 0x3ffcb47b, 0x998bbfcf, 0x5c2c8141, 0xfa5b8af5, 0xcbb39068, + 0x6dc49bdc, 0x9b8ceb35, 0x3dfbe081, 0x0c13fa1c, 0xaa64f1a8, + 0x6fc3cf26, 0xc9b4c492, 0xf85cde0f, 0x5e2bd5bb, 0x440b7579, + 0xe27c7ecd, 0xd3946450, 0x75e36fe4, 0xb044516a, 0x16335ade, + 0x27db4043, 0x81ac4bf7, 0x77e43b1e, 0xd19330aa, 0xe07b2a37, + 0x460c2183, 0x83ab1f0d, 0x25dc14b9, 0x14340e24, 0xb2430590, + 0x23d5e9b7, 0x85a2e203, 0xb44af89e, 0x123df32a, 0xd79acda4, + 0x71edc610, 0x4005dc8d, 0xe672d739, 0x103aa7d0, 0xb64dac64, + 0x87a5b6f9, 0x21d2bd4d, 0xe47583c3, 0x42028877, 0x73ea92ea, + 0xd59d995e, 0x8bb64ce5, 0x2dc14751, 0x1c295dcc, 0xba5e5678, + 0x7ff968f6, 0xd98e6342, 0xe86679df, 0x4e11726b, 0xb8590282, + 0x1e2e0936, 0x2fc613ab, 0x89b1181f, 0x4c162691, 0xea612d25, + 0xdb8937b8, 0x7dfe3c0c, 0xec68d02b, 0x4a1fdb9f, 0x7bf7c102, + 0xdd80cab6, 0x1827f438, 0xbe50ff8c, 0x8fb8e511, 0x29cfeea5, + 0xdf879e4c, 0x79f095f8, 0x48188f65, 0xee6f84d1, 0x2bc8ba5f, + 0x8dbfb1eb, 0xbc57ab76, 0x1a20a0c2, 0x8816eaf2, 0x2e61e146, + 0x1f89fbdb, 0xb9fef06f, 0x7c59cee1, 0xda2ec555, 0xebc6dfc8, + 0x4db1d47c, 0xbbf9a495, 0x1d8eaf21, 0x2c66b5bc, 0x8a11be08, + 0x4fb68086, 0xe9c18b32, 0xd82991af, 0x7e5e9a1b, 0xefc8763c, + 0x49bf7d88, 0x78576715, 0xde206ca1, 0x1b87522f, 0xbdf0599b, + 0x8c184306, 0x2a6f48b2, 0xdc27385b, 0x7a5033ef, 0x4bb82972, + 0xedcf22c6, 0x28681c48, 0x8e1f17fc, 0xbff70d61, 0x198006d5, + 0x47abd36e, 0xe1dcd8da, 0xd034c247, 0x7643c9f3, 0xb3e4f77d, + 0x1593fcc9, 0x247be654, 0x820cede0, 0x74449d09, 0xd23396bd, + 0xe3db8c20, 0x45ac8794, 0x800bb91a, 0x267cb2ae, 0x1794a833, + 0xb1e3a387, 0x20754fa0, 0x86024414, 0xb7ea5e89, 0x119d553d, + 0xd43a6bb3, 0x724d6007, 0x43a57a9a, 0xe5d2712e, 0x139a01c7, + 0xb5ed0a73, 0x840510ee, 0x22721b5a, 0xe7d525d4, 0x41a22e60, + 0x704a34fd, 0xd63d3f49, 0xcc1d9f8b, 0x6a6a943f, 0x5b828ea2, + 0xfdf58516, 0x3852bb98, 0x9e25b02c, 0xafcdaab1, 0x09baa105, + 0xfff2d1ec, 0x5985da58, 0x686dc0c5, 0xce1acb71, 0x0bbdf5ff, + 0xadcafe4b, 0x9c22e4d6, 0x3a55ef62, 0xabc30345, 0x0db408f1, + 0x3c5c126c, 0x9a2b19d8, 0x5f8c2756, 0xf9fb2ce2, 0xc813367f, + 0x6e643dcb, 0x982c4d22, 0x3e5b4696, 0x0fb35c0b, 0xa9c457bf, + 0x6c636931, 0xca146285, 0xfbfc7818, 0x5d8b73ac, 0x03a0a617, + 0xa5d7ada3, 0x943fb73e, 0x3248bc8a, 0xf7ef8204, 0x519889b0, + 0x6070932d, 0xc6079899, 0x304fe870, 0x9638e3c4, 0xa7d0f959, + 0x01a7f2ed, 0xc400cc63, 0x6277c7d7, 0x539fdd4a, 0xf5e8d6fe, + 0x647e3ad9, 0xc209316d, 0xf3e12bf0, 0x55962044, 0x90311eca, + 0x3646157e, 0x07ae0fe3, 0xa1d90457, 0x579174be, 0xf1e67f0a, + 0xc00e6597, 0x66796e23, 0xa3de50ad, 0x05a95b19, 0x34414184, + 0x92364a30}, + {0x00000000, 0xcb5cd3a5, 0x4dc8a10b, 0x869472ae, 0x9b914216, + 0x50cd91b3, 0xd659e31d, 0x1d0530b8, 0xec53826d, 0x270f51c8, + 0xa19b2366, 0x6ac7f0c3, 0x77c2c07b, 0xbc9e13de, 0x3a0a6170, + 0xf156b2d5, 0x03d6029b, 0xc88ad13e, 0x4e1ea390, 0x85427035, + 0x9847408d, 0x531b9328, 0xd58fe186, 0x1ed33223, 0xef8580f6, + 0x24d95353, 0xa24d21fd, 0x6911f258, 0x7414c2e0, 0xbf481145, + 0x39dc63eb, 0xf280b04e, 0x07ac0536, 0xccf0d693, 0x4a64a43d, + 0x81387798, 0x9c3d4720, 0x57619485, 0xd1f5e62b, 0x1aa9358e, + 0xebff875b, 0x20a354fe, 0xa6372650, 0x6d6bf5f5, 0x706ec54d, + 0xbb3216e8, 0x3da66446, 0xf6fab7e3, 0x047a07ad, 0xcf26d408, + 0x49b2a6a6, 0x82ee7503, 0x9feb45bb, 0x54b7961e, 0xd223e4b0, + 0x197f3715, 0xe82985c0, 0x23755665, 0xa5e124cb, 0x6ebdf76e, + 0x73b8c7d6, 0xb8e41473, 0x3e7066dd, 0xf52cb578, 0x0f580a6c, + 0xc404d9c9, 0x4290ab67, 0x89cc78c2, 0x94c9487a, 0x5f959bdf, + 0xd901e971, 0x125d3ad4, 0xe30b8801, 0x28575ba4, 0xaec3290a, + 0x659ffaaf, 0x789aca17, 0xb3c619b2, 0x35526b1c, 0xfe0eb8b9, + 0x0c8e08f7, 0xc7d2db52, 0x4146a9fc, 0x8a1a7a59, 0x971f4ae1, + 0x5c439944, 0xdad7ebea, 0x118b384f, 0xe0dd8a9a, 0x2b81593f, + 0xad152b91, 0x6649f834, 0x7b4cc88c, 0xb0101b29, 0x36846987, + 0xfdd8ba22, 0x08f40f5a, 0xc3a8dcff, 0x453cae51, 0x8e607df4, + 0x93654d4c, 0x58399ee9, 0xdeadec47, 0x15f13fe2, 0xe4a78d37, + 0x2ffb5e92, 0xa96f2c3c, 0x6233ff99, 0x7f36cf21, 0xb46a1c84, + 0x32fe6e2a, 0xf9a2bd8f, 0x0b220dc1, 0xc07ede64, 0x46eaacca, + 0x8db67f6f, 0x90b34fd7, 0x5bef9c72, 0xdd7beedc, 0x16273d79, + 0xe7718fac, 0x2c2d5c09, 0xaab92ea7, 0x61e5fd02, 0x7ce0cdba, + 0xb7bc1e1f, 0x31286cb1, 0xfa74bf14, 0x1eb014d8, 0xd5ecc77d, + 0x5378b5d3, 0x98246676, 0x852156ce, 0x4e7d856b, 0xc8e9f7c5, + 0x03b52460, 0xf2e396b5, 0x39bf4510, 0xbf2b37be, 0x7477e41b, + 0x6972d4a3, 0xa22e0706, 0x24ba75a8, 0xefe6a60d, 0x1d661643, + 0xd63ac5e6, 0x50aeb748, 0x9bf264ed, 0x86f75455, 0x4dab87f0, + 0xcb3ff55e, 0x006326fb, 0xf135942e, 0x3a69478b, 0xbcfd3525, + 0x77a1e680, 0x6aa4d638, 0xa1f8059d, 0x276c7733, 0xec30a496, + 0x191c11ee, 0xd240c24b, 0x54d4b0e5, 0x9f886340, 0x828d53f8, + 0x49d1805d, 0xcf45f2f3, 0x04192156, 0xf54f9383, 0x3e134026, + 0xb8873288, 0x73dbe12d, 0x6eded195, 0xa5820230, 0x2316709e, + 0xe84aa33b, 0x1aca1375, 0xd196c0d0, 0x5702b27e, 0x9c5e61db, + 0x815b5163, 0x4a0782c6, 0xcc93f068, 0x07cf23cd, 0xf6999118, + 0x3dc542bd, 0xbb513013, 0x700de3b6, 0x6d08d30e, 0xa65400ab, + 0x20c07205, 0xeb9ca1a0, 0x11e81eb4, 0xdab4cd11, 0x5c20bfbf, + 0x977c6c1a, 0x8a795ca2, 0x41258f07, 0xc7b1fda9, 0x0ced2e0c, + 0xfdbb9cd9, 0x36e74f7c, 0xb0733dd2, 0x7b2fee77, 0x662adecf, + 0xad760d6a, 0x2be27fc4, 0xe0beac61, 0x123e1c2f, 0xd962cf8a, + 0x5ff6bd24, 0x94aa6e81, 0x89af5e39, 0x42f38d9c, 0xc467ff32, + 0x0f3b2c97, 0xfe6d9e42, 0x35314de7, 0xb3a53f49, 0x78f9ecec, + 0x65fcdc54, 0xaea00ff1, 0x28347d5f, 0xe368aefa, 0x16441b82, + 0xdd18c827, 0x5b8cba89, 0x90d0692c, 0x8dd55994, 0x46898a31, + 0xc01df89f, 0x0b412b3a, 0xfa1799ef, 0x314b4a4a, 0xb7df38e4, + 0x7c83eb41, 0x6186dbf9, 0xaada085c, 0x2c4e7af2, 0xe712a957, + 0x15921919, 0xdececabc, 0x585ab812, 0x93066bb7, 0x8e035b0f, + 0x455f88aa, 0xc3cbfa04, 0x089729a1, 0xf9c19b74, 0x329d48d1, + 0xb4093a7f, 0x7f55e9da, 0x6250d962, 0xa90c0ac7, 0x2f987869, + 0xe4c4abcc}, + {0x00000000, 0x3d6029b0, 0x7ac05360, 0x47a07ad0, 0xf580a6c0, + 0xc8e08f70, 0x8f40f5a0, 0xb220dc10, 0x30704bc1, 0x0d106271, + 0x4ab018a1, 0x77d03111, 0xc5f0ed01, 0xf890c4b1, 0xbf30be61, + 0x825097d1, 0x60e09782, 0x5d80be32, 0x1a20c4e2, 0x2740ed52, + 0x95603142, 0xa80018f2, 0xefa06222, 0xd2c04b92, 0x5090dc43, + 0x6df0f5f3, 0x2a508f23, 0x1730a693, 0xa5107a83, 0x98705333, + 0xdfd029e3, 0xe2b00053, 0xc1c12f04, 0xfca106b4, 0xbb017c64, + 0x866155d4, 0x344189c4, 0x0921a074, 0x4e81daa4, 0x73e1f314, + 0xf1b164c5, 0xccd14d75, 0x8b7137a5, 0xb6111e15, 0x0431c205, + 0x3951ebb5, 0x7ef19165, 0x4391b8d5, 0xa121b886, 0x9c419136, + 0xdbe1ebe6, 0xe681c256, 0x54a11e46, 0x69c137f6, 0x2e614d26, + 0x13016496, 0x9151f347, 0xac31daf7, 0xeb91a027, 0xd6f18997, + 0x64d15587, 0x59b17c37, 0x1e1106e7, 0x23712f57, 0x58f35849, + 0x659371f9, 0x22330b29, 0x1f532299, 0xad73fe89, 0x9013d739, + 0xd7b3ade9, 0xead38459, 0x68831388, 0x55e33a38, 0x124340e8, + 0x2f236958, 0x9d03b548, 0xa0639cf8, 0xe7c3e628, 0xdaa3cf98, + 0x3813cfcb, 0x0573e67b, 0x42d39cab, 0x7fb3b51b, 0xcd93690b, + 0xf0f340bb, 0xb7533a6b, 0x8a3313db, 0x0863840a, 0x3503adba, + 0x72a3d76a, 0x4fc3feda, 0xfde322ca, 0xc0830b7a, 0x872371aa, + 0xba43581a, 0x9932774d, 0xa4525efd, 0xe3f2242d, 0xde920d9d, + 0x6cb2d18d, 0x51d2f83d, 0x167282ed, 0x2b12ab5d, 0xa9423c8c, + 0x9422153c, 0xd3826fec, 0xeee2465c, 0x5cc29a4c, 0x61a2b3fc, + 0x2602c92c, 0x1b62e09c, 0xf9d2e0cf, 0xc4b2c97f, 0x8312b3af, + 0xbe729a1f, 0x0c52460f, 0x31326fbf, 0x7692156f, 0x4bf23cdf, + 0xc9a2ab0e, 0xf4c282be, 0xb362f86e, 0x8e02d1de, 0x3c220dce, + 0x0142247e, 0x46e25eae, 0x7b82771e, 0xb1e6b092, 0x8c869922, + 0xcb26e3f2, 0xf646ca42, 0x44661652, 0x79063fe2, 0x3ea64532, + 0x03c66c82, 0x8196fb53, 0xbcf6d2e3, 0xfb56a833, 0xc6368183, + 0x74165d93, 0x49767423, 0x0ed60ef3, 0x33b62743, 0xd1062710, + 0xec660ea0, 0xabc67470, 0x96a65dc0, 0x248681d0, 0x19e6a860, + 0x5e46d2b0, 0x6326fb00, 0xe1766cd1, 0xdc164561, 0x9bb63fb1, + 0xa6d61601, 0x14f6ca11, 0x2996e3a1, 0x6e369971, 0x5356b0c1, + 0x70279f96, 0x4d47b626, 0x0ae7ccf6, 0x3787e546, 0x85a73956, + 0xb8c710e6, 0xff676a36, 0xc2074386, 0x4057d457, 0x7d37fde7, + 0x3a978737, 0x07f7ae87, 0xb5d77297, 0x88b75b27, 0xcf1721f7, + 0xf2770847, 0x10c70814, 0x2da721a4, 0x6a075b74, 0x576772c4, + 0xe547aed4, 0xd8278764, 0x9f87fdb4, 0xa2e7d404, 0x20b743d5, + 0x1dd76a65, 0x5a7710b5, 0x67173905, 0xd537e515, 0xe857cca5, + 0xaff7b675, 0x92979fc5, 0xe915e8db, 0xd475c16b, 0x93d5bbbb, + 0xaeb5920b, 0x1c954e1b, 0x21f567ab, 0x66551d7b, 0x5b3534cb, + 0xd965a31a, 0xe4058aaa, 0xa3a5f07a, 0x9ec5d9ca, 0x2ce505da, + 0x11852c6a, 0x562556ba, 0x6b457f0a, 0x89f57f59, 0xb49556e9, + 0xf3352c39, 0xce550589, 0x7c75d999, 0x4115f029, 0x06b58af9, + 0x3bd5a349, 0xb9853498, 0x84e51d28, 0xc34567f8, 0xfe254e48, + 0x4c059258, 0x7165bbe8, 0x36c5c138, 0x0ba5e888, 0x28d4c7df, + 0x15b4ee6f, 0x521494bf, 0x6f74bd0f, 0xdd54611f, 0xe03448af, + 0xa794327f, 0x9af41bcf, 0x18a48c1e, 0x25c4a5ae, 0x6264df7e, + 0x5f04f6ce, 0xed242ade, 0xd044036e, 0x97e479be, 0xaa84500e, + 0x4834505d, 0x755479ed, 0x32f4033d, 0x0f942a8d, 0xbdb4f69d, + 0x80d4df2d, 0xc774a5fd, 0xfa148c4d, 0x78441b9c, 0x4524322c, + 0x028448fc, 0x3fe4614c, 0x8dc4bd5c, 0xb0a494ec, 0xf704ee3c, + 0xca64c78c}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x00000000, 0xb029603d, 0x6053c07a, 0xd07aa047, 0xc0a680f5, + 0x708fe0c8, 0xa0f5408f, 0x10dc20b2, 0xc14b7030, 0x7162100d, + 0xa118b04a, 0x1131d077, 0x01edf0c5, 0xb1c490f8, 0x61be30bf, + 0xd1975082, 0x8297e060, 0x32be805d, 0xe2c4201a, 0x52ed4027, + 0x42316095, 0xf21800a8, 0x2262a0ef, 0x924bc0d2, 0x43dc9050, + 0xf3f5f06d, 0x238f502a, 0x93a63017, 0x837a10a5, 0x33537098, + 0xe329d0df, 0x5300b0e2, 0x042fc1c1, 0xb406a1fc, 0x647c01bb, + 0xd4556186, 0xc4894134, 0x74a02109, 0xa4da814e, 0x14f3e173, + 0xc564b1f1, 0x754dd1cc, 0xa537718b, 0x151e11b6, 0x05c23104, + 0xb5eb5139, 0x6591f17e, 0xd5b89143, 0x86b821a1, 0x3691419c, + 0xe6ebe1db, 0x56c281e6, 0x461ea154, 0xf637c169, 0x264d612e, + 0x96640113, 0x47f35191, 0xf7da31ac, 0x27a091eb, 0x9789f1d6, + 0x8755d164, 0x377cb159, 0xe706111e, 0x572f7123, 0x4958f358, + 0xf9719365, 0x290b3322, 0x9922531f, 0x89fe73ad, 0x39d71390, + 0xe9adb3d7, 0x5984d3ea, 0x88138368, 0x383ae355, 0xe8404312, + 0x5869232f, 0x48b5039d, 0xf89c63a0, 0x28e6c3e7, 0x98cfa3da, + 0xcbcf1338, 0x7be67305, 0xab9cd342, 0x1bb5b37f, 0x0b6993cd, + 0xbb40f3f0, 0x6b3a53b7, 0xdb13338a, 0x0a846308, 0xbaad0335, + 0x6ad7a372, 0xdafec34f, 0xca22e3fd, 0x7a0b83c0, 0xaa712387, + 0x1a5843ba, 0x4d773299, 0xfd5e52a4, 0x2d24f2e3, 0x9d0d92de, + 0x8dd1b26c, 0x3df8d251, 0xed827216, 0x5dab122b, 0x8c3c42a9, + 0x3c152294, 0xec6f82d3, 0x5c46e2ee, 0x4c9ac25c, 0xfcb3a261, + 0x2cc90226, 0x9ce0621b, 0xcfe0d2f9, 0x7fc9b2c4, 0xafb31283, + 0x1f9a72be, 0x0f46520c, 0xbf6f3231, 0x6f159276, 0xdf3cf24b, + 0x0eaba2c9, 0xbe82c2f4, 0x6ef862b3, 0xded1028e, 0xce0d223c, + 0x7e244201, 0xae5ee246, 0x1e77827b, 0x92b0e6b1, 0x2299868c, + 0xf2e326cb, 0x42ca46f6, 0x52166644, 0xe23f0679, 0x3245a63e, + 0x826cc603, 0x53fb9681, 0xe3d2f6bc, 0x33a856fb, 0x838136c6, + 0x935d1674, 0x23747649, 0xf30ed60e, 0x4327b633, 0x102706d1, + 0xa00e66ec, 0x7074c6ab, 0xc05da696, 0xd0818624, 0x60a8e619, + 0xb0d2465e, 0x00fb2663, 0xd16c76e1, 0x614516dc, 0xb13fb69b, + 0x0116d6a6, 0x11caf614, 0xa1e39629, 0x7199366e, 0xc1b05653, + 0x969f2770, 0x26b6474d, 0xf6cce70a, 0x46e58737, 0x5639a785, + 0xe610c7b8, 0x366a67ff, 0x864307c2, 0x57d45740, 0xe7fd377d, + 0x3787973a, 0x87aef707, 0x9772d7b5, 0x275bb788, 0xf72117cf, + 0x470877f2, 0x1408c710, 0xa421a72d, 0x745b076a, 0xc4726757, + 0xd4ae47e5, 0x648727d8, 0xb4fd879f, 0x04d4e7a2, 0xd543b720, + 0x656ad71d, 0xb510775a, 0x05391767, 0x15e537d5, 0xa5cc57e8, + 0x75b6f7af, 0xc59f9792, 0xdbe815e9, 0x6bc175d4, 0xbbbbd593, + 0x0b92b5ae, 0x1b4e951c, 0xab67f521, 0x7b1d5566, 0xcb34355b, + 0x1aa365d9, 0xaa8a05e4, 0x7af0a5a3, 0xcad9c59e, 0xda05e52c, + 0x6a2c8511, 0xba562556, 0x0a7f456b, 0x597ff589, 0xe95695b4, + 0x392c35f3, 0x890555ce, 0x99d9757c, 0x29f01541, 0xf98ab506, + 0x49a3d53b, 0x983485b9, 0x281de584, 0xf86745c3, 0x484e25fe, + 0x5892054c, 0xe8bb6571, 0x38c1c536, 0x88e8a50b, 0xdfc7d428, + 0x6feeb415, 0xbf941452, 0x0fbd746f, 0x1f6154dd, 0xaf4834e0, + 0x7f3294a7, 0xcf1bf49a, 0x1e8ca418, 0xaea5c425, 0x7edf6462, + 0xcef6045f, 0xde2a24ed, 0x6e0344d0, 0xbe79e497, 0x0e5084aa, + 0x5d503448, 0xed795475, 0x3d03f432, 0x8d2a940f, 0x9df6b4bd, + 0x2ddfd480, 0xfda574c7, 0x4d8c14fa, 0x9c1b4478, 0x2c322445, + 0xfc488402, 0x4c61e43f, 0x5cbdc48d, 0xec94a4b0, 0x3cee04f7, + 0x8cc764ca}, + {0x00000000, 0xa5d35ccb, 0x0ba1c84d, 0xae729486, 0x1642919b, + 0xb391cd50, 0x1de359d6, 0xb830051d, 0x6d8253ec, 0xc8510f27, + 0x66239ba1, 0xc3f0c76a, 0x7bc0c277, 0xde139ebc, 0x70610a3a, + 0xd5b256f1, 0x9b02d603, 0x3ed18ac8, 0x90a31e4e, 0x35704285, + 0x8d404798, 0x28931b53, 0x86e18fd5, 0x2332d31e, 0xf68085ef, + 0x5353d924, 0xfd214da2, 0x58f21169, 0xe0c21474, 0x451148bf, + 0xeb63dc39, 0x4eb080f2, 0x3605ac07, 0x93d6f0cc, 0x3da4644a, + 0x98773881, 0x20473d9c, 0x85946157, 0x2be6f5d1, 0x8e35a91a, + 0x5b87ffeb, 0xfe54a320, 0x502637a6, 0xf5f56b6d, 0x4dc56e70, + 0xe81632bb, 0x4664a63d, 0xe3b7faf6, 0xad077a04, 0x08d426cf, + 0xa6a6b249, 0x0375ee82, 0xbb45eb9f, 0x1e96b754, 0xb0e423d2, + 0x15377f19, 0xc08529e8, 0x65567523, 0xcb24e1a5, 0x6ef7bd6e, + 0xd6c7b873, 0x7314e4b8, 0xdd66703e, 0x78b52cf5, 0x6c0a580f, + 0xc9d904c4, 0x67ab9042, 0xc278cc89, 0x7a48c994, 0xdf9b955f, + 0x71e901d9, 0xd43a5d12, 0x01880be3, 0xa45b5728, 0x0a29c3ae, + 0xaffa9f65, 0x17ca9a78, 0xb219c6b3, 0x1c6b5235, 0xb9b80efe, + 0xf7088e0c, 0x52dbd2c7, 0xfca94641, 0x597a1a8a, 0xe14a1f97, + 0x4499435c, 0xeaebd7da, 0x4f388b11, 0x9a8adde0, 0x3f59812b, + 0x912b15ad, 0x34f84966, 0x8cc84c7b, 0x291b10b0, 0x87698436, + 0x22bad8fd, 0x5a0ff408, 0xffdca8c3, 0x51ae3c45, 0xf47d608e, + 0x4c4d6593, 0xe99e3958, 0x47ecadde, 0xe23ff115, 0x378da7e4, + 0x925efb2f, 0x3c2c6fa9, 0x99ff3362, 0x21cf367f, 0x841c6ab4, + 0x2a6efe32, 0x8fbda2f9, 0xc10d220b, 0x64de7ec0, 0xcaacea46, + 0x6f7fb68d, 0xd74fb390, 0x729cef5b, 0xdcee7bdd, 0x793d2716, + 0xac8f71e7, 0x095c2d2c, 0xa72eb9aa, 0x02fde561, 0xbacde07c, + 0x1f1ebcb7, 0xb16c2831, 0x14bf74fa, 0xd814b01e, 0x7dc7ecd5, + 0xd3b57853, 0x76662498, 0xce562185, 0x6b857d4e, 0xc5f7e9c8, + 0x6024b503, 0xb596e3f2, 0x1045bf39, 0xbe372bbf, 0x1be47774, + 0xa3d47269, 0x06072ea2, 0xa875ba24, 0x0da6e6ef, 0x4316661d, + 0xe6c53ad6, 0x48b7ae50, 0xed64f29b, 0x5554f786, 0xf087ab4d, + 0x5ef53fcb, 0xfb266300, 0x2e9435f1, 0x8b47693a, 0x2535fdbc, + 0x80e6a177, 0x38d6a46a, 0x9d05f8a1, 0x33776c27, 0x96a430ec, + 0xee111c19, 0x4bc240d2, 0xe5b0d454, 0x4063889f, 0xf8538d82, + 0x5d80d149, 0xf3f245cf, 0x56211904, 0x83934ff5, 0x2640133e, + 0x883287b8, 0x2de1db73, 0x95d1de6e, 0x300282a5, 0x9e701623, + 0x3ba34ae8, 0x7513ca1a, 0xd0c096d1, 0x7eb20257, 0xdb615e9c, + 0x63515b81, 0xc682074a, 0x68f093cc, 0xcd23cf07, 0x189199f6, + 0xbd42c53d, 0x133051bb, 0xb6e30d70, 0x0ed3086d, 0xab0054a6, + 0x0572c020, 0xa0a19ceb, 0xb41ee811, 0x11cdb4da, 0xbfbf205c, + 0x1a6c7c97, 0xa25c798a, 0x078f2541, 0xa9fdb1c7, 0x0c2eed0c, + 0xd99cbbfd, 0x7c4fe736, 0xd23d73b0, 0x77ee2f7b, 0xcfde2a66, + 0x6a0d76ad, 0xc47fe22b, 0x61acbee0, 0x2f1c3e12, 0x8acf62d9, + 0x24bdf65f, 0x816eaa94, 0x395eaf89, 0x9c8df342, 0x32ff67c4, + 0x972c3b0f, 0x429e6dfe, 0xe74d3135, 0x493fa5b3, 0xececf978, + 0x54dcfc65, 0xf10fa0ae, 0x5f7d3428, 0xfaae68e3, 0x821b4416, + 0x27c818dd, 0x89ba8c5b, 0x2c69d090, 0x9459d58d, 0x318a8946, + 0x9ff81dc0, 0x3a2b410b, 0xef9917fa, 0x4a4a4b31, 0xe438dfb7, + 0x41eb837c, 0xf9db8661, 0x5c08daaa, 0xf27a4e2c, 0x57a912e7, + 0x19199215, 0xbccacede, 0x12b85a58, 0xb76b0693, 0x0f5b038e, + 0xaa885f45, 0x04facbc3, 0xa1299708, 0x749bc1f9, 0xd1489d32, + 0x7f3a09b4, 0xdae9557f, 0x62d95062, 0xc70a0ca9, 0x6978982f, + 0xccabc4e4}, + {0x00000000, 0xb40b77a6, 0x29119f97, 0x9d1ae831, 0x13244ff4, + 0xa72f3852, 0x3a35d063, 0x8e3ea7c5, 0x674eef33, 0xd3459895, + 0x4e5f70a4, 0xfa540702, 0x746aa0c7, 0xc061d761, 0x5d7b3f50, + 0xe97048f6, 0xce9cde67, 0x7a97a9c1, 0xe78d41f0, 0x53863656, + 0xddb89193, 0x69b3e635, 0xf4a90e04, 0x40a279a2, 0xa9d23154, + 0x1dd946f2, 0x80c3aec3, 0x34c8d965, 0xbaf67ea0, 0x0efd0906, + 0x93e7e137, 0x27ec9691, 0x9c39bdcf, 0x2832ca69, 0xb5282258, + 0x012355fe, 0x8f1df23b, 0x3b16859d, 0xa60c6dac, 0x12071a0a, + 0xfb7752fc, 0x4f7c255a, 0xd266cd6b, 0x666dbacd, 0xe8531d08, + 0x5c586aae, 0xc142829f, 0x7549f539, 0x52a563a8, 0xe6ae140e, + 0x7bb4fc3f, 0xcfbf8b99, 0x41812c5c, 0xf58a5bfa, 0x6890b3cb, + 0xdc9bc46d, 0x35eb8c9b, 0x81e0fb3d, 0x1cfa130c, 0xa8f164aa, + 0x26cfc36f, 0x92c4b4c9, 0x0fde5cf8, 0xbbd52b5e, 0x79750b44, + 0xcd7e7ce2, 0x506494d3, 0xe46fe375, 0x6a5144b0, 0xde5a3316, + 0x4340db27, 0xf74bac81, 0x1e3be477, 0xaa3093d1, 0x372a7be0, + 0x83210c46, 0x0d1fab83, 0xb914dc25, 0x240e3414, 0x900543b2, + 0xb7e9d523, 0x03e2a285, 0x9ef84ab4, 0x2af33d12, 0xa4cd9ad7, + 0x10c6ed71, 0x8ddc0540, 0x39d772e6, 0xd0a73a10, 0x64ac4db6, + 0xf9b6a587, 0x4dbdd221, 0xc38375e4, 0x77880242, 0xea92ea73, + 0x5e999dd5, 0xe54cb68b, 0x5147c12d, 0xcc5d291c, 0x78565eba, + 0xf668f97f, 0x42638ed9, 0xdf7966e8, 0x6b72114e, 0x820259b8, + 0x36092e1e, 0xab13c62f, 0x1f18b189, 0x9126164c, 0x252d61ea, + 0xb83789db, 0x0c3cfe7d, 0x2bd068ec, 0x9fdb1f4a, 0x02c1f77b, + 0xb6ca80dd, 0x38f42718, 0x8cff50be, 0x11e5b88f, 0xa5eecf29, + 0x4c9e87df, 0xf895f079, 0x658f1848, 0xd1846fee, 0x5fbac82b, + 0xebb1bf8d, 0x76ab57bc, 0xc2a0201a, 0xf2ea1688, 0x46e1612e, + 0xdbfb891f, 0x6ff0feb9, 0xe1ce597c, 0x55c52eda, 0xc8dfc6eb, + 0x7cd4b14d, 0x95a4f9bb, 0x21af8e1d, 0xbcb5662c, 0x08be118a, + 0x8680b64f, 0x328bc1e9, 0xaf9129d8, 0x1b9a5e7e, 0x3c76c8ef, + 0x887dbf49, 0x15675778, 0xa16c20de, 0x2f52871b, 0x9b59f0bd, + 0x0643188c, 0xb2486f2a, 0x5b3827dc, 0xef33507a, 0x7229b84b, + 0xc622cfed, 0x481c6828, 0xfc171f8e, 0x610df7bf, 0xd5068019, + 0x6ed3ab47, 0xdad8dce1, 0x47c234d0, 0xf3c94376, 0x7df7e4b3, + 0xc9fc9315, 0x54e67b24, 0xe0ed0c82, 0x099d4474, 0xbd9633d2, + 0x208cdbe3, 0x9487ac45, 0x1ab90b80, 0xaeb27c26, 0x33a89417, + 0x87a3e3b1, 0xa04f7520, 0x14440286, 0x895eeab7, 0x3d559d11, + 0xb36b3ad4, 0x07604d72, 0x9a7aa543, 0x2e71d2e5, 0xc7019a13, + 0x730aedb5, 0xee100584, 0x5a1b7222, 0xd425d5e7, 0x602ea241, + 0xfd344a70, 0x493f3dd6, 0x8b9f1dcc, 0x3f946a6a, 0xa28e825b, + 0x1685f5fd, 0x98bb5238, 0x2cb0259e, 0xb1aacdaf, 0x05a1ba09, + 0xecd1f2ff, 0x58da8559, 0xc5c06d68, 0x71cb1ace, 0xfff5bd0b, + 0x4bfecaad, 0xd6e4229c, 0x62ef553a, 0x4503c3ab, 0xf108b40d, + 0x6c125c3c, 0xd8192b9a, 0x56278c5f, 0xe22cfbf9, 0x7f3613c8, + 0xcb3d646e, 0x224d2c98, 0x96465b3e, 0x0b5cb30f, 0xbf57c4a9, + 0x3169636c, 0x856214ca, 0x1878fcfb, 0xac738b5d, 0x17a6a003, + 0xa3add7a5, 0x3eb73f94, 0x8abc4832, 0x0482eff7, 0xb0899851, + 0x2d937060, 0x999807c6, 0x70e84f30, 0xc4e33896, 0x59f9d0a7, + 0xedf2a701, 0x63cc00c4, 0xd7c77762, 0x4add9f53, 0xfed6e8f5, + 0xd93a7e64, 0x6d3109c2, 0xf02be1f3, 0x44209655, 0xca1e3190, + 0x7e154636, 0xe30fae07, 0x5704d9a1, 0xbe749157, 0x0a7fe6f1, + 0x97650ec0, 0x236e7966, 0xad50dea3, 0x195ba905, 0x84414134, + 0x304a3692}, + {0x00000000, 0x9e00aacc, 0x7d072542, 0xe3078f8e, 0xfa0e4a84, + 0x640ee048, 0x87096fc6, 0x1909c50a, 0xb51be5d3, 0x2b1b4f1f, + 0xc81cc091, 0x561c6a5d, 0x4f15af57, 0xd115059b, 0x32128a15, + 0xac1220d9, 0x2b31bb7c, 0xb53111b0, 0x56369e3e, 0xc83634f2, + 0xd13ff1f8, 0x4f3f5b34, 0xac38d4ba, 0x32387e76, 0x9e2a5eaf, + 0x002af463, 0xe32d7bed, 0x7d2dd121, 0x6424142b, 0xfa24bee7, + 0x19233169, 0x87239ba5, 0x566276f9, 0xc862dc35, 0x2b6553bb, + 0xb565f977, 0xac6c3c7d, 0x326c96b1, 0xd16b193f, 0x4f6bb3f3, + 0xe379932a, 0x7d7939e6, 0x9e7eb668, 0x007e1ca4, 0x1977d9ae, + 0x87777362, 0x6470fcec, 0xfa705620, 0x7d53cd85, 0xe3536749, + 0x0054e8c7, 0x9e54420b, 0x875d8701, 0x195d2dcd, 0xfa5aa243, + 0x645a088f, 0xc8482856, 0x5648829a, 0xb54f0d14, 0x2b4fa7d8, + 0x324662d2, 0xac46c81e, 0x4f414790, 0xd141ed5c, 0xedc29d29, + 0x73c237e5, 0x90c5b86b, 0x0ec512a7, 0x17ccd7ad, 0x89cc7d61, + 0x6acbf2ef, 0xf4cb5823, 0x58d978fa, 0xc6d9d236, 0x25de5db8, + 0xbbdef774, 0xa2d7327e, 0x3cd798b2, 0xdfd0173c, 0x41d0bdf0, + 0xc6f32655, 0x58f38c99, 0xbbf40317, 0x25f4a9db, 0x3cfd6cd1, + 0xa2fdc61d, 0x41fa4993, 0xdffae35f, 0x73e8c386, 0xede8694a, + 0x0eefe6c4, 0x90ef4c08, 0x89e68902, 0x17e623ce, 0xf4e1ac40, + 0x6ae1068c, 0xbba0ebd0, 0x25a0411c, 0xc6a7ce92, 0x58a7645e, + 0x41aea154, 0xdfae0b98, 0x3ca98416, 0xa2a92eda, 0x0ebb0e03, + 0x90bba4cf, 0x73bc2b41, 0xedbc818d, 0xf4b54487, 0x6ab5ee4b, + 0x89b261c5, 0x17b2cb09, 0x909150ac, 0x0e91fa60, 0xed9675ee, + 0x7396df22, 0x6a9f1a28, 0xf49fb0e4, 0x17983f6a, 0x899895a6, + 0x258ab57f, 0xbb8a1fb3, 0x588d903d, 0xc68d3af1, 0xdf84fffb, + 0x41845537, 0xa283dab9, 0x3c837075, 0xda853b53, 0x4485919f, + 0xa7821e11, 0x3982b4dd, 0x208b71d7, 0xbe8bdb1b, 0x5d8c5495, + 0xc38cfe59, 0x6f9ede80, 0xf19e744c, 0x1299fbc2, 0x8c99510e, + 0x95909404, 0x0b903ec8, 0xe897b146, 0x76971b8a, 0xf1b4802f, + 0x6fb42ae3, 0x8cb3a56d, 0x12b30fa1, 0x0bbacaab, 0x95ba6067, + 0x76bdefe9, 0xe8bd4525, 0x44af65fc, 0xdaafcf30, 0x39a840be, + 0xa7a8ea72, 0xbea12f78, 0x20a185b4, 0xc3a60a3a, 0x5da6a0f6, + 0x8ce74daa, 0x12e7e766, 0xf1e068e8, 0x6fe0c224, 0x76e9072e, + 0xe8e9ade2, 0x0bee226c, 0x95ee88a0, 0x39fca879, 0xa7fc02b5, + 0x44fb8d3b, 0xdafb27f7, 0xc3f2e2fd, 0x5df24831, 0xbef5c7bf, + 0x20f56d73, 0xa7d6f6d6, 0x39d65c1a, 0xdad1d394, 0x44d17958, + 0x5dd8bc52, 0xc3d8169e, 0x20df9910, 0xbedf33dc, 0x12cd1305, + 0x8ccdb9c9, 0x6fca3647, 0xf1ca9c8b, 0xe8c35981, 0x76c3f34d, + 0x95c47cc3, 0x0bc4d60f, 0x3747a67a, 0xa9470cb6, 0x4a408338, + 0xd44029f4, 0xcd49ecfe, 0x53494632, 0xb04ec9bc, 0x2e4e6370, + 0x825c43a9, 0x1c5ce965, 0xff5b66eb, 0x615bcc27, 0x7852092d, + 0xe652a3e1, 0x05552c6f, 0x9b5586a3, 0x1c761d06, 0x8276b7ca, + 0x61713844, 0xff719288, 0xe6785782, 0x7878fd4e, 0x9b7f72c0, + 0x057fd80c, 0xa96df8d5, 0x376d5219, 0xd46add97, 0x4a6a775b, + 0x5363b251, 0xcd63189d, 0x2e649713, 0xb0643ddf, 0x6125d083, + 0xff257a4f, 0x1c22f5c1, 0x82225f0d, 0x9b2b9a07, 0x052b30cb, + 0xe62cbf45, 0x782c1589, 0xd43e3550, 0x4a3e9f9c, 0xa9391012, + 0x3739bade, 0x2e307fd4, 0xb030d518, 0x53375a96, 0xcd37f05a, + 0x4a146bff, 0xd414c133, 0x37134ebd, 0xa913e471, 0xb01a217b, + 0x2e1a8bb7, 0xcd1d0439, 0x531daef5, 0xff0f8e2c, 0x610f24e0, + 0x8208ab6e, 0x1c0801a2, 0x0501c4a8, 0x9b016e64, 0x7806e1ea, + 0xe6064b26}}; + +#endif + +#endif + +#if N == 3 + +#if W == 8 + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0x81256527, 0xd93bcc0f, 0x581ea928, 0x69069e5f, + 0xe823fb78, 0xb03d5250, 0x31183777, 0xd20d3cbe, 0x53285999, + 0x0b36f0b1, 0x8a139596, 0xbb0ba2e1, 0x3a2ec7c6, 0x62306eee, + 0xe3150bc9, 0x7f6b7f3d, 0xfe4e1a1a, 0xa650b332, 0x2775d615, + 0x166de162, 0x97488445, 0xcf562d6d, 0x4e73484a, 0xad664383, + 0x2c4326a4, 0x745d8f8c, 0xf578eaab, 0xc460dddc, 0x4545b8fb, + 0x1d5b11d3, 0x9c7e74f4, 0xfed6fe7a, 0x7ff39b5d, 0x27ed3275, + 0xa6c85752, 0x97d06025, 0x16f50502, 0x4eebac2a, 0xcfcec90d, + 0x2cdbc2c4, 0xadfea7e3, 0xf5e00ecb, 0x74c56bec, 0x45dd5c9b, + 0xc4f839bc, 0x9ce69094, 0x1dc3f5b3, 0x81bd8147, 0x0098e460, + 0x58864d48, 0xd9a3286f, 0xe8bb1f18, 0x699e7a3f, 0x3180d317, + 0xb0a5b630, 0x53b0bdf9, 0xd295d8de, 0x8a8b71f6, 0x0bae14d1, + 0x3ab623a6, 0xbb934681, 0xe38defa9, 0x62a88a8e, 0x26dcfab5, + 0xa7f99f92, 0xffe736ba, 0x7ec2539d, 0x4fda64ea, 0xceff01cd, + 0x96e1a8e5, 0x17c4cdc2, 0xf4d1c60b, 0x75f4a32c, 0x2dea0a04, + 0xaccf6f23, 0x9dd75854, 0x1cf23d73, 0x44ec945b, 0xc5c9f17c, + 0x59b78588, 0xd892e0af, 0x808c4987, 0x01a92ca0, 0x30b11bd7, + 0xb1947ef0, 0xe98ad7d8, 0x68afb2ff, 0x8bbab936, 0x0a9fdc11, + 0x52817539, 0xd3a4101e, 0xe2bc2769, 0x6399424e, 0x3b87eb66, + 0xbaa28e41, 0xd80a04cf, 0x592f61e8, 0x0131c8c0, 0x8014ade7, + 0xb10c9a90, 0x3029ffb7, 0x6837569f, 0xe91233b8, 0x0a073871, + 0x8b225d56, 0xd33cf47e, 0x52199159, 0x6301a62e, 0xe224c309, + 0xba3a6a21, 0x3b1f0f06, 0xa7617bf2, 0x26441ed5, 0x7e5ab7fd, + 0xff7fd2da, 0xce67e5ad, 0x4f42808a, 0x175c29a2, 0x96794c85, + 0x756c474c, 0xf449226b, 0xac578b43, 0x2d72ee64, 0x1c6ad913, + 0x9d4fbc34, 0xc551151c, 0x4474703b, 0x4db9f56a, 0xcc9c904d, + 0x94823965, 0x15a75c42, 0x24bf6b35, 0xa59a0e12, 0xfd84a73a, + 0x7ca1c21d, 0x9fb4c9d4, 0x1e91acf3, 0x468f05db, 0xc7aa60fc, + 0xf6b2578b, 0x779732ac, 0x2f899b84, 0xaeacfea3, 0x32d28a57, + 0xb3f7ef70, 0xebe94658, 0x6acc237f, 0x5bd41408, 0xdaf1712f, + 0x82efd807, 0x03cabd20, 0xe0dfb6e9, 0x61fad3ce, 0x39e47ae6, + 0xb8c11fc1, 0x89d928b6, 0x08fc4d91, 0x50e2e4b9, 0xd1c7819e, + 0xb36f0b10, 0x324a6e37, 0x6a54c71f, 0xeb71a238, 0xda69954f, + 0x5b4cf068, 0x03525940, 0x82773c67, 0x616237ae, 0xe0475289, + 0xb859fba1, 0x397c9e86, 0x0864a9f1, 0x8941ccd6, 0xd15f65fe, + 0x507a00d9, 0xcc04742d, 0x4d21110a, 0x153fb822, 0x941add05, + 0xa502ea72, 0x24278f55, 0x7c39267d, 0xfd1c435a, 0x1e094893, + 0x9f2c2db4, 0xc732849c, 0x4617e1bb, 0x770fd6cc, 0xf62ab3eb, + 0xae341ac3, 0x2f117fe4, 0x6b650fdf, 0xea406af8, 0xb25ec3d0, + 0x337ba6f7, 0x02639180, 0x8346f4a7, 0xdb585d8f, 0x5a7d38a8, + 0xb9683361, 0x384d5646, 0x6053ff6e, 0xe1769a49, 0xd06ead3e, + 0x514bc819, 0x09556131, 0x88700416, 0x140e70e2, 0x952b15c5, + 0xcd35bced, 0x4c10d9ca, 0x7d08eebd, 0xfc2d8b9a, 0xa43322b2, + 0x25164795, 0xc6034c5c, 0x4726297b, 0x1f388053, 0x9e1de574, + 0xaf05d203, 0x2e20b724, 0x763e1e0c, 0xf71b7b2b, 0x95b3f1a5, + 0x14969482, 0x4c883daa, 0xcdad588d, 0xfcb56ffa, 0x7d900add, + 0x258ea3f5, 0xa4abc6d2, 0x47becd1b, 0xc69ba83c, 0x9e850114, + 0x1fa06433, 0x2eb85344, 0xaf9d3663, 0xf7839f4b, 0x76a6fa6c, + 0xead88e98, 0x6bfdebbf, 0x33e34297, 0xb2c627b0, 0x83de10c7, + 0x02fb75e0, 0x5ae5dcc8, 0xdbc0b9ef, 0x38d5b226, 0xb9f0d701, + 0xe1ee7e29, 0x60cb1b0e, 0x51d32c79, 0xd0f6495e, 0x88e8e076, + 0x09cd8551}, + {0x00000000, 0x9b73ead4, 0xed96d3e9, 0x76e5393d, 0x005ca193, + 0x9b2f4b47, 0xedca727a, 0x76b998ae, 0x00b94326, 0x9bcaa9f2, + 0xed2f90cf, 0x765c7a1b, 0x00e5e2b5, 0x9b960861, 0xed73315c, + 0x7600db88, 0x0172864c, 0x9a016c98, 0xece455a5, 0x7797bf71, + 0x012e27df, 0x9a5dcd0b, 0xecb8f436, 0x77cb1ee2, 0x01cbc56a, + 0x9ab82fbe, 0xec5d1683, 0x772efc57, 0x019764f9, 0x9ae48e2d, + 0xec01b710, 0x77725dc4, 0x02e50c98, 0x9996e64c, 0xef73df71, + 0x740035a5, 0x02b9ad0b, 0x99ca47df, 0xef2f7ee2, 0x745c9436, + 0x025c4fbe, 0x992fa56a, 0xefca9c57, 0x74b97683, 0x0200ee2d, + 0x997304f9, 0xef963dc4, 0x74e5d710, 0x03978ad4, 0x98e46000, + 0xee01593d, 0x7572b3e9, 0x03cb2b47, 0x98b8c193, 0xee5df8ae, + 0x752e127a, 0x032ec9f2, 0x985d2326, 0xeeb81a1b, 0x75cbf0cf, + 0x03726861, 0x980182b5, 0xeee4bb88, 0x7597515c, 0x05ca1930, + 0x9eb9f3e4, 0xe85ccad9, 0x732f200d, 0x0596b8a3, 0x9ee55277, + 0xe8006b4a, 0x7373819e, 0x05735a16, 0x9e00b0c2, 0xe8e589ff, + 0x7396632b, 0x052ffb85, 0x9e5c1151, 0xe8b9286c, 0x73cac2b8, + 0x04b89f7c, 0x9fcb75a8, 0xe92e4c95, 0x725da641, 0x04e43eef, + 0x9f97d43b, 0xe972ed06, 0x720107d2, 0x0401dc5a, 0x9f72368e, + 0xe9970fb3, 0x72e4e567, 0x045d7dc9, 0x9f2e971d, 0xe9cbae20, + 0x72b844f4, 0x072f15a8, 0x9c5cff7c, 0xeab9c641, 0x71ca2c95, + 0x0773b43b, 0x9c005eef, 0xeae567d2, 0x71968d06, 0x0796568e, + 0x9ce5bc5a, 0xea008567, 0x71736fb3, 0x07caf71d, 0x9cb91dc9, + 0xea5c24f4, 0x712fce20, 0x065d93e4, 0x9d2e7930, 0xebcb400d, + 0x70b8aad9, 0x06013277, 0x9d72d8a3, 0xeb97e19e, 0x70e40b4a, + 0x06e4d0c2, 0x9d973a16, 0xeb72032b, 0x7001e9ff, 0x06b87151, + 0x9dcb9b85, 0xeb2ea2b8, 0x705d486c, 0x0b943260, 0x90e7d8b4, + 0xe602e189, 0x7d710b5d, 0x0bc893f3, 0x90bb7927, 0xe65e401a, + 0x7d2daace, 0x0b2d7146, 0x905e9b92, 0xe6bba2af, 0x7dc8487b, + 0x0b71d0d5, 0x90023a01, 0xe6e7033c, 0x7d94e9e8, 0x0ae6b42c, + 0x91955ef8, 0xe77067c5, 0x7c038d11, 0x0aba15bf, 0x91c9ff6b, + 0xe72cc656, 0x7c5f2c82, 0x0a5ff70a, 0x912c1dde, 0xe7c924e3, + 0x7cbace37, 0x0a035699, 0x9170bc4d, 0xe7958570, 0x7ce66fa4, + 0x09713ef8, 0x9202d42c, 0xe4e7ed11, 0x7f9407c5, 0x092d9f6b, + 0x925e75bf, 0xe4bb4c82, 0x7fc8a656, 0x09c87dde, 0x92bb970a, + 0xe45eae37, 0x7f2d44e3, 0x0994dc4d, 0x92e73699, 0xe4020fa4, + 0x7f71e570, 0x0803b8b4, 0x93705260, 0xe5956b5d, 0x7ee68189, + 0x085f1927, 0x932cf3f3, 0xe5c9cace, 0x7eba201a, 0x08bafb92, + 0x93c91146, 0xe52c287b, 0x7e5fc2af, 0x08e65a01, 0x9395b0d5, + 0xe57089e8, 0x7e03633c, 0x0e5e2b50, 0x952dc184, 0xe3c8f8b9, + 0x78bb126d, 0x0e028ac3, 0x95716017, 0xe394592a, 0x78e7b3fe, + 0x0ee76876, 0x959482a2, 0xe371bb9f, 0x7802514b, 0x0ebbc9e5, + 0x95c82331, 0xe32d1a0c, 0x785ef0d8, 0x0f2cad1c, 0x945f47c8, + 0xe2ba7ef5, 0x79c99421, 0x0f700c8f, 0x9403e65b, 0xe2e6df66, + 0x799535b2, 0x0f95ee3a, 0x94e604ee, 0xe2033dd3, 0x7970d707, + 0x0fc94fa9, 0x94baa57d, 0xe25f9c40, 0x792c7694, 0x0cbb27c8, + 0x97c8cd1c, 0xe12df421, 0x7a5e1ef5, 0x0ce7865b, 0x97946c8f, + 0xe17155b2, 0x7a02bf66, 0x0c0264ee, 0x97718e3a, 0xe194b707, + 0x7ae75dd3, 0x0c5ec57d, 0x972d2fa9, 0xe1c81694, 0x7abbfc40, + 0x0dc9a184, 0x96ba4b50, 0xe05f726d, 0x7b2c98b9, 0x0d950017, + 0x96e6eac3, 0xe003d3fe, 0x7b70392a, 0x0d70e2a2, 0x96030876, + 0xe0e6314b, 0x7b95db9f, 0x0d2c4331, 0x965fa9e5, 0xe0ba90d8, + 0x7bc97a0c}, + {0x00000000, 0x172864c0, 0x2e50c980, 0x3978ad40, 0x5ca19300, + 0x4b89f7c0, 0x72f15a80, 0x65d93e40, 0xb9432600, 0xae6b42c0, + 0x9713ef80, 0x803b8b40, 0xe5e2b500, 0xf2cad1c0, 0xcbb27c80, + 0xdc9a1840, 0xa9f74a41, 0xbedf2e81, 0x87a783c1, 0x908fe701, + 0xf556d941, 0xe27ebd81, 0xdb0610c1, 0xcc2e7401, 0x10b46c41, + 0x079c0881, 0x3ee4a5c1, 0x29ccc101, 0x4c15ff41, 0x5b3d9b81, + 0x624536c1, 0x756d5201, 0x889f92c3, 0x9fb7f603, 0xa6cf5b43, + 0xb1e73f83, 0xd43e01c3, 0xc3166503, 0xfa6ec843, 0xed46ac83, + 0x31dcb4c3, 0x26f4d003, 0x1f8c7d43, 0x08a41983, 0x6d7d27c3, + 0x7a554303, 0x432dee43, 0x54058a83, 0x2168d882, 0x3640bc42, + 0x0f381102, 0x181075c2, 0x7dc94b82, 0x6ae12f42, 0x53998202, + 0x44b1e6c2, 0x982bfe82, 0x8f039a42, 0xb67b3702, 0xa15353c2, + 0xc48a6d82, 0xd3a20942, 0xeadaa402, 0xfdf2c0c2, 0xca4e23c7, + 0xdd664707, 0xe41eea47, 0xf3368e87, 0x96efb0c7, 0x81c7d407, + 0xb8bf7947, 0xaf971d87, 0x730d05c7, 0x64256107, 0x5d5dcc47, + 0x4a75a887, 0x2fac96c7, 0x3884f207, 0x01fc5f47, 0x16d43b87, + 0x63b96986, 0x74910d46, 0x4de9a006, 0x5ac1c4c6, 0x3f18fa86, + 0x28309e46, 0x11483306, 0x066057c6, 0xdafa4f86, 0xcdd22b46, + 0xf4aa8606, 0xe382e2c6, 0x865bdc86, 0x9173b846, 0xa80b1506, + 0xbf2371c6, 0x42d1b104, 0x55f9d5c4, 0x6c817884, 0x7ba91c44, + 0x1e702204, 0x095846c4, 0x3020eb84, 0x27088f44, 0xfb929704, + 0xecbaf3c4, 0xd5c25e84, 0xc2ea3a44, 0xa7330404, 0xb01b60c4, + 0x8963cd84, 0x9e4ba944, 0xeb26fb45, 0xfc0e9f85, 0xc57632c5, + 0xd25e5605, 0xb7876845, 0xa0af0c85, 0x99d7a1c5, 0x8effc505, + 0x5265dd45, 0x454db985, 0x7c3514c5, 0x6b1d7005, 0x0ec44e45, + 0x19ec2a85, 0x209487c5, 0x37bce305, 0x4fed41cf, 0x58c5250f, + 0x61bd884f, 0x7695ec8f, 0x134cd2cf, 0x0464b60f, 0x3d1c1b4f, + 0x2a347f8f, 0xf6ae67cf, 0xe186030f, 0xd8feae4f, 0xcfd6ca8f, + 0xaa0ff4cf, 0xbd27900f, 0x845f3d4f, 0x9377598f, 0xe61a0b8e, + 0xf1326f4e, 0xc84ac20e, 0xdf62a6ce, 0xbabb988e, 0xad93fc4e, + 0x94eb510e, 0x83c335ce, 0x5f592d8e, 0x4871494e, 0x7109e40e, + 0x662180ce, 0x03f8be8e, 0x14d0da4e, 0x2da8770e, 0x3a8013ce, + 0xc772d30c, 0xd05ab7cc, 0xe9221a8c, 0xfe0a7e4c, 0x9bd3400c, + 0x8cfb24cc, 0xb583898c, 0xa2abed4c, 0x7e31f50c, 0x691991cc, + 0x50613c8c, 0x4749584c, 0x2290660c, 0x35b802cc, 0x0cc0af8c, + 0x1be8cb4c, 0x6e85994d, 0x79adfd8d, 0x40d550cd, 0x57fd340d, + 0x32240a4d, 0x250c6e8d, 0x1c74c3cd, 0x0b5ca70d, 0xd7c6bf4d, + 0xc0eedb8d, 0xf99676cd, 0xeebe120d, 0x8b672c4d, 0x9c4f488d, + 0xa537e5cd, 0xb21f810d, 0x85a36208, 0x928b06c8, 0xabf3ab88, + 0xbcdbcf48, 0xd902f108, 0xce2a95c8, 0xf7523888, 0xe07a5c48, + 0x3ce04408, 0x2bc820c8, 0x12b08d88, 0x0598e948, 0x6041d708, + 0x7769b3c8, 0x4e111e88, 0x59397a48, 0x2c542849, 0x3b7c4c89, + 0x0204e1c9, 0x152c8509, 0x70f5bb49, 0x67dddf89, 0x5ea572c9, + 0x498d1609, 0x95170e49, 0x823f6a89, 0xbb47c7c9, 0xac6fa309, + 0xc9b69d49, 0xde9ef989, 0xe7e654c9, 0xf0ce3009, 0x0d3cf0cb, + 0x1a14940b, 0x236c394b, 0x34445d8b, 0x519d63cb, 0x46b5070b, + 0x7fcdaa4b, 0x68e5ce8b, 0xb47fd6cb, 0xa357b20b, 0x9a2f1f4b, + 0x8d077b8b, 0xe8de45cb, 0xfff6210b, 0xc68e8c4b, 0xd1a6e88b, + 0xa4cbba8a, 0xb3e3de4a, 0x8a9b730a, 0x9db317ca, 0xf86a298a, + 0xef424d4a, 0xd63ae00a, 0xc11284ca, 0x1d889c8a, 0x0aa0f84a, + 0x33d8550a, 0x24f031ca, 0x41290f8a, 0x56016b4a, 0x6f79c60a, + 0x7851a2ca}, + {0x00000000, 0x9fda839e, 0xe4c4017d, 0x7b1e82e3, 0x12f904bb, + 0x8d238725, 0xf63d05c6, 0x69e78658, 0x25f20976, 0xba288ae8, + 0xc136080b, 0x5eec8b95, 0x370b0dcd, 0xa8d18e53, 0xd3cf0cb0, + 0x4c158f2e, 0x4be412ec, 0xd43e9172, 0xaf201391, 0x30fa900f, + 0x591d1657, 0xc6c795c9, 0xbdd9172a, 0x220394b4, 0x6e161b9a, + 0xf1cc9804, 0x8ad21ae7, 0x15089979, 0x7cef1f21, 0xe3359cbf, + 0x982b1e5c, 0x07f19dc2, 0x97c825d8, 0x0812a646, 0x730c24a5, + 0xecd6a73b, 0x85312163, 0x1aeba2fd, 0x61f5201e, 0xfe2fa380, + 0xb23a2cae, 0x2de0af30, 0x56fe2dd3, 0xc924ae4d, 0xa0c32815, + 0x3f19ab8b, 0x44072968, 0xdbddaaf6, 0xdc2c3734, 0x43f6b4aa, + 0x38e83649, 0xa732b5d7, 0xced5338f, 0x510fb011, 0x2a1132f2, + 0xb5cbb16c, 0xf9de3e42, 0x6604bddc, 0x1d1a3f3f, 0x82c0bca1, + 0xeb273af9, 0x74fdb967, 0x0fe33b84, 0x9039b81a, 0xf4e14df1, + 0x6b3bce6f, 0x10254c8c, 0x8fffcf12, 0xe618494a, 0x79c2cad4, + 0x02dc4837, 0x9d06cba9, 0xd1134487, 0x4ec9c719, 0x35d745fa, + 0xaa0dc664, 0xc3ea403c, 0x5c30c3a2, 0x272e4141, 0xb8f4c2df, + 0xbf055f1d, 0x20dfdc83, 0x5bc15e60, 0xc41bddfe, 0xadfc5ba6, + 0x3226d838, 0x49385adb, 0xd6e2d945, 0x9af7566b, 0x052dd5f5, + 0x7e335716, 0xe1e9d488, 0x880e52d0, 0x17d4d14e, 0x6cca53ad, + 0xf310d033, 0x63296829, 0xfcf3ebb7, 0x87ed6954, 0x1837eaca, + 0x71d06c92, 0xee0aef0c, 0x95146def, 0x0aceee71, 0x46db615f, + 0xd901e2c1, 0xa21f6022, 0x3dc5e3bc, 0x542265e4, 0xcbf8e67a, + 0xb0e66499, 0x2f3ce707, 0x28cd7ac5, 0xb717f95b, 0xcc097bb8, + 0x53d3f826, 0x3a347e7e, 0xa5eefde0, 0xdef07f03, 0x412afc9d, + 0x0d3f73b3, 0x92e5f02d, 0xe9fb72ce, 0x7621f150, 0x1fc67708, + 0x801cf496, 0xfb027675, 0x64d8f5eb, 0x32b39da3, 0xad691e3d, + 0xd6779cde, 0x49ad1f40, 0x204a9918, 0xbf901a86, 0xc48e9865, + 0x5b541bfb, 0x174194d5, 0x889b174b, 0xf38595a8, 0x6c5f1636, + 0x05b8906e, 0x9a6213f0, 0xe17c9113, 0x7ea6128d, 0x79578f4f, + 0xe68d0cd1, 0x9d938e32, 0x02490dac, 0x6bae8bf4, 0xf474086a, + 0x8f6a8a89, 0x10b00917, 0x5ca58639, 0xc37f05a7, 0xb8618744, + 0x27bb04da, 0x4e5c8282, 0xd186011c, 0xaa9883ff, 0x35420061, + 0xa57bb87b, 0x3aa13be5, 0x41bfb906, 0xde653a98, 0xb782bcc0, + 0x28583f5e, 0x5346bdbd, 0xcc9c3e23, 0x8089b10d, 0x1f533293, + 0x644db070, 0xfb9733ee, 0x9270b5b6, 0x0daa3628, 0x76b4b4cb, + 0xe96e3755, 0xee9faa97, 0x71452909, 0x0a5babea, 0x95812874, + 0xfc66ae2c, 0x63bc2db2, 0x18a2af51, 0x87782ccf, 0xcb6da3e1, + 0x54b7207f, 0x2fa9a29c, 0xb0732102, 0xd994a75a, 0x464e24c4, + 0x3d50a627, 0xa28a25b9, 0xc652d052, 0x598853cc, 0x2296d12f, + 0xbd4c52b1, 0xd4abd4e9, 0x4b715777, 0x306fd594, 0xafb5560a, + 0xe3a0d924, 0x7c7a5aba, 0x0764d859, 0x98be5bc7, 0xf159dd9f, + 0x6e835e01, 0x159ddce2, 0x8a475f7c, 0x8db6c2be, 0x126c4120, + 0x6972c3c3, 0xf6a8405d, 0x9f4fc605, 0x0095459b, 0x7b8bc778, + 0xe45144e6, 0xa844cbc8, 0x379e4856, 0x4c80cab5, 0xd35a492b, + 0xbabdcf73, 0x25674ced, 0x5e79ce0e, 0xc1a34d90, 0x519af58a, + 0xce407614, 0xb55ef4f7, 0x2a847769, 0x4363f131, 0xdcb972af, + 0xa7a7f04c, 0x387d73d2, 0x7468fcfc, 0xebb27f62, 0x90acfd81, + 0x0f767e1f, 0x6691f847, 0xf94b7bd9, 0x8255f93a, 0x1d8f7aa4, + 0x1a7ee766, 0x85a464f8, 0xfebae61b, 0x61606585, 0x0887e3dd, + 0x975d6043, 0xec43e2a0, 0x7399613e, 0x3f8cee10, 0xa0566d8e, + 0xdb48ef6d, 0x44926cf3, 0x2d75eaab, 0xb2af6935, 0xc9b1ebd6, + 0x566b6848}, + {0x00000000, 0x65673b46, 0xcace768c, 0xafa94dca, 0x4eedeb59, + 0x2b8ad01f, 0x84239dd5, 0xe144a693, 0x9ddbd6b2, 0xf8bcedf4, + 0x5715a03e, 0x32729b78, 0xd3363deb, 0xb65106ad, 0x19f84b67, + 0x7c9f7021, 0xe0c6ab25, 0x85a19063, 0x2a08dda9, 0x4f6fe6ef, + 0xae2b407c, 0xcb4c7b3a, 0x64e536f0, 0x01820db6, 0x7d1d7d97, + 0x187a46d1, 0xb7d30b1b, 0xd2b4305d, 0x33f096ce, 0x5697ad88, + 0xf93ee042, 0x9c59db04, 0x1afc500b, 0x7f9b6b4d, 0xd0322687, + 0xb5551dc1, 0x5411bb52, 0x31768014, 0x9edfcdde, 0xfbb8f698, + 0x872786b9, 0xe240bdff, 0x4de9f035, 0x288ecb73, 0xc9ca6de0, + 0xacad56a6, 0x03041b6c, 0x6663202a, 0xfa3afb2e, 0x9f5dc068, + 0x30f48da2, 0x5593b6e4, 0xb4d71077, 0xd1b02b31, 0x7e1966fb, + 0x1b7e5dbd, 0x67e12d9c, 0x028616da, 0xad2f5b10, 0xc8486056, + 0x290cc6c5, 0x4c6bfd83, 0xe3c2b049, 0x86a58b0f, 0x35f8a016, + 0x509f9b50, 0xff36d69a, 0x9a51eddc, 0x7b154b4f, 0x1e727009, + 0xb1db3dc3, 0xd4bc0685, 0xa82376a4, 0xcd444de2, 0x62ed0028, + 0x078a3b6e, 0xe6ce9dfd, 0x83a9a6bb, 0x2c00eb71, 0x4967d037, + 0xd53e0b33, 0xb0593075, 0x1ff07dbf, 0x7a9746f9, 0x9bd3e06a, + 0xfeb4db2c, 0x511d96e6, 0x347aada0, 0x48e5dd81, 0x2d82e6c7, + 0x822bab0d, 0xe74c904b, 0x060836d8, 0x636f0d9e, 0xccc64054, + 0xa9a17b12, 0x2f04f01d, 0x4a63cb5b, 0xe5ca8691, 0x80adbdd7, + 0x61e91b44, 0x048e2002, 0xab276dc8, 0xce40568e, 0xb2df26af, + 0xd7b81de9, 0x78115023, 0x1d766b65, 0xfc32cdf6, 0x9955f6b0, + 0x36fcbb7a, 0x539b803c, 0xcfc25b38, 0xaaa5607e, 0x050c2db4, + 0x606b16f2, 0x812fb061, 0xe4488b27, 0x4be1c6ed, 0x2e86fdab, + 0x52198d8a, 0x377eb6cc, 0x98d7fb06, 0xfdb0c040, 0x1cf466d3, + 0x79935d95, 0xd63a105f, 0xb35d2b19, 0x6bf1402c, 0x0e967b6a, + 0xa13f36a0, 0xc4580de6, 0x251cab75, 0x407b9033, 0xefd2ddf9, + 0x8ab5e6bf, 0xf62a969e, 0x934dadd8, 0x3ce4e012, 0x5983db54, + 0xb8c77dc7, 0xdda04681, 0x72090b4b, 0x176e300d, 0x8b37eb09, + 0xee50d04f, 0x41f99d85, 0x249ea6c3, 0xc5da0050, 0xa0bd3b16, + 0x0f1476dc, 0x6a734d9a, 0x16ec3dbb, 0x738b06fd, 0xdc224b37, + 0xb9457071, 0x5801d6e2, 0x3d66eda4, 0x92cfa06e, 0xf7a89b28, + 0x710d1027, 0x146a2b61, 0xbbc366ab, 0xdea45ded, 0x3fe0fb7e, + 0x5a87c038, 0xf52e8df2, 0x9049b6b4, 0xecd6c695, 0x89b1fdd3, + 0x2618b019, 0x437f8b5f, 0xa23b2dcc, 0xc75c168a, 0x68f55b40, + 0x0d926006, 0x91cbbb02, 0xf4ac8044, 0x5b05cd8e, 0x3e62f6c8, + 0xdf26505b, 0xba416b1d, 0x15e826d7, 0x708f1d91, 0x0c106db0, + 0x697756f6, 0xc6de1b3c, 0xa3b9207a, 0x42fd86e9, 0x279abdaf, + 0x8833f065, 0xed54cb23, 0x5e09e03a, 0x3b6edb7c, 0x94c796b6, + 0xf1a0adf0, 0x10e40b63, 0x75833025, 0xda2a7def, 0xbf4d46a9, + 0xc3d23688, 0xa6b50dce, 0x091c4004, 0x6c7b7b42, 0x8d3fddd1, + 0xe858e697, 0x47f1ab5d, 0x2296901b, 0xbecf4b1f, 0xdba87059, + 0x74013d93, 0x116606d5, 0xf022a046, 0x95459b00, 0x3aecd6ca, + 0x5f8bed8c, 0x23149dad, 0x4673a6eb, 0xe9daeb21, 0x8cbdd067, + 0x6df976f4, 0x089e4db2, 0xa7370078, 0xc2503b3e, 0x44f5b031, + 0x21928b77, 0x8e3bc6bd, 0xeb5cfdfb, 0x0a185b68, 0x6f7f602e, + 0xc0d62de4, 0xa5b116a2, 0xd92e6683, 0xbc495dc5, 0x13e0100f, + 0x76872b49, 0x97c38dda, 0xf2a4b69c, 0x5d0dfb56, 0x386ac010, + 0xa4331b14, 0xc1542052, 0x6efd6d98, 0x0b9a56de, 0xeadef04d, + 0x8fb9cb0b, 0x201086c1, 0x4577bd87, 0x39e8cda6, 0x5c8ff6e0, + 0xf326bb2a, 0x9641806c, 0x770526ff, 0x12621db9, 0xbdcb5073, + 0xd8ac6b35}, + {0x00000000, 0xd7e28058, 0x74b406f1, 0xa35686a9, 0xe9680de2, + 0x3e8a8dba, 0x9ddc0b13, 0x4a3e8b4b, 0x09a11d85, 0xde439ddd, + 0x7d151b74, 0xaaf79b2c, 0xe0c91067, 0x372b903f, 0x947d1696, + 0x439f96ce, 0x13423b0a, 0xc4a0bb52, 0x67f63dfb, 0xb014bda3, + 0xfa2a36e8, 0x2dc8b6b0, 0x8e9e3019, 0x597cb041, 0x1ae3268f, + 0xcd01a6d7, 0x6e57207e, 0xb9b5a026, 0xf38b2b6d, 0x2469ab35, + 0x873f2d9c, 0x50ddadc4, 0x26847614, 0xf166f64c, 0x523070e5, + 0x85d2f0bd, 0xcfec7bf6, 0x180efbae, 0xbb587d07, 0x6cbafd5f, + 0x2f256b91, 0xf8c7ebc9, 0x5b916d60, 0x8c73ed38, 0xc64d6673, + 0x11afe62b, 0xb2f96082, 0x651be0da, 0x35c64d1e, 0xe224cd46, + 0x41724bef, 0x9690cbb7, 0xdcae40fc, 0x0b4cc0a4, 0xa81a460d, + 0x7ff8c655, 0x3c67509b, 0xeb85d0c3, 0x48d3566a, 0x9f31d632, + 0xd50f5d79, 0x02eddd21, 0xa1bb5b88, 0x7659dbd0, 0x4d08ec28, + 0x9aea6c70, 0x39bcead9, 0xee5e6a81, 0xa460e1ca, 0x73826192, + 0xd0d4e73b, 0x07366763, 0x44a9f1ad, 0x934b71f5, 0x301df75c, + 0xe7ff7704, 0xadc1fc4f, 0x7a237c17, 0xd975fabe, 0x0e977ae6, + 0x5e4ad722, 0x89a8577a, 0x2afed1d3, 0xfd1c518b, 0xb722dac0, + 0x60c05a98, 0xc396dc31, 0x14745c69, 0x57ebcaa7, 0x80094aff, + 0x235fcc56, 0xf4bd4c0e, 0xbe83c745, 0x6961471d, 0xca37c1b4, + 0x1dd541ec, 0x6b8c9a3c, 0xbc6e1a64, 0x1f389ccd, 0xc8da1c95, + 0x82e497de, 0x55061786, 0xf650912f, 0x21b21177, 0x622d87b9, + 0xb5cf07e1, 0x16998148, 0xc17b0110, 0x8b458a5b, 0x5ca70a03, + 0xfff18caa, 0x28130cf2, 0x78cea136, 0xaf2c216e, 0x0c7aa7c7, + 0xdb98279f, 0x91a6acd4, 0x46442c8c, 0xe512aa25, 0x32f02a7d, + 0x716fbcb3, 0xa68d3ceb, 0x05dbba42, 0xd2393a1a, 0x9807b151, + 0x4fe53109, 0xecb3b7a0, 0x3b5137f8, 0x9a11d850, 0x4df35808, + 0xeea5dea1, 0x39475ef9, 0x7379d5b2, 0xa49b55ea, 0x07cdd343, + 0xd02f531b, 0x93b0c5d5, 0x4452458d, 0xe704c324, 0x30e6437c, + 0x7ad8c837, 0xad3a486f, 0x0e6ccec6, 0xd98e4e9e, 0x8953e35a, + 0x5eb16302, 0xfde7e5ab, 0x2a0565f3, 0x603beeb8, 0xb7d96ee0, + 0x148fe849, 0xc36d6811, 0x80f2fedf, 0x57107e87, 0xf446f82e, + 0x23a47876, 0x699af33d, 0xbe787365, 0x1d2ef5cc, 0xcacc7594, + 0xbc95ae44, 0x6b772e1c, 0xc821a8b5, 0x1fc328ed, 0x55fda3a6, + 0x821f23fe, 0x2149a557, 0xf6ab250f, 0xb534b3c1, 0x62d63399, + 0xc180b530, 0x16623568, 0x5c5cbe23, 0x8bbe3e7b, 0x28e8b8d2, + 0xff0a388a, 0xafd7954e, 0x78351516, 0xdb6393bf, 0x0c8113e7, + 0x46bf98ac, 0x915d18f4, 0x320b9e5d, 0xe5e91e05, 0xa67688cb, + 0x71940893, 0xd2c28e3a, 0x05200e62, 0x4f1e8529, 0x98fc0571, + 0x3baa83d8, 0xec480380, 0xd7193478, 0x00fbb420, 0xa3ad3289, + 0x744fb2d1, 0x3e71399a, 0xe993b9c2, 0x4ac53f6b, 0x9d27bf33, + 0xdeb829fd, 0x095aa9a5, 0xaa0c2f0c, 0x7deeaf54, 0x37d0241f, + 0xe032a447, 0x436422ee, 0x9486a2b6, 0xc45b0f72, 0x13b98f2a, + 0xb0ef0983, 0x670d89db, 0x2d330290, 0xfad182c8, 0x59870461, + 0x8e658439, 0xcdfa12f7, 0x1a1892af, 0xb94e1406, 0x6eac945e, + 0x24921f15, 0xf3709f4d, 0x502619e4, 0x87c499bc, 0xf19d426c, + 0x267fc234, 0x8529449d, 0x52cbc4c5, 0x18f54f8e, 0xcf17cfd6, + 0x6c41497f, 0xbba3c927, 0xf83c5fe9, 0x2fdedfb1, 0x8c885918, + 0x5b6ad940, 0x1154520b, 0xc6b6d253, 0x65e054fa, 0xb202d4a2, + 0xe2df7966, 0x353df93e, 0x966b7f97, 0x4189ffcf, 0x0bb77484, + 0xdc55f4dc, 0x7f037275, 0xa8e1f22d, 0xeb7e64e3, 0x3c9ce4bb, + 0x9fca6212, 0x4828e24a, 0x02166901, 0xd5f4e959, 0x76a26ff0, + 0xa140efa8}, + {0x00000000, 0xef52b6e1, 0x05d46b83, 0xea86dd62, 0x0ba8d706, + 0xe4fa61e7, 0x0e7cbc85, 0xe12e0a64, 0x1751ae0c, 0xf80318ed, + 0x1285c58f, 0xfdd7736e, 0x1cf9790a, 0xf3abcfeb, 0x192d1289, + 0xf67fa468, 0x2ea35c18, 0xc1f1eaf9, 0x2b77379b, 0xc425817a, + 0x250b8b1e, 0xca593dff, 0x20dfe09d, 0xcf8d567c, 0x39f2f214, + 0xd6a044f5, 0x3c269997, 0xd3742f76, 0x325a2512, 0xdd0893f3, + 0x378e4e91, 0xd8dcf870, 0x5d46b830, 0xb2140ed1, 0x5892d3b3, + 0xb7c06552, 0x56ee6f36, 0xb9bcd9d7, 0x533a04b5, 0xbc68b254, + 0x4a17163c, 0xa545a0dd, 0x4fc37dbf, 0xa091cb5e, 0x41bfc13a, + 0xaeed77db, 0x446baab9, 0xab391c58, 0x73e5e428, 0x9cb752c9, + 0x76318fab, 0x9963394a, 0x784d332e, 0x971f85cf, 0x7d9958ad, + 0x92cbee4c, 0x64b44a24, 0x8be6fcc5, 0x616021a7, 0x8e329746, + 0x6f1c9d22, 0x804e2bc3, 0x6ac8f6a1, 0x859a4040, 0xba8d7060, + 0x55dfc681, 0xbf591be3, 0x500bad02, 0xb125a766, 0x5e771187, + 0xb4f1cce5, 0x5ba37a04, 0xaddcde6c, 0x428e688d, 0xa808b5ef, + 0x475a030e, 0xa674096a, 0x4926bf8b, 0xa3a062e9, 0x4cf2d408, + 0x942e2c78, 0x7b7c9a99, 0x91fa47fb, 0x7ea8f11a, 0x9f86fb7e, + 0x70d44d9f, 0x9a5290fd, 0x7500261c, 0x837f8274, 0x6c2d3495, + 0x86abe9f7, 0x69f95f16, 0x88d75572, 0x6785e393, 0x8d033ef1, + 0x62518810, 0xe7cbc850, 0x08997eb1, 0xe21fa3d3, 0x0d4d1532, + 0xec631f56, 0x0331a9b7, 0xe9b774d5, 0x06e5c234, 0xf09a665c, + 0x1fc8d0bd, 0xf54e0ddf, 0x1a1cbb3e, 0xfb32b15a, 0x146007bb, + 0xfee6dad9, 0x11b46c38, 0xc9689448, 0x263a22a9, 0xccbcffcb, + 0x23ee492a, 0xc2c0434e, 0x2d92f5af, 0xc71428cd, 0x28469e2c, + 0xde393a44, 0x316b8ca5, 0xdbed51c7, 0x34bfe726, 0xd591ed42, + 0x3ac35ba3, 0xd04586c1, 0x3f173020, 0xae6be681, 0x41395060, + 0xabbf8d02, 0x44ed3be3, 0xa5c33187, 0x4a918766, 0xa0175a04, + 0x4f45ece5, 0xb93a488d, 0x5668fe6c, 0xbcee230e, 0x53bc95ef, + 0xb2929f8b, 0x5dc0296a, 0xb746f408, 0x581442e9, 0x80c8ba99, + 0x6f9a0c78, 0x851cd11a, 0x6a4e67fb, 0x8b606d9f, 0x6432db7e, + 0x8eb4061c, 0x61e6b0fd, 0x97991495, 0x78cba274, 0x924d7f16, + 0x7d1fc9f7, 0x9c31c393, 0x73637572, 0x99e5a810, 0x76b71ef1, + 0xf32d5eb1, 0x1c7fe850, 0xf6f93532, 0x19ab83d3, 0xf88589b7, + 0x17d73f56, 0xfd51e234, 0x120354d5, 0xe47cf0bd, 0x0b2e465c, + 0xe1a89b3e, 0x0efa2ddf, 0xefd427bb, 0x0086915a, 0xea004c38, + 0x0552fad9, 0xdd8e02a9, 0x32dcb448, 0xd85a692a, 0x3708dfcb, + 0xd626d5af, 0x3974634e, 0xd3f2be2c, 0x3ca008cd, 0xcadfaca5, + 0x258d1a44, 0xcf0bc726, 0x205971c7, 0xc1777ba3, 0x2e25cd42, + 0xc4a31020, 0x2bf1a6c1, 0x14e696e1, 0xfbb42000, 0x1132fd62, + 0xfe604b83, 0x1f4e41e7, 0xf01cf706, 0x1a9a2a64, 0xf5c89c85, + 0x03b738ed, 0xece58e0c, 0x0663536e, 0xe931e58f, 0x081fefeb, + 0xe74d590a, 0x0dcb8468, 0xe2993289, 0x3a45caf9, 0xd5177c18, + 0x3f91a17a, 0xd0c3179b, 0x31ed1dff, 0xdebfab1e, 0x3439767c, + 0xdb6bc09d, 0x2d1464f5, 0xc246d214, 0x28c00f76, 0xc792b997, + 0x26bcb3f3, 0xc9ee0512, 0x2368d870, 0xcc3a6e91, 0x49a02ed1, + 0xa6f29830, 0x4c744552, 0xa326f3b3, 0x4208f9d7, 0xad5a4f36, + 0x47dc9254, 0xa88e24b5, 0x5ef180dd, 0xb1a3363c, 0x5b25eb5e, + 0xb4775dbf, 0x555957db, 0xba0be13a, 0x508d3c58, 0xbfdf8ab9, + 0x670372c9, 0x8851c428, 0x62d7194a, 0x8d85afab, 0x6caba5cf, + 0x83f9132e, 0x697fce4c, 0x862d78ad, 0x7052dcc5, 0x9f006a24, + 0x7586b746, 0x9ad401a7, 0x7bfa0bc3, 0x94a8bd22, 0x7e2e6040, + 0x917cd6a1}, + {0x00000000, 0x87a6cb43, 0xd43c90c7, 0x539a5b84, 0x730827cf, + 0xf4aeec8c, 0xa734b708, 0x20927c4b, 0xe6104f9e, 0x61b684dd, + 0x322cdf59, 0xb58a141a, 0x95186851, 0x12bea312, 0x4124f896, + 0xc68233d5, 0x1751997d, 0x90f7523e, 0xc36d09ba, 0x44cbc2f9, + 0x6459beb2, 0xe3ff75f1, 0xb0652e75, 0x37c3e536, 0xf141d6e3, + 0x76e71da0, 0x257d4624, 0xa2db8d67, 0x8249f12c, 0x05ef3a6f, + 0x567561eb, 0xd1d3aaa8, 0x2ea332fa, 0xa905f9b9, 0xfa9fa23d, + 0x7d39697e, 0x5dab1535, 0xda0dde76, 0x899785f2, 0x0e314eb1, + 0xc8b37d64, 0x4f15b627, 0x1c8feda3, 0x9b2926e0, 0xbbbb5aab, + 0x3c1d91e8, 0x6f87ca6c, 0xe821012f, 0x39f2ab87, 0xbe5460c4, + 0xedce3b40, 0x6a68f003, 0x4afa8c48, 0xcd5c470b, 0x9ec61c8f, + 0x1960d7cc, 0xdfe2e419, 0x58442f5a, 0x0bde74de, 0x8c78bf9d, + 0xaceac3d6, 0x2b4c0895, 0x78d65311, 0xff709852, 0x5d4665f4, + 0xdae0aeb7, 0x897af533, 0x0edc3e70, 0x2e4e423b, 0xa9e88978, + 0xfa72d2fc, 0x7dd419bf, 0xbb562a6a, 0x3cf0e129, 0x6f6abaad, + 0xe8cc71ee, 0xc85e0da5, 0x4ff8c6e6, 0x1c629d62, 0x9bc45621, + 0x4a17fc89, 0xcdb137ca, 0x9e2b6c4e, 0x198da70d, 0x391fdb46, + 0xbeb91005, 0xed234b81, 0x6a8580c2, 0xac07b317, 0x2ba17854, + 0x783b23d0, 0xff9de893, 0xdf0f94d8, 0x58a95f9b, 0x0b33041f, + 0x8c95cf5c, 0x73e5570e, 0xf4439c4d, 0xa7d9c7c9, 0x207f0c8a, + 0x00ed70c1, 0x874bbb82, 0xd4d1e006, 0x53772b45, 0x95f51890, + 0x1253d3d3, 0x41c98857, 0xc66f4314, 0xe6fd3f5f, 0x615bf41c, + 0x32c1af98, 0xb56764db, 0x64b4ce73, 0xe3120530, 0xb0885eb4, + 0x372e95f7, 0x17bce9bc, 0x901a22ff, 0xc380797b, 0x4426b238, + 0x82a481ed, 0x05024aae, 0x5698112a, 0xd13eda69, 0xf1aca622, + 0x760a6d61, 0x259036e5, 0xa236fda6, 0xba8ccbe8, 0x3d2a00ab, + 0x6eb05b2f, 0xe916906c, 0xc984ec27, 0x4e222764, 0x1db87ce0, + 0x9a1eb7a3, 0x5c9c8476, 0xdb3a4f35, 0x88a014b1, 0x0f06dff2, + 0x2f94a3b9, 0xa83268fa, 0xfba8337e, 0x7c0ef83d, 0xaddd5295, + 0x2a7b99d6, 0x79e1c252, 0xfe470911, 0xded5755a, 0x5973be19, + 0x0ae9e59d, 0x8d4f2ede, 0x4bcd1d0b, 0xcc6bd648, 0x9ff18dcc, + 0x1857468f, 0x38c53ac4, 0xbf63f187, 0xecf9aa03, 0x6b5f6140, + 0x942ff912, 0x13893251, 0x401369d5, 0xc7b5a296, 0xe727dedd, + 0x6081159e, 0x331b4e1a, 0xb4bd8559, 0x723fb68c, 0xf5997dcf, + 0xa603264b, 0x21a5ed08, 0x01379143, 0x86915a00, 0xd50b0184, + 0x52adcac7, 0x837e606f, 0x04d8ab2c, 0x5742f0a8, 0xd0e43beb, + 0xf07647a0, 0x77d08ce3, 0x244ad767, 0xa3ec1c24, 0x656e2ff1, + 0xe2c8e4b2, 0xb152bf36, 0x36f47475, 0x1666083e, 0x91c0c37d, + 0xc25a98f9, 0x45fc53ba, 0xe7caae1c, 0x606c655f, 0x33f63edb, + 0xb450f598, 0x94c289d3, 0x13644290, 0x40fe1914, 0xc758d257, + 0x01dae182, 0x867c2ac1, 0xd5e67145, 0x5240ba06, 0x72d2c64d, + 0xf5740d0e, 0xa6ee568a, 0x21489dc9, 0xf09b3761, 0x773dfc22, + 0x24a7a7a6, 0xa3016ce5, 0x839310ae, 0x0435dbed, 0x57af8069, + 0xd0094b2a, 0x168b78ff, 0x912db3bc, 0xc2b7e838, 0x4511237b, + 0x65835f30, 0xe2259473, 0xb1bfcff7, 0x361904b4, 0xc9699ce6, + 0x4ecf57a5, 0x1d550c21, 0x9af3c762, 0xba61bb29, 0x3dc7706a, + 0x6e5d2bee, 0xe9fbe0ad, 0x2f79d378, 0xa8df183b, 0xfb4543bf, + 0x7ce388fc, 0x5c71f4b7, 0xdbd73ff4, 0x884d6470, 0x0febaf33, + 0xde38059b, 0x599eced8, 0x0a04955c, 0x8da25e1f, 0xad302254, + 0x2a96e917, 0x790cb293, 0xfeaa79d0, 0x38284a05, 0xbf8e8146, + 0xec14dac2, 0x6bb21181, 0x4b206dca, 0xcc86a689, 0x9f1cfd0d, + 0x18ba364e}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x0000000000000000, 0x43cba68700000000, 0xc7903cd400000000, + 0x845b9a5300000000, 0xcf27087300000000, 0x8cecaef400000000, + 0x08b734a700000000, 0x4b7c922000000000, 0x9e4f10e600000000, + 0xdd84b66100000000, 0x59df2c3200000000, 0x1a148ab500000000, + 0x5168189500000000, 0x12a3be1200000000, 0x96f8244100000000, + 0xd53382c600000000, 0x7d99511700000000, 0x3e52f79000000000, + 0xba096dc300000000, 0xf9c2cb4400000000, 0xb2be596400000000, + 0xf175ffe300000000, 0x752e65b000000000, 0x36e5c33700000000, + 0xe3d641f100000000, 0xa01de77600000000, 0x24467d2500000000, + 0x678ddba200000000, 0x2cf1498200000000, 0x6f3aef0500000000, + 0xeb61755600000000, 0xa8aad3d100000000, 0xfa32a32e00000000, + 0xb9f905a900000000, 0x3da29ffa00000000, 0x7e69397d00000000, + 0x3515ab5d00000000, 0x76de0dda00000000, 0xf285978900000000, + 0xb14e310e00000000, 0x647db3c800000000, 0x27b6154f00000000, + 0xa3ed8f1c00000000, 0xe026299b00000000, 0xab5abbbb00000000, + 0xe8911d3c00000000, 0x6cca876f00000000, 0x2f0121e800000000, + 0x87abf23900000000, 0xc46054be00000000, 0x403bceed00000000, + 0x03f0686a00000000, 0x488cfa4a00000000, 0x0b475ccd00000000, + 0x8f1cc69e00000000, 0xccd7601900000000, 0x19e4e2df00000000, + 0x5a2f445800000000, 0xde74de0b00000000, 0x9dbf788c00000000, + 0xd6c3eaac00000000, 0x95084c2b00000000, 0x1153d67800000000, + 0x529870ff00000000, 0xf465465d00000000, 0xb7aee0da00000000, + 0x33f57a8900000000, 0x703edc0e00000000, 0x3b424e2e00000000, + 0x7889e8a900000000, 0xfcd272fa00000000, 0xbf19d47d00000000, + 0x6a2a56bb00000000, 0x29e1f03c00000000, 0xadba6a6f00000000, + 0xee71cce800000000, 0xa50d5ec800000000, 0xe6c6f84f00000000, + 0x629d621c00000000, 0x2156c49b00000000, 0x89fc174a00000000, + 0xca37b1cd00000000, 0x4e6c2b9e00000000, 0x0da78d1900000000, + 0x46db1f3900000000, 0x0510b9be00000000, 0x814b23ed00000000, + 0xc280856a00000000, 0x17b307ac00000000, 0x5478a12b00000000, + 0xd0233b7800000000, 0x93e89dff00000000, 0xd8940fdf00000000, + 0x9b5fa95800000000, 0x1f04330b00000000, 0x5ccf958c00000000, + 0x0e57e57300000000, 0x4d9c43f400000000, 0xc9c7d9a700000000, + 0x8a0c7f2000000000, 0xc170ed0000000000, 0x82bb4b8700000000, + 0x06e0d1d400000000, 0x452b775300000000, 0x9018f59500000000, + 0xd3d3531200000000, 0x5788c94100000000, 0x14436fc600000000, + 0x5f3ffde600000000, 0x1cf45b6100000000, 0x98afc13200000000, + 0xdb6467b500000000, 0x73ceb46400000000, 0x300512e300000000, + 0xb45e88b000000000, 0xf7952e3700000000, 0xbce9bc1700000000, + 0xff221a9000000000, 0x7b7980c300000000, 0x38b2264400000000, + 0xed81a48200000000, 0xae4a020500000000, 0x2a11985600000000, + 0x69da3ed100000000, 0x22a6acf100000000, 0x616d0a7600000000, + 0xe536902500000000, 0xa6fd36a200000000, 0xe8cb8cba00000000, + 0xab002a3d00000000, 0x2f5bb06e00000000, 0x6c9016e900000000, + 0x27ec84c900000000, 0x6427224e00000000, 0xe07cb81d00000000, + 0xa3b71e9a00000000, 0x76849c5c00000000, 0x354f3adb00000000, + 0xb114a08800000000, 0xf2df060f00000000, 0xb9a3942f00000000, + 0xfa6832a800000000, 0x7e33a8fb00000000, 0x3df80e7c00000000, + 0x9552ddad00000000, 0xd6997b2a00000000, 0x52c2e17900000000, + 0x110947fe00000000, 0x5a75d5de00000000, 0x19be735900000000, + 0x9de5e90a00000000, 0xde2e4f8d00000000, 0x0b1dcd4b00000000, + 0x48d66bcc00000000, 0xcc8df19f00000000, 0x8f46571800000000, + 0xc43ac53800000000, 0x87f163bf00000000, 0x03aaf9ec00000000, + 0x40615f6b00000000, 0x12f92f9400000000, 0x5132891300000000, + 0xd569134000000000, 0x96a2b5c700000000, 0xddde27e700000000, + 0x9e15816000000000, 0x1a4e1b3300000000, 0x5985bdb400000000, + 0x8cb63f7200000000, 0xcf7d99f500000000, 0x4b2603a600000000, + 0x08eda52100000000, 0x4391370100000000, 0x005a918600000000, + 0x84010bd500000000, 0xc7caad5200000000, 0x6f607e8300000000, + 0x2cabd80400000000, 0xa8f0425700000000, 0xeb3be4d000000000, + 0xa04776f000000000, 0xe38cd07700000000, 0x67d74a2400000000, + 0x241ceca300000000, 0xf12f6e6500000000, 0xb2e4c8e200000000, + 0x36bf52b100000000, 0x7574f43600000000, 0x3e08661600000000, + 0x7dc3c09100000000, 0xf9985ac200000000, 0xba53fc4500000000, + 0x1caecae700000000, 0x5f656c6000000000, 0xdb3ef63300000000, + 0x98f550b400000000, 0xd389c29400000000, 0x9042641300000000, + 0x1419fe4000000000, 0x57d258c700000000, 0x82e1da0100000000, + 0xc12a7c8600000000, 0x4571e6d500000000, 0x06ba405200000000, + 0x4dc6d27200000000, 0x0e0d74f500000000, 0x8a56eea600000000, + 0xc99d482100000000, 0x61379bf000000000, 0x22fc3d7700000000, + 0xa6a7a72400000000, 0xe56c01a300000000, 0xae10938300000000, + 0xeddb350400000000, 0x6980af5700000000, 0x2a4b09d000000000, + 0xff788b1600000000, 0xbcb32d9100000000, 0x38e8b7c200000000, + 0x7b23114500000000, 0x305f836500000000, 0x739425e200000000, + 0xf7cfbfb100000000, 0xb404193600000000, 0xe69c69c900000000, + 0xa557cf4e00000000, 0x210c551d00000000, 0x62c7f39a00000000, + 0x29bb61ba00000000, 0x6a70c73d00000000, 0xee2b5d6e00000000, + 0xade0fbe900000000, 0x78d3792f00000000, 0x3b18dfa800000000, + 0xbf4345fb00000000, 0xfc88e37c00000000, 0xb7f4715c00000000, + 0xf43fd7db00000000, 0x70644d8800000000, 0x33afeb0f00000000, + 0x9b0538de00000000, 0xd8ce9e5900000000, 0x5c95040a00000000, + 0x1f5ea28d00000000, 0x542230ad00000000, 0x17e9962a00000000, + 0x93b20c7900000000, 0xd079aafe00000000, 0x054a283800000000, + 0x46818ebf00000000, 0xc2da14ec00000000, 0x8111b26b00000000, + 0xca6d204b00000000, 0x89a686cc00000000, 0x0dfd1c9f00000000, + 0x4e36ba1800000000}, + {0x0000000000000000, 0xe1b652ef00000000, 0x836bd40500000000, + 0x62dd86ea00000000, 0x06d7a80b00000000, 0xe761fae400000000, + 0x85bc7c0e00000000, 0x640a2ee100000000, 0x0cae511700000000, + 0xed1803f800000000, 0x8fc5851200000000, 0x6e73d7fd00000000, + 0x0a79f91c00000000, 0xebcfabf300000000, 0x89122d1900000000, + 0x68a47ff600000000, 0x185ca32e00000000, 0xf9eaf1c100000000, + 0x9b37772b00000000, 0x7a8125c400000000, 0x1e8b0b2500000000, + 0xff3d59ca00000000, 0x9de0df2000000000, 0x7c568dcf00000000, + 0x14f2f23900000000, 0xf544a0d600000000, 0x9799263c00000000, + 0x762f74d300000000, 0x12255a3200000000, 0xf39308dd00000000, + 0x914e8e3700000000, 0x70f8dcd800000000, 0x30b8465d00000000, + 0xd10e14b200000000, 0xb3d3925800000000, 0x5265c0b700000000, + 0x366fee5600000000, 0xd7d9bcb900000000, 0xb5043a5300000000, + 0x54b268bc00000000, 0x3c16174a00000000, 0xdda045a500000000, + 0xbf7dc34f00000000, 0x5ecb91a000000000, 0x3ac1bf4100000000, + 0xdb77edae00000000, 0xb9aa6b4400000000, 0x581c39ab00000000, + 0x28e4e57300000000, 0xc952b79c00000000, 0xab8f317600000000, + 0x4a39639900000000, 0x2e334d7800000000, 0xcf851f9700000000, + 0xad58997d00000000, 0x4ceecb9200000000, 0x244ab46400000000, + 0xc5fce68b00000000, 0xa721606100000000, 0x4697328e00000000, + 0x229d1c6f00000000, 0xc32b4e8000000000, 0xa1f6c86a00000000, + 0x40409a8500000000, 0x60708dba00000000, 0x81c6df5500000000, + 0xe31b59bf00000000, 0x02ad0b5000000000, 0x66a725b100000000, + 0x8711775e00000000, 0xe5ccf1b400000000, 0x047aa35b00000000, + 0x6cdedcad00000000, 0x8d688e4200000000, 0xefb508a800000000, + 0x0e035a4700000000, 0x6a0974a600000000, 0x8bbf264900000000, + 0xe962a0a300000000, 0x08d4f24c00000000, 0x782c2e9400000000, + 0x999a7c7b00000000, 0xfb47fa9100000000, 0x1af1a87e00000000, + 0x7efb869f00000000, 0x9f4dd47000000000, 0xfd90529a00000000, + 0x1c26007500000000, 0x74827f8300000000, 0x95342d6c00000000, + 0xf7e9ab8600000000, 0x165ff96900000000, 0x7255d78800000000, + 0x93e3856700000000, 0xf13e038d00000000, 0x1088516200000000, + 0x50c8cbe700000000, 0xb17e990800000000, 0xd3a31fe200000000, + 0x32154d0d00000000, 0x561f63ec00000000, 0xb7a9310300000000, + 0xd574b7e900000000, 0x34c2e50600000000, 0x5c669af000000000, + 0xbdd0c81f00000000, 0xdf0d4ef500000000, 0x3ebb1c1a00000000, + 0x5ab132fb00000000, 0xbb07601400000000, 0xd9dae6fe00000000, + 0x386cb41100000000, 0x489468c900000000, 0xa9223a2600000000, + 0xcbffbccc00000000, 0x2a49ee2300000000, 0x4e43c0c200000000, + 0xaff5922d00000000, 0xcd2814c700000000, 0x2c9e462800000000, + 0x443a39de00000000, 0xa58c6b3100000000, 0xc751eddb00000000, + 0x26e7bf3400000000, 0x42ed91d500000000, 0xa35bc33a00000000, + 0xc18645d000000000, 0x2030173f00000000, 0x81e66bae00000000, + 0x6050394100000000, 0x028dbfab00000000, 0xe33bed4400000000, + 0x8731c3a500000000, 0x6687914a00000000, 0x045a17a000000000, + 0xe5ec454f00000000, 0x8d483ab900000000, 0x6cfe685600000000, + 0x0e23eebc00000000, 0xef95bc5300000000, 0x8b9f92b200000000, + 0x6a29c05d00000000, 0x08f446b700000000, 0xe942145800000000, + 0x99bac88000000000, 0x780c9a6f00000000, 0x1ad11c8500000000, + 0xfb674e6a00000000, 0x9f6d608b00000000, 0x7edb326400000000, + 0x1c06b48e00000000, 0xfdb0e66100000000, 0x9514999700000000, + 0x74a2cb7800000000, 0x167f4d9200000000, 0xf7c91f7d00000000, + 0x93c3319c00000000, 0x7275637300000000, 0x10a8e59900000000, + 0xf11eb77600000000, 0xb15e2df300000000, 0x50e87f1c00000000, + 0x3235f9f600000000, 0xd383ab1900000000, 0xb78985f800000000, + 0x563fd71700000000, 0x34e251fd00000000, 0xd554031200000000, + 0xbdf07ce400000000, 0x5c462e0b00000000, 0x3e9ba8e100000000, + 0xdf2dfa0e00000000, 0xbb27d4ef00000000, 0x5a91860000000000, + 0x384c00ea00000000, 0xd9fa520500000000, 0xa9028edd00000000, + 0x48b4dc3200000000, 0x2a695ad800000000, 0xcbdf083700000000, + 0xafd526d600000000, 0x4e63743900000000, 0x2cbef2d300000000, + 0xcd08a03c00000000, 0xa5acdfca00000000, 0x441a8d2500000000, + 0x26c70bcf00000000, 0xc771592000000000, 0xa37b77c100000000, + 0x42cd252e00000000, 0x2010a3c400000000, 0xc1a6f12b00000000, + 0xe196e61400000000, 0x0020b4fb00000000, 0x62fd321100000000, + 0x834b60fe00000000, 0xe7414e1f00000000, 0x06f71cf000000000, + 0x642a9a1a00000000, 0x859cc8f500000000, 0xed38b70300000000, + 0x0c8ee5ec00000000, 0x6e53630600000000, 0x8fe531e900000000, + 0xebef1f0800000000, 0x0a594de700000000, 0x6884cb0d00000000, + 0x893299e200000000, 0xf9ca453a00000000, 0x187c17d500000000, + 0x7aa1913f00000000, 0x9b17c3d000000000, 0xff1ded3100000000, + 0x1eabbfde00000000, 0x7c76393400000000, 0x9dc06bdb00000000, + 0xf564142d00000000, 0x14d246c200000000, 0x760fc02800000000, + 0x97b992c700000000, 0xf3b3bc2600000000, 0x1205eec900000000, + 0x70d8682300000000, 0x916e3acc00000000, 0xd12ea04900000000, + 0x3098f2a600000000, 0x5245744c00000000, 0xb3f326a300000000, + 0xd7f9084200000000, 0x364f5aad00000000, 0x5492dc4700000000, + 0xb5248ea800000000, 0xdd80f15e00000000, 0x3c36a3b100000000, + 0x5eeb255b00000000, 0xbf5d77b400000000, 0xdb57595500000000, + 0x3ae10bba00000000, 0x583c8d5000000000, 0xb98adfbf00000000, + 0xc972036700000000, 0x28c4518800000000, 0x4a19d76200000000, + 0xabaf858d00000000, 0xcfa5ab6c00000000, 0x2e13f98300000000, + 0x4cce7f6900000000, 0xad782d8600000000, 0xc5dc527000000000, + 0x246a009f00000000, 0x46b7867500000000, 0xa701d49a00000000, + 0xc30bfa7b00000000, 0x22bda89400000000, 0x40602e7e00000000, + 0xa1d67c9100000000}, + {0x0000000000000000, 0x5880e2d700000000, 0xf106b47400000000, + 0xa98656a300000000, 0xe20d68e900000000, 0xba8d8a3e00000000, + 0x130bdc9d00000000, 0x4b8b3e4a00000000, 0x851da10900000000, + 0xdd9d43de00000000, 0x741b157d00000000, 0x2c9bf7aa00000000, + 0x6710c9e000000000, 0x3f902b3700000000, 0x96167d9400000000, + 0xce969f4300000000, 0x0a3b421300000000, 0x52bba0c400000000, + 0xfb3df66700000000, 0xa3bd14b000000000, 0xe8362afa00000000, + 0xb0b6c82d00000000, 0x19309e8e00000000, 0x41b07c5900000000, + 0x8f26e31a00000000, 0xd7a601cd00000000, 0x7e20576e00000000, + 0x26a0b5b900000000, 0x6d2b8bf300000000, 0x35ab692400000000, + 0x9c2d3f8700000000, 0xc4addd5000000000, 0x1476842600000000, + 0x4cf666f100000000, 0xe570305200000000, 0xbdf0d28500000000, + 0xf67beccf00000000, 0xaefb0e1800000000, 0x077d58bb00000000, + 0x5ffdba6c00000000, 0x916b252f00000000, 0xc9ebc7f800000000, + 0x606d915b00000000, 0x38ed738c00000000, 0x73664dc600000000, + 0x2be6af1100000000, 0x8260f9b200000000, 0xdae01b6500000000, + 0x1e4dc63500000000, 0x46cd24e200000000, 0xef4b724100000000, + 0xb7cb909600000000, 0xfc40aedc00000000, 0xa4c04c0b00000000, + 0x0d461aa800000000, 0x55c6f87f00000000, 0x9b50673c00000000, + 0xc3d085eb00000000, 0x6a56d34800000000, 0x32d6319f00000000, + 0x795d0fd500000000, 0x21dded0200000000, 0x885bbba100000000, + 0xd0db597600000000, 0x28ec084d00000000, 0x706cea9a00000000, + 0xd9eabc3900000000, 0x816a5eee00000000, 0xcae160a400000000, + 0x9261827300000000, 0x3be7d4d000000000, 0x6367360700000000, + 0xadf1a94400000000, 0xf5714b9300000000, 0x5cf71d3000000000, + 0x0477ffe700000000, 0x4ffcc1ad00000000, 0x177c237a00000000, + 0xbefa75d900000000, 0xe67a970e00000000, 0x22d74a5e00000000, + 0x7a57a88900000000, 0xd3d1fe2a00000000, 0x8b511cfd00000000, + 0xc0da22b700000000, 0x985ac06000000000, 0x31dc96c300000000, + 0x695c741400000000, 0xa7caeb5700000000, 0xff4a098000000000, + 0x56cc5f2300000000, 0x0e4cbdf400000000, 0x45c783be00000000, + 0x1d47616900000000, 0xb4c137ca00000000, 0xec41d51d00000000, + 0x3c9a8c6b00000000, 0x641a6ebc00000000, 0xcd9c381f00000000, + 0x951cdac800000000, 0xde97e48200000000, 0x8617065500000000, + 0x2f9150f600000000, 0x7711b22100000000, 0xb9872d6200000000, + 0xe107cfb500000000, 0x4881991600000000, 0x10017bc100000000, + 0x5b8a458b00000000, 0x030aa75c00000000, 0xaa8cf1ff00000000, + 0xf20c132800000000, 0x36a1ce7800000000, 0x6e212caf00000000, + 0xc7a77a0c00000000, 0x9f2798db00000000, 0xd4aca69100000000, + 0x8c2c444600000000, 0x25aa12e500000000, 0x7d2af03200000000, + 0xb3bc6f7100000000, 0xeb3c8da600000000, 0x42badb0500000000, + 0x1a3a39d200000000, 0x51b1079800000000, 0x0931e54f00000000, + 0xa0b7b3ec00000000, 0xf837513b00000000, 0x50d8119a00000000, + 0x0858f34d00000000, 0xa1dea5ee00000000, 0xf95e473900000000, + 0xb2d5797300000000, 0xea559ba400000000, 0x43d3cd0700000000, + 0x1b532fd000000000, 0xd5c5b09300000000, 0x8d45524400000000, + 0x24c304e700000000, 0x7c43e63000000000, 0x37c8d87a00000000, + 0x6f483aad00000000, 0xc6ce6c0e00000000, 0x9e4e8ed900000000, + 0x5ae3538900000000, 0x0263b15e00000000, 0xabe5e7fd00000000, + 0xf365052a00000000, 0xb8ee3b6000000000, 0xe06ed9b700000000, + 0x49e88f1400000000, 0x11686dc300000000, 0xdffef28000000000, + 0x877e105700000000, 0x2ef846f400000000, 0x7678a42300000000, + 0x3df39a6900000000, 0x657378be00000000, 0xccf52e1d00000000, + 0x9475ccca00000000, 0x44ae95bc00000000, 0x1c2e776b00000000, + 0xb5a821c800000000, 0xed28c31f00000000, 0xa6a3fd5500000000, + 0xfe231f8200000000, 0x57a5492100000000, 0x0f25abf600000000, + 0xc1b334b500000000, 0x9933d66200000000, 0x30b580c100000000, + 0x6835621600000000, 0x23be5c5c00000000, 0x7b3ebe8b00000000, + 0xd2b8e82800000000, 0x8a380aff00000000, 0x4e95d7af00000000, + 0x1615357800000000, 0xbf9363db00000000, 0xe713810c00000000, + 0xac98bf4600000000, 0xf4185d9100000000, 0x5d9e0b3200000000, + 0x051ee9e500000000, 0xcb8876a600000000, 0x9308947100000000, + 0x3a8ec2d200000000, 0x620e200500000000, 0x29851e4f00000000, + 0x7105fc9800000000, 0xd883aa3b00000000, 0x800348ec00000000, + 0x783419d700000000, 0x20b4fb0000000000, 0x8932ada300000000, + 0xd1b24f7400000000, 0x9a39713e00000000, 0xc2b993e900000000, + 0x6b3fc54a00000000, 0x33bf279d00000000, 0xfd29b8de00000000, + 0xa5a95a0900000000, 0x0c2f0caa00000000, 0x54afee7d00000000, + 0x1f24d03700000000, 0x47a432e000000000, 0xee22644300000000, + 0xb6a2869400000000, 0x720f5bc400000000, 0x2a8fb91300000000, + 0x8309efb000000000, 0xdb890d6700000000, 0x9002332d00000000, + 0xc882d1fa00000000, 0x6104875900000000, 0x3984658e00000000, + 0xf712facd00000000, 0xaf92181a00000000, 0x06144eb900000000, + 0x5e94ac6e00000000, 0x151f922400000000, 0x4d9f70f300000000, + 0xe419265000000000, 0xbc99c48700000000, 0x6c429df100000000, + 0x34c27f2600000000, 0x9d44298500000000, 0xc5c4cb5200000000, + 0x8e4ff51800000000, 0xd6cf17cf00000000, 0x7f49416c00000000, + 0x27c9a3bb00000000, 0xe95f3cf800000000, 0xb1dfde2f00000000, + 0x1859888c00000000, 0x40d96a5b00000000, 0x0b52541100000000, + 0x53d2b6c600000000, 0xfa54e06500000000, 0xa2d402b200000000, + 0x6679dfe200000000, 0x3ef93d3500000000, 0x977f6b9600000000, + 0xcfff894100000000, 0x8474b70b00000000, 0xdcf455dc00000000, + 0x7572037f00000000, 0x2df2e1a800000000, 0xe3647eeb00000000, + 0xbbe49c3c00000000, 0x1262ca9f00000000, 0x4ae2284800000000, + 0x0169160200000000, 0x59e9f4d500000000, 0xf06fa27600000000, + 0xa8ef40a100000000}, + {0x0000000000000000, 0x463b676500000000, 0x8c76ceca00000000, + 0xca4da9af00000000, 0x59ebed4e00000000, 0x1fd08a2b00000000, + 0xd59d238400000000, 0x93a644e100000000, 0xb2d6db9d00000000, + 0xf4edbcf800000000, 0x3ea0155700000000, 0x789b723200000000, + 0xeb3d36d300000000, 0xad0651b600000000, 0x674bf81900000000, + 0x21709f7c00000000, 0x25abc6e000000000, 0x6390a18500000000, + 0xa9dd082a00000000, 0xefe66f4f00000000, 0x7c402bae00000000, + 0x3a7b4ccb00000000, 0xf036e56400000000, 0xb60d820100000000, + 0x977d1d7d00000000, 0xd1467a1800000000, 0x1b0bd3b700000000, + 0x5d30b4d200000000, 0xce96f03300000000, 0x88ad975600000000, + 0x42e03ef900000000, 0x04db599c00000000, 0x0b50fc1a00000000, + 0x4d6b9b7f00000000, 0x872632d000000000, 0xc11d55b500000000, + 0x52bb115400000000, 0x1480763100000000, 0xdecddf9e00000000, + 0x98f6b8fb00000000, 0xb986278700000000, 0xffbd40e200000000, + 0x35f0e94d00000000, 0x73cb8e2800000000, 0xe06dcac900000000, + 0xa656adac00000000, 0x6c1b040300000000, 0x2a20636600000000, + 0x2efb3afa00000000, 0x68c05d9f00000000, 0xa28df43000000000, + 0xe4b6935500000000, 0x7710d7b400000000, 0x312bb0d100000000, + 0xfb66197e00000000, 0xbd5d7e1b00000000, 0x9c2de16700000000, + 0xda16860200000000, 0x105b2fad00000000, 0x566048c800000000, + 0xc5c60c2900000000, 0x83fd6b4c00000000, 0x49b0c2e300000000, + 0x0f8ba58600000000, 0x16a0f83500000000, 0x509b9f5000000000, + 0x9ad636ff00000000, 0xdced519a00000000, 0x4f4b157b00000000, + 0x0970721e00000000, 0xc33ddbb100000000, 0x8506bcd400000000, + 0xa47623a800000000, 0xe24d44cd00000000, 0x2800ed6200000000, + 0x6e3b8a0700000000, 0xfd9dcee600000000, 0xbba6a98300000000, + 0x71eb002c00000000, 0x37d0674900000000, 0x330b3ed500000000, + 0x753059b000000000, 0xbf7df01f00000000, 0xf946977a00000000, + 0x6ae0d39b00000000, 0x2cdbb4fe00000000, 0xe6961d5100000000, + 0xa0ad7a3400000000, 0x81dde54800000000, 0xc7e6822d00000000, + 0x0dab2b8200000000, 0x4b904ce700000000, 0xd836080600000000, + 0x9e0d6f6300000000, 0x5440c6cc00000000, 0x127ba1a900000000, + 0x1df0042f00000000, 0x5bcb634a00000000, 0x9186cae500000000, + 0xd7bdad8000000000, 0x441be96100000000, 0x02208e0400000000, + 0xc86d27ab00000000, 0x8e5640ce00000000, 0xaf26dfb200000000, + 0xe91db8d700000000, 0x2350117800000000, 0x656b761d00000000, + 0xf6cd32fc00000000, 0xb0f6559900000000, 0x7abbfc3600000000, + 0x3c809b5300000000, 0x385bc2cf00000000, 0x7e60a5aa00000000, + 0xb42d0c0500000000, 0xf2166b6000000000, 0x61b02f8100000000, + 0x278b48e400000000, 0xedc6e14b00000000, 0xabfd862e00000000, + 0x8a8d195200000000, 0xccb67e3700000000, 0x06fbd79800000000, + 0x40c0b0fd00000000, 0xd366f41c00000000, 0x955d937900000000, + 0x5f103ad600000000, 0x192b5db300000000, 0x2c40f16b00000000, + 0x6a7b960e00000000, 0xa0363fa100000000, 0xe60d58c400000000, + 0x75ab1c2500000000, 0x33907b4000000000, 0xf9ddd2ef00000000, + 0xbfe6b58a00000000, 0x9e962af600000000, 0xd8ad4d9300000000, + 0x12e0e43c00000000, 0x54db835900000000, 0xc77dc7b800000000, + 0x8146a0dd00000000, 0x4b0b097200000000, 0x0d306e1700000000, + 0x09eb378b00000000, 0x4fd050ee00000000, 0x859df94100000000, + 0xc3a69e2400000000, 0x5000dac500000000, 0x163bbda000000000, + 0xdc76140f00000000, 0x9a4d736a00000000, 0xbb3dec1600000000, + 0xfd068b7300000000, 0x374b22dc00000000, 0x717045b900000000, + 0xe2d6015800000000, 0xa4ed663d00000000, 0x6ea0cf9200000000, + 0x289ba8f700000000, 0x27100d7100000000, 0x612b6a1400000000, + 0xab66c3bb00000000, 0xed5da4de00000000, 0x7efbe03f00000000, + 0x38c0875a00000000, 0xf28d2ef500000000, 0xb4b6499000000000, + 0x95c6d6ec00000000, 0xd3fdb18900000000, 0x19b0182600000000, + 0x5f8b7f4300000000, 0xcc2d3ba200000000, 0x8a165cc700000000, + 0x405bf56800000000, 0x0660920d00000000, 0x02bbcb9100000000, + 0x4480acf400000000, 0x8ecd055b00000000, 0xc8f6623e00000000, + 0x5b5026df00000000, 0x1d6b41ba00000000, 0xd726e81500000000, + 0x911d8f7000000000, 0xb06d100c00000000, 0xf656776900000000, + 0x3c1bdec600000000, 0x7a20b9a300000000, 0xe986fd4200000000, + 0xafbd9a2700000000, 0x65f0338800000000, 0x23cb54ed00000000, + 0x3ae0095e00000000, 0x7cdb6e3b00000000, 0xb696c79400000000, + 0xf0ada0f100000000, 0x630be41000000000, 0x2530837500000000, + 0xef7d2ada00000000, 0xa9464dbf00000000, 0x8836d2c300000000, + 0xce0db5a600000000, 0x04401c0900000000, 0x427b7b6c00000000, + 0xd1dd3f8d00000000, 0x97e658e800000000, 0x5dabf14700000000, + 0x1b90962200000000, 0x1f4bcfbe00000000, 0x5970a8db00000000, + 0x933d017400000000, 0xd506661100000000, 0x46a022f000000000, + 0x009b459500000000, 0xcad6ec3a00000000, 0x8ced8b5f00000000, + 0xad9d142300000000, 0xeba6734600000000, 0x21ebdae900000000, + 0x67d0bd8c00000000, 0xf476f96d00000000, 0xb24d9e0800000000, + 0x780037a700000000, 0x3e3b50c200000000, 0x31b0f54400000000, + 0x778b922100000000, 0xbdc63b8e00000000, 0xfbfd5ceb00000000, + 0x685b180a00000000, 0x2e607f6f00000000, 0xe42dd6c000000000, + 0xa216b1a500000000, 0x83662ed900000000, 0xc55d49bc00000000, + 0x0f10e01300000000, 0x492b877600000000, 0xda8dc39700000000, + 0x9cb6a4f200000000, 0x56fb0d5d00000000, 0x10c06a3800000000, + 0x141b33a400000000, 0x522054c100000000, 0x986dfd6e00000000, + 0xde569a0b00000000, 0x4df0deea00000000, 0x0bcbb98f00000000, + 0xc186102000000000, 0x87bd774500000000, 0xa6cde83900000000, + 0xe0f68f5c00000000, 0x2abb26f300000000, 0x6c80419600000000, + 0xff26057700000000, 0xb91d621200000000, 0x7350cbbd00000000, + 0x356bacd800000000}, + {0x0000000000000000, 0x9e83da9f00000000, 0x7d01c4e400000000, + 0xe3821e7b00000000, 0xbb04f91200000000, 0x2587238d00000000, + 0xc6053df600000000, 0x5886e76900000000, 0x7609f22500000000, + 0xe88a28ba00000000, 0x0b0836c100000000, 0x958bec5e00000000, + 0xcd0d0b3700000000, 0x538ed1a800000000, 0xb00ccfd300000000, + 0x2e8f154c00000000, 0xec12e44b00000000, 0x72913ed400000000, + 0x911320af00000000, 0x0f90fa3000000000, 0x57161d5900000000, + 0xc995c7c600000000, 0x2a17d9bd00000000, 0xb494032200000000, + 0x9a1b166e00000000, 0x0498ccf100000000, 0xe71ad28a00000000, + 0x7999081500000000, 0x211fef7c00000000, 0xbf9c35e300000000, + 0x5c1e2b9800000000, 0xc29df10700000000, 0xd825c89700000000, + 0x46a6120800000000, 0xa5240c7300000000, 0x3ba7d6ec00000000, + 0x6321318500000000, 0xfda2eb1a00000000, 0x1e20f56100000000, + 0x80a32ffe00000000, 0xae2c3ab200000000, 0x30afe02d00000000, + 0xd32dfe5600000000, 0x4dae24c900000000, 0x1528c3a000000000, + 0x8bab193f00000000, 0x6829074400000000, 0xf6aadddb00000000, + 0x34372cdc00000000, 0xaab4f64300000000, 0x4936e83800000000, + 0xd7b532a700000000, 0x8f33d5ce00000000, 0x11b00f5100000000, + 0xf232112a00000000, 0x6cb1cbb500000000, 0x423edef900000000, + 0xdcbd046600000000, 0x3f3f1a1d00000000, 0xa1bcc08200000000, + 0xf93a27eb00000000, 0x67b9fd7400000000, 0x843be30f00000000, + 0x1ab8399000000000, 0xf14de1f400000000, 0x6fce3b6b00000000, + 0x8c4c251000000000, 0x12cfff8f00000000, 0x4a4918e600000000, + 0xd4cac27900000000, 0x3748dc0200000000, 0xa9cb069d00000000, + 0x874413d100000000, 0x19c7c94e00000000, 0xfa45d73500000000, + 0x64c60daa00000000, 0x3c40eac300000000, 0xa2c3305c00000000, + 0x41412e2700000000, 0xdfc2f4b800000000, 0x1d5f05bf00000000, + 0x83dcdf2000000000, 0x605ec15b00000000, 0xfedd1bc400000000, + 0xa65bfcad00000000, 0x38d8263200000000, 0xdb5a384900000000, + 0x45d9e2d600000000, 0x6b56f79a00000000, 0xf5d52d0500000000, + 0x1657337e00000000, 0x88d4e9e100000000, 0xd0520e8800000000, + 0x4ed1d41700000000, 0xad53ca6c00000000, 0x33d010f300000000, + 0x2968296300000000, 0xb7ebf3fc00000000, 0x5469ed8700000000, + 0xcaea371800000000, 0x926cd07100000000, 0x0cef0aee00000000, + 0xef6d149500000000, 0x71eece0a00000000, 0x5f61db4600000000, + 0xc1e201d900000000, 0x22601fa200000000, 0xbce3c53d00000000, + 0xe465225400000000, 0x7ae6f8cb00000000, 0x9964e6b000000000, + 0x07e73c2f00000000, 0xc57acd2800000000, 0x5bf917b700000000, + 0xb87b09cc00000000, 0x26f8d35300000000, 0x7e7e343a00000000, + 0xe0fdeea500000000, 0x037ff0de00000000, 0x9dfc2a4100000000, + 0xb3733f0d00000000, 0x2df0e59200000000, 0xce72fbe900000000, + 0x50f1217600000000, 0x0877c61f00000000, 0x96f41c8000000000, + 0x757602fb00000000, 0xebf5d86400000000, 0xa39db33200000000, + 0x3d1e69ad00000000, 0xde9c77d600000000, 0x401fad4900000000, + 0x18994a2000000000, 0x861a90bf00000000, 0x65988ec400000000, + 0xfb1b545b00000000, 0xd594411700000000, 0x4b179b8800000000, + 0xa89585f300000000, 0x36165f6c00000000, 0x6e90b80500000000, + 0xf013629a00000000, 0x13917ce100000000, 0x8d12a67e00000000, + 0x4f8f577900000000, 0xd10c8de600000000, 0x328e939d00000000, + 0xac0d490200000000, 0xf48bae6b00000000, 0x6a0874f400000000, + 0x898a6a8f00000000, 0x1709b01000000000, 0x3986a55c00000000, + 0xa7057fc300000000, 0x448761b800000000, 0xda04bb2700000000, + 0x82825c4e00000000, 0x1c0186d100000000, 0xff8398aa00000000, + 0x6100423500000000, 0x7bb87ba500000000, 0xe53ba13a00000000, + 0x06b9bf4100000000, 0x983a65de00000000, 0xc0bc82b700000000, + 0x5e3f582800000000, 0xbdbd465300000000, 0x233e9ccc00000000, + 0x0db1898000000000, 0x9332531f00000000, 0x70b04d6400000000, + 0xee3397fb00000000, 0xb6b5709200000000, 0x2836aa0d00000000, + 0xcbb4b47600000000, 0x55376ee900000000, 0x97aa9fee00000000, + 0x0929457100000000, 0xeaab5b0a00000000, 0x7428819500000000, + 0x2cae66fc00000000, 0xb22dbc6300000000, 0x51afa21800000000, + 0xcf2c788700000000, 0xe1a36dcb00000000, 0x7f20b75400000000, + 0x9ca2a92f00000000, 0x022173b000000000, 0x5aa794d900000000, + 0xc4244e4600000000, 0x27a6503d00000000, 0xb9258aa200000000, + 0x52d052c600000000, 0xcc53885900000000, 0x2fd1962200000000, + 0xb1524cbd00000000, 0xe9d4abd400000000, 0x7757714b00000000, + 0x94d56f3000000000, 0x0a56b5af00000000, 0x24d9a0e300000000, + 0xba5a7a7c00000000, 0x59d8640700000000, 0xc75bbe9800000000, + 0x9fdd59f100000000, 0x015e836e00000000, 0xe2dc9d1500000000, + 0x7c5f478a00000000, 0xbec2b68d00000000, 0x20416c1200000000, + 0xc3c3726900000000, 0x5d40a8f600000000, 0x05c64f9f00000000, + 0x9b45950000000000, 0x78c78b7b00000000, 0xe64451e400000000, + 0xc8cb44a800000000, 0x56489e3700000000, 0xb5ca804c00000000, + 0x2b495ad300000000, 0x73cfbdba00000000, 0xed4c672500000000, + 0x0ece795e00000000, 0x904da3c100000000, 0x8af59a5100000000, + 0x147640ce00000000, 0xf7f45eb500000000, 0x6977842a00000000, + 0x31f1634300000000, 0xaf72b9dc00000000, 0x4cf0a7a700000000, + 0xd2737d3800000000, 0xfcfc687400000000, 0x627fb2eb00000000, + 0x81fdac9000000000, 0x1f7e760f00000000, 0x47f8916600000000, + 0xd97b4bf900000000, 0x3af9558200000000, 0xa47a8f1d00000000, + 0x66e77e1a00000000, 0xf864a48500000000, 0x1be6bafe00000000, + 0x8565606100000000, 0xdde3870800000000, 0x43605d9700000000, + 0xa0e243ec00000000, 0x3e61997300000000, 0x10ee8c3f00000000, + 0x8e6d56a000000000, 0x6def48db00000000, 0xf36c924400000000, + 0xabea752d00000000, 0x3569afb200000000, 0xd6ebb1c900000000, + 0x48686b5600000000}, + {0x0000000000000000, 0xc064281700000000, 0x80c9502e00000000, + 0x40ad783900000000, 0x0093a15c00000000, 0xc0f7894b00000000, + 0x805af17200000000, 0x403ed96500000000, 0x002643b900000000, + 0xc0426bae00000000, 0x80ef139700000000, 0x408b3b8000000000, + 0x00b5e2e500000000, 0xc0d1caf200000000, 0x807cb2cb00000000, + 0x40189adc00000000, 0x414af7a900000000, 0x812edfbe00000000, + 0xc183a78700000000, 0x01e78f9000000000, 0x41d956f500000000, + 0x81bd7ee200000000, 0xc11006db00000000, 0x01742ecc00000000, + 0x416cb41000000000, 0x81089c0700000000, 0xc1a5e43e00000000, + 0x01c1cc2900000000, 0x41ff154c00000000, 0x819b3d5b00000000, + 0xc136456200000000, 0x01526d7500000000, 0xc3929f8800000000, + 0x03f6b79f00000000, 0x435bcfa600000000, 0x833fe7b100000000, + 0xc3013ed400000000, 0x036516c300000000, 0x43c86efa00000000, + 0x83ac46ed00000000, 0xc3b4dc3100000000, 0x03d0f42600000000, + 0x437d8c1f00000000, 0x8319a40800000000, 0xc3277d6d00000000, + 0x0343557a00000000, 0x43ee2d4300000000, 0x838a055400000000, + 0x82d8682100000000, 0x42bc403600000000, 0x0211380f00000000, + 0xc275101800000000, 0x824bc97d00000000, 0x422fe16a00000000, + 0x0282995300000000, 0xc2e6b14400000000, 0x82fe2b9800000000, + 0x429a038f00000000, 0x02377bb600000000, 0xc25353a100000000, + 0x826d8ac400000000, 0x4209a2d300000000, 0x02a4daea00000000, + 0xc2c0f2fd00000000, 0xc7234eca00000000, 0x074766dd00000000, + 0x47ea1ee400000000, 0x878e36f300000000, 0xc7b0ef9600000000, + 0x07d4c78100000000, 0x4779bfb800000000, 0x871d97af00000000, + 0xc7050d7300000000, 0x0761256400000000, 0x47cc5d5d00000000, + 0x87a8754a00000000, 0xc796ac2f00000000, 0x07f2843800000000, + 0x475ffc0100000000, 0x873bd41600000000, 0x8669b96300000000, + 0x460d917400000000, 0x06a0e94d00000000, 0xc6c4c15a00000000, + 0x86fa183f00000000, 0x469e302800000000, 0x0633481100000000, + 0xc657600600000000, 0x864ffada00000000, 0x462bd2cd00000000, + 0x0686aaf400000000, 0xc6e282e300000000, 0x86dc5b8600000000, + 0x46b8739100000000, 0x06150ba800000000, 0xc67123bf00000000, + 0x04b1d14200000000, 0xc4d5f95500000000, 0x8478816c00000000, + 0x441ca97b00000000, 0x0422701e00000000, 0xc446580900000000, + 0x84eb203000000000, 0x448f082700000000, 0x049792fb00000000, + 0xc4f3baec00000000, 0x845ec2d500000000, 0x443aeac200000000, + 0x040433a700000000, 0xc4601bb000000000, 0x84cd638900000000, + 0x44a94b9e00000000, 0x45fb26eb00000000, 0x859f0efc00000000, + 0xc53276c500000000, 0x05565ed200000000, 0x456887b700000000, + 0x850cafa000000000, 0xc5a1d79900000000, 0x05c5ff8e00000000, + 0x45dd655200000000, 0x85b94d4500000000, 0xc514357c00000000, + 0x05701d6b00000000, 0x454ec40e00000000, 0x852aec1900000000, + 0xc587942000000000, 0x05e3bc3700000000, 0xcf41ed4f00000000, + 0x0f25c55800000000, 0x4f88bd6100000000, 0x8fec957600000000, + 0xcfd24c1300000000, 0x0fb6640400000000, 0x4f1b1c3d00000000, + 0x8f7f342a00000000, 0xcf67aef600000000, 0x0f0386e100000000, + 0x4faefed800000000, 0x8fcad6cf00000000, 0xcff40faa00000000, + 0x0f9027bd00000000, 0x4f3d5f8400000000, 0x8f59779300000000, + 0x8e0b1ae600000000, 0x4e6f32f100000000, 0x0ec24ac800000000, + 0xcea662df00000000, 0x8e98bbba00000000, 0x4efc93ad00000000, + 0x0e51eb9400000000, 0xce35c38300000000, 0x8e2d595f00000000, + 0x4e49714800000000, 0x0ee4097100000000, 0xce80216600000000, + 0x8ebef80300000000, 0x4edad01400000000, 0x0e77a82d00000000, + 0xce13803a00000000, 0x0cd372c700000000, 0xccb75ad000000000, + 0x8c1a22e900000000, 0x4c7e0afe00000000, 0x0c40d39b00000000, + 0xcc24fb8c00000000, 0x8c8983b500000000, 0x4cedaba200000000, + 0x0cf5317e00000000, 0xcc91196900000000, 0x8c3c615000000000, + 0x4c58494700000000, 0x0c66902200000000, 0xcc02b83500000000, + 0x8cafc00c00000000, 0x4ccbe81b00000000, 0x4d99856e00000000, + 0x8dfdad7900000000, 0xcd50d54000000000, 0x0d34fd5700000000, + 0x4d0a243200000000, 0x8d6e0c2500000000, 0xcdc3741c00000000, + 0x0da75c0b00000000, 0x4dbfc6d700000000, 0x8ddbeec000000000, + 0xcd7696f900000000, 0x0d12beee00000000, 0x4d2c678b00000000, + 0x8d484f9c00000000, 0xcde537a500000000, 0x0d811fb200000000, + 0x0862a38500000000, 0xc8068b9200000000, 0x88abf3ab00000000, + 0x48cfdbbc00000000, 0x08f102d900000000, 0xc8952ace00000000, + 0x883852f700000000, 0x485c7ae000000000, 0x0844e03c00000000, + 0xc820c82b00000000, 0x888db01200000000, 0x48e9980500000000, + 0x08d7416000000000, 0xc8b3697700000000, 0x881e114e00000000, + 0x487a395900000000, 0x4928542c00000000, 0x894c7c3b00000000, + 0xc9e1040200000000, 0x09852c1500000000, 0x49bbf57000000000, + 0x89dfdd6700000000, 0xc972a55e00000000, 0x09168d4900000000, + 0x490e179500000000, 0x896a3f8200000000, 0xc9c747bb00000000, + 0x09a36fac00000000, 0x499db6c900000000, 0x89f99ede00000000, + 0xc954e6e700000000, 0x0930cef000000000, 0xcbf03c0d00000000, + 0x0b94141a00000000, 0x4b396c2300000000, 0x8b5d443400000000, + 0xcb639d5100000000, 0x0b07b54600000000, 0x4baacd7f00000000, + 0x8bcee56800000000, 0xcbd67fb400000000, 0x0bb257a300000000, + 0x4b1f2f9a00000000, 0x8b7b078d00000000, 0xcb45dee800000000, + 0x0b21f6ff00000000, 0x4b8c8ec600000000, 0x8be8a6d100000000, + 0x8abacba400000000, 0x4adee3b300000000, 0x0a739b8a00000000, + 0xca17b39d00000000, 0x8a296af800000000, 0x4a4d42ef00000000, + 0x0ae03ad600000000, 0xca8412c100000000, 0x8a9c881d00000000, + 0x4af8a00a00000000, 0x0a55d83300000000, 0xca31f02400000000, + 0x8a0f294100000000, 0x4a6b015600000000, 0x0ac6796f00000000, + 0xcaa2517800000000}, + {0x0000000000000000, 0xd4ea739b00000000, 0xe9d396ed00000000, + 0x3d39e57600000000, 0x93a15c0000000000, 0x474b2f9b00000000, + 0x7a72caed00000000, 0xae98b97600000000, 0x2643b90000000000, + 0xf2a9ca9b00000000, 0xcf902fed00000000, 0x1b7a5c7600000000, + 0xb5e2e50000000000, 0x6108969b00000000, 0x5c3173ed00000000, + 0x88db007600000000, 0x4c86720100000000, 0x986c019a00000000, + 0xa555e4ec00000000, 0x71bf977700000000, 0xdf272e0100000000, + 0x0bcd5d9a00000000, 0x36f4b8ec00000000, 0xe21ecb7700000000, + 0x6ac5cb0100000000, 0xbe2fb89a00000000, 0x83165dec00000000, + 0x57fc2e7700000000, 0xf964970100000000, 0x2d8ee49a00000000, + 0x10b701ec00000000, 0xc45d727700000000, 0x980ce50200000000, + 0x4ce6969900000000, 0x71df73ef00000000, 0xa535007400000000, + 0x0badb90200000000, 0xdf47ca9900000000, 0xe27e2fef00000000, + 0x36945c7400000000, 0xbe4f5c0200000000, 0x6aa52f9900000000, + 0x579ccaef00000000, 0x8376b97400000000, 0x2dee000200000000, + 0xf904739900000000, 0xc43d96ef00000000, 0x10d7e57400000000, + 0xd48a970300000000, 0x0060e49800000000, 0x3d5901ee00000000, + 0xe9b3727500000000, 0x472bcb0300000000, 0x93c1b89800000000, + 0xaef85dee00000000, 0x7a122e7500000000, 0xf2c92e0300000000, + 0x26235d9800000000, 0x1b1ab8ee00000000, 0xcff0cb7500000000, + 0x6168720300000000, 0xb582019800000000, 0x88bbe4ee00000000, + 0x5c51977500000000, 0x3019ca0500000000, 0xe4f3b99e00000000, + 0xd9ca5ce800000000, 0x0d202f7300000000, 0xa3b8960500000000, + 0x7752e59e00000000, 0x4a6b00e800000000, 0x9e81737300000000, + 0x165a730500000000, 0xc2b0009e00000000, 0xff89e5e800000000, + 0x2b63967300000000, 0x85fb2f0500000000, 0x51115c9e00000000, + 0x6c28b9e800000000, 0xb8c2ca7300000000, 0x7c9fb80400000000, + 0xa875cb9f00000000, 0x954c2ee900000000, 0x41a65d7200000000, + 0xef3ee40400000000, 0x3bd4979f00000000, 0x06ed72e900000000, + 0xd207017200000000, 0x5adc010400000000, 0x8e36729f00000000, + 0xb30f97e900000000, 0x67e5e47200000000, 0xc97d5d0400000000, + 0x1d972e9f00000000, 0x20aecbe900000000, 0xf444b87200000000, + 0xa8152f0700000000, 0x7cff5c9c00000000, 0x41c6b9ea00000000, + 0x952cca7100000000, 0x3bb4730700000000, 0xef5e009c00000000, + 0xd267e5ea00000000, 0x068d967100000000, 0x8e56960700000000, + 0x5abce59c00000000, 0x678500ea00000000, 0xb36f737100000000, + 0x1df7ca0700000000, 0xc91db99c00000000, 0xf4245cea00000000, + 0x20ce2f7100000000, 0xe4935d0600000000, 0x30792e9d00000000, + 0x0d40cbeb00000000, 0xd9aab87000000000, 0x7732010600000000, + 0xa3d8729d00000000, 0x9ee197eb00000000, 0x4a0be47000000000, + 0xc2d0e40600000000, 0x163a979d00000000, 0x2b0372eb00000000, + 0xffe9017000000000, 0x5171b80600000000, 0x859bcb9d00000000, + 0xb8a22eeb00000000, 0x6c485d7000000000, 0x6032940b00000000, + 0xb4d8e79000000000, 0x89e102e600000000, 0x5d0b717d00000000, + 0xf393c80b00000000, 0x2779bb9000000000, 0x1a405ee600000000, + 0xceaa2d7d00000000, 0x46712d0b00000000, 0x929b5e9000000000, + 0xafa2bbe600000000, 0x7b48c87d00000000, 0xd5d0710b00000000, + 0x013a029000000000, 0x3c03e7e600000000, 0xe8e9947d00000000, + 0x2cb4e60a00000000, 0xf85e959100000000, 0xc56770e700000000, + 0x118d037c00000000, 0xbf15ba0a00000000, 0x6bffc99100000000, + 0x56c62ce700000000, 0x822c5f7c00000000, 0x0af75f0a00000000, + 0xde1d2c9100000000, 0xe324c9e700000000, 0x37ceba7c00000000, + 0x9956030a00000000, 0x4dbc709100000000, 0x708595e700000000, + 0xa46fe67c00000000, 0xf83e710900000000, 0x2cd4029200000000, + 0x11ede7e400000000, 0xc507947f00000000, 0x6b9f2d0900000000, + 0xbf755e9200000000, 0x824cbbe400000000, 0x56a6c87f00000000, + 0xde7dc80900000000, 0x0a97bb9200000000, 0x37ae5ee400000000, + 0xe3442d7f00000000, 0x4ddc940900000000, 0x9936e79200000000, + 0xa40f02e400000000, 0x70e5717f00000000, 0xb4b8030800000000, + 0x6052709300000000, 0x5d6b95e500000000, 0x8981e67e00000000, + 0x27195f0800000000, 0xf3f32c9300000000, 0xcecac9e500000000, + 0x1a20ba7e00000000, 0x92fbba0800000000, 0x4611c99300000000, + 0x7b282ce500000000, 0xafc25f7e00000000, 0x015ae60800000000, + 0xd5b0959300000000, 0xe88970e500000000, 0x3c63037e00000000, + 0x502b5e0e00000000, 0x84c12d9500000000, 0xb9f8c8e300000000, + 0x6d12bb7800000000, 0xc38a020e00000000, 0x1760719500000000, + 0x2a5994e300000000, 0xfeb3e77800000000, 0x7668e70e00000000, + 0xa282949500000000, 0x9fbb71e300000000, 0x4b51027800000000, + 0xe5c9bb0e00000000, 0x3123c89500000000, 0x0c1a2de300000000, + 0xd8f05e7800000000, 0x1cad2c0f00000000, 0xc8475f9400000000, + 0xf57ebae200000000, 0x2194c97900000000, 0x8f0c700f00000000, + 0x5be6039400000000, 0x66dfe6e200000000, 0xb235957900000000, + 0x3aee950f00000000, 0xee04e69400000000, 0xd33d03e200000000, + 0x07d7707900000000, 0xa94fc90f00000000, 0x7da5ba9400000000, + 0x409c5fe200000000, 0x94762c7900000000, 0xc827bb0c00000000, + 0x1ccdc89700000000, 0x21f42de100000000, 0xf51e5e7a00000000, + 0x5b86e70c00000000, 0x8f6c949700000000, 0xb25571e100000000, + 0x66bf027a00000000, 0xee64020c00000000, 0x3a8e719700000000, + 0x07b794e100000000, 0xd35de77a00000000, 0x7dc55e0c00000000, + 0xa92f2d9700000000, 0x9416c8e100000000, 0x40fcbb7a00000000, + 0x84a1c90d00000000, 0x504bba9600000000, 0x6d725fe000000000, + 0xb9982c7b00000000, 0x1700950d00000000, 0xc3eae69600000000, + 0xfed303e000000000, 0x2a39707b00000000, 0xa2e2700d00000000, + 0x7608039600000000, 0x4b31e6e000000000, 0x9fdb957b00000000, + 0x31432c0d00000000, 0xe5a95f9600000000, 0xd890bae000000000, + 0x0c7ac97b00000000}, + {0x0000000000000000, 0x2765258100000000, 0x0fcc3bd900000000, + 0x28a91e5800000000, 0x5f9e066900000000, 0x78fb23e800000000, + 0x50523db000000000, 0x7737183100000000, 0xbe3c0dd200000000, + 0x9959285300000000, 0xb1f0360b00000000, 0x9695138a00000000, + 0xe1a20bbb00000000, 0xc6c72e3a00000000, 0xee6e306200000000, + 0xc90b15e300000000, 0x3d7f6b7f00000000, 0x1a1a4efe00000000, + 0x32b350a600000000, 0x15d6752700000000, 0x62e16d1600000000, + 0x4584489700000000, 0x6d2d56cf00000000, 0x4a48734e00000000, + 0x834366ad00000000, 0xa426432c00000000, 0x8c8f5d7400000000, + 0xabea78f500000000, 0xdcdd60c400000000, 0xfbb8454500000000, + 0xd3115b1d00000000, 0xf4747e9c00000000, 0x7afed6fe00000000, + 0x5d9bf37f00000000, 0x7532ed2700000000, 0x5257c8a600000000, + 0x2560d09700000000, 0x0205f51600000000, 0x2aaceb4e00000000, + 0x0dc9cecf00000000, 0xc4c2db2c00000000, 0xe3a7fead00000000, + 0xcb0ee0f500000000, 0xec6bc57400000000, 0x9b5cdd4500000000, + 0xbc39f8c400000000, 0x9490e69c00000000, 0xb3f5c31d00000000, + 0x4781bd8100000000, 0x60e4980000000000, 0x484d865800000000, + 0x6f28a3d900000000, 0x181fbbe800000000, 0x3f7a9e6900000000, + 0x17d3803100000000, 0x30b6a5b000000000, 0xf9bdb05300000000, + 0xded895d200000000, 0xf6718b8a00000000, 0xd114ae0b00000000, + 0xa623b63a00000000, 0x814693bb00000000, 0xa9ef8de300000000, + 0x8e8aa86200000000, 0xb5fadc2600000000, 0x929ff9a700000000, + 0xba36e7ff00000000, 0x9d53c27e00000000, 0xea64da4f00000000, + 0xcd01ffce00000000, 0xe5a8e19600000000, 0xc2cdc41700000000, + 0x0bc6d1f400000000, 0x2ca3f47500000000, 0x040aea2d00000000, + 0x236fcfac00000000, 0x5458d79d00000000, 0x733df21c00000000, + 0x5b94ec4400000000, 0x7cf1c9c500000000, 0x8885b75900000000, + 0xafe092d800000000, 0x87498c8000000000, 0xa02ca90100000000, + 0xd71bb13000000000, 0xf07e94b100000000, 0xd8d78ae900000000, + 0xffb2af6800000000, 0x36b9ba8b00000000, 0x11dc9f0a00000000, + 0x3975815200000000, 0x1e10a4d300000000, 0x6927bce200000000, + 0x4e42996300000000, 0x66eb873b00000000, 0x418ea2ba00000000, + 0xcf040ad800000000, 0xe8612f5900000000, 0xc0c8310100000000, + 0xe7ad148000000000, 0x909a0cb100000000, 0xb7ff293000000000, + 0x9f56376800000000, 0xb83312e900000000, 0x7138070a00000000, + 0x565d228b00000000, 0x7ef43cd300000000, 0x5991195200000000, + 0x2ea6016300000000, 0x09c324e200000000, 0x216a3aba00000000, + 0x060f1f3b00000000, 0xf27b61a700000000, 0xd51e442600000000, + 0xfdb75a7e00000000, 0xdad27fff00000000, 0xade567ce00000000, + 0x8a80424f00000000, 0xa2295c1700000000, 0x854c799600000000, + 0x4c476c7500000000, 0x6b2249f400000000, 0x438b57ac00000000, + 0x64ee722d00000000, 0x13d96a1c00000000, 0x34bc4f9d00000000, + 0x1c1551c500000000, 0x3b70744400000000, 0x6af5b94d00000000, + 0x4d909ccc00000000, 0x6539829400000000, 0x425ca71500000000, + 0x356bbf2400000000, 0x120e9aa500000000, 0x3aa784fd00000000, + 0x1dc2a17c00000000, 0xd4c9b49f00000000, 0xf3ac911e00000000, + 0xdb058f4600000000, 0xfc60aac700000000, 0x8b57b2f600000000, + 0xac32977700000000, 0x849b892f00000000, 0xa3feacae00000000, + 0x578ad23200000000, 0x70eff7b300000000, 0x5846e9eb00000000, + 0x7f23cc6a00000000, 0x0814d45b00000000, 0x2f71f1da00000000, + 0x07d8ef8200000000, 0x20bdca0300000000, 0xe9b6dfe000000000, + 0xced3fa6100000000, 0xe67ae43900000000, 0xc11fc1b800000000, + 0xb628d98900000000, 0x914dfc0800000000, 0xb9e4e25000000000, + 0x9e81c7d100000000, 0x100b6fb300000000, 0x376e4a3200000000, + 0x1fc7546a00000000, 0x38a271eb00000000, 0x4f9569da00000000, + 0x68f04c5b00000000, 0x4059520300000000, 0x673c778200000000, + 0xae37626100000000, 0x895247e000000000, 0xa1fb59b800000000, + 0x869e7c3900000000, 0xf1a9640800000000, 0xd6cc418900000000, + 0xfe655fd100000000, 0xd9007a5000000000, 0x2d7404cc00000000, + 0x0a11214d00000000, 0x22b83f1500000000, 0x05dd1a9400000000, + 0x72ea02a500000000, 0x558f272400000000, 0x7d26397c00000000, + 0x5a431cfd00000000, 0x9348091e00000000, 0xb42d2c9f00000000, + 0x9c8432c700000000, 0xbbe1174600000000, 0xccd60f7700000000, + 0xebb32af600000000, 0xc31a34ae00000000, 0xe47f112f00000000, + 0xdf0f656b00000000, 0xf86a40ea00000000, 0xd0c35eb200000000, + 0xf7a67b3300000000, 0x8091630200000000, 0xa7f4468300000000, + 0x8f5d58db00000000, 0xa8387d5a00000000, 0x613368b900000000, + 0x46564d3800000000, 0x6eff536000000000, 0x499a76e100000000, + 0x3ead6ed000000000, 0x19c84b5100000000, 0x3161550900000000, + 0x1604708800000000, 0xe2700e1400000000, 0xc5152b9500000000, + 0xedbc35cd00000000, 0xcad9104c00000000, 0xbdee087d00000000, + 0x9a8b2dfc00000000, 0xb22233a400000000, 0x9547162500000000, + 0x5c4c03c600000000, 0x7b29264700000000, 0x5380381f00000000, + 0x74e51d9e00000000, 0x03d205af00000000, 0x24b7202e00000000, + 0x0c1e3e7600000000, 0x2b7b1bf700000000, 0xa5f1b39500000000, + 0x8294961400000000, 0xaa3d884c00000000, 0x8d58adcd00000000, + 0xfa6fb5fc00000000, 0xdd0a907d00000000, 0xf5a38e2500000000, + 0xd2c6aba400000000, 0x1bcdbe4700000000, 0x3ca89bc600000000, + 0x1401859e00000000, 0x3364a01f00000000, 0x4453b82e00000000, + 0x63369daf00000000, 0x4b9f83f700000000, 0x6cfaa67600000000, + 0x988ed8ea00000000, 0xbfebfd6b00000000, 0x9742e33300000000, + 0xb027c6b200000000, 0xc710de8300000000, 0xe075fb0200000000, + 0xc8dce55a00000000, 0xefb9c0db00000000, 0x26b2d53800000000, + 0x01d7f0b900000000, 0x297eeee100000000, 0x0e1bcb6000000000, + 0x792cd35100000000, 0x5e49f6d000000000, 0x76e0e88800000000, + 0x5185cd0900000000}}; + +#else /* W == 4 */ + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0x9ba54c6f, 0xec3b9e9f, 0x779ed2f0, 0x03063b7f, + 0x98a37710, 0xef3da5e0, 0x7498e98f, 0x060c76fe, 0x9da93a91, + 0xea37e861, 0x7192a40e, 0x050a4d81, 0x9eaf01ee, 0xe931d31e, + 0x72949f71, 0x0c18edfc, 0x97bda193, 0xe0237363, 0x7b863f0c, + 0x0f1ed683, 0x94bb9aec, 0xe325481c, 0x78800473, 0x0a149b02, + 0x91b1d76d, 0xe62f059d, 0x7d8a49f2, 0x0912a07d, 0x92b7ec12, + 0xe5293ee2, 0x7e8c728d, 0x1831dbf8, 0x83949797, 0xf40a4567, + 0x6faf0908, 0x1b37e087, 0x8092ace8, 0xf70c7e18, 0x6ca93277, + 0x1e3dad06, 0x8598e169, 0xf2063399, 0x69a37ff6, 0x1d3b9679, + 0x869eda16, 0xf10008e6, 0x6aa54489, 0x14293604, 0x8f8c7a6b, + 0xf812a89b, 0x63b7e4f4, 0x172f0d7b, 0x8c8a4114, 0xfb1493e4, + 0x60b1df8b, 0x122540fa, 0x89800c95, 0xfe1ede65, 0x65bb920a, + 0x11237b85, 0x8a8637ea, 0xfd18e51a, 0x66bda975, 0x3063b7f0, + 0xabc6fb9f, 0xdc58296f, 0x47fd6500, 0x33658c8f, 0xa8c0c0e0, + 0xdf5e1210, 0x44fb5e7f, 0x366fc10e, 0xadca8d61, 0xda545f91, + 0x41f113fe, 0x3569fa71, 0xaeccb61e, 0xd95264ee, 0x42f72881, + 0x3c7b5a0c, 0xa7de1663, 0xd040c493, 0x4be588fc, 0x3f7d6173, + 0xa4d82d1c, 0xd346ffec, 0x48e3b383, 0x3a772cf2, 0xa1d2609d, + 0xd64cb26d, 0x4de9fe02, 0x3971178d, 0xa2d45be2, 0xd54a8912, + 0x4eefc57d, 0x28526c08, 0xb3f72067, 0xc469f297, 0x5fccbef8, + 0x2b545777, 0xb0f11b18, 0xc76fc9e8, 0x5cca8587, 0x2e5e1af6, + 0xb5fb5699, 0xc2658469, 0x59c0c806, 0x2d582189, 0xb6fd6de6, + 0xc163bf16, 0x5ac6f379, 0x244a81f4, 0xbfefcd9b, 0xc8711f6b, + 0x53d45304, 0x274cba8b, 0xbce9f6e4, 0xcb772414, 0x50d2687b, + 0x2246f70a, 0xb9e3bb65, 0xce7d6995, 0x55d825fa, 0x2140cc75, + 0xbae5801a, 0xcd7b52ea, 0x56de1e85, 0x60c76fe0, 0xfb62238f, + 0x8cfcf17f, 0x1759bd10, 0x63c1549f, 0xf86418f0, 0x8ffaca00, + 0x145f866f, 0x66cb191e, 0xfd6e5571, 0x8af08781, 0x1155cbee, + 0x65cd2261, 0xfe686e0e, 0x89f6bcfe, 0x1253f091, 0x6cdf821c, + 0xf77ace73, 0x80e41c83, 0x1b4150ec, 0x6fd9b963, 0xf47cf50c, + 0x83e227fc, 0x18476b93, 0x6ad3f4e2, 0xf176b88d, 0x86e86a7d, + 0x1d4d2612, 0x69d5cf9d, 0xf27083f2, 0x85ee5102, 0x1e4b1d6d, + 0x78f6b418, 0xe353f877, 0x94cd2a87, 0x0f6866e8, 0x7bf08f67, + 0xe055c308, 0x97cb11f8, 0x0c6e5d97, 0x7efac2e6, 0xe55f8e89, + 0x92c15c79, 0x09641016, 0x7dfcf999, 0xe659b5f6, 0x91c76706, + 0x0a622b69, 0x74ee59e4, 0xef4b158b, 0x98d5c77b, 0x03708b14, + 0x77e8629b, 0xec4d2ef4, 0x9bd3fc04, 0x0076b06b, 0x72e22f1a, + 0xe9476375, 0x9ed9b185, 0x057cfdea, 0x71e41465, 0xea41580a, + 0x9ddf8afa, 0x067ac695, 0x50a4d810, 0xcb01947f, 0xbc9f468f, + 0x273a0ae0, 0x53a2e36f, 0xc807af00, 0xbf997df0, 0x243c319f, + 0x56a8aeee, 0xcd0de281, 0xba933071, 0x21367c1e, 0x55ae9591, + 0xce0bd9fe, 0xb9950b0e, 0x22304761, 0x5cbc35ec, 0xc7197983, + 0xb087ab73, 0x2b22e71c, 0x5fba0e93, 0xc41f42fc, 0xb381900c, + 0x2824dc63, 0x5ab04312, 0xc1150f7d, 0xb68bdd8d, 0x2d2e91e2, + 0x59b6786d, 0xc2133402, 0xb58de6f2, 0x2e28aa9d, 0x489503e8, + 0xd3304f87, 0xa4ae9d77, 0x3f0bd118, 0x4b933897, 0xd03674f8, + 0xa7a8a608, 0x3c0dea67, 0x4e997516, 0xd53c3979, 0xa2a2eb89, + 0x3907a7e6, 0x4d9f4e69, 0xd63a0206, 0xa1a4d0f6, 0x3a019c99, + 0x448dee14, 0xdf28a27b, 0xa8b6708b, 0x33133ce4, 0x478bd56b, + 0xdc2e9904, 0xabb04bf4, 0x3015079b, 0x428198ea, 0xd924d485, + 0xaeba0675, 0x351f4a1a, 0x4187a395, 0xda22effa, 0xadbc3d0a, + 0x36197165}, + {0x00000000, 0xc18edfc0, 0x586cb9c1, 0x99e26601, 0xb0d97382, + 0x7157ac42, 0xe8b5ca43, 0x293b1583, 0xbac3e145, 0x7b4d3e85, + 0xe2af5884, 0x23218744, 0x0a1a92c7, 0xcb944d07, 0x52762b06, + 0x93f8f4c6, 0xaef6c4cb, 0x6f781b0b, 0xf69a7d0a, 0x3714a2ca, + 0x1e2fb749, 0xdfa16889, 0x46430e88, 0x87cdd148, 0x1435258e, + 0xd5bbfa4e, 0x4c599c4f, 0x8dd7438f, 0xa4ec560c, 0x656289cc, + 0xfc80efcd, 0x3d0e300d, 0x869c8fd7, 0x47125017, 0xdef03616, + 0x1f7ee9d6, 0x3645fc55, 0xf7cb2395, 0x6e294594, 0xafa79a54, + 0x3c5f6e92, 0xfdd1b152, 0x6433d753, 0xa5bd0893, 0x8c861d10, + 0x4d08c2d0, 0xd4eaa4d1, 0x15647b11, 0x286a4b1c, 0xe9e494dc, + 0x7006f2dd, 0xb1882d1d, 0x98b3389e, 0x593de75e, 0xc0df815f, + 0x01515e9f, 0x92a9aa59, 0x53277599, 0xcac51398, 0x0b4bcc58, + 0x2270d9db, 0xe3fe061b, 0x7a1c601a, 0xbb92bfda, 0xd64819ef, + 0x17c6c62f, 0x8e24a02e, 0x4faa7fee, 0x66916a6d, 0xa71fb5ad, + 0x3efdd3ac, 0xff730c6c, 0x6c8bf8aa, 0xad05276a, 0x34e7416b, + 0xf5699eab, 0xdc528b28, 0x1ddc54e8, 0x843e32e9, 0x45b0ed29, + 0x78bedd24, 0xb93002e4, 0x20d264e5, 0xe15cbb25, 0xc867aea6, + 0x09e97166, 0x900b1767, 0x5185c8a7, 0xc27d3c61, 0x03f3e3a1, + 0x9a1185a0, 0x5b9f5a60, 0x72a44fe3, 0xb32a9023, 0x2ac8f622, + 0xeb4629e2, 0x50d49638, 0x915a49f8, 0x08b82ff9, 0xc936f039, + 0xe00de5ba, 0x21833a7a, 0xb8615c7b, 0x79ef83bb, 0xea17777d, + 0x2b99a8bd, 0xb27bcebc, 0x73f5117c, 0x5ace04ff, 0x9b40db3f, + 0x02a2bd3e, 0xc32c62fe, 0xfe2252f3, 0x3fac8d33, 0xa64eeb32, + 0x67c034f2, 0x4efb2171, 0x8f75feb1, 0x169798b0, 0xd7194770, + 0x44e1b3b6, 0x856f6c76, 0x1c8d0a77, 0xdd03d5b7, 0xf438c034, + 0x35b61ff4, 0xac5479f5, 0x6ddaa635, 0x77e1359f, 0xb66fea5f, + 0x2f8d8c5e, 0xee03539e, 0xc738461d, 0x06b699dd, 0x9f54ffdc, + 0x5eda201c, 0xcd22d4da, 0x0cac0b1a, 0x954e6d1b, 0x54c0b2db, + 0x7dfba758, 0xbc757898, 0x25971e99, 0xe419c159, 0xd917f154, + 0x18992e94, 0x817b4895, 0x40f59755, 0x69ce82d6, 0xa8405d16, + 0x31a23b17, 0xf02ce4d7, 0x63d41011, 0xa25acfd1, 0x3bb8a9d0, + 0xfa367610, 0xd30d6393, 0x1283bc53, 0x8b61da52, 0x4aef0592, + 0xf17dba48, 0x30f36588, 0xa9110389, 0x689fdc49, 0x41a4c9ca, + 0x802a160a, 0x19c8700b, 0xd846afcb, 0x4bbe5b0d, 0x8a3084cd, + 0x13d2e2cc, 0xd25c3d0c, 0xfb67288f, 0x3ae9f74f, 0xa30b914e, + 0x62854e8e, 0x5f8b7e83, 0x9e05a143, 0x07e7c742, 0xc6691882, + 0xef520d01, 0x2edcd2c1, 0xb73eb4c0, 0x76b06b00, 0xe5489fc6, + 0x24c64006, 0xbd242607, 0x7caaf9c7, 0x5591ec44, 0x941f3384, + 0x0dfd5585, 0xcc738a45, 0xa1a92c70, 0x6027f3b0, 0xf9c595b1, + 0x384b4a71, 0x11705ff2, 0xd0fe8032, 0x491ce633, 0x889239f3, + 0x1b6acd35, 0xdae412f5, 0x430674f4, 0x8288ab34, 0xabb3beb7, + 0x6a3d6177, 0xf3df0776, 0x3251d8b6, 0x0f5fe8bb, 0xced1377b, + 0x5733517a, 0x96bd8eba, 0xbf869b39, 0x7e0844f9, 0xe7ea22f8, + 0x2664fd38, 0xb59c09fe, 0x7412d63e, 0xedf0b03f, 0x2c7e6fff, + 0x05457a7c, 0xc4cba5bc, 0x5d29c3bd, 0x9ca71c7d, 0x2735a3a7, + 0xe6bb7c67, 0x7f591a66, 0xbed7c5a6, 0x97ecd025, 0x56620fe5, + 0xcf8069e4, 0x0e0eb624, 0x9df642e2, 0x5c789d22, 0xc59afb23, + 0x041424e3, 0x2d2f3160, 0xeca1eea0, 0x754388a1, 0xb4cd5761, + 0x89c3676c, 0x484db8ac, 0xd1afdead, 0x1021016d, 0x391a14ee, + 0xf894cb2e, 0x6176ad2f, 0xa0f872ef, 0x33008629, 0xf28e59e9, + 0x6b6c3fe8, 0xaae2e028, 0x83d9f5ab, 0x42572a6b, 0xdbb54c6a, + 0x1a3b93aa}, + {0x00000000, 0xefc26b3e, 0x04f5d03d, 0xeb37bb03, 0x09eba07a, + 0xe629cb44, 0x0d1e7047, 0xe2dc1b79, 0x13d740f4, 0xfc152bca, + 0x172290c9, 0xf8e0fbf7, 0x1a3ce08e, 0xf5fe8bb0, 0x1ec930b3, + 0xf10b5b8d, 0x27ae81e8, 0xc86cead6, 0x235b51d5, 0xcc993aeb, + 0x2e452192, 0xc1874aac, 0x2ab0f1af, 0xc5729a91, 0x3479c11c, + 0xdbbbaa22, 0x308c1121, 0xdf4e7a1f, 0x3d926166, 0xd2500a58, + 0x3967b15b, 0xd6a5da65, 0x4f5d03d0, 0xa09f68ee, 0x4ba8d3ed, + 0xa46ab8d3, 0x46b6a3aa, 0xa974c894, 0x42437397, 0xad8118a9, + 0x5c8a4324, 0xb348281a, 0x587f9319, 0xb7bdf827, 0x5561e35e, + 0xbaa38860, 0x51943363, 0xbe56585d, 0x68f38238, 0x8731e906, + 0x6c065205, 0x83c4393b, 0x61182242, 0x8eda497c, 0x65edf27f, + 0x8a2f9941, 0x7b24c2cc, 0x94e6a9f2, 0x7fd112f1, 0x901379cf, + 0x72cf62b6, 0x9d0d0988, 0x763ab28b, 0x99f8d9b5, 0x9eba07a0, + 0x71786c9e, 0x9a4fd79d, 0x758dbca3, 0x9751a7da, 0x7893cce4, + 0x93a477e7, 0x7c661cd9, 0x8d6d4754, 0x62af2c6a, 0x89989769, + 0x665afc57, 0x8486e72e, 0x6b448c10, 0x80733713, 0x6fb15c2d, + 0xb9148648, 0x56d6ed76, 0xbde15675, 0x52233d4b, 0xb0ff2632, + 0x5f3d4d0c, 0xb40af60f, 0x5bc89d31, 0xaac3c6bc, 0x4501ad82, + 0xae361681, 0x41f47dbf, 0xa32866c6, 0x4cea0df8, 0xa7ddb6fb, + 0x481fddc5, 0xd1e70470, 0x3e256f4e, 0xd512d44d, 0x3ad0bf73, + 0xd80ca40a, 0x37cecf34, 0xdcf97437, 0x333b1f09, 0xc2304484, + 0x2df22fba, 0xc6c594b9, 0x2907ff87, 0xcbdbe4fe, 0x24198fc0, + 0xcf2e34c3, 0x20ec5ffd, 0xf6498598, 0x198beea6, 0xf2bc55a5, + 0x1d7e3e9b, 0xffa225e2, 0x10604edc, 0xfb57f5df, 0x14959ee1, + 0xe59ec56c, 0x0a5cae52, 0xe16b1551, 0x0ea97e6f, 0xec756516, + 0x03b70e28, 0xe880b52b, 0x0742de15, 0xe6050901, 0x09c7623f, + 0xe2f0d93c, 0x0d32b202, 0xefeea97b, 0x002cc245, 0xeb1b7946, + 0x04d91278, 0xf5d249f5, 0x1a1022cb, 0xf12799c8, 0x1ee5f2f6, + 0xfc39e98f, 0x13fb82b1, 0xf8cc39b2, 0x170e528c, 0xc1ab88e9, + 0x2e69e3d7, 0xc55e58d4, 0x2a9c33ea, 0xc8402893, 0x278243ad, + 0xccb5f8ae, 0x23779390, 0xd27cc81d, 0x3dbea323, 0xd6891820, + 0x394b731e, 0xdb976867, 0x34550359, 0xdf62b85a, 0x30a0d364, + 0xa9580ad1, 0x469a61ef, 0xadaddaec, 0x426fb1d2, 0xa0b3aaab, + 0x4f71c195, 0xa4467a96, 0x4b8411a8, 0xba8f4a25, 0x554d211b, + 0xbe7a9a18, 0x51b8f126, 0xb364ea5f, 0x5ca68161, 0xb7913a62, + 0x5853515c, 0x8ef68b39, 0x6134e007, 0x8a035b04, 0x65c1303a, + 0x871d2b43, 0x68df407d, 0x83e8fb7e, 0x6c2a9040, 0x9d21cbcd, + 0x72e3a0f3, 0x99d41bf0, 0x761670ce, 0x94ca6bb7, 0x7b080089, + 0x903fbb8a, 0x7ffdd0b4, 0x78bf0ea1, 0x977d659f, 0x7c4ade9c, + 0x9388b5a2, 0x7154aedb, 0x9e96c5e5, 0x75a17ee6, 0x9a6315d8, + 0x6b684e55, 0x84aa256b, 0x6f9d9e68, 0x805ff556, 0x6283ee2f, + 0x8d418511, 0x66763e12, 0x89b4552c, 0x5f118f49, 0xb0d3e477, + 0x5be45f74, 0xb426344a, 0x56fa2f33, 0xb938440d, 0x520fff0e, + 0xbdcd9430, 0x4cc6cfbd, 0xa304a483, 0x48331f80, 0xa7f174be, + 0x452d6fc7, 0xaaef04f9, 0x41d8bffa, 0xae1ad4c4, 0x37e20d71, + 0xd820664f, 0x3317dd4c, 0xdcd5b672, 0x3e09ad0b, 0xd1cbc635, + 0x3afc7d36, 0xd53e1608, 0x24354d85, 0xcbf726bb, 0x20c09db8, + 0xcf02f686, 0x2ddeedff, 0xc21c86c1, 0x292b3dc2, 0xc6e956fc, + 0x104c8c99, 0xff8ee7a7, 0x14b95ca4, 0xfb7b379a, 0x19a72ce3, + 0xf66547dd, 0x1d52fcde, 0xf29097e0, 0x039bcc6d, 0xec59a753, + 0x076e1c50, 0xe8ac776e, 0x0a706c17, 0xe5b20729, 0x0e85bc2a, + 0xe147d714}, + {0x00000000, 0x177b1443, 0x2ef62886, 0x398d3cc5, 0x5dec510c, + 0x4a97454f, 0x731a798a, 0x64616dc9, 0xbbd8a218, 0xaca3b65b, + 0x952e8a9e, 0x82559edd, 0xe634f314, 0xf14fe757, 0xc8c2db92, + 0xdfb9cfd1, 0xacc04271, 0xbbbb5632, 0x82366af7, 0x954d7eb4, + 0xf12c137d, 0xe657073e, 0xdfda3bfb, 0xc8a12fb8, 0x1718e069, + 0x0063f42a, 0x39eec8ef, 0x2e95dcac, 0x4af4b165, 0x5d8fa526, + 0x640299e3, 0x73798da0, 0x82f182a3, 0x958a96e0, 0xac07aa25, + 0xbb7cbe66, 0xdf1dd3af, 0xc866c7ec, 0xf1ebfb29, 0xe690ef6a, + 0x392920bb, 0x2e5234f8, 0x17df083d, 0x00a41c7e, 0x64c571b7, + 0x73be65f4, 0x4a335931, 0x5d484d72, 0x2e31c0d2, 0x394ad491, + 0x00c7e854, 0x17bcfc17, 0x73dd91de, 0x64a6859d, 0x5d2bb958, + 0x4a50ad1b, 0x95e962ca, 0x82927689, 0xbb1f4a4c, 0xac645e0f, + 0xc80533c6, 0xdf7e2785, 0xe6f31b40, 0xf1880f03, 0xde920307, + 0xc9e91744, 0xf0642b81, 0xe71f3fc2, 0x837e520b, 0x94054648, + 0xad887a8d, 0xbaf36ece, 0x654aa11f, 0x7231b55c, 0x4bbc8999, + 0x5cc79dda, 0x38a6f013, 0x2fdde450, 0x1650d895, 0x012bccd6, + 0x72524176, 0x65295535, 0x5ca469f0, 0x4bdf7db3, 0x2fbe107a, + 0x38c50439, 0x014838fc, 0x16332cbf, 0xc98ae36e, 0xdef1f72d, + 0xe77ccbe8, 0xf007dfab, 0x9466b262, 0x831da621, 0xba909ae4, + 0xadeb8ea7, 0x5c6381a4, 0x4b1895e7, 0x7295a922, 0x65eebd61, + 0x018fd0a8, 0x16f4c4eb, 0x2f79f82e, 0x3802ec6d, 0xe7bb23bc, + 0xf0c037ff, 0xc94d0b3a, 0xde361f79, 0xba5772b0, 0xad2c66f3, + 0x94a15a36, 0x83da4e75, 0xf0a3c3d5, 0xe7d8d796, 0xde55eb53, + 0xc92eff10, 0xad4f92d9, 0xba34869a, 0x83b9ba5f, 0x94c2ae1c, + 0x4b7b61cd, 0x5c00758e, 0x658d494b, 0x72f65d08, 0x169730c1, + 0x01ec2482, 0x38611847, 0x2f1a0c04, 0x6655004f, 0x712e140c, + 0x48a328c9, 0x5fd83c8a, 0x3bb95143, 0x2cc24500, 0x154f79c5, + 0x02346d86, 0xdd8da257, 0xcaf6b614, 0xf37b8ad1, 0xe4009e92, + 0x8061f35b, 0x971ae718, 0xae97dbdd, 0xb9eccf9e, 0xca95423e, + 0xddee567d, 0xe4636ab8, 0xf3187efb, 0x97791332, 0x80020771, + 0xb98f3bb4, 0xaef42ff7, 0x714de026, 0x6636f465, 0x5fbbc8a0, + 0x48c0dce3, 0x2ca1b12a, 0x3bdaa569, 0x025799ac, 0x152c8def, + 0xe4a482ec, 0xf3df96af, 0xca52aa6a, 0xdd29be29, 0xb948d3e0, + 0xae33c7a3, 0x97befb66, 0x80c5ef25, 0x5f7c20f4, 0x480734b7, + 0x718a0872, 0x66f11c31, 0x029071f8, 0x15eb65bb, 0x2c66597e, + 0x3b1d4d3d, 0x4864c09d, 0x5f1fd4de, 0x6692e81b, 0x71e9fc58, + 0x15889191, 0x02f385d2, 0x3b7eb917, 0x2c05ad54, 0xf3bc6285, + 0xe4c776c6, 0xdd4a4a03, 0xca315e40, 0xae503389, 0xb92b27ca, + 0x80a61b0f, 0x97dd0f4c, 0xb8c70348, 0xafbc170b, 0x96312bce, + 0x814a3f8d, 0xe52b5244, 0xf2504607, 0xcbdd7ac2, 0xdca66e81, + 0x031fa150, 0x1464b513, 0x2de989d6, 0x3a929d95, 0x5ef3f05c, + 0x4988e41f, 0x7005d8da, 0x677ecc99, 0x14074139, 0x037c557a, + 0x3af169bf, 0x2d8a7dfc, 0x49eb1035, 0x5e900476, 0x671d38b3, + 0x70662cf0, 0xafdfe321, 0xb8a4f762, 0x8129cba7, 0x9652dfe4, + 0xf233b22d, 0xe548a66e, 0xdcc59aab, 0xcbbe8ee8, 0x3a3681eb, + 0x2d4d95a8, 0x14c0a96d, 0x03bbbd2e, 0x67dad0e7, 0x70a1c4a4, + 0x492cf861, 0x5e57ec22, 0x81ee23f3, 0x969537b0, 0xaf180b75, + 0xb8631f36, 0xdc0272ff, 0xcb7966bc, 0xf2f45a79, 0xe58f4e3a, + 0x96f6c39a, 0x818dd7d9, 0xb800eb1c, 0xaf7bff5f, 0xcb1a9296, + 0xdc6186d5, 0xe5ecba10, 0xf297ae53, 0x2d2e6182, 0x3a5575c1, + 0x03d84904, 0x14a35d47, 0x70c2308e, 0x67b924cd, 0x5e341808, + 0x494f0c4b}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x00000000, 0x43147b17, 0x8628f62e, 0xc53c8d39, 0x0c51ec5d, + 0x4f45974a, 0x8a791a73, 0xc96d6164, 0x18a2d8bb, 0x5bb6a3ac, + 0x9e8a2e95, 0xdd9e5582, 0x14f334e6, 0x57e74ff1, 0x92dbc2c8, + 0xd1cfb9df, 0x7142c0ac, 0x3256bbbb, 0xf76a3682, 0xb47e4d95, + 0x7d132cf1, 0x3e0757e6, 0xfb3bdadf, 0xb82fa1c8, 0x69e01817, + 0x2af46300, 0xefc8ee39, 0xacdc952e, 0x65b1f44a, 0x26a58f5d, + 0xe3990264, 0xa08d7973, 0xa382f182, 0xe0968a95, 0x25aa07ac, + 0x66be7cbb, 0xafd31ddf, 0xecc766c8, 0x29fbebf1, 0x6aef90e6, + 0xbb202939, 0xf834522e, 0x3d08df17, 0x7e1ca400, 0xb771c564, + 0xf465be73, 0x3159334a, 0x724d485d, 0xd2c0312e, 0x91d44a39, + 0x54e8c700, 0x17fcbc17, 0xde91dd73, 0x9d85a664, 0x58b92b5d, + 0x1bad504a, 0xca62e995, 0x89769282, 0x4c4a1fbb, 0x0f5e64ac, + 0xc63305c8, 0x85277edf, 0x401bf3e6, 0x030f88f1, 0x070392de, + 0x4417e9c9, 0x812b64f0, 0xc23f1fe7, 0x0b527e83, 0x48460594, + 0x8d7a88ad, 0xce6ef3ba, 0x1fa14a65, 0x5cb53172, 0x9989bc4b, + 0xda9dc75c, 0x13f0a638, 0x50e4dd2f, 0x95d85016, 0xd6cc2b01, + 0x76415272, 0x35552965, 0xf069a45c, 0xb37ddf4b, 0x7a10be2f, + 0x3904c538, 0xfc384801, 0xbf2c3316, 0x6ee38ac9, 0x2df7f1de, + 0xe8cb7ce7, 0xabdf07f0, 0x62b26694, 0x21a61d83, 0xe49a90ba, + 0xa78eebad, 0xa481635c, 0xe795184b, 0x22a99572, 0x61bdee65, + 0xa8d08f01, 0xebc4f416, 0x2ef8792f, 0x6dec0238, 0xbc23bbe7, + 0xff37c0f0, 0x3a0b4dc9, 0x791f36de, 0xb07257ba, 0xf3662cad, + 0x365aa194, 0x754eda83, 0xd5c3a3f0, 0x96d7d8e7, 0x53eb55de, + 0x10ff2ec9, 0xd9924fad, 0x9a8634ba, 0x5fbab983, 0x1caec294, + 0xcd617b4b, 0x8e75005c, 0x4b498d65, 0x085df672, 0xc1309716, + 0x8224ec01, 0x47186138, 0x040c1a2f, 0x4f005566, 0x0c142e71, + 0xc928a348, 0x8a3cd85f, 0x4351b93b, 0x0045c22c, 0xc5794f15, + 0x866d3402, 0x57a28ddd, 0x14b6f6ca, 0xd18a7bf3, 0x929e00e4, + 0x5bf36180, 0x18e71a97, 0xdddb97ae, 0x9ecfecb9, 0x3e4295ca, + 0x7d56eedd, 0xb86a63e4, 0xfb7e18f3, 0x32137997, 0x71070280, + 0xb43b8fb9, 0xf72ff4ae, 0x26e04d71, 0x65f43666, 0xa0c8bb5f, + 0xe3dcc048, 0x2ab1a12c, 0x69a5da3b, 0xac995702, 0xef8d2c15, + 0xec82a4e4, 0xaf96dff3, 0x6aaa52ca, 0x29be29dd, 0xe0d348b9, + 0xa3c733ae, 0x66fbbe97, 0x25efc580, 0xf4207c5f, 0xb7340748, + 0x72088a71, 0x311cf166, 0xf8719002, 0xbb65eb15, 0x7e59662c, + 0x3d4d1d3b, 0x9dc06448, 0xded41f5f, 0x1be89266, 0x58fce971, + 0x91918815, 0xd285f302, 0x17b97e3b, 0x54ad052c, 0x8562bcf3, + 0xc676c7e4, 0x034a4add, 0x405e31ca, 0x893350ae, 0xca272bb9, + 0x0f1ba680, 0x4c0fdd97, 0x4803c7b8, 0x0b17bcaf, 0xce2b3196, + 0x8d3f4a81, 0x44522be5, 0x074650f2, 0xc27addcb, 0x816ea6dc, + 0x50a11f03, 0x13b56414, 0xd689e92d, 0x959d923a, 0x5cf0f35e, + 0x1fe48849, 0xdad80570, 0x99cc7e67, 0x39410714, 0x7a557c03, + 0xbf69f13a, 0xfc7d8a2d, 0x3510eb49, 0x7604905e, 0xb3381d67, + 0xf02c6670, 0x21e3dfaf, 0x62f7a4b8, 0xa7cb2981, 0xe4df5296, + 0x2db233f2, 0x6ea648e5, 0xab9ac5dc, 0xe88ebecb, 0xeb81363a, + 0xa8954d2d, 0x6da9c014, 0x2ebdbb03, 0xe7d0da67, 0xa4c4a170, + 0x61f82c49, 0x22ec575e, 0xf323ee81, 0xb0379596, 0x750b18af, + 0x361f63b8, 0xff7202dc, 0xbc6679cb, 0x795af4f2, 0x3a4e8fe5, + 0x9ac3f696, 0xd9d78d81, 0x1ceb00b8, 0x5fff7baf, 0x96921acb, + 0xd58661dc, 0x10baece5, 0x53ae97f2, 0x82612e2d, 0xc175553a, + 0x0449d803, 0x475da314, 0x8e30c270, 0xcd24b967, 0x0818345e, + 0x4b0c4f49}, + {0x00000000, 0x3e6bc2ef, 0x3dd0f504, 0x03bb37eb, 0x7aa0eb09, + 0x44cb29e6, 0x47701e0d, 0x791bdce2, 0xf440d713, 0xca2b15fc, + 0xc9902217, 0xf7fbe0f8, 0x8ee03c1a, 0xb08bfef5, 0xb330c91e, + 0x8d5b0bf1, 0xe881ae27, 0xd6ea6cc8, 0xd5515b23, 0xeb3a99cc, + 0x9221452e, 0xac4a87c1, 0xaff1b02a, 0x919a72c5, 0x1cc17934, + 0x22aabbdb, 0x21118c30, 0x1f7a4edf, 0x6661923d, 0x580a50d2, + 0x5bb16739, 0x65daa5d6, 0xd0035d4f, 0xee689fa0, 0xedd3a84b, + 0xd3b86aa4, 0xaaa3b646, 0x94c874a9, 0x97734342, 0xa91881ad, + 0x24438a5c, 0x1a2848b3, 0x19937f58, 0x27f8bdb7, 0x5ee36155, + 0x6088a3ba, 0x63339451, 0x5d5856be, 0x3882f368, 0x06e93187, + 0x0552066c, 0x3b39c483, 0x42221861, 0x7c49da8e, 0x7ff2ed65, + 0x41992f8a, 0xccc2247b, 0xf2a9e694, 0xf112d17f, 0xcf791390, + 0xb662cf72, 0x88090d9d, 0x8bb23a76, 0xb5d9f899, 0xa007ba9e, + 0x9e6c7871, 0x9dd74f9a, 0xa3bc8d75, 0xdaa75197, 0xe4cc9378, + 0xe777a493, 0xd91c667c, 0x54476d8d, 0x6a2caf62, 0x69979889, + 0x57fc5a66, 0x2ee78684, 0x108c446b, 0x13377380, 0x2d5cb16f, + 0x488614b9, 0x76edd656, 0x7556e1bd, 0x4b3d2352, 0x3226ffb0, + 0x0c4d3d5f, 0x0ff60ab4, 0x319dc85b, 0xbcc6c3aa, 0x82ad0145, + 0x811636ae, 0xbf7df441, 0xc66628a3, 0xf80dea4c, 0xfbb6dda7, + 0xc5dd1f48, 0x7004e7d1, 0x4e6f253e, 0x4dd412d5, 0x73bfd03a, + 0x0aa40cd8, 0x34cfce37, 0x3774f9dc, 0x091f3b33, 0x844430c2, + 0xba2ff22d, 0xb994c5c6, 0x87ff0729, 0xfee4dbcb, 0xc08f1924, + 0xc3342ecf, 0xfd5fec20, 0x988549f6, 0xa6ee8b19, 0xa555bcf2, + 0x9b3e7e1d, 0xe225a2ff, 0xdc4e6010, 0xdff557fb, 0xe19e9514, + 0x6cc59ee5, 0x52ae5c0a, 0x51156be1, 0x6f7ea90e, 0x166575ec, + 0x280eb703, 0x2bb580e8, 0x15de4207, 0x010905e6, 0x3f62c709, + 0x3cd9f0e2, 0x02b2320d, 0x7ba9eeef, 0x45c22c00, 0x46791beb, + 0x7812d904, 0xf549d2f5, 0xcb22101a, 0xc89927f1, 0xf6f2e51e, + 0x8fe939fc, 0xb182fb13, 0xb239ccf8, 0x8c520e17, 0xe988abc1, + 0xd7e3692e, 0xd4585ec5, 0xea339c2a, 0x932840c8, 0xad438227, + 0xaef8b5cc, 0x90937723, 0x1dc87cd2, 0x23a3be3d, 0x201889d6, + 0x1e734b39, 0x676897db, 0x59035534, 0x5ab862df, 0x64d3a030, + 0xd10a58a9, 0xef619a46, 0xecdaadad, 0xd2b16f42, 0xabaab3a0, + 0x95c1714f, 0x967a46a4, 0xa811844b, 0x254a8fba, 0x1b214d55, + 0x189a7abe, 0x26f1b851, 0x5fea64b3, 0x6181a65c, 0x623a91b7, + 0x5c515358, 0x398bf68e, 0x07e03461, 0x045b038a, 0x3a30c165, + 0x432b1d87, 0x7d40df68, 0x7efbe883, 0x40902a6c, 0xcdcb219d, + 0xf3a0e372, 0xf01bd499, 0xce701676, 0xb76bca94, 0x8900087b, + 0x8abb3f90, 0xb4d0fd7f, 0xa10ebf78, 0x9f657d97, 0x9cde4a7c, + 0xa2b58893, 0xdbae5471, 0xe5c5969e, 0xe67ea175, 0xd815639a, + 0x554e686b, 0x6b25aa84, 0x689e9d6f, 0x56f55f80, 0x2fee8362, + 0x1185418d, 0x123e7666, 0x2c55b489, 0x498f115f, 0x77e4d3b0, + 0x745fe45b, 0x4a3426b4, 0x332ffa56, 0x0d4438b9, 0x0eff0f52, + 0x3094cdbd, 0xbdcfc64c, 0x83a404a3, 0x801f3348, 0xbe74f1a7, + 0xc76f2d45, 0xf904efaa, 0xfabfd841, 0xc4d41aae, 0x710de237, + 0x4f6620d8, 0x4cdd1733, 0x72b6d5dc, 0x0bad093e, 0x35c6cbd1, + 0x367dfc3a, 0x08163ed5, 0x854d3524, 0xbb26f7cb, 0xb89dc020, + 0x86f602cf, 0xffedde2d, 0xc1861cc2, 0xc23d2b29, 0xfc56e9c6, + 0x998c4c10, 0xa7e78eff, 0xa45cb914, 0x9a377bfb, 0xe32ca719, + 0xdd4765f6, 0xdefc521d, 0xe09790f2, 0x6dcc9b03, 0x53a759ec, + 0x501c6e07, 0x6e77ace8, 0x176c700a, 0x2907b2e5, 0x2abc850e, + 0x14d747e1}, + {0x00000000, 0xc0df8ec1, 0xc1b96c58, 0x0166e299, 0x8273d9b0, + 0x42ac5771, 0x43cab5e8, 0x83153b29, 0x45e1c3ba, 0x853e4d7b, + 0x8458afe2, 0x44872123, 0xc7921a0a, 0x074d94cb, 0x062b7652, + 0xc6f4f893, 0xcbc4f6ae, 0x0b1b786f, 0x0a7d9af6, 0xcaa21437, + 0x49b72f1e, 0x8968a1df, 0x880e4346, 0x48d1cd87, 0x8e253514, + 0x4efabbd5, 0x4f9c594c, 0x8f43d78d, 0x0c56eca4, 0xcc896265, + 0xcdef80fc, 0x0d300e3d, 0xd78f9c86, 0x17501247, 0x1636f0de, + 0xd6e97e1f, 0x55fc4536, 0x9523cbf7, 0x9445296e, 0x549aa7af, + 0x926e5f3c, 0x52b1d1fd, 0x53d73364, 0x9308bda5, 0x101d868c, + 0xd0c2084d, 0xd1a4ead4, 0x117b6415, 0x1c4b6a28, 0xdc94e4e9, + 0xddf20670, 0x1d2d88b1, 0x9e38b398, 0x5ee73d59, 0x5f81dfc0, + 0x9f5e5101, 0x59aaa992, 0x99752753, 0x9813c5ca, 0x58cc4b0b, + 0xdbd97022, 0x1b06fee3, 0x1a601c7a, 0xdabf92bb, 0xef1948d6, + 0x2fc6c617, 0x2ea0248e, 0xee7faa4f, 0x6d6a9166, 0xadb51fa7, + 0xacd3fd3e, 0x6c0c73ff, 0xaaf88b6c, 0x6a2705ad, 0x6b41e734, + 0xab9e69f5, 0x288b52dc, 0xe854dc1d, 0xe9323e84, 0x29edb045, + 0x24ddbe78, 0xe40230b9, 0xe564d220, 0x25bb5ce1, 0xa6ae67c8, + 0x6671e909, 0x67170b90, 0xa7c88551, 0x613c7dc2, 0xa1e3f303, + 0xa085119a, 0x605a9f5b, 0xe34fa472, 0x23902ab3, 0x22f6c82a, + 0xe22946eb, 0x3896d450, 0xf8495a91, 0xf92fb808, 0x39f036c9, + 0xbae50de0, 0x7a3a8321, 0x7b5c61b8, 0xbb83ef79, 0x7d7717ea, + 0xbda8992b, 0xbcce7bb2, 0x7c11f573, 0xff04ce5a, 0x3fdb409b, + 0x3ebda202, 0xfe622cc3, 0xf35222fe, 0x338dac3f, 0x32eb4ea6, + 0xf234c067, 0x7121fb4e, 0xb1fe758f, 0xb0989716, 0x704719d7, + 0xb6b3e144, 0x766c6f85, 0x770a8d1c, 0xb7d503dd, 0x34c038f4, + 0xf41fb635, 0xf57954ac, 0x35a6da6d, 0x9f35e177, 0x5fea6fb6, + 0x5e8c8d2f, 0x9e5303ee, 0x1d4638c7, 0xdd99b606, 0xdcff549f, + 0x1c20da5e, 0xdad422cd, 0x1a0bac0c, 0x1b6d4e95, 0xdbb2c054, + 0x58a7fb7d, 0x987875bc, 0x991e9725, 0x59c119e4, 0x54f117d9, + 0x942e9918, 0x95487b81, 0x5597f540, 0xd682ce69, 0x165d40a8, + 0x173ba231, 0xd7e42cf0, 0x1110d463, 0xd1cf5aa2, 0xd0a9b83b, + 0x107636fa, 0x93630dd3, 0x53bc8312, 0x52da618b, 0x9205ef4a, + 0x48ba7df1, 0x8865f330, 0x890311a9, 0x49dc9f68, 0xcac9a441, + 0x0a162a80, 0x0b70c819, 0xcbaf46d8, 0x0d5bbe4b, 0xcd84308a, + 0xcce2d213, 0x0c3d5cd2, 0x8f2867fb, 0x4ff7e93a, 0x4e910ba3, + 0x8e4e8562, 0x837e8b5f, 0x43a1059e, 0x42c7e707, 0x821869c6, + 0x010d52ef, 0xc1d2dc2e, 0xc0b43eb7, 0x006bb076, 0xc69f48e5, + 0x0640c624, 0x072624bd, 0xc7f9aa7c, 0x44ec9155, 0x84331f94, + 0x8555fd0d, 0x458a73cc, 0x702ca9a1, 0xb0f32760, 0xb195c5f9, + 0x714a4b38, 0xf25f7011, 0x3280fed0, 0x33e61c49, 0xf3399288, + 0x35cd6a1b, 0xf512e4da, 0xf4740643, 0x34ab8882, 0xb7beb3ab, + 0x77613d6a, 0x7607dff3, 0xb6d85132, 0xbbe85f0f, 0x7b37d1ce, + 0x7a513357, 0xba8ebd96, 0x399b86bf, 0xf944087e, 0xf822eae7, + 0x38fd6426, 0xfe099cb5, 0x3ed61274, 0x3fb0f0ed, 0xff6f7e2c, + 0x7c7a4505, 0xbca5cbc4, 0xbdc3295d, 0x7d1ca79c, 0xa7a33527, + 0x677cbbe6, 0x661a597f, 0xa6c5d7be, 0x25d0ec97, 0xe50f6256, + 0xe46980cf, 0x24b60e0e, 0xe242f69d, 0x229d785c, 0x23fb9ac5, + 0xe3241404, 0x60312f2d, 0xa0eea1ec, 0xa1884375, 0x6157cdb4, + 0x6c67c389, 0xacb84d48, 0xaddeafd1, 0x6d012110, 0xee141a39, + 0x2ecb94f8, 0x2fad7661, 0xef72f8a0, 0x29860033, 0xe9598ef2, + 0xe83f6c6b, 0x28e0e2aa, 0xabf5d983, 0x6b2a5742, 0x6a4cb5db, + 0xaa933b1a}, + {0x00000000, 0x6f4ca59b, 0x9f9e3bec, 0xf0d29e77, 0x7f3b0603, + 0x1077a398, 0xe0a53def, 0x8fe99874, 0xfe760c06, 0x913aa99d, + 0x61e837ea, 0x0ea49271, 0x814d0a05, 0xee01af9e, 0x1ed331e9, + 0x719f9472, 0xfced180c, 0x93a1bd97, 0x637323e0, 0x0c3f867b, + 0x83d61e0f, 0xec9abb94, 0x1c4825e3, 0x73048078, 0x029b140a, + 0x6dd7b191, 0x9d052fe6, 0xf2498a7d, 0x7da01209, 0x12ecb792, + 0xe23e29e5, 0x8d728c7e, 0xf8db3118, 0x97979483, 0x67450af4, + 0x0809af6f, 0x87e0371b, 0xe8ac9280, 0x187e0cf7, 0x7732a96c, + 0x06ad3d1e, 0x69e19885, 0x993306f2, 0xf67fa369, 0x79963b1d, + 0x16da9e86, 0xe60800f1, 0x8944a56a, 0x04362914, 0x6b7a8c8f, + 0x9ba812f8, 0xf4e4b763, 0x7b0d2f17, 0x14418a8c, 0xe49314fb, + 0x8bdfb160, 0xfa402512, 0x950c8089, 0x65de1efe, 0x0a92bb65, + 0x857b2311, 0xea37868a, 0x1ae518fd, 0x75a9bd66, 0xf0b76330, + 0x9ffbc6ab, 0x6f2958dc, 0x0065fd47, 0x8f8c6533, 0xe0c0c0a8, + 0x10125edf, 0x7f5efb44, 0x0ec16f36, 0x618dcaad, 0x915f54da, + 0xfe13f141, 0x71fa6935, 0x1eb6ccae, 0xee6452d9, 0x8128f742, + 0x0c5a7b3c, 0x6316dea7, 0x93c440d0, 0xfc88e54b, 0x73617d3f, + 0x1c2dd8a4, 0xecff46d3, 0x83b3e348, 0xf22c773a, 0x9d60d2a1, + 0x6db24cd6, 0x02fee94d, 0x8d177139, 0xe25bd4a2, 0x12894ad5, + 0x7dc5ef4e, 0x086c5228, 0x6720f7b3, 0x97f269c4, 0xf8becc5f, + 0x7757542b, 0x181bf1b0, 0xe8c96fc7, 0x8785ca5c, 0xf61a5e2e, + 0x9956fbb5, 0x698465c2, 0x06c8c059, 0x8921582d, 0xe66dfdb6, + 0x16bf63c1, 0x79f3c65a, 0xf4814a24, 0x9bcdefbf, 0x6b1f71c8, + 0x0453d453, 0x8bba4c27, 0xe4f6e9bc, 0x142477cb, 0x7b68d250, + 0x0af74622, 0x65bbe3b9, 0x95697dce, 0xfa25d855, 0x75cc4021, + 0x1a80e5ba, 0xea527bcd, 0x851ede56, 0xe06fc760, 0x8f2362fb, + 0x7ff1fc8c, 0x10bd5917, 0x9f54c163, 0xf01864f8, 0x00cafa8f, + 0x6f865f14, 0x1e19cb66, 0x71556efd, 0x8187f08a, 0xeecb5511, + 0x6122cd65, 0x0e6e68fe, 0xfebcf689, 0x91f05312, 0x1c82df6c, + 0x73ce7af7, 0x831ce480, 0xec50411b, 0x63b9d96f, 0x0cf57cf4, + 0xfc27e283, 0x936b4718, 0xe2f4d36a, 0x8db876f1, 0x7d6ae886, + 0x12264d1d, 0x9dcfd569, 0xf28370f2, 0x0251ee85, 0x6d1d4b1e, + 0x18b4f678, 0x77f853e3, 0x872acd94, 0xe866680f, 0x678ff07b, + 0x08c355e0, 0xf811cb97, 0x975d6e0c, 0xe6c2fa7e, 0x898e5fe5, + 0x795cc192, 0x16106409, 0x99f9fc7d, 0xf6b559e6, 0x0667c791, + 0x692b620a, 0xe459ee74, 0x8b154bef, 0x7bc7d598, 0x148b7003, + 0x9b62e877, 0xf42e4dec, 0x04fcd39b, 0x6bb07600, 0x1a2fe272, + 0x756347e9, 0x85b1d99e, 0xeafd7c05, 0x6514e471, 0x0a5841ea, + 0xfa8adf9d, 0x95c67a06, 0x10d8a450, 0x7f9401cb, 0x8f469fbc, + 0xe00a3a27, 0x6fe3a253, 0x00af07c8, 0xf07d99bf, 0x9f313c24, + 0xeeaea856, 0x81e20dcd, 0x713093ba, 0x1e7c3621, 0x9195ae55, + 0xfed90bce, 0x0e0b95b9, 0x61473022, 0xec35bc5c, 0x837919c7, + 0x73ab87b0, 0x1ce7222b, 0x930eba5f, 0xfc421fc4, 0x0c9081b3, + 0x63dc2428, 0x1243b05a, 0x7d0f15c1, 0x8ddd8bb6, 0xe2912e2d, + 0x6d78b659, 0x023413c2, 0xf2e68db5, 0x9daa282e, 0xe8039548, + 0x874f30d3, 0x779daea4, 0x18d10b3f, 0x9738934b, 0xf87436d0, + 0x08a6a8a7, 0x67ea0d3c, 0x1675994e, 0x79393cd5, 0x89eba2a2, + 0xe6a70739, 0x694e9f4d, 0x06023ad6, 0xf6d0a4a1, 0x999c013a, + 0x14ee8d44, 0x7ba228df, 0x8b70b6a8, 0xe43c1333, 0x6bd58b47, + 0x04992edc, 0xf44bb0ab, 0x9b071530, 0xea988142, 0x85d424d9, + 0x7506baae, 0x1a4a1f35, 0x95a38741, 0xfaef22da, 0x0a3dbcad, + 0x65711936}}; + +#endif + +#endif + +#if N == 4 + +#if W == 8 + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0xf1da05aa, 0x38c50d15, 0xc91f08bf, 0x718a1a2a, + 0x80501f80, 0x494f173f, 0xb8951295, 0xe3143454, 0x12ce31fe, + 0xdbd13941, 0x2a0b3ceb, 0x929e2e7e, 0x63442bd4, 0xaa5b236b, + 0x5b8126c1, 0x1d596ee9, 0xec836b43, 0x259c63fc, 0xd4466656, + 0x6cd374c3, 0x9d097169, 0x541679d6, 0xa5cc7c7c, 0xfe4d5abd, + 0x0f975f17, 0xc68857a8, 0x37525202, 0x8fc74097, 0x7e1d453d, + 0xb7024d82, 0x46d84828, 0x3ab2ddd2, 0xcb68d878, 0x0277d0c7, + 0xf3add56d, 0x4b38c7f8, 0xbae2c252, 0x73fdcaed, 0x8227cf47, + 0xd9a6e986, 0x287cec2c, 0xe163e493, 0x10b9e139, 0xa82cf3ac, + 0x59f6f606, 0x90e9feb9, 0x6133fb13, 0x27ebb33b, 0xd631b691, + 0x1f2ebe2e, 0xeef4bb84, 0x5661a911, 0xa7bbacbb, 0x6ea4a404, + 0x9f7ea1ae, 0xc4ff876f, 0x352582c5, 0xfc3a8a7a, 0x0de08fd0, + 0xb5759d45, 0x44af98ef, 0x8db09050, 0x7c6a95fa, 0x7565bba4, + 0x84bfbe0e, 0x4da0b6b1, 0xbc7ab31b, 0x04efa18e, 0xf535a424, + 0x3c2aac9b, 0xcdf0a931, 0x96718ff0, 0x67ab8a5a, 0xaeb482e5, + 0x5f6e874f, 0xe7fb95da, 0x16219070, 0xdf3e98cf, 0x2ee49d65, + 0x683cd54d, 0x99e6d0e7, 0x50f9d858, 0xa123ddf2, 0x19b6cf67, + 0xe86ccacd, 0x2173c272, 0xd0a9c7d8, 0x8b28e119, 0x7af2e4b3, + 0xb3edec0c, 0x4237e9a6, 0xfaa2fb33, 0x0b78fe99, 0xc267f626, + 0x33bdf38c, 0x4fd76676, 0xbe0d63dc, 0x77126b63, 0x86c86ec9, + 0x3e5d7c5c, 0xcf8779f6, 0x06987149, 0xf74274e3, 0xacc35222, + 0x5d195788, 0x94065f37, 0x65dc5a9d, 0xdd494808, 0x2c934da2, + 0xe58c451d, 0x145640b7, 0x528e089f, 0xa3540d35, 0x6a4b058a, + 0x9b910020, 0x230412b5, 0xd2de171f, 0x1bc11fa0, 0xea1b1a0a, + 0xb19a3ccb, 0x40403961, 0x895f31de, 0x78853474, 0xc01026e1, + 0x31ca234b, 0xf8d52bf4, 0x090f2e5e, 0xeacb7748, 0x1b1172e2, + 0xd20e7a5d, 0x23d47ff7, 0x9b416d62, 0x6a9b68c8, 0xa3846077, + 0x525e65dd, 0x09df431c, 0xf80546b6, 0x311a4e09, 0xc0c04ba3, + 0x78555936, 0x898f5c9c, 0x40905423, 0xb14a5189, 0xf79219a1, + 0x06481c0b, 0xcf5714b4, 0x3e8d111e, 0x8618038b, 0x77c20621, + 0xbedd0e9e, 0x4f070b34, 0x14862df5, 0xe55c285f, 0x2c4320e0, + 0xdd99254a, 0x650c37df, 0x94d63275, 0x5dc93aca, 0xac133f60, + 0xd079aa9a, 0x21a3af30, 0xe8bca78f, 0x1966a225, 0xa1f3b0b0, + 0x5029b51a, 0x9936bda5, 0x68ecb80f, 0x336d9ece, 0xc2b79b64, + 0x0ba893db, 0xfa729671, 0x42e784e4, 0xb33d814e, 0x7a2289f1, + 0x8bf88c5b, 0xcd20c473, 0x3cfac1d9, 0xf5e5c966, 0x043fcccc, + 0xbcaade59, 0x4d70dbf3, 0x846fd34c, 0x75b5d6e6, 0x2e34f027, + 0xdfeef58d, 0x16f1fd32, 0xe72bf898, 0x5fbeea0d, 0xae64efa7, + 0x677be718, 0x96a1e2b2, 0x9faeccec, 0x6e74c946, 0xa76bc1f9, + 0x56b1c453, 0xee24d6c6, 0x1ffed36c, 0xd6e1dbd3, 0x273bde79, + 0x7cbaf8b8, 0x8d60fd12, 0x447ff5ad, 0xb5a5f007, 0x0d30e292, + 0xfceae738, 0x35f5ef87, 0xc42fea2d, 0x82f7a205, 0x732da7af, + 0xba32af10, 0x4be8aaba, 0xf37db82f, 0x02a7bd85, 0xcbb8b53a, + 0x3a62b090, 0x61e39651, 0x903993fb, 0x59269b44, 0xa8fc9eee, + 0x10698c7b, 0xe1b389d1, 0x28ac816e, 0xd97684c4, 0xa51c113e, + 0x54c61494, 0x9dd91c2b, 0x6c031981, 0xd4960b14, 0x254c0ebe, + 0xec530601, 0x1d8903ab, 0x4608256a, 0xb7d220c0, 0x7ecd287f, + 0x8f172dd5, 0x37823f40, 0xc6583aea, 0x0f473255, 0xfe9d37ff, + 0xb8457fd7, 0x499f7a7d, 0x808072c2, 0x715a7768, 0xc9cf65fd, + 0x38156057, 0xf10a68e8, 0x00d06d42, 0x5b514b83, 0xaa8b4e29, + 0x63944696, 0x924e433c, 0x2adb51a9, 0xdb015403, 0x121e5cbc, + 0xe3c45916}, + {0x00000000, 0x0ee7e8d1, 0x1dcfd1a2, 0x13283973, 0x3b9fa344, + 0x35784b95, 0x265072e6, 0x28b79a37, 0x773f4688, 0x79d8ae59, + 0x6af0972a, 0x64177ffb, 0x4ca0e5cc, 0x42470d1d, 0x516f346e, + 0x5f88dcbf, 0xee7e8d10, 0xe09965c1, 0xf3b15cb2, 0xfd56b463, + 0xd5e12e54, 0xdb06c685, 0xc82efff6, 0xc6c91727, 0x9941cb98, + 0x97a62349, 0x848e1a3a, 0x8a69f2eb, 0xa2de68dc, 0xac39800d, + 0xbf11b97e, 0xb1f651af, 0x078c1c61, 0x096bf4b0, 0x1a43cdc3, + 0x14a42512, 0x3c13bf25, 0x32f457f4, 0x21dc6e87, 0x2f3b8656, + 0x70b35ae9, 0x7e54b238, 0x6d7c8b4b, 0x639b639a, 0x4b2cf9ad, + 0x45cb117c, 0x56e3280f, 0x5804c0de, 0xe9f29171, 0xe71579a0, + 0xf43d40d3, 0xfadaa802, 0xd26d3235, 0xdc8adae4, 0xcfa2e397, + 0xc1450b46, 0x9ecdd7f9, 0x902a3f28, 0x8302065b, 0x8de5ee8a, + 0xa55274bd, 0xabb59c6c, 0xb89da51f, 0xb67a4dce, 0x0f1838c2, + 0x01ffd013, 0x12d7e960, 0x1c3001b1, 0x34879b86, 0x3a607357, + 0x29484a24, 0x27afa2f5, 0x78277e4a, 0x76c0969b, 0x65e8afe8, + 0x6b0f4739, 0x43b8dd0e, 0x4d5f35df, 0x5e770cac, 0x5090e47d, + 0xe166b5d2, 0xef815d03, 0xfca96470, 0xf24e8ca1, 0xdaf91696, + 0xd41efe47, 0xc736c734, 0xc9d12fe5, 0x9659f35a, 0x98be1b8b, + 0x8b9622f8, 0x8571ca29, 0xadc6501e, 0xa321b8cf, 0xb00981bc, + 0xbeee696d, 0x089424a3, 0x0673cc72, 0x155bf501, 0x1bbc1dd0, + 0x330b87e7, 0x3dec6f36, 0x2ec45645, 0x2023be94, 0x7fab622b, + 0x714c8afa, 0x6264b389, 0x6c835b58, 0x4434c16f, 0x4ad329be, + 0x59fb10cd, 0x571cf81c, 0xe6eaa9b3, 0xe80d4162, 0xfb257811, + 0xf5c290c0, 0xdd750af7, 0xd392e226, 0xc0badb55, 0xce5d3384, + 0x91d5ef3b, 0x9f3207ea, 0x8c1a3e99, 0x82fdd648, 0xaa4a4c7f, + 0xa4ada4ae, 0xb7859ddd, 0xb962750c, 0x1e307184, 0x10d79955, + 0x03ffa026, 0x0d1848f7, 0x25afd2c0, 0x2b483a11, 0x38600362, + 0x3687ebb3, 0x690f370c, 0x67e8dfdd, 0x74c0e6ae, 0x7a270e7f, + 0x52909448, 0x5c777c99, 0x4f5f45ea, 0x41b8ad3b, 0xf04efc94, + 0xfea91445, 0xed812d36, 0xe366c5e7, 0xcbd15fd0, 0xc536b701, + 0xd61e8e72, 0xd8f966a3, 0x8771ba1c, 0x899652cd, 0x9abe6bbe, + 0x9459836f, 0xbcee1958, 0xb209f189, 0xa121c8fa, 0xafc6202b, + 0x19bc6de5, 0x175b8534, 0x0473bc47, 0x0a945496, 0x2223cea1, + 0x2cc42670, 0x3fec1f03, 0x310bf7d2, 0x6e832b6d, 0x6064c3bc, + 0x734cfacf, 0x7dab121e, 0x551c8829, 0x5bfb60f8, 0x48d3598b, + 0x4634b15a, 0xf7c2e0f5, 0xf9250824, 0xea0d3157, 0xe4ead986, + 0xcc5d43b1, 0xc2baab60, 0xd1929213, 0xdf757ac2, 0x80fda67d, + 0x8e1a4eac, 0x9d3277df, 0x93d59f0e, 0xbb620539, 0xb585ede8, + 0xa6add49b, 0xa84a3c4a, 0x11284946, 0x1fcfa197, 0x0ce798e4, + 0x02007035, 0x2ab7ea02, 0x245002d3, 0x37783ba0, 0x399fd371, + 0x66170fce, 0x68f0e71f, 0x7bd8de6c, 0x753f36bd, 0x5d88ac8a, + 0x536f445b, 0x40477d28, 0x4ea095f9, 0xff56c456, 0xf1b12c87, + 0xe29915f4, 0xec7efd25, 0xc4c96712, 0xca2e8fc3, 0xd906b6b0, + 0xd7e15e61, 0x886982de, 0x868e6a0f, 0x95a6537c, 0x9b41bbad, + 0xb3f6219a, 0xbd11c94b, 0xae39f038, 0xa0de18e9, 0x16a45527, + 0x1843bdf6, 0x0b6b8485, 0x058c6c54, 0x2d3bf663, 0x23dc1eb2, + 0x30f427c1, 0x3e13cf10, 0x619b13af, 0x6f7cfb7e, 0x7c54c20d, + 0x72b32adc, 0x5a04b0eb, 0x54e3583a, 0x47cb6149, 0x492c8998, + 0xf8dad837, 0xf63d30e6, 0xe5150995, 0xebf2e144, 0xc3457b73, + 0xcda293a2, 0xde8aaad1, 0xd06d4200, 0x8fe59ebf, 0x8102766e, + 0x922a4f1d, 0x9ccda7cc, 0xb47a3dfb, 0xba9dd52a, 0xa9b5ec59, + 0xa7520488}, + {0x00000000, 0x3c60e308, 0x78c1c610, 0x44a12518, 0xf1838c20, + 0xcde36f28, 0x89424a30, 0xb522a938, 0x38761e01, 0x0416fd09, + 0x40b7d811, 0x7cd73b19, 0xc9f59221, 0xf5957129, 0xb1345431, + 0x8d54b739, 0x70ec3c02, 0x4c8cdf0a, 0x082dfa12, 0x344d191a, + 0x816fb022, 0xbd0f532a, 0xf9ae7632, 0xc5ce953a, 0x489a2203, + 0x74fac10b, 0x305be413, 0x0c3b071b, 0xb919ae23, 0x85794d2b, + 0xc1d86833, 0xfdb88b3b, 0xe1d87804, 0xddb89b0c, 0x9919be14, + 0xa5795d1c, 0x105bf424, 0x2c3b172c, 0x689a3234, 0x54fad13c, + 0xd9ae6605, 0xe5ce850d, 0xa16fa015, 0x9d0f431d, 0x282dea25, + 0x144d092d, 0x50ec2c35, 0x6c8ccf3d, 0x91344406, 0xad54a70e, + 0xe9f58216, 0xd595611e, 0x60b7c826, 0x5cd72b2e, 0x18760e36, + 0x2416ed3e, 0xa9425a07, 0x9522b90f, 0xd1839c17, 0xede37f1f, + 0x58c1d627, 0x64a1352f, 0x20001037, 0x1c60f33f, 0x18c1f649, + 0x24a11541, 0x60003059, 0x5c60d351, 0xe9427a69, 0xd5229961, + 0x9183bc79, 0xade35f71, 0x20b7e848, 0x1cd70b40, 0x58762e58, + 0x6416cd50, 0xd1346468, 0xed548760, 0xa9f5a278, 0x95954170, + 0x682dca4b, 0x544d2943, 0x10ec0c5b, 0x2c8cef53, 0x99ae466b, + 0xa5cea563, 0xe16f807b, 0xdd0f6373, 0x505bd44a, 0x6c3b3742, + 0x289a125a, 0x14faf152, 0xa1d8586a, 0x9db8bb62, 0xd9199e7a, + 0xe5797d72, 0xf9198e4d, 0xc5796d45, 0x81d8485d, 0xbdb8ab55, + 0x089a026d, 0x34fae165, 0x705bc47d, 0x4c3b2775, 0xc16f904c, + 0xfd0f7344, 0xb9ae565c, 0x85ceb554, 0x30ec1c6c, 0x0c8cff64, + 0x482dda7c, 0x744d3974, 0x89f5b24f, 0xb5955147, 0xf134745f, + 0xcd549757, 0x78763e6f, 0x4416dd67, 0x00b7f87f, 0x3cd71b77, + 0xb183ac4e, 0x8de34f46, 0xc9426a5e, 0xf5228956, 0x4000206e, + 0x7c60c366, 0x38c1e67e, 0x04a10576, 0x3183ec92, 0x0de30f9a, + 0x49422a82, 0x7522c98a, 0xc00060b2, 0xfc6083ba, 0xb8c1a6a2, + 0x84a145aa, 0x09f5f293, 0x3595119b, 0x71343483, 0x4d54d78b, + 0xf8767eb3, 0xc4169dbb, 0x80b7b8a3, 0xbcd75bab, 0x416fd090, + 0x7d0f3398, 0x39ae1680, 0x05cef588, 0xb0ec5cb0, 0x8c8cbfb8, + 0xc82d9aa0, 0xf44d79a8, 0x7919ce91, 0x45792d99, 0x01d80881, + 0x3db8eb89, 0x889a42b1, 0xb4faa1b9, 0xf05b84a1, 0xcc3b67a9, + 0xd05b9496, 0xec3b779e, 0xa89a5286, 0x94fab18e, 0x21d818b6, + 0x1db8fbbe, 0x5919dea6, 0x65793dae, 0xe82d8a97, 0xd44d699f, + 0x90ec4c87, 0xac8caf8f, 0x19ae06b7, 0x25cee5bf, 0x616fc0a7, + 0x5d0f23af, 0xa0b7a894, 0x9cd74b9c, 0xd8766e84, 0xe4168d8c, + 0x513424b4, 0x6d54c7bc, 0x29f5e2a4, 0x159501ac, 0x98c1b695, + 0xa4a1559d, 0xe0007085, 0xdc60938d, 0x69423ab5, 0x5522d9bd, + 0x1183fca5, 0x2de31fad, 0x29421adb, 0x1522f9d3, 0x5183dccb, + 0x6de33fc3, 0xd8c196fb, 0xe4a175f3, 0xa00050eb, 0x9c60b3e3, + 0x113404da, 0x2d54e7d2, 0x69f5c2ca, 0x559521c2, 0xe0b788fa, + 0xdcd76bf2, 0x98764eea, 0xa416ade2, 0x59ae26d9, 0x65cec5d1, + 0x216fe0c9, 0x1d0f03c1, 0xa82daaf9, 0x944d49f1, 0xd0ec6ce9, + 0xec8c8fe1, 0x61d838d8, 0x5db8dbd0, 0x1919fec8, 0x25791dc0, + 0x905bb4f8, 0xac3b57f0, 0xe89a72e8, 0xd4fa91e0, 0xc89a62df, + 0xf4fa81d7, 0xb05ba4cf, 0x8c3b47c7, 0x3919eeff, 0x05790df7, + 0x41d828ef, 0x7db8cbe7, 0xf0ec7cde, 0xcc8c9fd6, 0x882dbace, + 0xb44d59c6, 0x016ff0fe, 0x3d0f13f6, 0x79ae36ee, 0x45ced5e6, + 0xb8765edd, 0x8416bdd5, 0xc0b798cd, 0xfcd77bc5, 0x49f5d2fd, + 0x759531f5, 0x313414ed, 0x0d54f7e5, 0x800040dc, 0xbc60a3d4, + 0xf8c186cc, 0xc4a165c4, 0x7183ccfc, 0x4de32ff4, 0x09420aec, + 0x3522e9e4}, + {0x00000000, 0x6307d924, 0xc60fb248, 0xa5086b6c, 0x576e62d1, + 0x3469bbf5, 0x9161d099, 0xf26609bd, 0xaedcc5a2, 0xcddb1c86, + 0x68d377ea, 0x0bd4aece, 0xf9b2a773, 0x9ab57e57, 0x3fbd153b, + 0x5cbacc1f, 0x86c88d05, 0xe5cf5421, 0x40c73f4d, 0x23c0e669, + 0xd1a6efd4, 0xb2a136f0, 0x17a95d9c, 0x74ae84b8, 0x281448a7, + 0x4b139183, 0xee1bfaef, 0x8d1c23cb, 0x7f7a2a76, 0x1c7df352, + 0xb975983e, 0xda72411a, 0xd6e01c4b, 0xb5e7c56f, 0x10efae03, + 0x73e87727, 0x818e7e9a, 0xe289a7be, 0x4781ccd2, 0x248615f6, + 0x783cd9e9, 0x1b3b00cd, 0xbe336ba1, 0xdd34b285, 0x2f52bb38, + 0x4c55621c, 0xe95d0970, 0x8a5ad054, 0x5028914e, 0x332f486a, + 0x96272306, 0xf520fa22, 0x0746f39f, 0x64412abb, 0xc14941d7, + 0xa24e98f3, 0xfef454ec, 0x9df38dc8, 0x38fbe6a4, 0x5bfc3f80, + 0xa99a363d, 0xca9def19, 0x6f958475, 0x0c925d51, 0x76b13ed7, + 0x15b6e7f3, 0xb0be8c9f, 0xd3b955bb, 0x21df5c06, 0x42d88522, + 0xe7d0ee4e, 0x84d7376a, 0xd86dfb75, 0xbb6a2251, 0x1e62493d, + 0x7d659019, 0x8f0399a4, 0xec044080, 0x490c2bec, 0x2a0bf2c8, + 0xf079b3d2, 0x937e6af6, 0x3676019a, 0x5571d8be, 0xa717d103, + 0xc4100827, 0x6118634b, 0x021fba6f, 0x5ea57670, 0x3da2af54, + 0x98aac438, 0xfbad1d1c, 0x09cb14a1, 0x6acccd85, 0xcfc4a6e9, + 0xacc37fcd, 0xa051229c, 0xc356fbb8, 0x665e90d4, 0x055949f0, + 0xf73f404d, 0x94389969, 0x3130f205, 0x52372b21, 0x0e8de73e, + 0x6d8a3e1a, 0xc8825576, 0xab858c52, 0x59e385ef, 0x3ae45ccb, + 0x9fec37a7, 0xfcebee83, 0x2699af99, 0x459e76bd, 0xe0961dd1, + 0x8391c4f5, 0x71f7cd48, 0x12f0146c, 0xb7f87f00, 0xd4ffa624, + 0x88456a3b, 0xeb42b31f, 0x4e4ad873, 0x2d4d0157, 0xdf2b08ea, + 0xbc2cd1ce, 0x1924baa2, 0x7a236386, 0xed627dae, 0x8e65a48a, + 0x2b6dcfe6, 0x486a16c2, 0xba0c1f7f, 0xd90bc65b, 0x7c03ad37, + 0x1f047413, 0x43beb80c, 0x20b96128, 0x85b10a44, 0xe6b6d360, + 0x14d0dadd, 0x77d703f9, 0xd2df6895, 0xb1d8b1b1, 0x6baaf0ab, + 0x08ad298f, 0xada542e3, 0xcea29bc7, 0x3cc4927a, 0x5fc34b5e, + 0xfacb2032, 0x99ccf916, 0xc5763509, 0xa671ec2d, 0x03798741, + 0x607e5e65, 0x921857d8, 0xf11f8efc, 0x5417e590, 0x37103cb4, + 0x3b8261e5, 0x5885b8c1, 0xfd8dd3ad, 0x9e8a0a89, 0x6cec0334, + 0x0febda10, 0xaae3b17c, 0xc9e46858, 0x955ea447, 0xf6597d63, + 0x5351160f, 0x3056cf2b, 0xc230c696, 0xa1371fb2, 0x043f74de, + 0x6738adfa, 0xbd4aece0, 0xde4d35c4, 0x7b455ea8, 0x1842878c, + 0xea248e31, 0x89235715, 0x2c2b3c79, 0x4f2ce55d, 0x13962942, + 0x7091f066, 0xd5999b0a, 0xb69e422e, 0x44f84b93, 0x27ff92b7, + 0x82f7f9db, 0xe1f020ff, 0x9bd34379, 0xf8d49a5d, 0x5ddcf131, + 0x3edb2815, 0xccbd21a8, 0xafbaf88c, 0x0ab293e0, 0x69b54ac4, + 0x350f86db, 0x56085fff, 0xf3003493, 0x9007edb7, 0x6261e40a, + 0x01663d2e, 0xa46e5642, 0xc7698f66, 0x1d1bce7c, 0x7e1c1758, + 0xdb147c34, 0xb813a510, 0x4a75acad, 0x29727589, 0x8c7a1ee5, + 0xef7dc7c1, 0xb3c70bde, 0xd0c0d2fa, 0x75c8b996, 0x16cf60b2, + 0xe4a9690f, 0x87aeb02b, 0x22a6db47, 0x41a10263, 0x4d335f32, + 0x2e348616, 0x8b3ced7a, 0xe83b345e, 0x1a5d3de3, 0x795ae4c7, + 0xdc528fab, 0xbf55568f, 0xe3ef9a90, 0x80e843b4, 0x25e028d8, + 0x46e7f1fc, 0xb481f841, 0xd7862165, 0x728e4a09, 0x1189932d, + 0xcbfbd237, 0xa8fc0b13, 0x0df4607f, 0x6ef3b95b, 0x9c95b0e6, + 0xff9269c2, 0x5a9a02ae, 0x399ddb8a, 0x65271795, 0x0620ceb1, + 0xa328a5dd, 0xc02f7cf9, 0x32497544, 0x514eac60, 0xf446c70c, + 0x97411e28}, + {0x00000000, 0x01b5fd1d, 0x036bfa3a, 0x02de0727, 0x06d7f474, + 0x07620969, 0x05bc0e4e, 0x0409f353, 0x0dafe8e8, 0x0c1a15f5, + 0x0ec412d2, 0x0f71efcf, 0x0b781c9c, 0x0acde181, 0x0813e6a6, + 0x09a61bbb, 0x1b5fd1d0, 0x1aea2ccd, 0x18342bea, 0x1981d6f7, + 0x1d8825a4, 0x1c3dd8b9, 0x1ee3df9e, 0x1f562283, 0x16f03938, + 0x1745c425, 0x159bc302, 0x142e3e1f, 0x1027cd4c, 0x11923051, + 0x134c3776, 0x12f9ca6b, 0x36bfa3a0, 0x370a5ebd, 0x35d4599a, + 0x3461a487, 0x306857d4, 0x31ddaac9, 0x3303adee, 0x32b650f3, + 0x3b104b48, 0x3aa5b655, 0x387bb172, 0x39ce4c6f, 0x3dc7bf3c, + 0x3c724221, 0x3eac4506, 0x3f19b81b, 0x2de07270, 0x2c558f6d, + 0x2e8b884a, 0x2f3e7557, 0x2b378604, 0x2a827b19, 0x285c7c3e, + 0x29e98123, 0x204f9a98, 0x21fa6785, 0x232460a2, 0x22919dbf, + 0x26986eec, 0x272d93f1, 0x25f394d6, 0x244669cb, 0x6d7f4740, + 0x6ccaba5d, 0x6e14bd7a, 0x6fa14067, 0x6ba8b334, 0x6a1d4e29, + 0x68c3490e, 0x6976b413, 0x60d0afa8, 0x616552b5, 0x63bb5592, + 0x620ea88f, 0x66075bdc, 0x67b2a6c1, 0x656ca1e6, 0x64d95cfb, + 0x76209690, 0x77956b8d, 0x754b6caa, 0x74fe91b7, 0x70f762e4, + 0x71429ff9, 0x739c98de, 0x722965c3, 0x7b8f7e78, 0x7a3a8365, + 0x78e48442, 0x7951795f, 0x7d588a0c, 0x7ced7711, 0x7e337036, + 0x7f868d2b, 0x5bc0e4e0, 0x5a7519fd, 0x58ab1eda, 0x591ee3c7, + 0x5d171094, 0x5ca2ed89, 0x5e7ceaae, 0x5fc917b3, 0x566f0c08, + 0x57daf115, 0x5504f632, 0x54b10b2f, 0x50b8f87c, 0x510d0561, + 0x53d30246, 0x5266ff5b, 0x409f3530, 0x412ac82d, 0x43f4cf0a, + 0x42413217, 0x4648c144, 0x47fd3c59, 0x45233b7e, 0x4496c663, + 0x4d30ddd8, 0x4c8520c5, 0x4e5b27e2, 0x4feedaff, 0x4be729ac, + 0x4a52d4b1, 0x488cd396, 0x49392e8b, 0xdafe8e80, 0xdb4b739d, + 0xd99574ba, 0xd82089a7, 0xdc297af4, 0xdd9c87e9, 0xdf4280ce, + 0xdef77dd3, 0xd7516668, 0xd6e49b75, 0xd43a9c52, 0xd58f614f, + 0xd186921c, 0xd0336f01, 0xd2ed6826, 0xd358953b, 0xc1a15f50, + 0xc014a24d, 0xc2caa56a, 0xc37f5877, 0xc776ab24, 0xc6c35639, + 0xc41d511e, 0xc5a8ac03, 0xcc0eb7b8, 0xcdbb4aa5, 0xcf654d82, + 0xced0b09f, 0xcad943cc, 0xcb6cbed1, 0xc9b2b9f6, 0xc80744eb, + 0xec412d20, 0xedf4d03d, 0xef2ad71a, 0xee9f2a07, 0xea96d954, + 0xeb232449, 0xe9fd236e, 0xe848de73, 0xe1eec5c8, 0xe05b38d5, + 0xe2853ff2, 0xe330c2ef, 0xe73931bc, 0xe68ccca1, 0xe452cb86, + 0xe5e7369b, 0xf71efcf0, 0xf6ab01ed, 0xf47506ca, 0xf5c0fbd7, + 0xf1c90884, 0xf07cf599, 0xf2a2f2be, 0xf3170fa3, 0xfab11418, + 0xfb04e905, 0xf9daee22, 0xf86f133f, 0xfc66e06c, 0xfdd31d71, + 0xff0d1a56, 0xfeb8e74b, 0xb781c9c0, 0xb63434dd, 0xb4ea33fa, + 0xb55fcee7, 0xb1563db4, 0xb0e3c0a9, 0xb23dc78e, 0xb3883a93, + 0xba2e2128, 0xbb9bdc35, 0xb945db12, 0xb8f0260f, 0xbcf9d55c, + 0xbd4c2841, 0xbf922f66, 0xbe27d27b, 0xacde1810, 0xad6be50d, + 0xafb5e22a, 0xae001f37, 0xaa09ec64, 0xabbc1179, 0xa962165e, + 0xa8d7eb43, 0xa171f0f8, 0xa0c40de5, 0xa21a0ac2, 0xa3aff7df, + 0xa7a6048c, 0xa613f991, 0xa4cdfeb6, 0xa57803ab, 0x813e6a60, + 0x808b977d, 0x8255905a, 0x83e06d47, 0x87e99e14, 0x865c6309, + 0x8482642e, 0x85379933, 0x8c918288, 0x8d247f95, 0x8ffa78b2, + 0x8e4f85af, 0x8a4676fc, 0x8bf38be1, 0x892d8cc6, 0x889871db, + 0x9a61bbb0, 0x9bd446ad, 0x990a418a, 0x98bfbc97, 0x9cb64fc4, + 0x9d03b2d9, 0x9fddb5fe, 0x9e6848e3, 0x97ce5358, 0x967bae45, + 0x94a5a962, 0x9510547f, 0x9119a72c, 0x90ac5a31, 0x92725d16, + 0x93c7a00b}, + {0x00000000, 0x6e8c1b41, 0xdd183682, 0xb3942dc3, 0x61416b45, + 0x0fcd7004, 0xbc595dc7, 0xd2d54686, 0xc282d68a, 0xac0ecdcb, + 0x1f9ae008, 0x7116fb49, 0xa3c3bdcf, 0xcd4fa68e, 0x7edb8b4d, + 0x1057900c, 0x5e74ab55, 0x30f8b014, 0x836c9dd7, 0xede08696, + 0x3f35c010, 0x51b9db51, 0xe22df692, 0x8ca1edd3, 0x9cf67ddf, + 0xf27a669e, 0x41ee4b5d, 0x2f62501c, 0xfdb7169a, 0x933b0ddb, + 0x20af2018, 0x4e233b59, 0xbce956aa, 0xd2654deb, 0x61f16028, + 0x0f7d7b69, 0xdda83def, 0xb32426ae, 0x00b00b6d, 0x6e3c102c, + 0x7e6b8020, 0x10e79b61, 0xa373b6a2, 0xcdffade3, 0x1f2aeb65, + 0x71a6f024, 0xc232dde7, 0xacbec6a6, 0xe29dfdff, 0x8c11e6be, + 0x3f85cb7d, 0x5109d03c, 0x83dc96ba, 0xed508dfb, 0x5ec4a038, + 0x3048bb79, 0x201f2b75, 0x4e933034, 0xfd071df7, 0x938b06b6, + 0x415e4030, 0x2fd25b71, 0x9c4676b2, 0xf2ca6df3, 0xa2a3ab15, + 0xcc2fb054, 0x7fbb9d97, 0x113786d6, 0xc3e2c050, 0xad6edb11, + 0x1efaf6d2, 0x7076ed93, 0x60217d9f, 0x0ead66de, 0xbd394b1d, + 0xd3b5505c, 0x016016da, 0x6fec0d9b, 0xdc782058, 0xb2f43b19, + 0xfcd70040, 0x925b1b01, 0x21cf36c2, 0x4f432d83, 0x9d966b05, + 0xf31a7044, 0x408e5d87, 0x2e0246c6, 0x3e55d6ca, 0x50d9cd8b, + 0xe34de048, 0x8dc1fb09, 0x5f14bd8f, 0x3198a6ce, 0x820c8b0d, + 0xec80904c, 0x1e4afdbf, 0x70c6e6fe, 0xc352cb3d, 0xadded07c, + 0x7f0b96fa, 0x11878dbb, 0xa213a078, 0xcc9fbb39, 0xdcc82b35, + 0xb2443074, 0x01d01db7, 0x6f5c06f6, 0xbd894070, 0xd3055b31, + 0x609176f2, 0x0e1d6db3, 0x403e56ea, 0x2eb24dab, 0x9d266068, + 0xf3aa7b29, 0x217f3daf, 0x4ff326ee, 0xfc670b2d, 0x92eb106c, + 0x82bc8060, 0xec309b21, 0x5fa4b6e2, 0x3128ada3, 0xe3fdeb25, + 0x8d71f064, 0x3ee5dda7, 0x5069c6e6, 0x9e36506b, 0xf0ba4b2a, + 0x432e66e9, 0x2da27da8, 0xff773b2e, 0x91fb206f, 0x226f0dac, + 0x4ce316ed, 0x5cb486e1, 0x32389da0, 0x81acb063, 0xef20ab22, + 0x3df5eda4, 0x5379f6e5, 0xe0eddb26, 0x8e61c067, 0xc042fb3e, + 0xaecee07f, 0x1d5acdbc, 0x73d6d6fd, 0xa103907b, 0xcf8f8b3a, + 0x7c1ba6f9, 0x1297bdb8, 0x02c02db4, 0x6c4c36f5, 0xdfd81b36, + 0xb1540077, 0x638146f1, 0x0d0d5db0, 0xbe997073, 0xd0156b32, + 0x22df06c1, 0x4c531d80, 0xffc73043, 0x914b2b02, 0x439e6d84, + 0x2d1276c5, 0x9e865b06, 0xf00a4047, 0xe05dd04b, 0x8ed1cb0a, + 0x3d45e6c9, 0x53c9fd88, 0x811cbb0e, 0xef90a04f, 0x5c048d8c, + 0x328896cd, 0x7cabad94, 0x1227b6d5, 0xa1b39b16, 0xcf3f8057, + 0x1deac6d1, 0x7366dd90, 0xc0f2f053, 0xae7eeb12, 0xbe297b1e, + 0xd0a5605f, 0x63314d9c, 0x0dbd56dd, 0xdf68105b, 0xb1e40b1a, + 0x027026d9, 0x6cfc3d98, 0x3c95fb7e, 0x5219e03f, 0xe18dcdfc, + 0x8f01d6bd, 0x5dd4903b, 0x33588b7a, 0x80cca6b9, 0xee40bdf8, + 0xfe172df4, 0x909b36b5, 0x230f1b76, 0x4d830037, 0x9f5646b1, + 0xf1da5df0, 0x424e7033, 0x2cc26b72, 0x62e1502b, 0x0c6d4b6a, + 0xbff966a9, 0xd1757de8, 0x03a03b6e, 0x6d2c202f, 0xdeb80dec, + 0xb03416ad, 0xa06386a1, 0xceef9de0, 0x7d7bb023, 0x13f7ab62, + 0xc122ede4, 0xafaef6a5, 0x1c3adb66, 0x72b6c027, 0x807cadd4, + 0xeef0b695, 0x5d649b56, 0x33e88017, 0xe13dc691, 0x8fb1ddd0, + 0x3c25f013, 0x52a9eb52, 0x42fe7b5e, 0x2c72601f, 0x9fe64ddc, + 0xf16a569d, 0x23bf101b, 0x4d330b5a, 0xfea72699, 0x902b3dd8, + 0xde080681, 0xb0841dc0, 0x03103003, 0x6d9c2b42, 0xbf496dc4, + 0xd1c57685, 0x62515b46, 0x0cdd4007, 0x1c8ad00b, 0x7206cb4a, + 0xc192e689, 0xaf1efdc8, 0x7dcbbb4e, 0x1347a00f, 0xa0d38dcc, + 0xce5f968d}, + {0x00000000, 0xe71da697, 0x154a4b6f, 0xf257edf8, 0x2a9496de, + 0xcd893049, 0x3fdeddb1, 0xd8c37b26, 0x55292dbc, 0xb2348b2b, + 0x406366d3, 0xa77ec044, 0x7fbdbb62, 0x98a01df5, 0x6af7f00d, + 0x8dea569a, 0xaa525b78, 0x4d4ffdef, 0xbf181017, 0x5805b680, + 0x80c6cda6, 0x67db6b31, 0x958c86c9, 0x7291205e, 0xff7b76c4, + 0x1866d053, 0xea313dab, 0x0d2c9b3c, 0xd5efe01a, 0x32f2468d, + 0xc0a5ab75, 0x27b80de2, 0x8fd5b0b1, 0x68c81626, 0x9a9ffbde, + 0x7d825d49, 0xa541266f, 0x425c80f8, 0xb00b6d00, 0x5716cb97, + 0xdafc9d0d, 0x3de13b9a, 0xcfb6d662, 0x28ab70f5, 0xf0680bd3, + 0x1775ad44, 0xe52240bc, 0x023fe62b, 0x2587ebc9, 0xc29a4d5e, + 0x30cda0a6, 0xd7d00631, 0x0f137d17, 0xe80edb80, 0x1a593678, + 0xfd4490ef, 0x70aec675, 0x97b360e2, 0x65e48d1a, 0x82f92b8d, + 0x5a3a50ab, 0xbd27f63c, 0x4f701bc4, 0xa86dbd53, 0xc4da6723, + 0x23c7c1b4, 0xd1902c4c, 0x368d8adb, 0xee4ef1fd, 0x0953576a, + 0xfb04ba92, 0x1c191c05, 0x91f34a9f, 0x76eeec08, 0x84b901f0, + 0x63a4a767, 0xbb67dc41, 0x5c7a7ad6, 0xae2d972e, 0x493031b9, + 0x6e883c5b, 0x89959acc, 0x7bc27734, 0x9cdfd1a3, 0x441caa85, + 0xa3010c12, 0x5156e1ea, 0xb64b477d, 0x3ba111e7, 0xdcbcb770, + 0x2eeb5a88, 0xc9f6fc1f, 0x11358739, 0xf62821ae, 0x047fcc56, + 0xe3626ac1, 0x4b0fd792, 0xac127105, 0x5e459cfd, 0xb9583a6a, + 0x619b414c, 0x8686e7db, 0x74d10a23, 0x93ccacb4, 0x1e26fa2e, + 0xf93b5cb9, 0x0b6cb141, 0xec7117d6, 0x34b26cf0, 0xd3afca67, + 0x21f8279f, 0xc6e58108, 0xe15d8cea, 0x06402a7d, 0xf417c785, + 0x130a6112, 0xcbc91a34, 0x2cd4bca3, 0xde83515b, 0x399ef7cc, + 0xb474a156, 0x536907c1, 0xa13eea39, 0x46234cae, 0x9ee03788, + 0x79fd911f, 0x8baa7ce7, 0x6cb7da70, 0x52c5c807, 0xb5d86e90, + 0x478f8368, 0xa09225ff, 0x78515ed9, 0x9f4cf84e, 0x6d1b15b6, + 0x8a06b321, 0x07ece5bb, 0xe0f1432c, 0x12a6aed4, 0xf5bb0843, + 0x2d787365, 0xca65d5f2, 0x3832380a, 0xdf2f9e9d, 0xf897937f, + 0x1f8a35e8, 0xedddd810, 0x0ac07e87, 0xd20305a1, 0x351ea336, + 0xc7494ece, 0x2054e859, 0xadbebec3, 0x4aa31854, 0xb8f4f5ac, + 0x5fe9533b, 0x872a281d, 0x60378e8a, 0x92606372, 0x757dc5e5, + 0xdd1078b6, 0x3a0dde21, 0xc85a33d9, 0x2f47954e, 0xf784ee68, + 0x109948ff, 0xe2cea507, 0x05d30390, 0x8839550a, 0x6f24f39d, + 0x9d731e65, 0x7a6eb8f2, 0xa2adc3d4, 0x45b06543, 0xb7e788bb, + 0x50fa2e2c, 0x774223ce, 0x905f8559, 0x620868a1, 0x8515ce36, + 0x5dd6b510, 0xbacb1387, 0x489cfe7f, 0xaf8158e8, 0x226b0e72, + 0xc576a8e5, 0x3721451d, 0xd03ce38a, 0x08ff98ac, 0xefe23e3b, + 0x1db5d3c3, 0xfaa87554, 0x961faf24, 0x710209b3, 0x8355e44b, + 0x644842dc, 0xbc8b39fa, 0x5b969f6d, 0xa9c17295, 0x4edcd402, + 0xc3368298, 0x242b240f, 0xd67cc9f7, 0x31616f60, 0xe9a21446, + 0x0ebfb2d1, 0xfce85f29, 0x1bf5f9be, 0x3c4df45c, 0xdb5052cb, + 0x2907bf33, 0xce1a19a4, 0x16d96282, 0xf1c4c415, 0x039329ed, + 0xe48e8f7a, 0x6964d9e0, 0x8e797f77, 0x7c2e928f, 0x9b333418, + 0x43f04f3e, 0xa4ede9a9, 0x56ba0451, 0xb1a7a2c6, 0x19ca1f95, + 0xfed7b902, 0x0c8054fa, 0xeb9df26d, 0x335e894b, 0xd4432fdc, + 0x2614c224, 0xc10964b3, 0x4ce33229, 0xabfe94be, 0x59a97946, + 0xbeb4dfd1, 0x6677a4f7, 0x816a0260, 0x733def98, 0x9420490f, + 0xb39844ed, 0x5485e27a, 0xa6d20f82, 0x41cfa915, 0x990cd233, + 0x7e1174a4, 0x8c46995c, 0x6b5b3fcb, 0xe6b16951, 0x01accfc6, + 0xf3fb223e, 0x14e684a9, 0xcc25ff8f, 0x2b385918, 0xd96fb4e0, + 0x3e721277}, + {0x00000000, 0xa58b900e, 0x9066265d, 0x35edb653, 0xfbbd4afb, + 0x5e36daf5, 0x6bdb6ca6, 0xce50fca8, 0x2c0b93b7, 0x898003b9, + 0xbc6db5ea, 0x19e625e4, 0xd7b6d94c, 0x723d4942, 0x47d0ff11, + 0xe25b6f1f, 0x5817276e, 0xfd9cb760, 0xc8710133, 0x6dfa913d, + 0xa3aa6d95, 0x0621fd9b, 0x33cc4bc8, 0x9647dbc6, 0x741cb4d9, + 0xd19724d7, 0xe47a9284, 0x41f1028a, 0x8fa1fe22, 0x2a2a6e2c, + 0x1fc7d87f, 0xba4c4871, 0xb02e4edc, 0x15a5ded2, 0x20486881, + 0x85c3f88f, 0x4b930427, 0xee189429, 0xdbf5227a, 0x7e7eb274, + 0x9c25dd6b, 0x39ae4d65, 0x0c43fb36, 0xa9c86b38, 0x67989790, + 0xc213079e, 0xf7feb1cd, 0x527521c3, 0xe83969b2, 0x4db2f9bc, + 0x785f4fef, 0xddd4dfe1, 0x13842349, 0xb60fb347, 0x83e20514, + 0x2669951a, 0xc432fa05, 0x61b96a0b, 0x5454dc58, 0xf1df4c56, + 0x3f8fb0fe, 0x9a0420f0, 0xafe996a3, 0x0a6206ad, 0xbb2d9bf9, + 0x1ea60bf7, 0x2b4bbda4, 0x8ec02daa, 0x4090d102, 0xe51b410c, + 0xd0f6f75f, 0x757d6751, 0x9726084e, 0x32ad9840, 0x07402e13, + 0xa2cbbe1d, 0x6c9b42b5, 0xc910d2bb, 0xfcfd64e8, 0x5976f4e6, + 0xe33abc97, 0x46b12c99, 0x735c9aca, 0xd6d70ac4, 0x1887f66c, + 0xbd0c6662, 0x88e1d031, 0x2d6a403f, 0xcf312f20, 0x6ababf2e, + 0x5f57097d, 0xfadc9973, 0x348c65db, 0x9107f5d5, 0xa4ea4386, + 0x0161d388, 0x0b03d525, 0xae88452b, 0x9b65f378, 0x3eee6376, + 0xf0be9fde, 0x55350fd0, 0x60d8b983, 0xc553298d, 0x27084692, + 0x8283d69c, 0xb76e60cf, 0x12e5f0c1, 0xdcb50c69, 0x793e9c67, + 0x4cd32a34, 0xe958ba3a, 0x5314f24b, 0xf69f6245, 0xc372d416, + 0x66f94418, 0xa8a9b8b0, 0x0d2228be, 0x38cf9eed, 0x9d440ee3, + 0x7f1f61fc, 0xda94f1f2, 0xef7947a1, 0x4af2d7af, 0x84a22b07, + 0x2129bb09, 0x14c40d5a, 0xb14f9d54, 0xad2a31b3, 0x08a1a1bd, + 0x3d4c17ee, 0x98c787e0, 0x56977b48, 0xf31ceb46, 0xc6f15d15, + 0x637acd1b, 0x8121a204, 0x24aa320a, 0x11478459, 0xb4cc1457, + 0x7a9ce8ff, 0xdf1778f1, 0xeafacea2, 0x4f715eac, 0xf53d16dd, + 0x50b686d3, 0x655b3080, 0xc0d0a08e, 0x0e805c26, 0xab0bcc28, + 0x9ee67a7b, 0x3b6dea75, 0xd936856a, 0x7cbd1564, 0x4950a337, + 0xecdb3339, 0x228bcf91, 0x87005f9f, 0xb2ede9cc, 0x176679c2, + 0x1d047f6f, 0xb88fef61, 0x8d625932, 0x28e9c93c, 0xe6b93594, + 0x4332a59a, 0x76df13c9, 0xd35483c7, 0x310fecd8, 0x94847cd6, + 0xa169ca85, 0x04e25a8b, 0xcab2a623, 0x6f39362d, 0x5ad4807e, + 0xff5f1070, 0x45135801, 0xe098c80f, 0xd5757e5c, 0x70feee52, + 0xbeae12fa, 0x1b2582f4, 0x2ec834a7, 0x8b43a4a9, 0x6918cbb6, + 0xcc935bb8, 0xf97eedeb, 0x5cf57de5, 0x92a5814d, 0x372e1143, + 0x02c3a710, 0xa748371e, 0x1607aa4a, 0xb38c3a44, 0x86618c17, + 0x23ea1c19, 0xedbae0b1, 0x483170bf, 0x7ddcc6ec, 0xd85756e2, + 0x3a0c39fd, 0x9f87a9f3, 0xaa6a1fa0, 0x0fe18fae, 0xc1b17306, + 0x643ae308, 0x51d7555b, 0xf45cc555, 0x4e108d24, 0xeb9b1d2a, + 0xde76ab79, 0x7bfd3b77, 0xb5adc7df, 0x102657d1, 0x25cbe182, + 0x8040718c, 0x621b1e93, 0xc7908e9d, 0xf27d38ce, 0x57f6a8c0, + 0x99a65468, 0x3c2dc466, 0x09c07235, 0xac4be23b, 0xa629e496, + 0x03a27498, 0x364fc2cb, 0x93c452c5, 0x5d94ae6d, 0xf81f3e63, + 0xcdf28830, 0x6879183e, 0x8a227721, 0x2fa9e72f, 0x1a44517c, + 0xbfcfc172, 0x719f3dda, 0xd414add4, 0xe1f91b87, 0x44728b89, + 0xfe3ec3f8, 0x5bb553f6, 0x6e58e5a5, 0xcbd375ab, 0x05838903, + 0xa008190d, 0x95e5af5e, 0x306e3f50, 0xd235504f, 0x77bec041, + 0x42537612, 0xe7d8e61c, 0x29881ab4, 0x8c038aba, 0xb9ee3ce9, + 0x1c65ace7}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x0000000000000000, 0x0e908ba500000000, 0x5d26669000000000, + 0x53b6ed3500000000, 0xfb4abdfb00000000, 0xf5da365e00000000, + 0xa66cdb6b00000000, 0xa8fc50ce00000000, 0xb7930b2c00000000, + 0xb903808900000000, 0xeab56dbc00000000, 0xe425e61900000000, + 0x4cd9b6d700000000, 0x42493d7200000000, 0x11ffd04700000000, + 0x1f6f5be200000000, 0x6e27175800000000, 0x60b79cfd00000000, + 0x330171c800000000, 0x3d91fa6d00000000, 0x956daaa300000000, + 0x9bfd210600000000, 0xc84bcc3300000000, 0xc6db479600000000, + 0xd9b41c7400000000, 0xd72497d100000000, 0x84927ae400000000, + 0x8a02f14100000000, 0x22fea18f00000000, 0x2c6e2a2a00000000, + 0x7fd8c71f00000000, 0x71484cba00000000, 0xdc4e2eb000000000, + 0xd2dea51500000000, 0x8168482000000000, 0x8ff8c38500000000, + 0x2704934b00000000, 0x299418ee00000000, 0x7a22f5db00000000, + 0x74b27e7e00000000, 0x6bdd259c00000000, 0x654dae3900000000, + 0x36fb430c00000000, 0x386bc8a900000000, 0x9097986700000000, + 0x9e0713c200000000, 0xcdb1fef700000000, 0xc321755200000000, + 0xb26939e800000000, 0xbcf9b24d00000000, 0xef4f5f7800000000, + 0xe1dfd4dd00000000, 0x4923841300000000, 0x47b30fb600000000, + 0x1405e28300000000, 0x1a95692600000000, 0x05fa32c400000000, + 0x0b6ab96100000000, 0x58dc545400000000, 0x564cdff100000000, + 0xfeb08f3f00000000, 0xf020049a00000000, 0xa396e9af00000000, + 0xad06620a00000000, 0xf99b2dbb00000000, 0xf70ba61e00000000, + 0xa4bd4b2b00000000, 0xaa2dc08e00000000, 0x02d1904000000000, + 0x0c411be500000000, 0x5ff7f6d000000000, 0x51677d7500000000, + 0x4e08269700000000, 0x4098ad3200000000, 0x132e400700000000, + 0x1dbecba200000000, 0xb5429b6c00000000, 0xbbd210c900000000, + 0xe864fdfc00000000, 0xe6f4765900000000, 0x97bc3ae300000000, + 0x992cb14600000000, 0xca9a5c7300000000, 0xc40ad7d600000000, + 0x6cf6871800000000, 0x62660cbd00000000, 0x31d0e18800000000, + 0x3f406a2d00000000, 0x202f31cf00000000, 0x2ebfba6a00000000, + 0x7d09575f00000000, 0x7399dcfa00000000, 0xdb658c3400000000, + 0xd5f5079100000000, 0x8643eaa400000000, 0x88d3610100000000, + 0x25d5030b00000000, 0x2b4588ae00000000, 0x78f3659b00000000, + 0x7663ee3e00000000, 0xde9fbef000000000, 0xd00f355500000000, + 0x83b9d86000000000, 0x8d2953c500000000, 0x9246082700000000, + 0x9cd6838200000000, 0xcf606eb700000000, 0xc1f0e51200000000, + 0x690cb5dc00000000, 0x679c3e7900000000, 0x342ad34c00000000, + 0x3aba58e900000000, 0x4bf2145300000000, 0x45629ff600000000, + 0x16d472c300000000, 0x1844f96600000000, 0xb0b8a9a800000000, + 0xbe28220d00000000, 0xed9ecf3800000000, 0xe30e449d00000000, + 0xfc611f7f00000000, 0xf2f194da00000000, 0xa14779ef00000000, + 0xafd7f24a00000000, 0x072ba28400000000, 0x09bb292100000000, + 0x5a0dc41400000000, 0x549d4fb100000000, 0xb3312aad00000000, + 0xbda1a10800000000, 0xee174c3d00000000, 0xe087c79800000000, + 0x487b975600000000, 0x46eb1cf300000000, 0x155df1c600000000, + 0x1bcd7a6300000000, 0x04a2218100000000, 0x0a32aa2400000000, + 0x5984471100000000, 0x5714ccb400000000, 0xffe89c7a00000000, + 0xf17817df00000000, 0xa2cefaea00000000, 0xac5e714f00000000, + 0xdd163df500000000, 0xd386b65000000000, 0x80305b6500000000, + 0x8ea0d0c000000000, 0x265c800e00000000, 0x28cc0bab00000000, + 0x7b7ae69e00000000, 0x75ea6d3b00000000, 0x6a8536d900000000, + 0x6415bd7c00000000, 0x37a3504900000000, 0x3933dbec00000000, + 0x91cf8b2200000000, 0x9f5f008700000000, 0xcce9edb200000000, + 0xc279661700000000, 0x6f7f041d00000000, 0x61ef8fb800000000, + 0x3259628d00000000, 0x3cc9e92800000000, 0x9435b9e600000000, + 0x9aa5324300000000, 0xc913df7600000000, 0xc78354d300000000, + 0xd8ec0f3100000000, 0xd67c849400000000, 0x85ca69a100000000, + 0x8b5ae20400000000, 0x23a6b2ca00000000, 0x2d36396f00000000, + 0x7e80d45a00000000, 0x70105fff00000000, 0x0158134500000000, + 0x0fc898e000000000, 0x5c7e75d500000000, 0x52eefe7000000000, + 0xfa12aebe00000000, 0xf482251b00000000, 0xa734c82e00000000, + 0xa9a4438b00000000, 0xb6cb186900000000, 0xb85b93cc00000000, + 0xebed7ef900000000, 0xe57df55c00000000, 0x4d81a59200000000, + 0x43112e3700000000, 0x10a7c30200000000, 0x1e3748a700000000, + 0x4aaa071600000000, 0x443a8cb300000000, 0x178c618600000000, + 0x191cea2300000000, 0xb1e0baed00000000, 0xbf70314800000000, + 0xecc6dc7d00000000, 0xe25657d800000000, 0xfd390c3a00000000, + 0xf3a9879f00000000, 0xa01f6aaa00000000, 0xae8fe10f00000000, + 0x0673b1c100000000, 0x08e33a6400000000, 0x5b55d75100000000, + 0x55c55cf400000000, 0x248d104e00000000, 0x2a1d9beb00000000, + 0x79ab76de00000000, 0x773bfd7b00000000, 0xdfc7adb500000000, + 0xd157261000000000, 0x82e1cb2500000000, 0x8c71408000000000, + 0x931e1b6200000000, 0x9d8e90c700000000, 0xce387df200000000, + 0xc0a8f65700000000, 0x6854a69900000000, 0x66c42d3c00000000, + 0x3572c00900000000, 0x3be24bac00000000, 0x96e429a600000000, + 0x9874a20300000000, 0xcbc24f3600000000, 0xc552c49300000000, + 0x6dae945d00000000, 0x633e1ff800000000, 0x3088f2cd00000000, + 0x3e18796800000000, 0x2177228a00000000, 0x2fe7a92f00000000, + 0x7c51441a00000000, 0x72c1cfbf00000000, 0xda3d9f7100000000, + 0xd4ad14d400000000, 0x871bf9e100000000, 0x898b724400000000, + 0xf8c33efe00000000, 0xf653b55b00000000, 0xa5e5586e00000000, + 0xab75d3cb00000000, 0x0389830500000000, 0x0d1908a000000000, + 0x5eafe59500000000, 0x503f6e3000000000, 0x4f5035d200000000, + 0x41c0be7700000000, 0x1276534200000000, 0x1ce6d8e700000000, + 0xb41a882900000000, 0xba8a038c00000000, 0xe93ceeb900000000, + 0xe7ac651c00000000}, + {0x0000000000000000, 0x97a61de700000000, 0x6f4b4a1500000000, + 0xf8ed57f200000000, 0xde96942a00000000, 0x493089cd00000000, + 0xb1ddde3f00000000, 0x267bc3d800000000, 0xbc2d295500000000, + 0x2b8b34b200000000, 0xd366634000000000, 0x44c07ea700000000, + 0x62bbbd7f00000000, 0xf51da09800000000, 0x0df0f76a00000000, + 0x9a56ea8d00000000, 0x785b52aa00000000, 0xeffd4f4d00000000, + 0x171018bf00000000, 0x80b6055800000000, 0xa6cdc68000000000, + 0x316bdb6700000000, 0xc9868c9500000000, 0x5e20917200000000, + 0xc4767bff00000000, 0x53d0661800000000, 0xab3d31ea00000000, + 0x3c9b2c0d00000000, 0x1ae0efd500000000, 0x8d46f23200000000, + 0x75aba5c000000000, 0xe20db82700000000, 0xb1b0d58f00000000, + 0x2616c86800000000, 0xdefb9f9a00000000, 0x495d827d00000000, + 0x6f2641a500000000, 0xf8805c4200000000, 0x006d0bb000000000, + 0x97cb165700000000, 0x0d9dfcda00000000, 0x9a3be13d00000000, + 0x62d6b6cf00000000, 0xf570ab2800000000, 0xd30b68f000000000, + 0x44ad751700000000, 0xbc4022e500000000, 0x2be63f0200000000, + 0xc9eb872500000000, 0x5e4d9ac200000000, 0xa6a0cd3000000000, + 0x3106d0d700000000, 0x177d130f00000000, 0x80db0ee800000000, + 0x7836591a00000000, 0xef9044fd00000000, 0x75c6ae7000000000, + 0xe260b39700000000, 0x1a8de46500000000, 0x8d2bf98200000000, + 0xab503a5a00000000, 0x3cf627bd00000000, 0xc41b704f00000000, + 0x53bd6da800000000, 0x2367dac400000000, 0xb4c1c72300000000, + 0x4c2c90d100000000, 0xdb8a8d3600000000, 0xfdf14eee00000000, + 0x6a57530900000000, 0x92ba04fb00000000, 0x051c191c00000000, + 0x9f4af39100000000, 0x08ecee7600000000, 0xf001b98400000000, + 0x67a7a46300000000, 0x41dc67bb00000000, 0xd67a7a5c00000000, + 0x2e972dae00000000, 0xb931304900000000, 0x5b3c886e00000000, + 0xcc9a958900000000, 0x3477c27b00000000, 0xa3d1df9c00000000, + 0x85aa1c4400000000, 0x120c01a300000000, 0xeae1565100000000, + 0x7d474bb600000000, 0xe711a13b00000000, 0x70b7bcdc00000000, + 0x885aeb2e00000000, 0x1ffcf6c900000000, 0x3987351100000000, + 0xae2128f600000000, 0x56cc7f0400000000, 0xc16a62e300000000, + 0x92d70f4b00000000, 0x057112ac00000000, 0xfd9c455e00000000, + 0x6a3a58b900000000, 0x4c419b6100000000, 0xdbe7868600000000, + 0x230ad17400000000, 0xb4accc9300000000, 0x2efa261e00000000, + 0xb95c3bf900000000, 0x41b16c0b00000000, 0xd61771ec00000000, + 0xf06cb23400000000, 0x67caafd300000000, 0x9f27f82100000000, + 0x0881e5c600000000, 0xea8c5de100000000, 0x7d2a400600000000, + 0x85c717f400000000, 0x12610a1300000000, 0x341ac9cb00000000, + 0xa3bcd42c00000000, 0x5b5183de00000000, 0xccf79e3900000000, + 0x56a174b400000000, 0xc107695300000000, 0x39ea3ea100000000, + 0xae4c234600000000, 0x8837e09e00000000, 0x1f91fd7900000000, + 0xe77caa8b00000000, 0x70dab76c00000000, 0x07c8c55200000000, + 0x906ed8b500000000, 0x68838f4700000000, 0xff2592a000000000, + 0xd95e517800000000, 0x4ef84c9f00000000, 0xb6151b6d00000000, + 0x21b3068a00000000, 0xbbe5ec0700000000, 0x2c43f1e000000000, + 0xd4aea61200000000, 0x4308bbf500000000, 0x6573782d00000000, + 0xf2d565ca00000000, 0x0a38323800000000, 0x9d9e2fdf00000000, + 0x7f9397f800000000, 0xe8358a1f00000000, 0x10d8dded00000000, + 0x877ec00a00000000, 0xa10503d200000000, 0x36a31e3500000000, + 0xce4e49c700000000, 0x59e8542000000000, 0xc3bebead00000000, + 0x5418a34a00000000, 0xacf5f4b800000000, 0x3b53e95f00000000, + 0x1d282a8700000000, 0x8a8e376000000000, 0x7263609200000000, + 0xe5c57d7500000000, 0xb67810dd00000000, 0x21de0d3a00000000, + 0xd9335ac800000000, 0x4e95472f00000000, 0x68ee84f700000000, + 0xff48991000000000, 0x07a5cee200000000, 0x9003d30500000000, + 0x0a55398800000000, 0x9df3246f00000000, 0x651e739d00000000, + 0xf2b86e7a00000000, 0xd4c3ada200000000, 0x4365b04500000000, + 0xbb88e7b700000000, 0x2c2efa5000000000, 0xce23427700000000, + 0x59855f9000000000, 0xa168086200000000, 0x36ce158500000000, + 0x10b5d65d00000000, 0x8713cbba00000000, 0x7ffe9c4800000000, + 0xe85881af00000000, 0x720e6b2200000000, 0xe5a876c500000000, + 0x1d45213700000000, 0x8ae33cd000000000, 0xac98ff0800000000, + 0x3b3ee2ef00000000, 0xc3d3b51d00000000, 0x5475a8fa00000000, + 0x24af1f9600000000, 0xb309027100000000, 0x4be4558300000000, + 0xdc42486400000000, 0xfa398bbc00000000, 0x6d9f965b00000000, + 0x9572c1a900000000, 0x02d4dc4e00000000, 0x988236c300000000, + 0x0f242b2400000000, 0xf7c97cd600000000, 0x606f613100000000, + 0x4614a2e900000000, 0xd1b2bf0e00000000, 0x295fe8fc00000000, + 0xbef9f51b00000000, 0x5cf44d3c00000000, 0xcb5250db00000000, + 0x33bf072900000000, 0xa4191ace00000000, 0x8262d91600000000, + 0x15c4c4f100000000, 0xed29930300000000, 0x7a8f8ee400000000, + 0xe0d9646900000000, 0x777f798e00000000, 0x8f922e7c00000000, + 0x1834339b00000000, 0x3e4ff04300000000, 0xa9e9eda400000000, + 0x5104ba5600000000, 0xc6a2a7b100000000, 0x951fca1900000000, + 0x02b9d7fe00000000, 0xfa54800c00000000, 0x6df29deb00000000, + 0x4b895e3300000000, 0xdc2f43d400000000, 0x24c2142600000000, + 0xb36409c100000000, 0x2932e34c00000000, 0xbe94feab00000000, + 0x4679a95900000000, 0xd1dfb4be00000000, 0xf7a4776600000000, + 0x60026a8100000000, 0x98ef3d7300000000, 0x0f49209400000000, + 0xed4498b300000000, 0x7ae2855400000000, 0x820fd2a600000000, + 0x15a9cf4100000000, 0x33d20c9900000000, 0xa474117e00000000, + 0x5c99468c00000000, 0xcb3f5b6b00000000, 0x5169b1e600000000, + 0xc6cfac0100000000, 0x3e22fbf300000000, 0xa984e61400000000, + 0x8fff25cc00000000, 0x1859382b00000000, 0xe0b46fd900000000, + 0x7712723e00000000}, + {0x0000000000000000, 0x411b8c6e00000000, 0x823618dd00000000, + 0xc32d94b300000000, 0x456b416100000000, 0x0470cd0f00000000, + 0xc75d59bc00000000, 0x8646d5d200000000, 0x8ad682c200000000, + 0xcbcd0eac00000000, 0x08e09a1f00000000, 0x49fb167100000000, + 0xcfbdc3a300000000, 0x8ea64fcd00000000, 0x4d8bdb7e00000000, + 0x0c90571000000000, 0x55ab745e00000000, 0x14b0f83000000000, + 0xd79d6c8300000000, 0x9686e0ed00000000, 0x10c0353f00000000, + 0x51dbb95100000000, 0x92f62de200000000, 0xd3eda18c00000000, + 0xdf7df69c00000000, 0x9e667af200000000, 0x5d4bee4100000000, + 0x1c50622f00000000, 0x9a16b7fd00000000, 0xdb0d3b9300000000, + 0x1820af2000000000, 0x593b234e00000000, 0xaa56e9bc00000000, + 0xeb4d65d200000000, 0x2860f16100000000, 0x697b7d0f00000000, + 0xef3da8dd00000000, 0xae2624b300000000, 0x6d0bb00000000000, + 0x2c103c6e00000000, 0x20806b7e00000000, 0x619be71000000000, + 0xa2b673a300000000, 0xe3adffcd00000000, 0x65eb2a1f00000000, + 0x24f0a67100000000, 0xe7dd32c200000000, 0xa6c6beac00000000, + 0xfffd9de200000000, 0xbee6118c00000000, 0x7dcb853f00000000, + 0x3cd0095100000000, 0xba96dc8300000000, 0xfb8d50ed00000000, + 0x38a0c45e00000000, 0x79bb483000000000, 0x752b1f2000000000, + 0x3430934e00000000, 0xf71d07fd00000000, 0xb6068b9300000000, + 0x30405e4100000000, 0x715bd22f00000000, 0xb276469c00000000, + 0xf36dcaf200000000, 0x15aba3a200000000, 0x54b02fcc00000000, + 0x979dbb7f00000000, 0xd686371100000000, 0x50c0e2c300000000, + 0x11db6ead00000000, 0xd2f6fa1e00000000, 0x93ed767000000000, + 0x9f7d216000000000, 0xde66ad0e00000000, 0x1d4b39bd00000000, + 0x5c50b5d300000000, 0xda16600100000000, 0x9b0dec6f00000000, + 0x582078dc00000000, 0x193bf4b200000000, 0x4000d7fc00000000, + 0x011b5b9200000000, 0xc236cf2100000000, 0x832d434f00000000, + 0x056b969d00000000, 0x44701af300000000, 0x875d8e4000000000, + 0xc646022e00000000, 0xcad6553e00000000, 0x8bcdd95000000000, + 0x48e04de300000000, 0x09fbc18d00000000, 0x8fbd145f00000000, + 0xcea6983100000000, 0x0d8b0c8200000000, 0x4c9080ec00000000, + 0xbffd4a1e00000000, 0xfee6c67000000000, 0x3dcb52c300000000, + 0x7cd0dead00000000, 0xfa960b7f00000000, 0xbb8d871100000000, + 0x78a013a200000000, 0x39bb9fcc00000000, 0x352bc8dc00000000, + 0x743044b200000000, 0xb71dd00100000000, 0xf6065c6f00000000, + 0x704089bd00000000, 0x315b05d300000000, 0xf276916000000000, + 0xb36d1d0e00000000, 0xea563e4000000000, 0xab4db22e00000000, + 0x6860269d00000000, 0x297baaf300000000, 0xaf3d7f2100000000, + 0xee26f34f00000000, 0x2d0b67fc00000000, 0x6c10eb9200000000, + 0x6080bc8200000000, 0x219b30ec00000000, 0xe2b6a45f00000000, + 0xa3ad283100000000, 0x25ebfde300000000, 0x64f0718d00000000, + 0xa7dde53e00000000, 0xe6c6695000000000, 0x6b50369e00000000, + 0x2a4bbaf000000000, 0xe9662e4300000000, 0xa87da22d00000000, + 0x2e3b77ff00000000, 0x6f20fb9100000000, 0xac0d6f2200000000, + 0xed16e34c00000000, 0xe186b45c00000000, 0xa09d383200000000, + 0x63b0ac8100000000, 0x22ab20ef00000000, 0xa4edf53d00000000, + 0xe5f6795300000000, 0x26dbede000000000, 0x67c0618e00000000, + 0x3efb42c000000000, 0x7fe0ceae00000000, 0xbccd5a1d00000000, + 0xfdd6d67300000000, 0x7b9003a100000000, 0x3a8b8fcf00000000, + 0xf9a61b7c00000000, 0xb8bd971200000000, 0xb42dc00200000000, + 0xf5364c6c00000000, 0x361bd8df00000000, 0x770054b100000000, + 0xf146816300000000, 0xb05d0d0d00000000, 0x737099be00000000, + 0x326b15d000000000, 0xc106df2200000000, 0x801d534c00000000, + 0x4330c7ff00000000, 0x022b4b9100000000, 0x846d9e4300000000, + 0xc576122d00000000, 0x065b869e00000000, 0x47400af000000000, + 0x4bd05de000000000, 0x0acbd18e00000000, 0xc9e6453d00000000, + 0x88fdc95300000000, 0x0ebb1c8100000000, 0x4fa090ef00000000, + 0x8c8d045c00000000, 0xcd96883200000000, 0x94adab7c00000000, + 0xd5b6271200000000, 0x169bb3a100000000, 0x57803fcf00000000, + 0xd1c6ea1d00000000, 0x90dd667300000000, 0x53f0f2c000000000, + 0x12eb7eae00000000, 0x1e7b29be00000000, 0x5f60a5d000000000, + 0x9c4d316300000000, 0xdd56bd0d00000000, 0x5b1068df00000000, + 0x1a0be4b100000000, 0xd926700200000000, 0x983dfc6c00000000, + 0x7efb953c00000000, 0x3fe0195200000000, 0xfccd8de100000000, + 0xbdd6018f00000000, 0x3b90d45d00000000, 0x7a8b583300000000, + 0xb9a6cc8000000000, 0xf8bd40ee00000000, 0xf42d17fe00000000, + 0xb5369b9000000000, 0x761b0f2300000000, 0x3700834d00000000, + 0xb146569f00000000, 0xf05ddaf100000000, 0x33704e4200000000, + 0x726bc22c00000000, 0x2b50e16200000000, 0x6a4b6d0c00000000, + 0xa966f9bf00000000, 0xe87d75d100000000, 0x6e3ba00300000000, + 0x2f202c6d00000000, 0xec0db8de00000000, 0xad1634b000000000, + 0xa18663a000000000, 0xe09defce00000000, 0x23b07b7d00000000, + 0x62abf71300000000, 0xe4ed22c100000000, 0xa5f6aeaf00000000, + 0x66db3a1c00000000, 0x27c0b67200000000, 0xd4ad7c8000000000, + 0x95b6f0ee00000000, 0x569b645d00000000, 0x1780e83300000000, + 0x91c63de100000000, 0xd0ddb18f00000000, 0x13f0253c00000000, + 0x52eba95200000000, 0x5e7bfe4200000000, 0x1f60722c00000000, + 0xdc4de69f00000000, 0x9d566af100000000, 0x1b10bf2300000000, + 0x5a0b334d00000000, 0x9926a7fe00000000, 0xd83d2b9000000000, + 0x810608de00000000, 0xc01d84b000000000, 0x0330100300000000, + 0x422b9c6d00000000, 0xc46d49bf00000000, 0x8576c5d100000000, + 0x465b516200000000, 0x0740dd0c00000000, 0x0bd08a1c00000000, + 0x4acb067200000000, 0x89e692c100000000, 0xc8fd1eaf00000000, + 0x4ebbcb7d00000000, 0x0fa0471300000000, 0xcc8dd3a000000000, + 0x8d965fce00000000}, + {0x0000000000000000, 0x1dfdb50100000000, 0x3afa6b0300000000, + 0x2707de0200000000, 0x74f4d70600000000, 0x6909620700000000, + 0x4e0ebc0500000000, 0x53f3090400000000, 0xe8e8af0d00000000, + 0xf5151a0c00000000, 0xd212c40e00000000, 0xcfef710f00000000, + 0x9c1c780b00000000, 0x81e1cd0a00000000, 0xa6e6130800000000, + 0xbb1ba60900000000, 0xd0d15f1b00000000, 0xcd2cea1a00000000, + 0xea2b341800000000, 0xf7d6811900000000, 0xa425881d00000000, + 0xb9d83d1c00000000, 0x9edfe31e00000000, 0x8322561f00000000, + 0x3839f01600000000, 0x25c4451700000000, 0x02c39b1500000000, + 0x1f3e2e1400000000, 0x4ccd271000000000, 0x5130921100000000, + 0x76374c1300000000, 0x6bcaf91200000000, 0xa0a3bf3600000000, + 0xbd5e0a3700000000, 0x9a59d43500000000, 0x87a4613400000000, + 0xd457683000000000, 0xc9aadd3100000000, 0xeead033300000000, + 0xf350b63200000000, 0x484b103b00000000, 0x55b6a53a00000000, + 0x72b17b3800000000, 0x6f4cce3900000000, 0x3cbfc73d00000000, + 0x2142723c00000000, 0x0645ac3e00000000, 0x1bb8193f00000000, + 0x7072e02d00000000, 0x6d8f552c00000000, 0x4a888b2e00000000, + 0x57753e2f00000000, 0x0486372b00000000, 0x197b822a00000000, + 0x3e7c5c2800000000, 0x2381e92900000000, 0x989a4f2000000000, + 0x8567fa2100000000, 0xa260242300000000, 0xbf9d912200000000, + 0xec6e982600000000, 0xf1932d2700000000, 0xd694f32500000000, + 0xcb69462400000000, 0x40477f6d00000000, 0x5dbaca6c00000000, + 0x7abd146e00000000, 0x6740a16f00000000, 0x34b3a86b00000000, + 0x294e1d6a00000000, 0x0e49c36800000000, 0x13b4766900000000, + 0xa8afd06000000000, 0xb552656100000000, 0x9255bb6300000000, + 0x8fa80e6200000000, 0xdc5b076600000000, 0xc1a6b26700000000, + 0xe6a16c6500000000, 0xfb5cd96400000000, 0x9096207600000000, + 0x8d6b957700000000, 0xaa6c4b7500000000, 0xb791fe7400000000, + 0xe462f77000000000, 0xf99f427100000000, 0xde989c7300000000, + 0xc365297200000000, 0x787e8f7b00000000, 0x65833a7a00000000, + 0x4284e47800000000, 0x5f79517900000000, 0x0c8a587d00000000, + 0x1177ed7c00000000, 0x3670337e00000000, 0x2b8d867f00000000, + 0xe0e4c05b00000000, 0xfd19755a00000000, 0xda1eab5800000000, + 0xc7e31e5900000000, 0x9410175d00000000, 0x89eda25c00000000, + 0xaeea7c5e00000000, 0xb317c95f00000000, 0x080c6f5600000000, + 0x15f1da5700000000, 0x32f6045500000000, 0x2f0bb15400000000, + 0x7cf8b85000000000, 0x61050d5100000000, 0x4602d35300000000, + 0x5bff665200000000, 0x30359f4000000000, 0x2dc82a4100000000, + 0x0acff44300000000, 0x1732414200000000, 0x44c1484600000000, + 0x593cfd4700000000, 0x7e3b234500000000, 0x63c6964400000000, + 0xd8dd304d00000000, 0xc520854c00000000, 0xe2275b4e00000000, + 0xffdaee4f00000000, 0xac29e74b00000000, 0xb1d4524a00000000, + 0x96d38c4800000000, 0x8b2e394900000000, 0x808efeda00000000, + 0x9d734bdb00000000, 0xba7495d900000000, 0xa78920d800000000, + 0xf47a29dc00000000, 0xe9879cdd00000000, 0xce8042df00000000, + 0xd37df7de00000000, 0x686651d700000000, 0x759be4d600000000, + 0x529c3ad400000000, 0x4f618fd500000000, 0x1c9286d100000000, + 0x016f33d000000000, 0x2668edd200000000, 0x3b9558d300000000, + 0x505fa1c100000000, 0x4da214c000000000, 0x6aa5cac200000000, + 0x77587fc300000000, 0x24ab76c700000000, 0x3956c3c600000000, + 0x1e511dc400000000, 0x03aca8c500000000, 0xb8b70ecc00000000, + 0xa54abbcd00000000, 0x824d65cf00000000, 0x9fb0d0ce00000000, + 0xcc43d9ca00000000, 0xd1be6ccb00000000, 0xf6b9b2c900000000, + 0xeb4407c800000000, 0x202d41ec00000000, 0x3dd0f4ed00000000, + 0x1ad72aef00000000, 0x072a9fee00000000, 0x54d996ea00000000, + 0x492423eb00000000, 0x6e23fde900000000, 0x73de48e800000000, + 0xc8c5eee100000000, 0xd5385be000000000, 0xf23f85e200000000, + 0xefc230e300000000, 0xbc3139e700000000, 0xa1cc8ce600000000, + 0x86cb52e400000000, 0x9b36e7e500000000, 0xf0fc1ef700000000, + 0xed01abf600000000, 0xca0675f400000000, 0xd7fbc0f500000000, + 0x8408c9f100000000, 0x99f57cf000000000, 0xbef2a2f200000000, + 0xa30f17f300000000, 0x1814b1fa00000000, 0x05e904fb00000000, + 0x22eedaf900000000, 0x3f136ff800000000, 0x6ce066fc00000000, + 0x711dd3fd00000000, 0x561a0dff00000000, 0x4be7b8fe00000000, + 0xc0c981b700000000, 0xdd3434b600000000, 0xfa33eab400000000, + 0xe7ce5fb500000000, 0xb43d56b100000000, 0xa9c0e3b000000000, + 0x8ec73db200000000, 0x933a88b300000000, 0x28212eba00000000, + 0x35dc9bbb00000000, 0x12db45b900000000, 0x0f26f0b800000000, + 0x5cd5f9bc00000000, 0x41284cbd00000000, 0x662f92bf00000000, + 0x7bd227be00000000, 0x1018deac00000000, 0x0de56bad00000000, + 0x2ae2b5af00000000, 0x371f00ae00000000, 0x64ec09aa00000000, + 0x7911bcab00000000, 0x5e1662a900000000, 0x43ebd7a800000000, + 0xf8f071a100000000, 0xe50dc4a000000000, 0xc20a1aa200000000, + 0xdff7afa300000000, 0x8c04a6a700000000, 0x91f913a600000000, + 0xb6fecda400000000, 0xab0378a500000000, 0x606a3e8100000000, + 0x7d978b8000000000, 0x5a90558200000000, 0x476de08300000000, + 0x149ee98700000000, 0x09635c8600000000, 0x2e64828400000000, + 0x3399378500000000, 0x8882918c00000000, 0x957f248d00000000, + 0xb278fa8f00000000, 0xaf854f8e00000000, 0xfc76468a00000000, + 0xe18bf38b00000000, 0xc68c2d8900000000, 0xdb71988800000000, + 0xb0bb619a00000000, 0xad46d49b00000000, 0x8a410a9900000000, + 0x97bcbf9800000000, 0xc44fb69c00000000, 0xd9b2039d00000000, + 0xfeb5dd9f00000000, 0xe348689e00000000, 0x5853ce9700000000, + 0x45ae7b9600000000, 0x62a9a59400000000, 0x7f54109500000000, + 0x2ca7199100000000, 0x315aac9000000000, 0x165d729200000000, + 0x0ba0c79300000000}, + {0x0000000000000000, 0x24d9076300000000, 0x48b20fc600000000, + 0x6c6b08a500000000, 0xd1626e5700000000, 0xf5bb693400000000, + 0x99d0619100000000, 0xbd0966f200000000, 0xa2c5dcae00000000, + 0x861cdbcd00000000, 0xea77d36800000000, 0xceaed40b00000000, + 0x73a7b2f900000000, 0x577eb59a00000000, 0x3b15bd3f00000000, + 0x1fccba5c00000000, 0x058dc88600000000, 0x2154cfe500000000, + 0x4d3fc74000000000, 0x69e6c02300000000, 0xd4efa6d100000000, + 0xf036a1b200000000, 0x9c5da91700000000, 0xb884ae7400000000, + 0xa748142800000000, 0x8391134b00000000, 0xeffa1bee00000000, + 0xcb231c8d00000000, 0x762a7a7f00000000, 0x52f37d1c00000000, + 0x3e9875b900000000, 0x1a4172da00000000, 0x4b1ce0d600000000, + 0x6fc5e7b500000000, 0x03aeef1000000000, 0x2777e87300000000, + 0x9a7e8e8100000000, 0xbea789e200000000, 0xd2cc814700000000, + 0xf615862400000000, 0xe9d93c7800000000, 0xcd003b1b00000000, + 0xa16b33be00000000, 0x85b234dd00000000, 0x38bb522f00000000, + 0x1c62554c00000000, 0x70095de900000000, 0x54d05a8a00000000, + 0x4e91285000000000, 0x6a482f3300000000, 0x0623279600000000, + 0x22fa20f500000000, 0x9ff3460700000000, 0xbb2a416400000000, + 0xd74149c100000000, 0xf3984ea200000000, 0xec54f4fe00000000, + 0xc88df39d00000000, 0xa4e6fb3800000000, 0x803ffc5b00000000, + 0x3d369aa900000000, 0x19ef9dca00000000, 0x7584956f00000000, + 0x515d920c00000000, 0xd73eb17600000000, 0xf3e7b61500000000, + 0x9f8cbeb000000000, 0xbb55b9d300000000, 0x065cdf2100000000, + 0x2285d84200000000, 0x4eeed0e700000000, 0x6a37d78400000000, + 0x75fb6dd800000000, 0x51226abb00000000, 0x3d49621e00000000, + 0x1990657d00000000, 0xa499038f00000000, 0x804004ec00000000, + 0xec2b0c4900000000, 0xc8f20b2a00000000, 0xd2b379f000000000, + 0xf66a7e9300000000, 0x9a01763600000000, 0xbed8715500000000, + 0x03d117a700000000, 0x270810c400000000, 0x4b63186100000000, + 0x6fba1f0200000000, 0x7076a55e00000000, 0x54afa23d00000000, + 0x38c4aa9800000000, 0x1c1dadfb00000000, 0xa114cb0900000000, + 0x85cdcc6a00000000, 0xe9a6c4cf00000000, 0xcd7fc3ac00000000, + 0x9c2251a000000000, 0xb8fb56c300000000, 0xd4905e6600000000, + 0xf049590500000000, 0x4d403ff700000000, 0x6999389400000000, + 0x05f2303100000000, 0x212b375200000000, 0x3ee78d0e00000000, + 0x1a3e8a6d00000000, 0x765582c800000000, 0x528c85ab00000000, + 0xef85e35900000000, 0xcb5ce43a00000000, 0xa737ec9f00000000, + 0x83eeebfc00000000, 0x99af992600000000, 0xbd769e4500000000, + 0xd11d96e000000000, 0xf5c4918300000000, 0x48cdf77100000000, + 0x6c14f01200000000, 0x007ff8b700000000, 0x24a6ffd400000000, + 0x3b6a458800000000, 0x1fb342eb00000000, 0x73d84a4e00000000, + 0x57014d2d00000000, 0xea082bdf00000000, 0xced12cbc00000000, + 0xa2ba241900000000, 0x8663237a00000000, 0xae7d62ed00000000, + 0x8aa4658e00000000, 0xe6cf6d2b00000000, 0xc2166a4800000000, + 0x7f1f0cba00000000, 0x5bc60bd900000000, 0x37ad037c00000000, + 0x1374041f00000000, 0x0cb8be4300000000, 0x2861b92000000000, + 0x440ab18500000000, 0x60d3b6e600000000, 0xdddad01400000000, + 0xf903d77700000000, 0x9568dfd200000000, 0xb1b1d8b100000000, + 0xabf0aa6b00000000, 0x8f29ad0800000000, 0xe342a5ad00000000, + 0xc79ba2ce00000000, 0x7a92c43c00000000, 0x5e4bc35f00000000, + 0x3220cbfa00000000, 0x16f9cc9900000000, 0x093576c500000000, + 0x2dec71a600000000, 0x4187790300000000, 0x655e7e6000000000, + 0xd857189200000000, 0xfc8e1ff100000000, 0x90e5175400000000, + 0xb43c103700000000, 0xe561823b00000000, 0xc1b8855800000000, + 0xadd38dfd00000000, 0x890a8a9e00000000, 0x3403ec6c00000000, + 0x10daeb0f00000000, 0x7cb1e3aa00000000, 0x5868e4c900000000, + 0x47a45e9500000000, 0x637d59f600000000, 0x0f16515300000000, + 0x2bcf563000000000, 0x96c630c200000000, 0xb21f37a100000000, + 0xde743f0400000000, 0xfaad386700000000, 0xe0ec4abd00000000, + 0xc4354dde00000000, 0xa85e457b00000000, 0x8c87421800000000, + 0x318e24ea00000000, 0x1557238900000000, 0x793c2b2c00000000, + 0x5de52c4f00000000, 0x4229961300000000, 0x66f0917000000000, + 0x0a9b99d500000000, 0x2e429eb600000000, 0x934bf84400000000, + 0xb792ff2700000000, 0xdbf9f78200000000, 0xff20f0e100000000, + 0x7943d39b00000000, 0x5d9ad4f800000000, 0x31f1dc5d00000000, + 0x1528db3e00000000, 0xa821bdcc00000000, 0x8cf8baaf00000000, + 0xe093b20a00000000, 0xc44ab56900000000, 0xdb860f3500000000, + 0xff5f085600000000, 0x933400f300000000, 0xb7ed079000000000, + 0x0ae4616200000000, 0x2e3d660100000000, 0x42566ea400000000, + 0x668f69c700000000, 0x7cce1b1d00000000, 0x58171c7e00000000, + 0x347c14db00000000, 0x10a513b800000000, 0xadac754a00000000, + 0x8975722900000000, 0xe51e7a8c00000000, 0xc1c77def00000000, + 0xde0bc7b300000000, 0xfad2c0d000000000, 0x96b9c87500000000, + 0xb260cf1600000000, 0x0f69a9e400000000, 0x2bb0ae8700000000, + 0x47dba62200000000, 0x6302a14100000000, 0x325f334d00000000, + 0x1686342e00000000, 0x7aed3c8b00000000, 0x5e343be800000000, + 0xe33d5d1a00000000, 0xc7e45a7900000000, 0xab8f52dc00000000, + 0x8f5655bf00000000, 0x909aefe300000000, 0xb443e88000000000, + 0xd828e02500000000, 0xfcf1e74600000000, 0x41f881b400000000, + 0x652186d700000000, 0x094a8e7200000000, 0x2d93891100000000, + 0x37d2fbcb00000000, 0x130bfca800000000, 0x7f60f40d00000000, + 0x5bb9f36e00000000, 0xe6b0959c00000000, 0xc26992ff00000000, + 0xae029a5a00000000, 0x8adb9d3900000000, 0x9517276500000000, + 0xb1ce200600000000, 0xdda528a300000000, 0xf97c2fc000000000, + 0x4475493200000000, 0x60ac4e5100000000, 0x0cc746f400000000, + 0x281e419700000000}, + {0x0000000000000000, 0x08e3603c00000000, 0x10c6c17800000000, + 0x1825a14400000000, 0x208c83f100000000, 0x286fe3cd00000000, + 0x304a428900000000, 0x38a922b500000000, 0x011e763800000000, + 0x09fd160400000000, 0x11d8b74000000000, 0x193bd77c00000000, + 0x2192f5c900000000, 0x297195f500000000, 0x315434b100000000, + 0x39b7548d00000000, 0x023cec7000000000, 0x0adf8c4c00000000, + 0x12fa2d0800000000, 0x1a194d3400000000, 0x22b06f8100000000, + 0x2a530fbd00000000, 0x3276aef900000000, 0x3a95cec500000000, + 0x03229a4800000000, 0x0bc1fa7400000000, 0x13e45b3000000000, + 0x1b073b0c00000000, 0x23ae19b900000000, 0x2b4d798500000000, + 0x3368d8c100000000, 0x3b8bb8fd00000000, 0x0478d8e100000000, + 0x0c9bb8dd00000000, 0x14be199900000000, 0x1c5d79a500000000, + 0x24f45b1000000000, 0x2c173b2c00000000, 0x34329a6800000000, + 0x3cd1fa5400000000, 0x0566aed900000000, 0x0d85cee500000000, + 0x15a06fa100000000, 0x1d430f9d00000000, 0x25ea2d2800000000, + 0x2d094d1400000000, 0x352cec5000000000, 0x3dcf8c6c00000000, + 0x0644349100000000, 0x0ea754ad00000000, 0x1682f5e900000000, + 0x1e6195d500000000, 0x26c8b76000000000, 0x2e2bd75c00000000, + 0x360e761800000000, 0x3eed162400000000, 0x075a42a900000000, + 0x0fb9229500000000, 0x179c83d100000000, 0x1f7fe3ed00000000, + 0x27d6c15800000000, 0x2f35a16400000000, 0x3710002000000000, + 0x3ff3601c00000000, 0x49f6c11800000000, 0x4115a12400000000, + 0x5930006000000000, 0x51d3605c00000000, 0x697a42e900000000, + 0x619922d500000000, 0x79bc839100000000, 0x715fe3ad00000000, + 0x48e8b72000000000, 0x400bd71c00000000, 0x582e765800000000, + 0x50cd166400000000, 0x686434d100000000, 0x608754ed00000000, + 0x78a2f5a900000000, 0x7041959500000000, 0x4bca2d6800000000, + 0x43294d5400000000, 0x5b0cec1000000000, 0x53ef8c2c00000000, + 0x6b46ae9900000000, 0x63a5cea500000000, 0x7b806fe100000000, + 0x73630fdd00000000, 0x4ad45b5000000000, 0x42373b6c00000000, + 0x5a129a2800000000, 0x52f1fa1400000000, 0x6a58d8a100000000, + 0x62bbb89d00000000, 0x7a9e19d900000000, 0x727d79e500000000, + 0x4d8e19f900000000, 0x456d79c500000000, 0x5d48d88100000000, + 0x55abb8bd00000000, 0x6d029a0800000000, 0x65e1fa3400000000, + 0x7dc45b7000000000, 0x75273b4c00000000, 0x4c906fc100000000, + 0x44730ffd00000000, 0x5c56aeb900000000, 0x54b5ce8500000000, + 0x6c1cec3000000000, 0x64ff8c0c00000000, 0x7cda2d4800000000, + 0x74394d7400000000, 0x4fb2f58900000000, 0x475195b500000000, + 0x5f7434f100000000, 0x579754cd00000000, 0x6f3e767800000000, + 0x67dd164400000000, 0x7ff8b70000000000, 0x771bd73c00000000, + 0x4eac83b100000000, 0x464fe38d00000000, 0x5e6a42c900000000, + 0x568922f500000000, 0x6e20004000000000, 0x66c3607c00000000, + 0x7ee6c13800000000, 0x7605a10400000000, 0x92ec833100000000, + 0x9a0fe30d00000000, 0x822a424900000000, 0x8ac9227500000000, + 0xb26000c000000000, 0xba8360fc00000000, 0xa2a6c1b800000000, + 0xaa45a18400000000, 0x93f2f50900000000, 0x9b11953500000000, + 0x8334347100000000, 0x8bd7544d00000000, 0xb37e76f800000000, + 0xbb9d16c400000000, 0xa3b8b78000000000, 0xab5bd7bc00000000, + 0x90d06f4100000000, 0x98330f7d00000000, 0x8016ae3900000000, + 0x88f5ce0500000000, 0xb05cecb000000000, 0xb8bf8c8c00000000, + 0xa09a2dc800000000, 0xa8794df400000000, 0x91ce197900000000, + 0x992d794500000000, 0x8108d80100000000, 0x89ebb83d00000000, + 0xb1429a8800000000, 0xb9a1fab400000000, 0xa1845bf000000000, + 0xa9673bcc00000000, 0x96945bd000000000, 0x9e773bec00000000, + 0x86529aa800000000, 0x8eb1fa9400000000, 0xb618d82100000000, + 0xbefbb81d00000000, 0xa6de195900000000, 0xae3d796500000000, + 0x978a2de800000000, 0x9f694dd400000000, 0x874cec9000000000, + 0x8faf8cac00000000, 0xb706ae1900000000, 0xbfe5ce2500000000, + 0xa7c06f6100000000, 0xaf230f5d00000000, 0x94a8b7a000000000, + 0x9c4bd79c00000000, 0x846e76d800000000, 0x8c8d16e400000000, + 0xb424345100000000, 0xbcc7546d00000000, 0xa4e2f52900000000, + 0xac01951500000000, 0x95b6c19800000000, 0x9d55a1a400000000, + 0x857000e000000000, 0x8d9360dc00000000, 0xb53a426900000000, + 0xbdd9225500000000, 0xa5fc831100000000, 0xad1fe32d00000000, + 0xdb1a422900000000, 0xd3f9221500000000, 0xcbdc835100000000, + 0xc33fe36d00000000, 0xfb96c1d800000000, 0xf375a1e400000000, + 0xeb5000a000000000, 0xe3b3609c00000000, 0xda04341100000000, + 0xd2e7542d00000000, 0xcac2f56900000000, 0xc221955500000000, + 0xfa88b7e000000000, 0xf26bd7dc00000000, 0xea4e769800000000, + 0xe2ad16a400000000, 0xd926ae5900000000, 0xd1c5ce6500000000, + 0xc9e06f2100000000, 0xc1030f1d00000000, 0xf9aa2da800000000, + 0xf1494d9400000000, 0xe96cecd000000000, 0xe18f8cec00000000, + 0xd838d86100000000, 0xd0dbb85d00000000, 0xc8fe191900000000, + 0xc01d792500000000, 0xf8b45b9000000000, 0xf0573bac00000000, + 0xe8729ae800000000, 0xe091fad400000000, 0xdf629ac800000000, + 0xd781faf400000000, 0xcfa45bb000000000, 0xc7473b8c00000000, + 0xffee193900000000, 0xf70d790500000000, 0xef28d84100000000, + 0xe7cbb87d00000000, 0xde7cecf000000000, 0xd69f8ccc00000000, + 0xceba2d8800000000, 0xc6594db400000000, 0xfef06f0100000000, + 0xf6130f3d00000000, 0xee36ae7900000000, 0xe6d5ce4500000000, + 0xdd5e76b800000000, 0xd5bd168400000000, 0xcd98b7c000000000, + 0xc57bd7fc00000000, 0xfdd2f54900000000, 0xf531957500000000, + 0xed14343100000000, 0xe5f7540d00000000, 0xdc40008000000000, + 0xd4a360bc00000000, 0xcc86c1f800000000, 0xc465a1c400000000, + 0xfccc837100000000, 0xf42fe34d00000000, 0xec0a420900000000, + 0xe4e9223500000000}, + {0x0000000000000000, 0xd1e8e70e00000000, 0xa2d1cf1d00000000, + 0x7339281300000000, 0x44a39f3b00000000, 0x954b783500000000, + 0xe672502600000000, 0x379ab72800000000, 0x88463f7700000000, + 0x59aed87900000000, 0x2a97f06a00000000, 0xfb7f176400000000, + 0xcce5a04c00000000, 0x1d0d474200000000, 0x6e346f5100000000, + 0xbfdc885f00000000, 0x108d7eee00000000, 0xc16599e000000000, + 0xb25cb1f300000000, 0x63b456fd00000000, 0x542ee1d500000000, + 0x85c606db00000000, 0xf6ff2ec800000000, 0x2717c9c600000000, + 0x98cb419900000000, 0x4923a69700000000, 0x3a1a8e8400000000, + 0xebf2698a00000000, 0xdc68dea200000000, 0x0d8039ac00000000, + 0x7eb911bf00000000, 0xaf51f6b100000000, 0x611c8c0700000000, + 0xb0f46b0900000000, 0xc3cd431a00000000, 0x1225a41400000000, + 0x25bf133c00000000, 0xf457f43200000000, 0x876edc2100000000, + 0x56863b2f00000000, 0xe95ab37000000000, 0x38b2547e00000000, + 0x4b8b7c6d00000000, 0x9a639b6300000000, 0xadf92c4b00000000, + 0x7c11cb4500000000, 0x0f28e35600000000, 0xdec0045800000000, + 0x7191f2e900000000, 0xa07915e700000000, 0xd3403df400000000, + 0x02a8dafa00000000, 0x35326dd200000000, 0xe4da8adc00000000, + 0x97e3a2cf00000000, 0x460b45c100000000, 0xf9d7cd9e00000000, + 0x283f2a9000000000, 0x5b06028300000000, 0x8aeee58d00000000, + 0xbd7452a500000000, 0x6c9cb5ab00000000, 0x1fa59db800000000, + 0xce4d7ab600000000, 0xc238180f00000000, 0x13d0ff0100000000, + 0x60e9d71200000000, 0xb101301c00000000, 0x869b873400000000, + 0x5773603a00000000, 0x244a482900000000, 0xf5a2af2700000000, + 0x4a7e277800000000, 0x9b96c07600000000, 0xe8afe86500000000, + 0x39470f6b00000000, 0x0eddb84300000000, 0xdf355f4d00000000, + 0xac0c775e00000000, 0x7de4905000000000, 0xd2b566e100000000, + 0x035d81ef00000000, 0x7064a9fc00000000, 0xa18c4ef200000000, + 0x9616f9da00000000, 0x47fe1ed400000000, 0x34c736c700000000, + 0xe52fd1c900000000, 0x5af3599600000000, 0x8b1bbe9800000000, + 0xf822968b00000000, 0x29ca718500000000, 0x1e50c6ad00000000, + 0xcfb821a300000000, 0xbc8109b000000000, 0x6d69eebe00000000, + 0xa324940800000000, 0x72cc730600000000, 0x01f55b1500000000, + 0xd01dbc1b00000000, 0xe7870b3300000000, 0x366fec3d00000000, + 0x4556c42e00000000, 0x94be232000000000, 0x2b62ab7f00000000, + 0xfa8a4c7100000000, 0x89b3646200000000, 0x585b836c00000000, + 0x6fc1344400000000, 0xbe29d34a00000000, 0xcd10fb5900000000, + 0x1cf81c5700000000, 0xb3a9eae600000000, 0x62410de800000000, + 0x117825fb00000000, 0xc090c2f500000000, 0xf70a75dd00000000, + 0x26e292d300000000, 0x55dbbac000000000, 0x84335dce00000000, + 0x3befd59100000000, 0xea07329f00000000, 0x993e1a8c00000000, + 0x48d6fd8200000000, 0x7f4c4aaa00000000, 0xaea4ada400000000, + 0xdd9d85b700000000, 0x0c7562b900000000, 0x8471301e00000000, + 0x5599d71000000000, 0x26a0ff0300000000, 0xf748180d00000000, + 0xc0d2af2500000000, 0x113a482b00000000, 0x6203603800000000, + 0xb3eb873600000000, 0x0c370f6900000000, 0xdddfe86700000000, + 0xaee6c07400000000, 0x7f0e277a00000000, 0x4894905200000000, + 0x997c775c00000000, 0xea455f4f00000000, 0x3badb84100000000, + 0x94fc4ef000000000, 0x4514a9fe00000000, 0x362d81ed00000000, + 0xe7c566e300000000, 0xd05fd1cb00000000, 0x01b736c500000000, + 0x728e1ed600000000, 0xa366f9d800000000, 0x1cba718700000000, + 0xcd52968900000000, 0xbe6bbe9a00000000, 0x6f83599400000000, + 0x5819eebc00000000, 0x89f109b200000000, 0xfac821a100000000, + 0x2b20c6af00000000, 0xe56dbc1900000000, 0x34855b1700000000, + 0x47bc730400000000, 0x9654940a00000000, 0xa1ce232200000000, + 0x7026c42c00000000, 0x031fec3f00000000, 0xd2f70b3100000000, + 0x6d2b836e00000000, 0xbcc3646000000000, 0xcffa4c7300000000, + 0x1e12ab7d00000000, 0x29881c5500000000, 0xf860fb5b00000000, + 0x8b59d34800000000, 0x5ab1344600000000, 0xf5e0c2f700000000, + 0x240825f900000000, 0x57310dea00000000, 0x86d9eae400000000, + 0xb1435dcc00000000, 0x60abbac200000000, 0x139292d100000000, + 0xc27a75df00000000, 0x7da6fd8000000000, 0xac4e1a8e00000000, + 0xdf77329d00000000, 0x0e9fd59300000000, 0x390562bb00000000, + 0xe8ed85b500000000, 0x9bd4ada600000000, 0x4a3c4aa800000000, + 0x4649281100000000, 0x97a1cf1f00000000, 0xe498e70c00000000, + 0x3570000200000000, 0x02eab72a00000000, 0xd302502400000000, + 0xa03b783700000000, 0x71d39f3900000000, 0xce0f176600000000, + 0x1fe7f06800000000, 0x6cded87b00000000, 0xbd363f7500000000, + 0x8aac885d00000000, 0x5b446f5300000000, 0x287d474000000000, + 0xf995a04e00000000, 0x56c456ff00000000, 0x872cb1f100000000, + 0xf41599e200000000, 0x25fd7eec00000000, 0x1267c9c400000000, + 0xc38f2eca00000000, 0xb0b606d900000000, 0x615ee1d700000000, + 0xde82698800000000, 0x0f6a8e8600000000, 0x7c53a69500000000, + 0xadbb419b00000000, 0x9a21f6b300000000, 0x4bc911bd00000000, + 0x38f039ae00000000, 0xe918dea000000000, 0x2755a41600000000, + 0xf6bd431800000000, 0x85846b0b00000000, 0x546c8c0500000000, + 0x63f63b2d00000000, 0xb21edc2300000000, 0xc127f43000000000, + 0x10cf133e00000000, 0xaf139b6100000000, 0x7efb7c6f00000000, + 0x0dc2547c00000000, 0xdc2ab37200000000, 0xebb0045a00000000, + 0x3a58e35400000000, 0x4961cb4700000000, 0x98892c4900000000, + 0x37d8daf800000000, 0xe6303df600000000, 0x950915e500000000, + 0x44e1f2eb00000000, 0x737b45c300000000, 0xa293a2cd00000000, + 0xd1aa8ade00000000, 0x00426dd000000000, 0xbf9ee58f00000000, + 0x6e76028100000000, 0x1d4f2a9200000000, 0xcca7cd9c00000000, + 0xfb3d7ab400000000, 0x2ad59dba00000000, 0x59ecb5a900000000, + 0x880452a700000000}, + {0x0000000000000000, 0xaa05daf100000000, 0x150dc53800000000, + 0xbf081fc900000000, 0x2a1a8a7100000000, 0x801f508000000000, + 0x3f174f4900000000, 0x951295b800000000, 0x543414e300000000, + 0xfe31ce1200000000, 0x4139d1db00000000, 0xeb3c0b2a00000000, + 0x7e2e9e9200000000, 0xd42b446300000000, 0x6b235baa00000000, + 0xc126815b00000000, 0xe96e591d00000000, 0x436b83ec00000000, + 0xfc639c2500000000, 0x566646d400000000, 0xc374d36c00000000, + 0x6971099d00000000, 0xd679165400000000, 0x7c7ccca500000000, + 0xbd5a4dfe00000000, 0x175f970f00000000, 0xa85788c600000000, + 0x0252523700000000, 0x9740c78f00000000, 0x3d451d7e00000000, + 0x824d02b700000000, 0x2848d84600000000, 0xd2ddb23a00000000, + 0x78d868cb00000000, 0xc7d0770200000000, 0x6dd5adf300000000, + 0xf8c7384b00000000, 0x52c2e2ba00000000, 0xedcafd7300000000, + 0x47cf278200000000, 0x86e9a6d900000000, 0x2cec7c2800000000, + 0x93e463e100000000, 0x39e1b91000000000, 0xacf32ca800000000, + 0x06f6f65900000000, 0xb9fee99000000000, 0x13fb336100000000, + 0x3bb3eb2700000000, 0x91b631d600000000, 0x2ebe2e1f00000000, + 0x84bbf4ee00000000, 0x11a9615600000000, 0xbbacbba700000000, + 0x04a4a46e00000000, 0xaea17e9f00000000, 0x6f87ffc400000000, + 0xc582253500000000, 0x7a8a3afc00000000, 0xd08fe00d00000000, + 0x459d75b500000000, 0xef98af4400000000, 0x5090b08d00000000, + 0xfa956a7c00000000, 0xa4bb657500000000, 0x0ebebf8400000000, + 0xb1b6a04d00000000, 0x1bb37abc00000000, 0x8ea1ef0400000000, + 0x24a435f500000000, 0x9bac2a3c00000000, 0x31a9f0cd00000000, + 0xf08f719600000000, 0x5a8aab6700000000, 0xe582b4ae00000000, + 0x4f876e5f00000000, 0xda95fbe700000000, 0x7090211600000000, + 0xcf983edf00000000, 0x659de42e00000000, 0x4dd53c6800000000, + 0xe7d0e69900000000, 0x58d8f95000000000, 0xf2dd23a100000000, + 0x67cfb61900000000, 0xcdca6ce800000000, 0x72c2732100000000, + 0xd8c7a9d000000000, 0x19e1288b00000000, 0xb3e4f27a00000000, + 0x0cecedb300000000, 0xa6e9374200000000, 0x33fba2fa00000000, + 0x99fe780b00000000, 0x26f667c200000000, 0x8cf3bd3300000000, + 0x7666d74f00000000, 0xdc630dbe00000000, 0x636b127700000000, + 0xc96ec88600000000, 0x5c7c5d3e00000000, 0xf67987cf00000000, + 0x4971980600000000, 0xe37442f700000000, 0x2252c3ac00000000, + 0x8857195d00000000, 0x375f069400000000, 0x9d5adc6500000000, + 0x084849dd00000000, 0xa24d932c00000000, 0x1d458ce500000000, + 0xb740561400000000, 0x9f088e5200000000, 0x350d54a300000000, + 0x8a054b6a00000000, 0x2000919b00000000, 0xb512042300000000, + 0x1f17ded200000000, 0xa01fc11b00000000, 0x0a1a1bea00000000, + 0xcb3c9ab100000000, 0x6139404000000000, 0xde315f8900000000, + 0x7434857800000000, 0xe12610c000000000, 0x4b23ca3100000000, + 0xf42bd5f800000000, 0x5e2e0f0900000000, 0x4877cbea00000000, + 0xe272111b00000000, 0x5d7a0ed200000000, 0xf77fd42300000000, + 0x626d419b00000000, 0xc8689b6a00000000, 0x776084a300000000, + 0xdd655e5200000000, 0x1c43df0900000000, 0xb64605f800000000, + 0x094e1a3100000000, 0xa34bc0c000000000, 0x3659557800000000, + 0x9c5c8f8900000000, 0x2354904000000000, 0x89514ab100000000, + 0xa11992f700000000, 0x0b1c480600000000, 0xb41457cf00000000, + 0x1e118d3e00000000, 0x8b03188600000000, 0x2106c27700000000, + 0x9e0eddbe00000000, 0x340b074f00000000, 0xf52d861400000000, + 0x5f285ce500000000, 0xe020432c00000000, 0x4a2599dd00000000, + 0xdf370c6500000000, 0x7532d69400000000, 0xca3ac95d00000000, + 0x603f13ac00000000, 0x9aaa79d000000000, 0x30afa32100000000, + 0x8fa7bce800000000, 0x25a2661900000000, 0xb0b0f3a100000000, + 0x1ab5295000000000, 0xa5bd369900000000, 0x0fb8ec6800000000, + 0xce9e6d3300000000, 0x649bb7c200000000, 0xdb93a80b00000000, + 0x719672fa00000000, 0xe484e74200000000, 0x4e813db300000000, + 0xf189227a00000000, 0x5b8cf88b00000000, 0x73c420cd00000000, + 0xd9c1fa3c00000000, 0x66c9e5f500000000, 0xcccc3f0400000000, + 0x59deaabc00000000, 0xf3db704d00000000, 0x4cd36f8400000000, + 0xe6d6b57500000000, 0x27f0342e00000000, 0x8df5eedf00000000, + 0x32fdf11600000000, 0x98f82be700000000, 0x0deabe5f00000000, + 0xa7ef64ae00000000, 0x18e77b6700000000, 0xb2e2a19600000000, + 0xecccae9f00000000, 0x46c9746e00000000, 0xf9c16ba700000000, + 0x53c4b15600000000, 0xc6d624ee00000000, 0x6cd3fe1f00000000, + 0xd3dbe1d600000000, 0x79de3b2700000000, 0xb8f8ba7c00000000, + 0x12fd608d00000000, 0xadf57f4400000000, 0x07f0a5b500000000, + 0x92e2300d00000000, 0x38e7eafc00000000, 0x87eff53500000000, + 0x2dea2fc400000000, 0x05a2f78200000000, 0xafa72d7300000000, + 0x10af32ba00000000, 0xbaaae84b00000000, 0x2fb87df300000000, + 0x85bda70200000000, 0x3ab5b8cb00000000, 0x90b0623a00000000, + 0x5196e36100000000, 0xfb93399000000000, 0x449b265900000000, + 0xee9efca800000000, 0x7b8c691000000000, 0xd189b3e100000000, + 0x6e81ac2800000000, 0xc48476d900000000, 0x3e111ca500000000, + 0x9414c65400000000, 0x2b1cd99d00000000, 0x8119036c00000000, + 0x140b96d400000000, 0xbe0e4c2500000000, 0x010653ec00000000, + 0xab03891d00000000, 0x6a25084600000000, 0xc020d2b700000000, + 0x7f28cd7e00000000, 0xd52d178f00000000, 0x403f823700000000, + 0xea3a58c600000000, 0x5532470f00000000, 0xff379dfe00000000, + 0xd77f45b800000000, 0x7d7a9f4900000000, 0xc272808000000000, + 0x68775a7100000000, 0xfd65cfc900000000, 0x5760153800000000, + 0xe8680af100000000, 0x426dd00000000000, 0x834b515b00000000, + 0x294e8baa00000000, 0x9646946300000000, 0x3c434e9200000000, + 0xa951db2a00000000, 0x035401db00000000, 0xbc5c1e1200000000, + 0x1659c4e300000000}}; + +#else /* W == 4 */ + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0xae689191, 0x87a02563, 0x29c8b4f2, 0xd4314c87, + 0x7a59dd16, 0x539169e4, 0xfdf9f875, 0x73139f4f, 0xdd7b0ede, + 0xf4b3ba2c, 0x5adb2bbd, 0xa722d3c8, 0x094a4259, 0x2082f6ab, + 0x8eea673a, 0xe6273e9e, 0x484faf0f, 0x61871bfd, 0xcfef8a6c, + 0x32167219, 0x9c7ee388, 0xb5b6577a, 0x1bdec6eb, 0x9534a1d1, + 0x3b5c3040, 0x129484b2, 0xbcfc1523, 0x4105ed56, 0xef6d7cc7, + 0xc6a5c835, 0x68cd59a4, 0x173f7b7d, 0xb957eaec, 0x909f5e1e, + 0x3ef7cf8f, 0xc30e37fa, 0x6d66a66b, 0x44ae1299, 0xeac68308, + 0x642ce432, 0xca4475a3, 0xe38cc151, 0x4de450c0, 0xb01da8b5, + 0x1e753924, 0x37bd8dd6, 0x99d51c47, 0xf11845e3, 0x5f70d472, + 0x76b86080, 0xd8d0f111, 0x25290964, 0x8b4198f5, 0xa2892c07, + 0x0ce1bd96, 0x820bdaac, 0x2c634b3d, 0x05abffcf, 0xabc36e5e, + 0x563a962b, 0xf85207ba, 0xd19ab348, 0x7ff222d9, 0x2e7ef6fa, + 0x8016676b, 0xa9ded399, 0x07b64208, 0xfa4fba7d, 0x54272bec, + 0x7def9f1e, 0xd3870e8f, 0x5d6d69b5, 0xf305f824, 0xdacd4cd6, + 0x74a5dd47, 0x895c2532, 0x2734b4a3, 0x0efc0051, 0xa09491c0, + 0xc859c864, 0x663159f5, 0x4ff9ed07, 0xe1917c96, 0x1c6884e3, + 0xb2001572, 0x9bc8a180, 0x35a03011, 0xbb4a572b, 0x1522c6ba, + 0x3cea7248, 0x9282e3d9, 0x6f7b1bac, 0xc1138a3d, 0xe8db3ecf, + 0x46b3af5e, 0x39418d87, 0x97291c16, 0xbee1a8e4, 0x10893975, + 0xed70c100, 0x43185091, 0x6ad0e463, 0xc4b875f2, 0x4a5212c8, + 0xe43a8359, 0xcdf237ab, 0x639aa63a, 0x9e635e4f, 0x300bcfde, + 0x19c37b2c, 0xb7abeabd, 0xdf66b319, 0x710e2288, 0x58c6967a, + 0xf6ae07eb, 0x0b57ff9e, 0xa53f6e0f, 0x8cf7dafd, 0x229f4b6c, + 0xac752c56, 0x021dbdc7, 0x2bd50935, 0x85bd98a4, 0x784460d1, + 0xd62cf140, 0xffe445b2, 0x518cd423, 0x5cfdedf4, 0xf2957c65, + 0xdb5dc897, 0x75355906, 0x88cca173, 0x26a430e2, 0x0f6c8410, + 0xa1041581, 0x2fee72bb, 0x8186e32a, 0xa84e57d8, 0x0626c649, + 0xfbdf3e3c, 0x55b7afad, 0x7c7f1b5f, 0xd2178ace, 0xbadad36a, + 0x14b242fb, 0x3d7af609, 0x93126798, 0x6eeb9fed, 0xc0830e7c, + 0xe94bba8e, 0x47232b1f, 0xc9c94c25, 0x67a1ddb4, 0x4e696946, + 0xe001f8d7, 0x1df800a2, 0xb3909133, 0x9a5825c1, 0x3430b450, + 0x4bc29689, 0xe5aa0718, 0xcc62b3ea, 0x620a227b, 0x9ff3da0e, + 0x319b4b9f, 0x1853ff6d, 0xb63b6efc, 0x38d109c6, 0x96b99857, + 0xbf712ca5, 0x1119bd34, 0xece04541, 0x4288d4d0, 0x6b406022, + 0xc528f1b3, 0xade5a817, 0x038d3986, 0x2a458d74, 0x842d1ce5, + 0x79d4e490, 0xd7bc7501, 0xfe74c1f3, 0x501c5062, 0xdef63758, + 0x709ea6c9, 0x5956123b, 0xf73e83aa, 0x0ac77bdf, 0xa4afea4e, + 0x8d675ebc, 0x230fcf2d, 0x72831b0e, 0xdceb8a9f, 0xf5233e6d, + 0x5b4baffc, 0xa6b25789, 0x08dac618, 0x211272ea, 0x8f7ae37b, + 0x01908441, 0xaff815d0, 0x8630a122, 0x285830b3, 0xd5a1c8c6, + 0x7bc95957, 0x5201eda5, 0xfc697c34, 0x94a42590, 0x3accb401, + 0x130400f3, 0xbd6c9162, 0x40956917, 0xeefdf886, 0xc7354c74, + 0x695ddde5, 0xe7b7badf, 0x49df2b4e, 0x60179fbc, 0xce7f0e2d, + 0x3386f658, 0x9dee67c9, 0xb426d33b, 0x1a4e42aa, 0x65bc6073, + 0xcbd4f1e2, 0xe21c4510, 0x4c74d481, 0xb18d2cf4, 0x1fe5bd65, + 0x362d0997, 0x98459806, 0x16afff3c, 0xb8c76ead, 0x910fda5f, + 0x3f674bce, 0xc29eb3bb, 0x6cf6222a, 0x453e96d8, 0xeb560749, + 0x839b5eed, 0x2df3cf7c, 0x043b7b8e, 0xaa53ea1f, 0x57aa126a, + 0xf9c283fb, 0xd00a3709, 0x7e62a698, 0xf088c1a2, 0x5ee05033, + 0x7728e4c1, 0xd9407550, 0x24b98d25, 0x8ad11cb4, 0xa319a846, + 0x0d7139d7}, + {0x00000000, 0xb9fbdbe8, 0xa886b191, 0x117d6a79, 0x8a7c6563, + 0x3387be8b, 0x22fad4f2, 0x9b010f1a, 0xcf89cc87, 0x7672176f, + 0x670f7d16, 0xdef4a6fe, 0x45f5a9e4, 0xfc0e720c, 0xed731875, + 0x5488c39d, 0x44629f4f, 0xfd9944a7, 0xece42ede, 0x551ff536, + 0xce1efa2c, 0x77e521c4, 0x66984bbd, 0xdf639055, 0x8beb53c8, + 0x32108820, 0x236de259, 0x9a9639b1, 0x019736ab, 0xb86ced43, + 0xa911873a, 0x10ea5cd2, 0x88c53e9e, 0x313ee576, 0x20438f0f, + 0x99b854e7, 0x02b95bfd, 0xbb428015, 0xaa3fea6c, 0x13c43184, + 0x474cf219, 0xfeb729f1, 0xefca4388, 0x56319860, 0xcd30977a, + 0x74cb4c92, 0x65b626eb, 0xdc4dfd03, 0xcca7a1d1, 0x755c7a39, + 0x64211040, 0xdddacba8, 0x46dbc4b2, 0xff201f5a, 0xee5d7523, + 0x57a6aecb, 0x032e6d56, 0xbad5b6be, 0xaba8dcc7, 0x1253072f, + 0x89520835, 0x30a9d3dd, 0x21d4b9a4, 0x982f624c, 0xcafb7b7d, + 0x7300a095, 0x627dcaec, 0xdb861104, 0x40871e1e, 0xf97cc5f6, + 0xe801af8f, 0x51fa7467, 0x0572b7fa, 0xbc896c12, 0xadf4066b, + 0x140fdd83, 0x8f0ed299, 0x36f50971, 0x27886308, 0x9e73b8e0, + 0x8e99e432, 0x37623fda, 0x261f55a3, 0x9fe48e4b, 0x04e58151, + 0xbd1e5ab9, 0xac6330c0, 0x1598eb28, 0x411028b5, 0xf8ebf35d, + 0xe9969924, 0x506d42cc, 0xcb6c4dd6, 0x7297963e, 0x63eafc47, + 0xda1127af, 0x423e45e3, 0xfbc59e0b, 0xeab8f472, 0x53432f9a, + 0xc8422080, 0x71b9fb68, 0x60c49111, 0xd93f4af9, 0x8db78964, + 0x344c528c, 0x253138f5, 0x9ccae31d, 0x07cbec07, 0xbe3037ef, + 0xaf4d5d96, 0x16b6867e, 0x065cdaac, 0xbfa70144, 0xaeda6b3d, + 0x1721b0d5, 0x8c20bfcf, 0x35db6427, 0x24a60e5e, 0x9d5dd5b6, + 0xc9d5162b, 0x702ecdc3, 0x6153a7ba, 0xd8a87c52, 0x43a97348, + 0xfa52a8a0, 0xeb2fc2d9, 0x52d41931, 0x4e87f0bb, 0xf77c2b53, + 0xe601412a, 0x5ffa9ac2, 0xc4fb95d8, 0x7d004e30, 0x6c7d2449, + 0xd586ffa1, 0x810e3c3c, 0x38f5e7d4, 0x29888dad, 0x90735645, + 0x0b72595f, 0xb28982b7, 0xa3f4e8ce, 0x1a0f3326, 0x0ae56ff4, + 0xb31eb41c, 0xa263de65, 0x1b98058d, 0x80990a97, 0x3962d17f, + 0x281fbb06, 0x91e460ee, 0xc56ca373, 0x7c97789b, 0x6dea12e2, + 0xd411c90a, 0x4f10c610, 0xf6eb1df8, 0xe7967781, 0x5e6dac69, + 0xc642ce25, 0x7fb915cd, 0x6ec47fb4, 0xd73fa45c, 0x4c3eab46, + 0xf5c570ae, 0xe4b81ad7, 0x5d43c13f, 0x09cb02a2, 0xb030d94a, + 0xa14db333, 0x18b668db, 0x83b767c1, 0x3a4cbc29, 0x2b31d650, + 0x92ca0db8, 0x8220516a, 0x3bdb8a82, 0x2aa6e0fb, 0x935d3b13, + 0x085c3409, 0xb1a7efe1, 0xa0da8598, 0x19215e70, 0x4da99ded, + 0xf4524605, 0xe52f2c7c, 0x5cd4f794, 0xc7d5f88e, 0x7e2e2366, + 0x6f53491f, 0xd6a892f7, 0x847c8bc6, 0x3d87502e, 0x2cfa3a57, + 0x9501e1bf, 0x0e00eea5, 0xb7fb354d, 0xa6865f34, 0x1f7d84dc, + 0x4bf54741, 0xf20e9ca9, 0xe373f6d0, 0x5a882d38, 0xc1892222, + 0x7872f9ca, 0x690f93b3, 0xd0f4485b, 0xc01e1489, 0x79e5cf61, + 0x6898a518, 0xd1637ef0, 0x4a6271ea, 0xf399aa02, 0xe2e4c07b, + 0x5b1f1b93, 0x0f97d80e, 0xb66c03e6, 0xa711699f, 0x1eeab277, + 0x85ebbd6d, 0x3c106685, 0x2d6d0cfc, 0x9496d714, 0x0cb9b558, + 0xb5426eb0, 0xa43f04c9, 0x1dc4df21, 0x86c5d03b, 0x3f3e0bd3, + 0x2e4361aa, 0x97b8ba42, 0xc33079df, 0x7acba237, 0x6bb6c84e, + 0xd24d13a6, 0x494c1cbc, 0xf0b7c754, 0xe1caad2d, 0x583176c5, + 0x48db2a17, 0xf120f1ff, 0xe05d9b86, 0x59a6406e, 0xc2a74f74, + 0x7b5c949c, 0x6a21fee5, 0xd3da250d, 0x8752e690, 0x3ea93d78, + 0x2fd45701, 0x962f8ce9, 0x0d2e83f3, 0xb4d5581b, 0xa5a83262, + 0x1c53e98a}, + {0x00000000, 0x9d0fe176, 0xe16ec4ad, 0x7c6125db, 0x19ac8f1b, + 0x84a36e6d, 0xf8c24bb6, 0x65cdaac0, 0x33591e36, 0xae56ff40, + 0xd237da9b, 0x4f383bed, 0x2af5912d, 0xb7fa705b, 0xcb9b5580, + 0x5694b4f6, 0x66b23c6c, 0xfbbddd1a, 0x87dcf8c1, 0x1ad319b7, + 0x7f1eb377, 0xe2115201, 0x9e7077da, 0x037f96ac, 0x55eb225a, + 0xc8e4c32c, 0xb485e6f7, 0x298a0781, 0x4c47ad41, 0xd1484c37, + 0xad2969ec, 0x3026889a, 0xcd6478d8, 0x506b99ae, 0x2c0abc75, + 0xb1055d03, 0xd4c8f7c3, 0x49c716b5, 0x35a6336e, 0xa8a9d218, + 0xfe3d66ee, 0x63328798, 0x1f53a243, 0x825c4335, 0xe791e9f5, + 0x7a9e0883, 0x06ff2d58, 0x9bf0cc2e, 0xabd644b4, 0x36d9a5c2, + 0x4ab88019, 0xd7b7616f, 0xb27acbaf, 0x2f752ad9, 0x53140f02, + 0xce1bee74, 0x988f5a82, 0x0580bbf4, 0x79e19e2f, 0xe4ee7f59, + 0x8123d599, 0x1c2c34ef, 0x604d1134, 0xfd42f042, 0x41b9f7f1, + 0xdcb61687, 0xa0d7335c, 0x3dd8d22a, 0x581578ea, 0xc51a999c, + 0xb97bbc47, 0x24745d31, 0x72e0e9c7, 0xefef08b1, 0x938e2d6a, + 0x0e81cc1c, 0x6b4c66dc, 0xf64387aa, 0x8a22a271, 0x172d4307, + 0x270bcb9d, 0xba042aeb, 0xc6650f30, 0x5b6aee46, 0x3ea74486, + 0xa3a8a5f0, 0xdfc9802b, 0x42c6615d, 0x1452d5ab, 0x895d34dd, + 0xf53c1106, 0x6833f070, 0x0dfe5ab0, 0x90f1bbc6, 0xec909e1d, + 0x719f7f6b, 0x8cdd8f29, 0x11d26e5f, 0x6db34b84, 0xf0bcaaf2, + 0x95710032, 0x087ee144, 0x741fc49f, 0xe91025e9, 0xbf84911f, + 0x228b7069, 0x5eea55b2, 0xc3e5b4c4, 0xa6281e04, 0x3b27ff72, + 0x4746daa9, 0xda493bdf, 0xea6fb345, 0x77605233, 0x0b0177e8, + 0x960e969e, 0xf3c33c5e, 0x6eccdd28, 0x12adf8f3, 0x8fa21985, + 0xd936ad73, 0x44394c05, 0x385869de, 0xa55788a8, 0xc09a2268, + 0x5d95c31e, 0x21f4e6c5, 0xbcfb07b3, 0x8373efe2, 0x1e7c0e94, + 0x621d2b4f, 0xff12ca39, 0x9adf60f9, 0x07d0818f, 0x7bb1a454, + 0xe6be4522, 0xb02af1d4, 0x2d2510a2, 0x51443579, 0xcc4bd40f, + 0xa9867ecf, 0x34899fb9, 0x48e8ba62, 0xd5e75b14, 0xe5c1d38e, + 0x78ce32f8, 0x04af1723, 0x99a0f655, 0xfc6d5c95, 0x6162bde3, + 0x1d039838, 0x800c794e, 0xd698cdb8, 0x4b972cce, 0x37f60915, + 0xaaf9e863, 0xcf3442a3, 0x523ba3d5, 0x2e5a860e, 0xb3556778, + 0x4e17973a, 0xd318764c, 0xaf795397, 0x3276b2e1, 0x57bb1821, + 0xcab4f957, 0xb6d5dc8c, 0x2bda3dfa, 0x7d4e890c, 0xe041687a, + 0x9c204da1, 0x012facd7, 0x64e20617, 0xf9ede761, 0x858cc2ba, + 0x188323cc, 0x28a5ab56, 0xb5aa4a20, 0xc9cb6ffb, 0x54c48e8d, + 0x3109244d, 0xac06c53b, 0xd067e0e0, 0x4d680196, 0x1bfcb560, + 0x86f35416, 0xfa9271cd, 0x679d90bb, 0x02503a7b, 0x9f5fdb0d, + 0xe33efed6, 0x7e311fa0, 0xc2ca1813, 0x5fc5f965, 0x23a4dcbe, + 0xbeab3dc8, 0xdb669708, 0x4669767e, 0x3a0853a5, 0xa707b2d3, + 0xf1930625, 0x6c9ce753, 0x10fdc288, 0x8df223fe, 0xe83f893e, + 0x75306848, 0x09514d93, 0x945eace5, 0xa478247f, 0x3977c509, + 0x4516e0d2, 0xd81901a4, 0xbdd4ab64, 0x20db4a12, 0x5cba6fc9, + 0xc1b58ebf, 0x97213a49, 0x0a2edb3f, 0x764ffee4, 0xeb401f92, + 0x8e8db552, 0x13825424, 0x6fe371ff, 0xf2ec9089, 0x0fae60cb, + 0x92a181bd, 0xeec0a466, 0x73cf4510, 0x1602efd0, 0x8b0d0ea6, + 0xf76c2b7d, 0x6a63ca0b, 0x3cf77efd, 0xa1f89f8b, 0xdd99ba50, + 0x40965b26, 0x255bf1e6, 0xb8541090, 0xc435354b, 0x593ad43d, + 0x691c5ca7, 0xf413bdd1, 0x8872980a, 0x157d797c, 0x70b0d3bc, + 0xedbf32ca, 0x91de1711, 0x0cd1f667, 0x5a454291, 0xc74aa3e7, + 0xbb2b863c, 0x2624674a, 0x43e9cd8a, 0xdee62cfc, 0xa2870927, + 0x3f88e851}, + {0x00000000, 0xdd96d985, 0x605cb54b, 0xbdca6cce, 0xc0b96a96, + 0x1d2fb313, 0xa0e5dfdd, 0x7d730658, 0x5a03d36d, 0x87950ae8, + 0x3a5f6626, 0xe7c9bfa3, 0x9abab9fb, 0x472c607e, 0xfae60cb0, + 0x2770d535, 0xb407a6da, 0x69917f5f, 0xd45b1391, 0x09cdca14, + 0x74becc4c, 0xa92815c9, 0x14e27907, 0xc974a082, 0xee0475b7, + 0x3392ac32, 0x8e58c0fc, 0x53ce1979, 0x2ebd1f21, 0xf32bc6a4, + 0x4ee1aa6a, 0x937773ef, 0xb37e4bf5, 0x6ee89270, 0xd322febe, + 0x0eb4273b, 0x73c72163, 0xae51f8e6, 0x139b9428, 0xce0d4dad, + 0xe97d9898, 0x34eb411d, 0x89212dd3, 0x54b7f456, 0x29c4f20e, + 0xf4522b8b, 0x49984745, 0x940e9ec0, 0x0779ed2f, 0xdaef34aa, + 0x67255864, 0xbab381e1, 0xc7c087b9, 0x1a565e3c, 0xa79c32f2, + 0x7a0aeb77, 0x5d7a3e42, 0x80ece7c7, 0x3d268b09, 0xe0b0528c, + 0x9dc354d4, 0x40558d51, 0xfd9fe19f, 0x2009381a, 0xbd8d91ab, + 0x601b482e, 0xddd124e0, 0x0047fd65, 0x7d34fb3d, 0xa0a222b8, + 0x1d684e76, 0xc0fe97f3, 0xe78e42c6, 0x3a189b43, 0x87d2f78d, + 0x5a442e08, 0x27372850, 0xfaa1f1d5, 0x476b9d1b, 0x9afd449e, + 0x098a3771, 0xd41ceef4, 0x69d6823a, 0xb4405bbf, 0xc9335de7, + 0x14a58462, 0xa96fe8ac, 0x74f93129, 0x5389e41c, 0x8e1f3d99, + 0x33d55157, 0xee4388d2, 0x93308e8a, 0x4ea6570f, 0xf36c3bc1, + 0x2efae244, 0x0ef3da5e, 0xd36503db, 0x6eaf6f15, 0xb339b690, + 0xce4ab0c8, 0x13dc694d, 0xae160583, 0x7380dc06, 0x54f00933, + 0x8966d0b6, 0x34acbc78, 0xe93a65fd, 0x944963a5, 0x49dfba20, + 0xf415d6ee, 0x29830f6b, 0xbaf47c84, 0x6762a501, 0xdaa8c9cf, + 0x073e104a, 0x7a4d1612, 0xa7dbcf97, 0x1a11a359, 0xc7877adc, + 0xe0f7afe9, 0x3d61766c, 0x80ab1aa2, 0x5d3dc327, 0x204ec57f, + 0xfdd81cfa, 0x40127034, 0x9d84a9b1, 0xa06a2517, 0x7dfcfc92, + 0xc036905c, 0x1da049d9, 0x60d34f81, 0xbd459604, 0x008ffaca, + 0xdd19234f, 0xfa69f67a, 0x27ff2fff, 0x9a354331, 0x47a39ab4, + 0x3ad09cec, 0xe7464569, 0x5a8c29a7, 0x871af022, 0x146d83cd, + 0xc9fb5a48, 0x74313686, 0xa9a7ef03, 0xd4d4e95b, 0x094230de, + 0xb4885c10, 0x691e8595, 0x4e6e50a0, 0x93f88925, 0x2e32e5eb, + 0xf3a43c6e, 0x8ed73a36, 0x5341e3b3, 0xee8b8f7d, 0x331d56f8, + 0x13146ee2, 0xce82b767, 0x7348dba9, 0xaede022c, 0xd3ad0474, + 0x0e3bddf1, 0xb3f1b13f, 0x6e6768ba, 0x4917bd8f, 0x9481640a, + 0x294b08c4, 0xf4ddd141, 0x89aed719, 0x54380e9c, 0xe9f26252, + 0x3464bbd7, 0xa713c838, 0x7a8511bd, 0xc74f7d73, 0x1ad9a4f6, + 0x67aaa2ae, 0xba3c7b2b, 0x07f617e5, 0xda60ce60, 0xfd101b55, + 0x2086c2d0, 0x9d4cae1e, 0x40da779b, 0x3da971c3, 0xe03fa846, + 0x5df5c488, 0x80631d0d, 0x1de7b4bc, 0xc0716d39, 0x7dbb01f7, + 0xa02dd872, 0xdd5ede2a, 0x00c807af, 0xbd026b61, 0x6094b2e4, + 0x47e467d1, 0x9a72be54, 0x27b8d29a, 0xfa2e0b1f, 0x875d0d47, + 0x5acbd4c2, 0xe701b80c, 0x3a976189, 0xa9e01266, 0x7476cbe3, + 0xc9bca72d, 0x142a7ea8, 0x695978f0, 0xb4cfa175, 0x0905cdbb, + 0xd493143e, 0xf3e3c10b, 0x2e75188e, 0x93bf7440, 0x4e29adc5, + 0x335aab9d, 0xeecc7218, 0x53061ed6, 0x8e90c753, 0xae99ff49, + 0x730f26cc, 0xcec54a02, 0x13539387, 0x6e2095df, 0xb3b64c5a, + 0x0e7c2094, 0xd3eaf911, 0xf49a2c24, 0x290cf5a1, 0x94c6996f, + 0x495040ea, 0x342346b2, 0xe9b59f37, 0x547ff3f9, 0x89e92a7c, + 0x1a9e5993, 0xc7088016, 0x7ac2ecd8, 0xa754355d, 0xda273305, + 0x07b1ea80, 0xba7b864e, 0x67ed5fcb, 0x409d8afe, 0x9d0b537b, + 0x20c13fb5, 0xfd57e630, 0x8024e068, 0x5db239ed, 0xe0785523, + 0x3dee8ca6}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x00000000, 0x85d996dd, 0x4bb55c60, 0xce6ccabd, 0x966ab9c0, + 0x13b32f1d, 0xdddfe5a0, 0x5806737d, 0x6dd3035a, 0xe80a9587, + 0x26665f3a, 0xa3bfc9e7, 0xfbb9ba9a, 0x7e602c47, 0xb00ce6fa, + 0x35d57027, 0xdaa607b4, 0x5f7f9169, 0x91135bd4, 0x14cacd09, + 0x4cccbe74, 0xc91528a9, 0x0779e214, 0x82a074c9, 0xb77504ee, + 0x32ac9233, 0xfcc0588e, 0x7919ce53, 0x211fbd2e, 0xa4c62bf3, + 0x6aaae14e, 0xef737793, 0xf54b7eb3, 0x7092e86e, 0xbefe22d3, + 0x3b27b40e, 0x6321c773, 0xe6f851ae, 0x28949b13, 0xad4d0dce, + 0x98987de9, 0x1d41eb34, 0xd32d2189, 0x56f4b754, 0x0ef2c429, + 0x8b2b52f4, 0x45479849, 0xc09e0e94, 0x2fed7907, 0xaa34efda, + 0x64582567, 0xe181b3ba, 0xb987c0c7, 0x3c5e561a, 0xf2329ca7, + 0x77eb0a7a, 0x423e7a5d, 0xc7e7ec80, 0x098b263d, 0x8c52b0e0, + 0xd454c39d, 0x518d5540, 0x9fe19ffd, 0x1a380920, 0xab918dbd, + 0x2e481b60, 0xe024d1dd, 0x65fd4700, 0x3dfb347d, 0xb822a2a0, + 0x764e681d, 0xf397fec0, 0xc6428ee7, 0x439b183a, 0x8df7d287, + 0x082e445a, 0x50283727, 0xd5f1a1fa, 0x1b9d6b47, 0x9e44fd9a, + 0x71378a09, 0xf4ee1cd4, 0x3a82d669, 0xbf5b40b4, 0xe75d33c9, + 0x6284a514, 0xace86fa9, 0x2931f974, 0x1ce48953, 0x993d1f8e, + 0x5751d533, 0xd28843ee, 0x8a8e3093, 0x0f57a64e, 0xc13b6cf3, + 0x44e2fa2e, 0x5edaf30e, 0xdb0365d3, 0x156faf6e, 0x90b639b3, + 0xc8b04ace, 0x4d69dc13, 0x830516ae, 0x06dc8073, 0x3309f054, + 0xb6d06689, 0x78bcac34, 0xfd653ae9, 0xa5634994, 0x20badf49, + 0xeed615f4, 0x6b0f8329, 0x847cf4ba, 0x01a56267, 0xcfc9a8da, + 0x4a103e07, 0x12164d7a, 0x97cfdba7, 0x59a3111a, 0xdc7a87c7, + 0xe9aff7e0, 0x6c76613d, 0xa21aab80, 0x27c33d5d, 0x7fc54e20, + 0xfa1cd8fd, 0x34701240, 0xb1a9849d, 0x17256aa0, 0x92fcfc7d, + 0x5c9036c0, 0xd949a01d, 0x814fd360, 0x049645bd, 0xcafa8f00, + 0x4f2319dd, 0x7af669fa, 0xff2fff27, 0x3143359a, 0xb49aa347, + 0xec9cd03a, 0x694546e7, 0xa7298c5a, 0x22f01a87, 0xcd836d14, + 0x485afbc9, 0x86363174, 0x03efa7a9, 0x5be9d4d4, 0xde304209, + 0x105c88b4, 0x95851e69, 0xa0506e4e, 0x2589f893, 0xebe5322e, + 0x6e3ca4f3, 0x363ad78e, 0xb3e34153, 0x7d8f8bee, 0xf8561d33, + 0xe26e1413, 0x67b782ce, 0xa9db4873, 0x2c02deae, 0x7404add3, + 0xf1dd3b0e, 0x3fb1f1b3, 0xba68676e, 0x8fbd1749, 0x0a648194, + 0xc4084b29, 0x41d1ddf4, 0x19d7ae89, 0x9c0e3854, 0x5262f2e9, + 0xd7bb6434, 0x38c813a7, 0xbd11857a, 0x737d4fc7, 0xf6a4d91a, + 0xaea2aa67, 0x2b7b3cba, 0xe517f607, 0x60ce60da, 0x551b10fd, + 0xd0c28620, 0x1eae4c9d, 0x9b77da40, 0xc371a93d, 0x46a83fe0, + 0x88c4f55d, 0x0d1d6380, 0xbcb4e71d, 0x396d71c0, 0xf701bb7d, + 0x72d82da0, 0x2ade5edd, 0xaf07c800, 0x616b02bd, 0xe4b29460, + 0xd167e447, 0x54be729a, 0x9ad2b827, 0x1f0b2efa, 0x470d5d87, + 0xc2d4cb5a, 0x0cb801e7, 0x8961973a, 0x6612e0a9, 0xe3cb7674, + 0x2da7bcc9, 0xa87e2a14, 0xf0785969, 0x75a1cfb4, 0xbbcd0509, + 0x3e1493d4, 0x0bc1e3f3, 0x8e18752e, 0x4074bf93, 0xc5ad294e, + 0x9dab5a33, 0x1872ccee, 0xd61e0653, 0x53c7908e, 0x49ff99ae, + 0xcc260f73, 0x024ac5ce, 0x87935313, 0xdf95206e, 0x5a4cb6b3, + 0x94207c0e, 0x11f9ead3, 0x242c9af4, 0xa1f50c29, 0x6f99c694, + 0xea405049, 0xb2462334, 0x379fb5e9, 0xf9f37f54, 0x7c2ae989, + 0x93599e1a, 0x168008c7, 0xd8ecc27a, 0x5d3554a7, 0x053327da, + 0x80eab107, 0x4e867bba, 0xcb5fed67, 0xfe8a9d40, 0x7b530b9d, + 0xb53fc120, 0x30e657fd, 0x68e02480, 0xed39b25d, 0x235578e0, + 0xa68cee3d}, + {0x00000000, 0x76e10f9d, 0xadc46ee1, 0xdb25617c, 0x1b8fac19, + 0x6d6ea384, 0xb64bc2f8, 0xc0aacd65, 0x361e5933, 0x40ff56ae, + 0x9bda37d2, 0xed3b384f, 0x2d91f52a, 0x5b70fab7, 0x80559bcb, + 0xf6b49456, 0x6c3cb266, 0x1addbdfb, 0xc1f8dc87, 0xb719d31a, + 0x77b31e7f, 0x015211e2, 0xda77709e, 0xac967f03, 0x5a22eb55, + 0x2cc3e4c8, 0xf7e685b4, 0x81078a29, 0x41ad474c, 0x374c48d1, + 0xec6929ad, 0x9a882630, 0xd87864cd, 0xae996b50, 0x75bc0a2c, + 0x035d05b1, 0xc3f7c8d4, 0xb516c749, 0x6e33a635, 0x18d2a9a8, + 0xee663dfe, 0x98873263, 0x43a2531f, 0x35435c82, 0xf5e991e7, + 0x83089e7a, 0x582dff06, 0x2eccf09b, 0xb444d6ab, 0xc2a5d936, + 0x1980b84a, 0x6f61b7d7, 0xafcb7ab2, 0xd92a752f, 0x020f1453, + 0x74ee1bce, 0x825a8f98, 0xf4bb8005, 0x2f9ee179, 0x597feee4, + 0x99d52381, 0xef342c1c, 0x34114d60, 0x42f042fd, 0xf1f7b941, + 0x8716b6dc, 0x5c33d7a0, 0x2ad2d83d, 0xea781558, 0x9c991ac5, + 0x47bc7bb9, 0x315d7424, 0xc7e9e072, 0xb108efef, 0x6a2d8e93, + 0x1ccc810e, 0xdc664c6b, 0xaa8743f6, 0x71a2228a, 0x07432d17, + 0x9dcb0b27, 0xeb2a04ba, 0x300f65c6, 0x46ee6a5b, 0x8644a73e, + 0xf0a5a8a3, 0x2b80c9df, 0x5d61c642, 0xabd55214, 0xdd345d89, + 0x06113cf5, 0x70f03368, 0xb05afe0d, 0xc6bbf190, 0x1d9e90ec, + 0x6b7f9f71, 0x298fdd8c, 0x5f6ed211, 0x844bb36d, 0xf2aabcf0, + 0x32007195, 0x44e17e08, 0x9fc41f74, 0xe92510e9, 0x1f9184bf, + 0x69708b22, 0xb255ea5e, 0xc4b4e5c3, 0x041e28a6, 0x72ff273b, + 0xa9da4647, 0xdf3b49da, 0x45b36fea, 0x33526077, 0xe877010b, + 0x9e960e96, 0x5e3cc3f3, 0x28ddcc6e, 0xf3f8ad12, 0x8519a28f, + 0x73ad36d9, 0x054c3944, 0xde695838, 0xa88857a5, 0x68229ac0, + 0x1ec3955d, 0xc5e6f421, 0xb307fbbc, 0xe2ef7383, 0x940e7c1e, + 0x4f2b1d62, 0x39ca12ff, 0xf960df9a, 0x8f81d007, 0x54a4b17b, + 0x2245bee6, 0xd4f12ab0, 0xa210252d, 0x79354451, 0x0fd44bcc, + 0xcf7e86a9, 0xb99f8934, 0x62bae848, 0x145be7d5, 0x8ed3c1e5, + 0xf832ce78, 0x2317af04, 0x55f6a099, 0x955c6dfc, 0xe3bd6261, + 0x3898031d, 0x4e790c80, 0xb8cd98d6, 0xce2c974b, 0x1509f637, + 0x63e8f9aa, 0xa34234cf, 0xd5a33b52, 0x0e865a2e, 0x786755b3, + 0x3a97174e, 0x4c7618d3, 0x975379af, 0xe1b27632, 0x2118bb57, + 0x57f9b4ca, 0x8cdcd5b6, 0xfa3dda2b, 0x0c894e7d, 0x7a6841e0, + 0xa14d209c, 0xd7ac2f01, 0x1706e264, 0x61e7edf9, 0xbac28c85, + 0xcc238318, 0x56aba528, 0x204aaab5, 0xfb6fcbc9, 0x8d8ec454, + 0x4d240931, 0x3bc506ac, 0xe0e067d0, 0x9601684d, 0x60b5fc1b, + 0x1654f386, 0xcd7192fa, 0xbb909d67, 0x7b3a5002, 0x0ddb5f9f, + 0xd6fe3ee3, 0xa01f317e, 0x1318cac2, 0x65f9c55f, 0xbedca423, + 0xc83dabbe, 0x089766db, 0x7e766946, 0xa553083a, 0xd3b207a7, + 0x250693f1, 0x53e79c6c, 0x88c2fd10, 0xfe23f28d, 0x3e893fe8, + 0x48683075, 0x934d5109, 0xe5ac5e94, 0x7f2478a4, 0x09c57739, + 0xd2e01645, 0xa40119d8, 0x64abd4bd, 0x124adb20, 0xc96fba5c, + 0xbf8eb5c1, 0x493a2197, 0x3fdb2e0a, 0xe4fe4f76, 0x921f40eb, + 0x52b58d8e, 0x24548213, 0xff71e36f, 0x8990ecf2, 0xcb60ae0f, + 0xbd81a192, 0x66a4c0ee, 0x1045cf73, 0xd0ef0216, 0xa60e0d8b, + 0x7d2b6cf7, 0x0bca636a, 0xfd7ef73c, 0x8b9ff8a1, 0x50ba99dd, + 0x265b9640, 0xe6f15b25, 0x901054b8, 0x4b3535c4, 0x3dd43a59, + 0xa75c1c69, 0xd1bd13f4, 0x0a987288, 0x7c797d15, 0xbcd3b070, + 0xca32bfed, 0x1117de91, 0x67f6d10c, 0x9142455a, 0xe7a34ac7, + 0x3c862bbb, 0x4a672426, 0x8acde943, 0xfc2ce6de, 0x270987a2, + 0x51e8883f}, + {0x00000000, 0xe8dbfbb9, 0x91b186a8, 0x796a7d11, 0x63657c8a, + 0x8bbe8733, 0xf2d4fa22, 0x1a0f019b, 0x87cc89cf, 0x6f177276, + 0x167d0f67, 0xfea6f4de, 0xe4a9f545, 0x0c720efc, 0x751873ed, + 0x9dc38854, 0x4f9f6244, 0xa74499fd, 0xde2ee4ec, 0x36f51f55, + 0x2cfa1ece, 0xc421e577, 0xbd4b9866, 0x559063df, 0xc853eb8b, + 0x20881032, 0x59e26d23, 0xb139969a, 0xab369701, 0x43ed6cb8, + 0x3a8711a9, 0xd25cea10, 0x9e3ec588, 0x76e53e31, 0x0f8f4320, + 0xe754b899, 0xfd5bb902, 0x158042bb, 0x6cea3faa, 0x8431c413, + 0x19f24c47, 0xf129b7fe, 0x8843caef, 0x60983156, 0x7a9730cd, + 0x924ccb74, 0xeb26b665, 0x03fd4ddc, 0xd1a1a7cc, 0x397a5c75, + 0x40102164, 0xa8cbdadd, 0xb2c4db46, 0x5a1f20ff, 0x23755dee, + 0xcbaea657, 0x566d2e03, 0xbeb6d5ba, 0xc7dca8ab, 0x2f075312, + 0x35085289, 0xddd3a930, 0xa4b9d421, 0x4c622f98, 0x7d7bfbca, + 0x95a00073, 0xecca7d62, 0x041186db, 0x1e1e8740, 0xf6c57cf9, + 0x8faf01e8, 0x6774fa51, 0xfab77205, 0x126c89bc, 0x6b06f4ad, + 0x83dd0f14, 0x99d20e8f, 0x7109f536, 0x08638827, 0xe0b8739e, + 0x32e4998e, 0xda3f6237, 0xa3551f26, 0x4b8ee49f, 0x5181e504, + 0xb95a1ebd, 0xc03063ac, 0x28eb9815, 0xb5281041, 0x5df3ebf8, + 0x249996e9, 0xcc426d50, 0xd64d6ccb, 0x3e969772, 0x47fcea63, + 0xaf2711da, 0xe3453e42, 0x0b9ec5fb, 0x72f4b8ea, 0x9a2f4353, + 0x802042c8, 0x68fbb971, 0x1191c460, 0xf94a3fd9, 0x6489b78d, + 0x8c524c34, 0xf5383125, 0x1de3ca9c, 0x07eccb07, 0xef3730be, + 0x965d4daf, 0x7e86b616, 0xacda5c06, 0x4401a7bf, 0x3d6bdaae, + 0xd5b02117, 0xcfbf208c, 0x2764db35, 0x5e0ea624, 0xb6d55d9d, + 0x2b16d5c9, 0xc3cd2e70, 0xbaa75361, 0x527ca8d8, 0x4873a943, + 0xa0a852fa, 0xd9c22feb, 0x3119d452, 0xbbf0874e, 0x532b7cf7, + 0x2a4101e6, 0xc29afa5f, 0xd895fbc4, 0x304e007d, 0x49247d6c, + 0xa1ff86d5, 0x3c3c0e81, 0xd4e7f538, 0xad8d8829, 0x45567390, + 0x5f59720b, 0xb78289b2, 0xcee8f4a3, 0x26330f1a, 0xf46fe50a, + 0x1cb41eb3, 0x65de63a2, 0x8d05981b, 0x970a9980, 0x7fd16239, + 0x06bb1f28, 0xee60e491, 0x73a36cc5, 0x9b78977c, 0xe212ea6d, + 0x0ac911d4, 0x10c6104f, 0xf81debf6, 0x817796e7, 0x69ac6d5e, + 0x25ce42c6, 0xcd15b97f, 0xb47fc46e, 0x5ca43fd7, 0x46ab3e4c, + 0xae70c5f5, 0xd71ab8e4, 0x3fc1435d, 0xa202cb09, 0x4ad930b0, + 0x33b34da1, 0xdb68b618, 0xc167b783, 0x29bc4c3a, 0x50d6312b, + 0xb80dca92, 0x6a512082, 0x828adb3b, 0xfbe0a62a, 0x133b5d93, + 0x09345c08, 0xe1efa7b1, 0x9885daa0, 0x705e2119, 0xed9da94d, + 0x054652f4, 0x7c2c2fe5, 0x94f7d45c, 0x8ef8d5c7, 0x66232e7e, + 0x1f49536f, 0xf792a8d6, 0xc68b7c84, 0x2e50873d, 0x573afa2c, + 0xbfe10195, 0xa5ee000e, 0x4d35fbb7, 0x345f86a6, 0xdc847d1f, + 0x4147f54b, 0xa99c0ef2, 0xd0f673e3, 0x382d885a, 0x222289c1, + 0xcaf97278, 0xb3930f69, 0x5b48f4d0, 0x89141ec0, 0x61cfe579, + 0x18a59868, 0xf07e63d1, 0xea71624a, 0x02aa99f3, 0x7bc0e4e2, + 0x931b1f5b, 0x0ed8970f, 0xe6036cb6, 0x9f6911a7, 0x77b2ea1e, + 0x6dbdeb85, 0x8566103c, 0xfc0c6d2d, 0x14d79694, 0x58b5b90c, + 0xb06e42b5, 0xc9043fa4, 0x21dfc41d, 0x3bd0c586, 0xd30b3e3f, + 0xaa61432e, 0x42bab897, 0xdf7930c3, 0x37a2cb7a, 0x4ec8b66b, + 0xa6134dd2, 0xbc1c4c49, 0x54c7b7f0, 0x2dadcae1, 0xc5763158, + 0x172adb48, 0xfff120f1, 0x869b5de0, 0x6e40a659, 0x744fa7c2, + 0x9c945c7b, 0xe5fe216a, 0x0d25dad3, 0x90e65287, 0x783da93e, + 0x0157d42f, 0xe98c2f96, 0xf3832e0d, 0x1b58d5b4, 0x6232a8a5, + 0x8ae9531c}, + {0x00000000, 0x919168ae, 0x6325a087, 0xf2b4c829, 0x874c31d4, + 0x16dd597a, 0xe4699153, 0x75f8f9fd, 0x4f9f1373, 0xde0e7bdd, + 0x2cbab3f4, 0xbd2bdb5a, 0xc8d322a7, 0x59424a09, 0xabf68220, + 0x3a67ea8e, 0x9e3e27e6, 0x0faf4f48, 0xfd1b8761, 0x6c8aefcf, + 0x19721632, 0x88e37e9c, 0x7a57b6b5, 0xebc6de1b, 0xd1a13495, + 0x40305c3b, 0xb2849412, 0x2315fcbc, 0x56ed0541, 0xc77c6def, + 0x35c8a5c6, 0xa459cd68, 0x7d7b3f17, 0xecea57b9, 0x1e5e9f90, + 0x8fcff73e, 0xfa370ec3, 0x6ba6666d, 0x9912ae44, 0x0883c6ea, + 0x32e42c64, 0xa37544ca, 0x51c18ce3, 0xc050e44d, 0xb5a81db0, + 0x2439751e, 0xd68dbd37, 0x471cd599, 0xe34518f1, 0x72d4705f, + 0x8060b876, 0x11f1d0d8, 0x64092925, 0xf598418b, 0x072c89a2, + 0x96bde10c, 0xacda0b82, 0x3d4b632c, 0xcfffab05, 0x5e6ec3ab, + 0x2b963a56, 0xba0752f8, 0x48b39ad1, 0xd922f27f, 0xfaf67e2e, + 0x6b671680, 0x99d3dea9, 0x0842b607, 0x7dba4ffa, 0xec2b2754, + 0x1e9fef7d, 0x8f0e87d3, 0xb5696d5d, 0x24f805f3, 0xd64ccdda, + 0x47dda574, 0x32255c89, 0xa3b43427, 0x5100fc0e, 0xc09194a0, + 0x64c859c8, 0xf5593166, 0x07edf94f, 0x967c91e1, 0xe384681c, + 0x721500b2, 0x80a1c89b, 0x1130a035, 0x2b574abb, 0xbac62215, + 0x4872ea3c, 0xd9e38292, 0xac1b7b6f, 0x3d8a13c1, 0xcf3edbe8, + 0x5eafb346, 0x878d4139, 0x161c2997, 0xe4a8e1be, 0x75398910, + 0x00c170ed, 0x91501843, 0x63e4d06a, 0xf275b8c4, 0xc812524a, + 0x59833ae4, 0xab37f2cd, 0x3aa69a63, 0x4f5e639e, 0xdecf0b30, + 0x2c7bc319, 0xbdeaabb7, 0x19b366df, 0x88220e71, 0x7a96c658, + 0xeb07aef6, 0x9eff570b, 0x0f6e3fa5, 0xfddaf78c, 0x6c4b9f22, + 0x562c75ac, 0xc7bd1d02, 0x3509d52b, 0xa498bd85, 0xd1604478, + 0x40f12cd6, 0xb245e4ff, 0x23d48c51, 0xf4edfd5c, 0x657c95f2, + 0x97c85ddb, 0x06593575, 0x73a1cc88, 0xe230a426, 0x10846c0f, + 0x811504a1, 0xbb72ee2f, 0x2ae38681, 0xd8574ea8, 0x49c62606, + 0x3c3edffb, 0xadafb755, 0x5f1b7f7c, 0xce8a17d2, 0x6ad3daba, + 0xfb42b214, 0x09f67a3d, 0x98671293, 0xed9feb6e, 0x7c0e83c0, + 0x8eba4be9, 0x1f2b2347, 0x254cc9c9, 0xb4dda167, 0x4669694e, + 0xd7f801e0, 0xa200f81d, 0x339190b3, 0xc125589a, 0x50b43034, + 0x8996c24b, 0x1807aae5, 0xeab362cc, 0x7b220a62, 0x0edaf39f, + 0x9f4b9b31, 0x6dff5318, 0xfc6e3bb6, 0xc609d138, 0x5798b996, + 0xa52c71bf, 0x34bd1911, 0x4145e0ec, 0xd0d48842, 0x2260406b, + 0xb3f128c5, 0x17a8e5ad, 0x86398d03, 0x748d452a, 0xe51c2d84, + 0x90e4d479, 0x0175bcd7, 0xf3c174fe, 0x62501c50, 0x5837f6de, + 0xc9a69e70, 0x3b125659, 0xaa833ef7, 0xdf7bc70a, 0x4eeaafa4, + 0xbc5e678d, 0x2dcf0f23, 0x0e1b8372, 0x9f8aebdc, 0x6d3e23f5, + 0xfcaf4b5b, 0x8957b2a6, 0x18c6da08, 0xea721221, 0x7be37a8f, + 0x41849001, 0xd015f8af, 0x22a13086, 0xb3305828, 0xc6c8a1d5, + 0x5759c97b, 0xa5ed0152, 0x347c69fc, 0x9025a494, 0x01b4cc3a, + 0xf3000413, 0x62916cbd, 0x17699540, 0x86f8fdee, 0x744c35c7, + 0xe5dd5d69, 0xdfbab7e7, 0x4e2bdf49, 0xbc9f1760, 0x2d0e7fce, + 0x58f68633, 0xc967ee9d, 0x3bd326b4, 0xaa424e1a, 0x7360bc65, + 0xe2f1d4cb, 0x10451ce2, 0x81d4744c, 0xf42c8db1, 0x65bde51f, + 0x97092d36, 0x06984598, 0x3cffaf16, 0xad6ec7b8, 0x5fda0f91, + 0xce4b673f, 0xbbb39ec2, 0x2a22f66c, 0xd8963e45, 0x490756eb, + 0xed5e9b83, 0x7ccff32d, 0x8e7b3b04, 0x1fea53aa, 0x6a12aa57, + 0xfb83c2f9, 0x09370ad0, 0x98a6627e, 0xa2c188f0, 0x3350e05e, + 0xc1e42877, 0x507540d9, 0x258db924, 0xb41cd18a, 0x46a819a3, + 0xd739710d}}; + +#endif + +#endif + +#if N == 5 + +#if W == 8 + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0xaf449247, 0x85f822cf, 0x2abcb088, 0xd08143df, + 0x7fc5d198, 0x55796110, 0xfa3df357, 0x7a7381ff, 0xd53713b8, + 0xff8ba330, 0x50cf3177, 0xaaf2c220, 0x05b65067, 0x2f0ae0ef, + 0x804e72a8, 0xf4e703fe, 0x5ba391b9, 0x711f2131, 0xde5bb376, + 0x24664021, 0x8b22d266, 0xa19e62ee, 0x0edaf0a9, 0x8e948201, + 0x21d01046, 0x0b6ca0ce, 0xa4283289, 0x5e15c1de, 0xf1515399, + 0xdbede311, 0x74a97156, 0x32bf01bd, 0x9dfb93fa, 0xb7472372, + 0x1803b135, 0xe23e4262, 0x4d7ad025, 0x67c660ad, 0xc882f2ea, + 0x48cc8042, 0xe7881205, 0xcd34a28d, 0x627030ca, 0x984dc39d, + 0x370951da, 0x1db5e152, 0xb2f17315, 0xc6580243, 0x691c9004, + 0x43a0208c, 0xece4b2cb, 0x16d9419c, 0xb99dd3db, 0x93216353, + 0x3c65f114, 0xbc2b83bc, 0x136f11fb, 0x39d3a173, 0x96973334, + 0x6caac063, 0xc3ee5224, 0xe952e2ac, 0x461670eb, 0x657e037a, + 0xca3a913d, 0xe08621b5, 0x4fc2b3f2, 0xb5ff40a5, 0x1abbd2e2, + 0x3007626a, 0x9f43f02d, 0x1f0d8285, 0xb04910c2, 0x9af5a04a, + 0x35b1320d, 0xcf8cc15a, 0x60c8531d, 0x4a74e395, 0xe53071d2, + 0x91990084, 0x3edd92c3, 0x1461224b, 0xbb25b00c, 0x4118435b, + 0xee5cd11c, 0xc4e06194, 0x6ba4f3d3, 0xebea817b, 0x44ae133c, + 0x6e12a3b4, 0xc15631f3, 0x3b6bc2a4, 0x942f50e3, 0xbe93e06b, + 0x11d7722c, 0x57c102c7, 0xf8859080, 0xd2392008, 0x7d7db24f, + 0x87404118, 0x2804d35f, 0x02b863d7, 0xadfcf190, 0x2db28338, + 0x82f6117f, 0xa84aa1f7, 0x070e33b0, 0xfd33c0e7, 0x527752a0, + 0x78cbe228, 0xd78f706f, 0xa3260139, 0x0c62937e, 0x26de23f6, + 0x899ab1b1, 0x73a742e6, 0xdce3d0a1, 0xf65f6029, 0x591bf26e, + 0xd95580c6, 0x76111281, 0x5cada209, 0xf3e9304e, 0x09d4c319, + 0xa690515e, 0x8c2ce1d6, 0x23687391, 0xcafc06f4, 0x65b894b3, + 0x4f04243b, 0xe040b67c, 0x1a7d452b, 0xb539d76c, 0x9f8567e4, + 0x30c1f5a3, 0xb08f870b, 0x1fcb154c, 0x3577a5c4, 0x9a333783, + 0x600ec4d4, 0xcf4a5693, 0xe5f6e61b, 0x4ab2745c, 0x3e1b050a, + 0x915f974d, 0xbbe327c5, 0x14a7b582, 0xee9a46d5, 0x41ded492, + 0x6b62641a, 0xc426f65d, 0x446884f5, 0xeb2c16b2, 0xc190a63a, + 0x6ed4347d, 0x94e9c72a, 0x3bad556d, 0x1111e5e5, 0xbe5577a2, + 0xf8430749, 0x5707950e, 0x7dbb2586, 0xd2ffb7c1, 0x28c24496, + 0x8786d6d1, 0xad3a6659, 0x027ef41e, 0x823086b6, 0x2d7414f1, + 0x07c8a479, 0xa88c363e, 0x52b1c569, 0xfdf5572e, 0xd749e7a6, + 0x780d75e1, 0x0ca404b7, 0xa3e096f0, 0x895c2678, 0x2618b43f, + 0xdc254768, 0x7361d52f, 0x59dd65a7, 0xf699f7e0, 0x76d78548, + 0xd993170f, 0xf32fa787, 0x5c6b35c0, 0xa656c697, 0x091254d0, + 0x23aee458, 0x8cea761f, 0xaf82058e, 0x00c697c9, 0x2a7a2741, + 0x853eb506, 0x7f034651, 0xd047d416, 0xfafb649e, 0x55bff6d9, + 0xd5f18471, 0x7ab51636, 0x5009a6be, 0xff4d34f9, 0x0570c7ae, + 0xaa3455e9, 0x8088e561, 0x2fcc7726, 0x5b650670, 0xf4219437, + 0xde9d24bf, 0x71d9b6f8, 0x8be445af, 0x24a0d7e8, 0x0e1c6760, + 0xa158f527, 0x2116878f, 0x8e5215c8, 0xa4eea540, 0x0baa3707, + 0xf197c450, 0x5ed35617, 0x746fe69f, 0xdb2b74d8, 0x9d3d0433, + 0x32799674, 0x18c526fc, 0xb781b4bb, 0x4dbc47ec, 0xe2f8d5ab, + 0xc8446523, 0x6700f764, 0xe74e85cc, 0x480a178b, 0x62b6a703, + 0xcdf23544, 0x37cfc613, 0x988b5454, 0xb237e4dc, 0x1d73769b, + 0x69da07cd, 0xc69e958a, 0xec222502, 0x4366b745, 0xb95b4412, + 0x161fd655, 0x3ca366dd, 0x93e7f49a, 0x13a98632, 0xbced1475, + 0x9651a4fd, 0x391536ba, 0xc328c5ed, 0x6c6c57aa, 0x46d0e722, + 0xe9947565}, + {0x00000000, 0x4e890ba9, 0x9d121752, 0xd39b1cfb, 0xe15528e5, + 0xafdc234c, 0x7c473fb7, 0x32ce341e, 0x19db578b, 0x57525c22, + 0x84c940d9, 0xca404b70, 0xf88e7f6e, 0xb60774c7, 0x659c683c, + 0x2b156395, 0x33b6af16, 0x7d3fa4bf, 0xaea4b844, 0xe02db3ed, + 0xd2e387f3, 0x9c6a8c5a, 0x4ff190a1, 0x01789b08, 0x2a6df89d, + 0x64e4f334, 0xb77fefcf, 0xf9f6e466, 0xcb38d078, 0x85b1dbd1, + 0x562ac72a, 0x18a3cc83, 0x676d5e2c, 0x29e45585, 0xfa7f497e, + 0xb4f642d7, 0x863876c9, 0xc8b17d60, 0x1b2a619b, 0x55a36a32, + 0x7eb609a7, 0x303f020e, 0xe3a41ef5, 0xad2d155c, 0x9fe32142, + 0xd16a2aeb, 0x02f13610, 0x4c783db9, 0x54dbf13a, 0x1a52fa93, + 0xc9c9e668, 0x8740edc1, 0xb58ed9df, 0xfb07d276, 0x289cce8d, + 0x6615c524, 0x4d00a6b1, 0x0389ad18, 0xd012b1e3, 0x9e9bba4a, + 0xac558e54, 0xe2dc85fd, 0x31479906, 0x7fce92af, 0xcedabc58, + 0x8053b7f1, 0x53c8ab0a, 0x1d41a0a3, 0x2f8f94bd, 0x61069f14, + 0xb29d83ef, 0xfc148846, 0xd701ebd3, 0x9988e07a, 0x4a13fc81, + 0x049af728, 0x3654c336, 0x78ddc89f, 0xab46d464, 0xe5cfdfcd, + 0xfd6c134e, 0xb3e518e7, 0x607e041c, 0x2ef70fb5, 0x1c393bab, + 0x52b03002, 0x812b2cf9, 0xcfa22750, 0xe4b744c5, 0xaa3e4f6c, + 0x79a55397, 0x372c583e, 0x05e26c20, 0x4b6b6789, 0x98f07b72, + 0xd67970db, 0xa9b7e274, 0xe73ee9dd, 0x34a5f526, 0x7a2cfe8f, + 0x48e2ca91, 0x066bc138, 0xd5f0ddc3, 0x9b79d66a, 0xb06cb5ff, + 0xfee5be56, 0x2d7ea2ad, 0x63f7a904, 0x51399d1a, 0x1fb096b3, + 0xcc2b8a48, 0x82a281e1, 0x9a014d62, 0xd48846cb, 0x07135a30, + 0x499a5199, 0x7b546587, 0x35dd6e2e, 0xe64672d5, 0xa8cf797c, + 0x83da1ae9, 0xcd531140, 0x1ec80dbb, 0x50410612, 0x628f320c, + 0x2c0639a5, 0xff9d255e, 0xb1142ef7, 0x46c47ef1, 0x084d7558, + 0xdbd669a3, 0x955f620a, 0xa7915614, 0xe9185dbd, 0x3a834146, + 0x740a4aef, 0x5f1f297a, 0x119622d3, 0xc20d3e28, 0x8c843581, + 0xbe4a019f, 0xf0c30a36, 0x235816cd, 0x6dd11d64, 0x7572d1e7, + 0x3bfbda4e, 0xe860c6b5, 0xa6e9cd1c, 0x9427f902, 0xdaaef2ab, + 0x0935ee50, 0x47bce5f9, 0x6ca9866c, 0x22208dc5, 0xf1bb913e, + 0xbf329a97, 0x8dfcae89, 0xc375a520, 0x10eeb9db, 0x5e67b272, + 0x21a920dd, 0x6f202b74, 0xbcbb378f, 0xf2323c26, 0xc0fc0838, + 0x8e750391, 0x5dee1f6a, 0x136714c3, 0x38727756, 0x76fb7cff, + 0xa5606004, 0xebe96bad, 0xd9275fb3, 0x97ae541a, 0x443548e1, + 0x0abc4348, 0x121f8fcb, 0x5c968462, 0x8f0d9899, 0xc1849330, + 0xf34aa72e, 0xbdc3ac87, 0x6e58b07c, 0x20d1bbd5, 0x0bc4d840, + 0x454dd3e9, 0x96d6cf12, 0xd85fc4bb, 0xea91f0a5, 0xa418fb0c, + 0x7783e7f7, 0x390aec5e, 0x881ec2a9, 0xc697c900, 0x150cd5fb, + 0x5b85de52, 0x694bea4c, 0x27c2e1e5, 0xf459fd1e, 0xbad0f6b7, + 0x91c59522, 0xdf4c9e8b, 0x0cd78270, 0x425e89d9, 0x7090bdc7, + 0x3e19b66e, 0xed82aa95, 0xa30ba13c, 0xbba86dbf, 0xf5216616, + 0x26ba7aed, 0x68337144, 0x5afd455a, 0x14744ef3, 0xc7ef5208, + 0x896659a1, 0xa2733a34, 0xecfa319d, 0x3f612d66, 0x71e826cf, + 0x432612d1, 0x0daf1978, 0xde340583, 0x90bd0e2a, 0xef739c85, + 0xa1fa972c, 0x72618bd7, 0x3ce8807e, 0x0e26b460, 0x40afbfc9, + 0x9334a332, 0xddbda89b, 0xf6a8cb0e, 0xb821c0a7, 0x6bbadc5c, + 0x2533d7f5, 0x17fde3eb, 0x5974e842, 0x8aeff4b9, 0xc466ff10, + 0xdcc53393, 0x924c383a, 0x41d724c1, 0x0f5e2f68, 0x3d901b76, + 0x731910df, 0xa0820c24, 0xee0b078d, 0xc51e6418, 0x8b976fb1, + 0x580c734a, 0x168578e3, 0x244b4cfd, 0x6ac24754, 0xb9595baf, + 0xf7d05006}, + {0x00000000, 0x8d88fde2, 0xc060fd85, 0x4de80067, 0x5bb0fd4b, + 0xd63800a9, 0x9bd000ce, 0x1658fd2c, 0xb761fa96, 0x3ae90774, + 0x77010713, 0xfa89faf1, 0xecd107dd, 0x6159fa3f, 0x2cb1fa58, + 0xa13907ba, 0xb5b2f36d, 0x383a0e8f, 0x75d20ee8, 0xf85af30a, + 0xee020e26, 0x638af3c4, 0x2e62f3a3, 0xa3ea0e41, 0x02d309fb, + 0x8f5bf419, 0xc2b3f47e, 0x4f3b099c, 0x5963f4b0, 0xd4eb0952, + 0x99030935, 0x148bf4d7, 0xb014e09b, 0x3d9c1d79, 0x70741d1e, + 0xfdfce0fc, 0xeba41dd0, 0x662ce032, 0x2bc4e055, 0xa64c1db7, + 0x07751a0d, 0x8afde7ef, 0xc715e788, 0x4a9d1a6a, 0x5cc5e746, + 0xd14d1aa4, 0x9ca51ac3, 0x112de721, 0x05a613f6, 0x882eee14, + 0xc5c6ee73, 0x484e1391, 0x5e16eebd, 0xd39e135f, 0x9e761338, + 0x13feeeda, 0xb2c7e960, 0x3f4f1482, 0x72a714e5, 0xff2fe907, + 0xe977142b, 0x64ffe9c9, 0x2917e9ae, 0xa49f144c, 0xbb58c777, + 0x36d03a95, 0x7b383af2, 0xf6b0c710, 0xe0e83a3c, 0x6d60c7de, + 0x2088c7b9, 0xad003a5b, 0x0c393de1, 0x81b1c003, 0xcc59c064, + 0x41d13d86, 0x5789c0aa, 0xda013d48, 0x97e93d2f, 0x1a61c0cd, + 0x0eea341a, 0x8362c9f8, 0xce8ac99f, 0x4302347d, 0x555ac951, + 0xd8d234b3, 0x953a34d4, 0x18b2c936, 0xb98bce8c, 0x3403336e, + 0x79eb3309, 0xf463ceeb, 0xe23b33c7, 0x6fb3ce25, 0x225bce42, + 0xafd333a0, 0x0b4c27ec, 0x86c4da0e, 0xcb2cda69, 0x46a4278b, + 0x50fcdaa7, 0xdd742745, 0x909c2722, 0x1d14dac0, 0xbc2ddd7a, + 0x31a52098, 0x7c4d20ff, 0xf1c5dd1d, 0xe79d2031, 0x6a15ddd3, + 0x27fdddb4, 0xaa752056, 0xbefed481, 0x33762963, 0x7e9e2904, + 0xf316d4e6, 0xe54e29ca, 0x68c6d428, 0x252ed44f, 0xa8a629ad, + 0x099f2e17, 0x8417d3f5, 0xc9ffd392, 0x44772e70, 0x522fd35c, + 0xdfa72ebe, 0x924f2ed9, 0x1fc7d33b, 0xadc088af, 0x2048754d, + 0x6da0752a, 0xe02888c8, 0xf67075e4, 0x7bf88806, 0x36108861, + 0xbb987583, 0x1aa17239, 0x97298fdb, 0xdac18fbc, 0x5749725e, + 0x41118f72, 0xcc997290, 0x817172f7, 0x0cf98f15, 0x18727bc2, + 0x95fa8620, 0xd8128647, 0x559a7ba5, 0x43c28689, 0xce4a7b6b, + 0x83a27b0c, 0x0e2a86ee, 0xaf138154, 0x229b7cb6, 0x6f737cd1, + 0xe2fb8133, 0xf4a37c1f, 0x792b81fd, 0x34c3819a, 0xb94b7c78, + 0x1dd46834, 0x905c95d6, 0xddb495b1, 0x503c6853, 0x4664957f, + 0xcbec689d, 0x860468fa, 0x0b8c9518, 0xaab592a2, 0x273d6f40, + 0x6ad56f27, 0xe75d92c5, 0xf1056fe9, 0x7c8d920b, 0x3165926c, + 0xbced6f8e, 0xa8669b59, 0x25ee66bb, 0x680666dc, 0xe58e9b3e, + 0xf3d66612, 0x7e5e9bf0, 0x33b69b97, 0xbe3e6675, 0x1f0761cf, + 0x928f9c2d, 0xdf679c4a, 0x52ef61a8, 0x44b79c84, 0xc93f6166, + 0x84d76101, 0x095f9ce3, 0x16984fd8, 0x9b10b23a, 0xd6f8b25d, + 0x5b704fbf, 0x4d28b293, 0xc0a04f71, 0x8d484f16, 0x00c0b2f4, + 0xa1f9b54e, 0x2c7148ac, 0x619948cb, 0xec11b529, 0xfa494805, + 0x77c1b5e7, 0x3a29b580, 0xb7a14862, 0xa32abcb5, 0x2ea24157, + 0x634a4130, 0xeec2bcd2, 0xf89a41fe, 0x7512bc1c, 0x38fabc7b, + 0xb5724199, 0x144b4623, 0x99c3bbc1, 0xd42bbba6, 0x59a34644, + 0x4ffbbb68, 0xc273468a, 0x8f9b46ed, 0x0213bb0f, 0xa68caf43, + 0x2b0452a1, 0x66ec52c6, 0xeb64af24, 0xfd3c5208, 0x70b4afea, + 0x3d5caf8d, 0xb0d4526f, 0x11ed55d5, 0x9c65a837, 0xd18da850, + 0x5c0555b2, 0x4a5da89e, 0xc7d5557c, 0x8a3d551b, 0x07b5a8f9, + 0x133e5c2e, 0x9eb6a1cc, 0xd35ea1ab, 0x5ed65c49, 0x488ea165, + 0xc5065c87, 0x88ee5ce0, 0x0566a102, 0xa45fa6b8, 0x29d75b5a, + 0x643f5b3d, 0xe9b7a6df, 0xffef5bf3, 0x7267a611, 0x3f8fa676, + 0xb2075b94}, + {0x00000000, 0x80f0171f, 0xda91287f, 0x5a613f60, 0x6e5356bf, + 0xeea341a0, 0xb4c27ec0, 0x343269df, 0xdca6ad7e, 0x5c56ba61, + 0x06378501, 0x86c7921e, 0xb2f5fbc1, 0x3205ecde, 0x6864d3be, + 0xe894c4a1, 0x623c5cbd, 0xe2cc4ba2, 0xb8ad74c2, 0x385d63dd, + 0x0c6f0a02, 0x8c9f1d1d, 0xd6fe227d, 0x560e3562, 0xbe9af1c3, + 0x3e6ae6dc, 0x640bd9bc, 0xe4fbcea3, 0xd0c9a77c, 0x5039b063, + 0x0a588f03, 0x8aa8981c, 0xc478b97a, 0x4488ae65, 0x1ee99105, + 0x9e19861a, 0xaa2befc5, 0x2adbf8da, 0x70bac7ba, 0xf04ad0a5, + 0x18de1404, 0x982e031b, 0xc24f3c7b, 0x42bf2b64, 0x768d42bb, + 0xf67d55a4, 0xac1c6ac4, 0x2cec7ddb, 0xa644e5c7, 0x26b4f2d8, + 0x7cd5cdb8, 0xfc25daa7, 0xc817b378, 0x48e7a467, 0x12869b07, + 0x92768c18, 0x7ae248b9, 0xfa125fa6, 0xa07360c6, 0x208377d9, + 0x14b11e06, 0x94410919, 0xce203679, 0x4ed02166, 0x538074b5, + 0xd37063aa, 0x89115cca, 0x09e14bd5, 0x3dd3220a, 0xbd233515, + 0xe7420a75, 0x67b21d6a, 0x8f26d9cb, 0x0fd6ced4, 0x55b7f1b4, + 0xd547e6ab, 0xe1758f74, 0x6185986b, 0x3be4a70b, 0xbb14b014, + 0x31bc2808, 0xb14c3f17, 0xeb2d0077, 0x6bdd1768, 0x5fef7eb7, + 0xdf1f69a8, 0x857e56c8, 0x058e41d7, 0xed1a8576, 0x6dea9269, + 0x378bad09, 0xb77bba16, 0x8349d3c9, 0x03b9c4d6, 0x59d8fbb6, + 0xd928eca9, 0x97f8cdcf, 0x1708dad0, 0x4d69e5b0, 0xcd99f2af, + 0xf9ab9b70, 0x795b8c6f, 0x233ab30f, 0xa3caa410, 0x4b5e60b1, + 0xcbae77ae, 0x91cf48ce, 0x113f5fd1, 0x250d360e, 0xa5fd2111, + 0xff9c1e71, 0x7f6c096e, 0xf5c49172, 0x7534866d, 0x2f55b90d, + 0xafa5ae12, 0x9b97c7cd, 0x1b67d0d2, 0x4106efb2, 0xc1f6f8ad, + 0x29623c0c, 0xa9922b13, 0xf3f31473, 0x7303036c, 0x47316ab3, + 0xc7c17dac, 0x9da042cc, 0x1d5055d3, 0xa700e96a, 0x27f0fe75, + 0x7d91c115, 0xfd61d60a, 0xc953bfd5, 0x49a3a8ca, 0x13c297aa, + 0x933280b5, 0x7ba64414, 0xfb56530b, 0xa1376c6b, 0x21c77b74, + 0x15f512ab, 0x950505b4, 0xcf643ad4, 0x4f942dcb, 0xc53cb5d7, + 0x45cca2c8, 0x1fad9da8, 0x9f5d8ab7, 0xab6fe368, 0x2b9ff477, + 0x71fecb17, 0xf10edc08, 0x199a18a9, 0x996a0fb6, 0xc30b30d6, + 0x43fb27c9, 0x77c94e16, 0xf7395909, 0xad586669, 0x2da87176, + 0x63785010, 0xe388470f, 0xb9e9786f, 0x39196f70, 0x0d2b06af, + 0x8ddb11b0, 0xd7ba2ed0, 0x574a39cf, 0xbfdefd6e, 0x3f2eea71, + 0x654fd511, 0xe5bfc20e, 0xd18dabd1, 0x517dbcce, 0x0b1c83ae, + 0x8bec94b1, 0x01440cad, 0x81b41bb2, 0xdbd524d2, 0x5b2533cd, + 0x6f175a12, 0xefe74d0d, 0xb586726d, 0x35766572, 0xdde2a1d3, + 0x5d12b6cc, 0x077389ac, 0x87839eb3, 0xb3b1f76c, 0x3341e073, + 0x6920df13, 0xe9d0c80c, 0xf4809ddf, 0x74708ac0, 0x2e11b5a0, + 0xaee1a2bf, 0x9ad3cb60, 0x1a23dc7f, 0x4042e31f, 0xc0b2f400, + 0x282630a1, 0xa8d627be, 0xf2b718de, 0x72470fc1, 0x4675661e, + 0xc6857101, 0x9ce44e61, 0x1c14597e, 0x96bcc162, 0x164cd67d, + 0x4c2de91d, 0xccddfe02, 0xf8ef97dd, 0x781f80c2, 0x227ebfa2, + 0xa28ea8bd, 0x4a1a6c1c, 0xcaea7b03, 0x908b4463, 0x107b537c, + 0x24493aa3, 0xa4b92dbc, 0xfed812dc, 0x7e2805c3, 0x30f824a5, + 0xb00833ba, 0xea690cda, 0x6a991bc5, 0x5eab721a, 0xde5b6505, + 0x843a5a65, 0x04ca4d7a, 0xec5e89db, 0x6cae9ec4, 0x36cfa1a4, + 0xb63fb6bb, 0x820ddf64, 0x02fdc87b, 0x589cf71b, 0xd86ce004, + 0x52c47818, 0xd2346f07, 0x88555067, 0x08a54778, 0x3c972ea7, + 0xbc6739b8, 0xe60606d8, 0x66f611c7, 0x8e62d566, 0x0e92c279, + 0x54f3fd19, 0xd403ea06, 0xe03183d9, 0x60c194c6, 0x3aa0aba6, + 0xba50bcb9}, + {0x00000000, 0x9570d495, 0xf190af6b, 0x64e07bfe, 0x38505897, + 0xad208c02, 0xc9c0f7fc, 0x5cb02369, 0x70a0b12e, 0xe5d065bb, + 0x81301e45, 0x1440cad0, 0x48f0e9b9, 0xdd803d2c, 0xb96046d2, + 0x2c109247, 0xe141625c, 0x7431b6c9, 0x10d1cd37, 0x85a119a2, + 0xd9113acb, 0x4c61ee5e, 0x288195a0, 0xbdf14135, 0x91e1d372, + 0x049107e7, 0x60717c19, 0xf501a88c, 0xa9b18be5, 0x3cc15f70, + 0x5821248e, 0xcd51f01b, 0x19f3c2f9, 0x8c83166c, 0xe8636d92, + 0x7d13b907, 0x21a39a6e, 0xb4d34efb, 0xd0333505, 0x4543e190, + 0x695373d7, 0xfc23a742, 0x98c3dcbc, 0x0db30829, 0x51032b40, + 0xc473ffd5, 0xa093842b, 0x35e350be, 0xf8b2a0a5, 0x6dc27430, + 0x09220fce, 0x9c52db5b, 0xc0e2f832, 0x55922ca7, 0x31725759, + 0xa40283cc, 0x8812118b, 0x1d62c51e, 0x7982bee0, 0xecf26a75, + 0xb042491c, 0x25329d89, 0x41d2e677, 0xd4a232e2, 0x33e785f2, + 0xa6975167, 0xc2772a99, 0x5707fe0c, 0x0bb7dd65, 0x9ec709f0, + 0xfa27720e, 0x6f57a69b, 0x434734dc, 0xd637e049, 0xb2d79bb7, + 0x27a74f22, 0x7b176c4b, 0xee67b8de, 0x8a87c320, 0x1ff717b5, + 0xd2a6e7ae, 0x47d6333b, 0x233648c5, 0xb6469c50, 0xeaf6bf39, + 0x7f866bac, 0x1b661052, 0x8e16c4c7, 0xa2065680, 0x37768215, + 0x5396f9eb, 0xc6e62d7e, 0x9a560e17, 0x0f26da82, 0x6bc6a17c, + 0xfeb675e9, 0x2a14470b, 0xbf64939e, 0xdb84e860, 0x4ef43cf5, + 0x12441f9c, 0x8734cb09, 0xe3d4b0f7, 0x76a46462, 0x5ab4f625, + 0xcfc422b0, 0xab24594e, 0x3e548ddb, 0x62e4aeb2, 0xf7947a27, + 0x937401d9, 0x0604d54c, 0xcb552557, 0x5e25f1c2, 0x3ac58a3c, + 0xafb55ea9, 0xf3057dc0, 0x6675a955, 0x0295d2ab, 0x97e5063e, + 0xbbf59479, 0x2e8540ec, 0x4a653b12, 0xdf15ef87, 0x83a5ccee, + 0x16d5187b, 0x72356385, 0xe745b710, 0x67cf0be4, 0xf2bfdf71, + 0x965fa48f, 0x032f701a, 0x5f9f5373, 0xcaef87e6, 0xae0ffc18, + 0x3b7f288d, 0x176fbaca, 0x821f6e5f, 0xe6ff15a1, 0x738fc134, + 0x2f3fe25d, 0xba4f36c8, 0xdeaf4d36, 0x4bdf99a3, 0x868e69b8, + 0x13febd2d, 0x771ec6d3, 0xe26e1246, 0xbede312f, 0x2baee5ba, + 0x4f4e9e44, 0xda3e4ad1, 0xf62ed896, 0x635e0c03, 0x07be77fd, + 0x92cea368, 0xce7e8001, 0x5b0e5494, 0x3fee2f6a, 0xaa9efbff, + 0x7e3cc91d, 0xeb4c1d88, 0x8fac6676, 0x1adcb2e3, 0x466c918a, + 0xd31c451f, 0xb7fc3ee1, 0x228cea74, 0x0e9c7833, 0x9becaca6, + 0xff0cd758, 0x6a7c03cd, 0x36cc20a4, 0xa3bcf431, 0xc75c8fcf, + 0x522c5b5a, 0x9f7dab41, 0x0a0d7fd4, 0x6eed042a, 0xfb9dd0bf, + 0xa72df3d6, 0x325d2743, 0x56bd5cbd, 0xc3cd8828, 0xefdd1a6f, + 0x7aadcefa, 0x1e4db504, 0x8b3d6191, 0xd78d42f8, 0x42fd966d, + 0x261ded93, 0xb36d3906, 0x54288e16, 0xc1585a83, 0xa5b8217d, + 0x30c8f5e8, 0x6c78d681, 0xf9080214, 0x9de879ea, 0x0898ad7f, + 0x24883f38, 0xb1f8ebad, 0xd5189053, 0x406844c6, 0x1cd867af, + 0x89a8b33a, 0xed48c8c4, 0x78381c51, 0xb569ec4a, 0x201938df, + 0x44f94321, 0xd18997b4, 0x8d39b4dd, 0x18496048, 0x7ca91bb6, + 0xe9d9cf23, 0xc5c95d64, 0x50b989f1, 0x3459f20f, 0xa129269a, + 0xfd9905f3, 0x68e9d166, 0x0c09aa98, 0x99797e0d, 0x4ddb4cef, + 0xd8ab987a, 0xbc4be384, 0x293b3711, 0x758b1478, 0xe0fbc0ed, + 0x841bbb13, 0x116b6f86, 0x3d7bfdc1, 0xa80b2954, 0xcceb52aa, + 0x599b863f, 0x052ba556, 0x905b71c3, 0xf4bb0a3d, 0x61cbdea8, + 0xac9a2eb3, 0x39eafa26, 0x5d0a81d8, 0xc87a554d, 0x94ca7624, + 0x01baa2b1, 0x655ad94f, 0xf02a0dda, 0xdc3a9f9d, 0x494a4b08, + 0x2daa30f6, 0xb8dae463, 0xe46ac70a, 0x711a139f, 0x15fa6861, + 0x808abcf4}, + {0x00000000, 0xcf9e17c8, 0x444d29d1, 0x8bd33e19, 0x889a53a2, + 0x4704446a, 0xccd77a73, 0x03496dbb, 0xca45a105, 0x05dbb6cd, + 0x8e0888d4, 0x41969f1c, 0x42dff2a7, 0x8d41e56f, 0x0692db76, + 0xc90cccbe, 0x4ffa444b, 0x80645383, 0x0bb76d9a, 0xc4297a52, + 0xc76017e9, 0x08fe0021, 0x832d3e38, 0x4cb329f0, 0x85bfe54e, + 0x4a21f286, 0xc1f2cc9f, 0x0e6cdb57, 0x0d25b6ec, 0xc2bba124, + 0x49689f3d, 0x86f688f5, 0x9ff48896, 0x506a9f5e, 0xdbb9a147, + 0x1427b68f, 0x176edb34, 0xd8f0ccfc, 0x5323f2e5, 0x9cbde52d, + 0x55b12993, 0x9a2f3e5b, 0x11fc0042, 0xde62178a, 0xdd2b7a31, + 0x12b56df9, 0x996653e0, 0x56f84428, 0xd00eccdd, 0x1f90db15, + 0x9443e50c, 0x5bddf2c4, 0x58949f7f, 0x970a88b7, 0x1cd9b6ae, + 0xd347a166, 0x1a4b6dd8, 0xd5d57a10, 0x5e064409, 0x919853c1, + 0x92d13e7a, 0x5d4f29b2, 0xd69c17ab, 0x19020063, 0xe498176d, + 0x2b0600a5, 0xa0d53ebc, 0x6f4b2974, 0x6c0244cf, 0xa39c5307, + 0x284f6d1e, 0xe7d17ad6, 0x2eddb668, 0xe143a1a0, 0x6a909fb9, + 0xa50e8871, 0xa647e5ca, 0x69d9f202, 0xe20acc1b, 0x2d94dbd3, + 0xab625326, 0x64fc44ee, 0xef2f7af7, 0x20b16d3f, 0x23f80084, + 0xec66174c, 0x67b52955, 0xa82b3e9d, 0x6127f223, 0xaeb9e5eb, + 0x256adbf2, 0xeaf4cc3a, 0xe9bda181, 0x2623b649, 0xadf08850, + 0x626e9f98, 0x7b6c9ffb, 0xb4f28833, 0x3f21b62a, 0xf0bfa1e2, + 0xf3f6cc59, 0x3c68db91, 0xb7bbe588, 0x7825f240, 0xb1293efe, + 0x7eb72936, 0xf564172f, 0x3afa00e7, 0x39b36d5c, 0xf62d7a94, + 0x7dfe448d, 0xb2605345, 0x3496dbb0, 0xfb08cc78, 0x70dbf261, + 0xbf45e5a9, 0xbc0c8812, 0x73929fda, 0xf841a1c3, 0x37dfb60b, + 0xfed37ab5, 0x314d6d7d, 0xba9e5364, 0x750044ac, 0x76492917, + 0xb9d73edf, 0x320400c6, 0xfd9a170e, 0x1241289b, 0xdddf3f53, + 0x560c014a, 0x99921682, 0x9adb7b39, 0x55456cf1, 0xde9652e8, + 0x11084520, 0xd804899e, 0x179a9e56, 0x9c49a04f, 0x53d7b787, + 0x509eda3c, 0x9f00cdf4, 0x14d3f3ed, 0xdb4de425, 0x5dbb6cd0, + 0x92257b18, 0x19f64501, 0xd66852c9, 0xd5213f72, 0x1abf28ba, + 0x916c16a3, 0x5ef2016b, 0x97fecdd5, 0x5860da1d, 0xd3b3e404, + 0x1c2df3cc, 0x1f649e77, 0xd0fa89bf, 0x5b29b7a6, 0x94b7a06e, + 0x8db5a00d, 0x422bb7c5, 0xc9f889dc, 0x06669e14, 0x052ff3af, + 0xcab1e467, 0x4162da7e, 0x8efccdb6, 0x47f00108, 0x886e16c0, + 0x03bd28d9, 0xcc233f11, 0xcf6a52aa, 0x00f44562, 0x8b277b7b, + 0x44b96cb3, 0xc24fe446, 0x0dd1f38e, 0x8602cd97, 0x499cda5f, + 0x4ad5b7e4, 0x854ba02c, 0x0e989e35, 0xc10689fd, 0x080a4543, + 0xc794528b, 0x4c476c92, 0x83d97b5a, 0x809016e1, 0x4f0e0129, + 0xc4dd3f30, 0x0b4328f8, 0xf6d93ff6, 0x3947283e, 0xb2941627, + 0x7d0a01ef, 0x7e436c54, 0xb1dd7b9c, 0x3a0e4585, 0xf590524d, + 0x3c9c9ef3, 0xf302893b, 0x78d1b722, 0xb74fa0ea, 0xb406cd51, + 0x7b98da99, 0xf04be480, 0x3fd5f348, 0xb9237bbd, 0x76bd6c75, + 0xfd6e526c, 0x32f045a4, 0x31b9281f, 0xfe273fd7, 0x75f401ce, + 0xba6a1606, 0x7366dab8, 0xbcf8cd70, 0x372bf369, 0xf8b5e4a1, + 0xfbfc891a, 0x34629ed2, 0xbfb1a0cb, 0x702fb703, 0x692db760, + 0xa6b3a0a8, 0x2d609eb1, 0xe2fe8979, 0xe1b7e4c2, 0x2e29f30a, + 0xa5facd13, 0x6a64dadb, 0xa3681665, 0x6cf601ad, 0xe7253fb4, + 0x28bb287c, 0x2bf245c7, 0xe46c520f, 0x6fbf6c16, 0xa0217bde, + 0x26d7f32b, 0xe949e4e3, 0x629adafa, 0xad04cd32, 0xae4da089, + 0x61d3b741, 0xea008958, 0x259e9e90, 0xec92522e, 0x230c45e6, + 0xa8df7bff, 0x67416c37, 0x6408018c, 0xab961644, 0x2045285d, + 0xefdb3f95}, + {0x00000000, 0x24825136, 0x4904a26c, 0x6d86f35a, 0x920944d8, + 0xb68b15ee, 0xdb0de6b4, 0xff8fb782, 0xff638ff1, 0xdbe1dec7, + 0xb6672d9d, 0x92e57cab, 0x6d6acb29, 0x49e89a1f, 0x246e6945, + 0x00ec3873, 0x25b619a3, 0x01344895, 0x6cb2bbcf, 0x4830eaf9, + 0xb7bf5d7b, 0x933d0c4d, 0xfebbff17, 0xda39ae21, 0xdad59652, + 0xfe57c764, 0x93d1343e, 0xb7536508, 0x48dcd28a, 0x6c5e83bc, + 0x01d870e6, 0x255a21d0, 0x4b6c3346, 0x6fee6270, 0x0268912a, + 0x26eac01c, 0xd965779e, 0xfde726a8, 0x9061d5f2, 0xb4e384c4, + 0xb40fbcb7, 0x908ded81, 0xfd0b1edb, 0xd9894fed, 0x2606f86f, + 0x0284a959, 0x6f025a03, 0x4b800b35, 0x6eda2ae5, 0x4a587bd3, + 0x27de8889, 0x035cd9bf, 0xfcd36e3d, 0xd8513f0b, 0xb5d7cc51, + 0x91559d67, 0x91b9a514, 0xb53bf422, 0xd8bd0778, 0xfc3f564e, + 0x03b0e1cc, 0x2732b0fa, 0x4ab443a0, 0x6e361296, 0x96d8668c, + 0xb25a37ba, 0xdfdcc4e0, 0xfb5e95d6, 0x04d12254, 0x20537362, + 0x4dd58038, 0x6957d10e, 0x69bbe97d, 0x4d39b84b, 0x20bf4b11, + 0x043d1a27, 0xfbb2ada5, 0xdf30fc93, 0xb2b60fc9, 0x96345eff, + 0xb36e7f2f, 0x97ec2e19, 0xfa6add43, 0xdee88c75, 0x21673bf7, + 0x05e56ac1, 0x6863999b, 0x4ce1c8ad, 0x4c0df0de, 0x688fa1e8, + 0x050952b2, 0x218b0384, 0xde04b406, 0xfa86e530, 0x9700166a, + 0xb382475c, 0xddb455ca, 0xf93604fc, 0x94b0f7a6, 0xb032a690, + 0x4fbd1112, 0x6b3f4024, 0x06b9b37e, 0x223be248, 0x22d7da3b, + 0x06558b0d, 0x6bd37857, 0x4f512961, 0xb0de9ee3, 0x945ccfd5, + 0xf9da3c8f, 0xdd586db9, 0xf8024c69, 0xdc801d5f, 0xb106ee05, + 0x9584bf33, 0x6a0b08b1, 0x4e895987, 0x230faadd, 0x078dfbeb, + 0x0761c398, 0x23e392ae, 0x4e6561f4, 0x6ae730c2, 0x95688740, + 0xb1ead676, 0xdc6c252c, 0xf8ee741a, 0xf6c1cb59, 0xd2439a6f, + 0xbfc56935, 0x9b473803, 0x64c88f81, 0x404adeb7, 0x2dcc2ded, + 0x094e7cdb, 0x09a244a8, 0x2d20159e, 0x40a6e6c4, 0x6424b7f2, + 0x9bab0070, 0xbf295146, 0xd2afa21c, 0xf62df32a, 0xd377d2fa, + 0xf7f583cc, 0x9a737096, 0xbef121a0, 0x417e9622, 0x65fcc714, + 0x087a344e, 0x2cf86578, 0x2c145d0b, 0x08960c3d, 0x6510ff67, + 0x4192ae51, 0xbe1d19d3, 0x9a9f48e5, 0xf719bbbf, 0xd39bea89, + 0xbdadf81f, 0x992fa929, 0xf4a95a73, 0xd02b0b45, 0x2fa4bcc7, + 0x0b26edf1, 0x66a01eab, 0x42224f9d, 0x42ce77ee, 0x664c26d8, + 0x0bcad582, 0x2f4884b4, 0xd0c73336, 0xf4456200, 0x99c3915a, + 0xbd41c06c, 0x981be1bc, 0xbc99b08a, 0xd11f43d0, 0xf59d12e6, + 0x0a12a564, 0x2e90f452, 0x43160708, 0x6794563e, 0x67786e4d, + 0x43fa3f7b, 0x2e7ccc21, 0x0afe9d17, 0xf5712a95, 0xd1f37ba3, + 0xbc7588f9, 0x98f7d9cf, 0x6019add5, 0x449bfce3, 0x291d0fb9, + 0x0d9f5e8f, 0xf210e90d, 0xd692b83b, 0xbb144b61, 0x9f961a57, + 0x9f7a2224, 0xbbf87312, 0xd67e8048, 0xf2fcd17e, 0x0d7366fc, + 0x29f137ca, 0x4477c490, 0x60f595a6, 0x45afb476, 0x612de540, + 0x0cab161a, 0x2829472c, 0xd7a6f0ae, 0xf324a198, 0x9ea252c2, + 0xba2003f4, 0xbacc3b87, 0x9e4e6ab1, 0xf3c899eb, 0xd74ac8dd, + 0x28c57f5f, 0x0c472e69, 0x61c1dd33, 0x45438c05, 0x2b759e93, + 0x0ff7cfa5, 0x62713cff, 0x46f36dc9, 0xb97cda4b, 0x9dfe8b7d, + 0xf0787827, 0xd4fa2911, 0xd4161162, 0xf0944054, 0x9d12b30e, + 0xb990e238, 0x461f55ba, 0x629d048c, 0x0f1bf7d6, 0x2b99a6e0, + 0x0ec38730, 0x2a41d606, 0x47c7255c, 0x6345746a, 0x9ccac3e8, + 0xb84892de, 0xd5ce6184, 0xf14c30b2, 0xf1a008c1, 0xd52259f7, + 0xb8a4aaad, 0x9c26fb9b, 0x63a94c19, 0x472b1d2f, 0x2aadee75, + 0x0e2fbf43}, + {0x00000000, 0x36f290f3, 0x6de521e6, 0x5b17b115, 0xdbca43cc, + 0xed38d33f, 0xb62f622a, 0x80ddf2d9, 0x6ce581d9, 0x5a17112a, + 0x0100a03f, 0x37f230cc, 0xb72fc215, 0x81dd52e6, 0xdacae3f3, + 0xec387300, 0xd9cb03b2, 0xef399341, 0xb42e2254, 0x82dcb2a7, + 0x0201407e, 0x34f3d08d, 0x6fe46198, 0x5916f16b, 0xb52e826b, + 0x83dc1298, 0xd8cba38d, 0xee39337e, 0x6ee4c1a7, 0x58165154, + 0x0301e041, 0x35f370b2, 0x68e70125, 0x5e1591d6, 0x050220c3, + 0x33f0b030, 0xb32d42e9, 0x85dfd21a, 0xdec8630f, 0xe83af3fc, + 0x040280fc, 0x32f0100f, 0x69e7a11a, 0x5f1531e9, 0xdfc8c330, + 0xe93a53c3, 0xb22de2d6, 0x84df7225, 0xb12c0297, 0x87de9264, + 0xdcc92371, 0xea3bb382, 0x6ae6415b, 0x5c14d1a8, 0x070360bd, + 0x31f1f04e, 0xddc9834e, 0xeb3b13bd, 0xb02ca2a8, 0x86de325b, + 0x0603c082, 0x30f15071, 0x6be6e164, 0x5d147197, 0xd1ce024a, + 0xe73c92b9, 0xbc2b23ac, 0x8ad9b35f, 0x0a044186, 0x3cf6d175, + 0x67e16060, 0x5113f093, 0xbd2b8393, 0x8bd91360, 0xd0cea275, + 0xe63c3286, 0x66e1c05f, 0x501350ac, 0x0b04e1b9, 0x3df6714a, + 0x080501f8, 0x3ef7910b, 0x65e0201e, 0x5312b0ed, 0xd3cf4234, + 0xe53dd2c7, 0xbe2a63d2, 0x88d8f321, 0x64e08021, 0x521210d2, + 0x0905a1c7, 0x3ff73134, 0xbf2ac3ed, 0x89d8531e, 0xd2cfe20b, + 0xe43d72f8, 0xb929036f, 0x8fdb939c, 0xd4cc2289, 0xe23eb27a, + 0x62e340a3, 0x5411d050, 0x0f066145, 0x39f4f1b6, 0xd5cc82b6, + 0xe33e1245, 0xb829a350, 0x8edb33a3, 0x0e06c17a, 0x38f45189, + 0x63e3e09c, 0x5511706f, 0x60e200dd, 0x5610902e, 0x0d07213b, + 0x3bf5b1c8, 0xbb284311, 0x8ddad3e2, 0xd6cd62f7, 0xe03ff204, + 0x0c078104, 0x3af511f7, 0x61e2a0e2, 0x57103011, 0xd7cdc2c8, + 0xe13f523b, 0xba28e32e, 0x8cda73dd, 0x78ed02d5, 0x4e1f9226, + 0x15082333, 0x23fab3c0, 0xa3274119, 0x95d5d1ea, 0xcec260ff, + 0xf830f00c, 0x1408830c, 0x22fa13ff, 0x79eda2ea, 0x4f1f3219, + 0xcfc2c0c0, 0xf9305033, 0xa227e126, 0x94d571d5, 0xa1260167, + 0x97d49194, 0xccc32081, 0xfa31b072, 0x7aec42ab, 0x4c1ed258, + 0x1709634d, 0x21fbf3be, 0xcdc380be, 0xfb31104d, 0xa026a158, + 0x96d431ab, 0x1609c372, 0x20fb5381, 0x7bece294, 0x4d1e7267, + 0x100a03f0, 0x26f89303, 0x7def2216, 0x4b1db2e5, 0xcbc0403c, + 0xfd32d0cf, 0xa62561da, 0x90d7f129, 0x7cef8229, 0x4a1d12da, + 0x110aa3cf, 0x27f8333c, 0xa725c1e5, 0x91d75116, 0xcac0e003, + 0xfc3270f0, 0xc9c10042, 0xff3390b1, 0xa42421a4, 0x92d6b157, + 0x120b438e, 0x24f9d37d, 0x7fee6268, 0x491cf29b, 0xa524819b, + 0x93d61168, 0xc8c1a07d, 0xfe33308e, 0x7eeec257, 0x481c52a4, + 0x130be3b1, 0x25f97342, 0xa923009f, 0x9fd1906c, 0xc4c62179, + 0xf234b18a, 0x72e94353, 0x441bd3a0, 0x1f0c62b5, 0x29fef246, + 0xc5c68146, 0xf33411b5, 0xa823a0a0, 0x9ed13053, 0x1e0cc28a, + 0x28fe5279, 0x73e9e36c, 0x451b739f, 0x70e8032d, 0x461a93de, + 0x1d0d22cb, 0x2bffb238, 0xab2240e1, 0x9dd0d012, 0xc6c76107, + 0xf035f1f4, 0x1c0d82f4, 0x2aff1207, 0x71e8a312, 0x471a33e1, + 0xc7c7c138, 0xf13551cb, 0xaa22e0de, 0x9cd0702d, 0xc1c401ba, + 0xf7369149, 0xac21205c, 0x9ad3b0af, 0x1a0e4276, 0x2cfcd285, + 0x77eb6390, 0x4119f363, 0xad218063, 0x9bd31090, 0xc0c4a185, + 0xf6363176, 0x76ebc3af, 0x4019535c, 0x1b0ee249, 0x2dfc72ba, + 0x180f0208, 0x2efd92fb, 0x75ea23ee, 0x4318b31d, 0xc3c541c4, + 0xf537d137, 0xae206022, 0x98d2f0d1, 0x74ea83d1, 0x42181322, + 0x190fa237, 0x2ffd32c4, 0xaf20c01d, 0x99d250ee, 0xc2c5e1fb, + 0xf4377108}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x0000000000000000, 0xf390f23600000000, 0xe621e56d00000000, + 0x15b1175b00000000, 0xcc43cadb00000000, 0x3fd338ed00000000, + 0x2a622fb600000000, 0xd9f2dd8000000000, 0xd981e56c00000000, + 0x2a11175a00000000, 0x3fa0000100000000, 0xcc30f23700000000, + 0x15c22fb700000000, 0xe652dd8100000000, 0xf3e3cada00000000, + 0x007338ec00000000, 0xb203cbd900000000, 0x419339ef00000000, + 0x54222eb400000000, 0xa7b2dc8200000000, 0x7e40010200000000, + 0x8dd0f33400000000, 0x9861e46f00000000, 0x6bf1165900000000, + 0x6b822eb500000000, 0x9812dc8300000000, 0x8da3cbd800000000, + 0x7e3339ee00000000, 0xa7c1e46e00000000, 0x5451165800000000, + 0x41e0010300000000, 0xb270f33500000000, 0x2501e76800000000, + 0xd691155e00000000, 0xc320020500000000, 0x30b0f03300000000, + 0xe9422db300000000, 0x1ad2df8500000000, 0x0f63c8de00000000, + 0xfcf33ae800000000, 0xfc80020400000000, 0x0f10f03200000000, + 0x1aa1e76900000000, 0xe931155f00000000, 0x30c3c8df00000000, + 0xc3533ae900000000, 0xd6e22db200000000, 0x2572df8400000000, + 0x97022cb100000000, 0x6492de8700000000, 0x7123c9dc00000000, + 0x82b33bea00000000, 0x5b41e66a00000000, 0xa8d1145c00000000, + 0xbd60030700000000, 0x4ef0f13100000000, 0x4e83c9dd00000000, + 0xbd133beb00000000, 0xa8a22cb000000000, 0x5b32de8600000000, + 0x82c0030600000000, 0x7150f13000000000, 0x64e1e66b00000000, + 0x9771145d00000000, 0x4a02ced100000000, 0xb9923ce700000000, + 0xac232bbc00000000, 0x5fb3d98a00000000, 0x8641040a00000000, + 0x75d1f63c00000000, 0x6060e16700000000, 0x93f0135100000000, + 0x93832bbd00000000, 0x6013d98b00000000, 0x75a2ced000000000, + 0x86323ce600000000, 0x5fc0e16600000000, 0xac50135000000000, + 0xb9e1040b00000000, 0x4a71f63d00000000, 0xf801050800000000, + 0x0b91f73e00000000, 0x1e20e06500000000, 0xedb0125300000000, + 0x3442cfd300000000, 0xc7d23de500000000, 0xd2632abe00000000, + 0x21f3d88800000000, 0x2180e06400000000, 0xd210125200000000, + 0xc7a1050900000000, 0x3431f73f00000000, 0xedc32abf00000000, + 0x1e53d88900000000, 0x0be2cfd200000000, 0xf8723de400000000, + 0x6f0329b900000000, 0x9c93db8f00000000, 0x8922ccd400000000, + 0x7ab23ee200000000, 0xa340e36200000000, 0x50d0115400000000, + 0x4561060f00000000, 0xb6f1f43900000000, 0xb682ccd500000000, + 0x45123ee300000000, 0x50a329b800000000, 0xa333db8e00000000, + 0x7ac1060e00000000, 0x8951f43800000000, 0x9ce0e36300000000, + 0x6f70115500000000, 0xdd00e26000000000, 0x2e90105600000000, + 0x3b21070d00000000, 0xc8b1f53b00000000, 0x114328bb00000000, + 0xe2d3da8d00000000, 0xf762cdd600000000, 0x04f23fe000000000, + 0x0481070c00000000, 0xf711f53a00000000, 0xe2a0e26100000000, + 0x1130105700000000, 0xc8c2cdd700000000, 0x3b523fe100000000, + 0x2ee328ba00000000, 0xdd73da8c00000000, 0xd502ed7800000000, + 0x26921f4e00000000, 0x3323081500000000, 0xc0b3fa2300000000, + 0x194127a300000000, 0xead1d59500000000, 0xff60c2ce00000000, + 0x0cf030f800000000, 0x0c83081400000000, 0xff13fa2200000000, + 0xeaa2ed7900000000, 0x19321f4f00000000, 0xc0c0c2cf00000000, + 0x335030f900000000, 0x26e127a200000000, 0xd571d59400000000, + 0x670126a100000000, 0x9491d49700000000, 0x8120c3cc00000000, + 0x72b031fa00000000, 0xab42ec7a00000000, 0x58d21e4c00000000, + 0x4d63091700000000, 0xbef3fb2100000000, 0xbe80c3cd00000000, + 0x4d1031fb00000000, 0x58a126a000000000, 0xab31d49600000000, + 0x72c3091600000000, 0x8153fb2000000000, 0x94e2ec7b00000000, + 0x67721e4d00000000, 0xf0030a1000000000, 0x0393f82600000000, + 0x1622ef7d00000000, 0xe5b21d4b00000000, 0x3c40c0cb00000000, + 0xcfd032fd00000000, 0xda6125a600000000, 0x29f1d79000000000, + 0x2982ef7c00000000, 0xda121d4a00000000, 0xcfa30a1100000000, + 0x3c33f82700000000, 0xe5c125a700000000, 0x1651d79100000000, + 0x03e0c0ca00000000, 0xf07032fc00000000, 0x4200c1c900000000, + 0xb19033ff00000000, 0xa42124a400000000, 0x57b1d69200000000, + 0x8e430b1200000000, 0x7dd3f92400000000, 0x6862ee7f00000000, + 0x9bf21c4900000000, 0x9b8124a500000000, 0x6811d69300000000, + 0x7da0c1c800000000, 0x8e3033fe00000000, 0x57c2ee7e00000000, + 0xa4521c4800000000, 0xb1e30b1300000000, 0x4273f92500000000, + 0x9f0023a900000000, 0x6c90d19f00000000, 0x7921c6c400000000, + 0x8ab134f200000000, 0x5343e97200000000, 0xa0d31b4400000000, + 0xb5620c1f00000000, 0x46f2fe2900000000, 0x4681c6c500000000, + 0xb51134f300000000, 0xa0a023a800000000, 0x5330d19e00000000, + 0x8ac20c1e00000000, 0x7952fe2800000000, 0x6ce3e97300000000, + 0x9f731b4500000000, 0x2d03e87000000000, 0xde931a4600000000, + 0xcb220d1d00000000, 0x38b2ff2b00000000, 0xe14022ab00000000, + 0x12d0d09d00000000, 0x0761c7c600000000, 0xf4f135f000000000, + 0xf4820d1c00000000, 0x0712ff2a00000000, 0x12a3e87100000000, + 0xe1331a4700000000, 0x38c1c7c700000000, 0xcb5135f100000000, + 0xdee022aa00000000, 0x2d70d09c00000000, 0xba01c4c100000000, + 0x499136f700000000, 0x5c2021ac00000000, 0xafb0d39a00000000, + 0x76420e1a00000000, 0x85d2fc2c00000000, 0x9063eb7700000000, + 0x63f3194100000000, 0x638021ad00000000, 0x9010d39b00000000, + 0x85a1c4c000000000, 0x763136f600000000, 0xafc3eb7600000000, + 0x5c53194000000000, 0x49e20e1b00000000, 0xba72fc2d00000000, + 0x08020f1800000000, 0xfb92fd2e00000000, 0xee23ea7500000000, + 0x1db3184300000000, 0xc441c5c300000000, 0x37d137f500000000, + 0x226020ae00000000, 0xd1f0d29800000000, 0xd183ea7400000000, + 0x2213184200000000, 0x37a20f1900000000, 0xc432fd2f00000000, + 0x1dc020af00000000, 0xee50d29900000000, 0xfbe1c5c200000000, + 0x087137f400000000}, + {0x0000000000000000, 0x3651822400000000, 0x6ca2044900000000, + 0x5af3866d00000000, 0xd844099200000000, 0xee158bb600000000, + 0xb4e60ddb00000000, 0x82b78fff00000000, 0xf18f63ff00000000, + 0xc7dee1db00000000, 0x9d2d67b600000000, 0xab7ce59200000000, + 0x29cb6a6d00000000, 0x1f9ae84900000000, 0x45696e2400000000, + 0x7338ec0000000000, 0xa319b62500000000, 0x9548340100000000, + 0xcfbbb26c00000000, 0xf9ea304800000000, 0x7b5dbfb700000000, + 0x4d0c3d9300000000, 0x17ffbbfe00000000, 0x21ae39da00000000, + 0x5296d5da00000000, 0x64c757fe00000000, 0x3e34d19300000000, + 0x086553b700000000, 0x8ad2dc4800000000, 0xbc835e6c00000000, + 0xe670d80100000000, 0xd0215a2500000000, 0x46336c4b00000000, + 0x7062ee6f00000000, 0x2a91680200000000, 0x1cc0ea2600000000, + 0x9e7765d900000000, 0xa826e7fd00000000, 0xf2d5619000000000, + 0xc484e3b400000000, 0xb7bc0fb400000000, 0x81ed8d9000000000, + 0xdb1e0bfd00000000, 0xed4f89d900000000, 0x6ff8062600000000, + 0x59a9840200000000, 0x035a026f00000000, 0x350b804b00000000, + 0xe52ada6e00000000, 0xd37b584a00000000, 0x8988de2700000000, + 0xbfd95c0300000000, 0x3d6ed3fc00000000, 0x0b3f51d800000000, + 0x51ccd7b500000000, 0x679d559100000000, 0x14a5b99100000000, + 0x22f43bb500000000, 0x7807bdd800000000, 0x4e563ffc00000000, + 0xcce1b00300000000, 0xfab0322700000000, 0xa043b44a00000000, + 0x9612366e00000000, 0x8c66d89600000000, 0xba375ab200000000, + 0xe0c4dcdf00000000, 0xd6955efb00000000, 0x5422d10400000000, + 0x6273532000000000, 0x3880d54d00000000, 0x0ed1576900000000, + 0x7de9bb6900000000, 0x4bb8394d00000000, 0x114bbf2000000000, + 0x271a3d0400000000, 0xa5adb2fb00000000, 0x93fc30df00000000, + 0xc90fb6b200000000, 0xff5e349600000000, 0x2f7f6eb300000000, + 0x192eec9700000000, 0x43dd6afa00000000, 0x758ce8de00000000, + 0xf73b672100000000, 0xc16ae50500000000, 0x9b99636800000000, + 0xadc8e14c00000000, 0xdef00d4c00000000, 0xe8a18f6800000000, + 0xb252090500000000, 0x84038b2100000000, 0x06b404de00000000, + 0x30e586fa00000000, 0x6a16009700000000, 0x5c4782b300000000, + 0xca55b4dd00000000, 0xfc0436f900000000, 0xa6f7b09400000000, + 0x90a632b000000000, 0x1211bd4f00000000, 0x24403f6b00000000, + 0x7eb3b90600000000, 0x48e23b2200000000, 0x3bdad72200000000, + 0x0d8b550600000000, 0x5778d36b00000000, 0x6129514f00000000, + 0xe39edeb000000000, 0xd5cf5c9400000000, 0x8f3cdaf900000000, + 0xb96d58dd00000000, 0x694c02f800000000, 0x5f1d80dc00000000, + 0x05ee06b100000000, 0x33bf849500000000, 0xb1080b6a00000000, + 0x8759894e00000000, 0xddaa0f2300000000, 0xebfb8d0700000000, + 0x98c3610700000000, 0xae92e32300000000, 0xf461654e00000000, + 0xc230e76a00000000, 0x4087689500000000, 0x76d6eab100000000, + 0x2c256cdc00000000, 0x1a74eef800000000, 0x59cbc1f600000000, + 0x6f9a43d200000000, 0x3569c5bf00000000, 0x0338479b00000000, + 0x818fc86400000000, 0xb7de4a4000000000, 0xed2dcc2d00000000, + 0xdb7c4e0900000000, 0xa844a20900000000, 0x9e15202d00000000, + 0xc4e6a64000000000, 0xf2b7246400000000, 0x7000ab9b00000000, + 0x465129bf00000000, 0x1ca2afd200000000, 0x2af32df600000000, + 0xfad277d300000000, 0xcc83f5f700000000, 0x9670739a00000000, + 0xa021f1be00000000, 0x22967e4100000000, 0x14c7fc6500000000, + 0x4e347a0800000000, 0x7865f82c00000000, 0x0b5d142c00000000, + 0x3d0c960800000000, 0x67ff106500000000, 0x51ae924100000000, + 0xd3191dbe00000000, 0xe5489f9a00000000, 0xbfbb19f700000000, + 0x89ea9bd300000000, 0x1ff8adbd00000000, 0x29a92f9900000000, + 0x735aa9f400000000, 0x450b2bd000000000, 0xc7bca42f00000000, + 0xf1ed260b00000000, 0xab1ea06600000000, 0x9d4f224200000000, + 0xee77ce4200000000, 0xd8264c6600000000, 0x82d5ca0b00000000, + 0xb484482f00000000, 0x3633c7d000000000, 0x006245f400000000, + 0x5a91c39900000000, 0x6cc041bd00000000, 0xbce11b9800000000, + 0x8ab099bc00000000, 0xd0431fd100000000, 0xe6129df500000000, + 0x64a5120a00000000, 0x52f4902e00000000, 0x0807164300000000, + 0x3e56946700000000, 0x4d6e786700000000, 0x7b3ffa4300000000, + 0x21cc7c2e00000000, 0x179dfe0a00000000, 0x952a71f500000000, + 0xa37bf3d100000000, 0xf98875bc00000000, 0xcfd9f79800000000, + 0xd5ad196000000000, 0xe3fc9b4400000000, 0xb90f1d2900000000, + 0x8f5e9f0d00000000, 0x0de910f200000000, 0x3bb892d600000000, + 0x614b14bb00000000, 0x571a969f00000000, 0x24227a9f00000000, + 0x1273f8bb00000000, 0x48807ed600000000, 0x7ed1fcf200000000, + 0xfc66730d00000000, 0xca37f12900000000, 0x90c4774400000000, + 0xa695f56000000000, 0x76b4af4500000000, 0x40e52d6100000000, + 0x1a16ab0c00000000, 0x2c47292800000000, 0xaef0a6d700000000, + 0x98a124f300000000, 0xc252a29e00000000, 0xf40320ba00000000, + 0x873bccba00000000, 0xb16a4e9e00000000, 0xeb99c8f300000000, + 0xddc84ad700000000, 0x5f7fc52800000000, 0x692e470c00000000, + 0x33ddc16100000000, 0x058c434500000000, 0x939e752b00000000, + 0xa5cff70f00000000, 0xff3c716200000000, 0xc96df34600000000, + 0x4bda7cb900000000, 0x7d8bfe9d00000000, 0x277878f000000000, + 0x1129fad400000000, 0x621116d400000000, 0x544094f000000000, + 0x0eb3129d00000000, 0x38e290b900000000, 0xba551f4600000000, + 0x8c049d6200000000, 0xd6f71b0f00000000, 0xe0a6992b00000000, + 0x3087c30e00000000, 0x06d6412a00000000, 0x5c25c74700000000, + 0x6a74456300000000, 0xe8c3ca9c00000000, 0xde9248b800000000, + 0x8461ced500000000, 0xb2304cf100000000, 0xc108a0f100000000, + 0xf75922d500000000, 0xadaaa4b800000000, 0x9bfb269c00000000, + 0x194ca96300000000, 0x2f1d2b4700000000, 0x75eead2a00000000, + 0x43bf2f0e00000000}, + {0x0000000000000000, 0xc8179ecf00000000, 0xd1294d4400000000, + 0x193ed38b00000000, 0xa2539a8800000000, 0x6a44044700000000, + 0x737ad7cc00000000, 0xbb6d490300000000, 0x05a145ca00000000, + 0xcdb6db0500000000, 0xd488088e00000000, 0x1c9f964100000000, + 0xa7f2df4200000000, 0x6fe5418d00000000, 0x76db920600000000, + 0xbecc0cc900000000, 0x4b44fa4f00000000, 0x8353648000000000, + 0x9a6db70b00000000, 0x527a29c400000000, 0xe91760c700000000, + 0x2100fe0800000000, 0x383e2d8300000000, 0xf029b34c00000000, + 0x4ee5bf8500000000, 0x86f2214a00000000, 0x9fccf2c100000000, + 0x57db6c0e00000000, 0xecb6250d00000000, 0x24a1bbc200000000, + 0x3d9f684900000000, 0xf588f68600000000, 0x9688f49f00000000, + 0x5e9f6a5000000000, 0x47a1b9db00000000, 0x8fb6271400000000, + 0x34db6e1700000000, 0xfcccf0d800000000, 0xe5f2235300000000, + 0x2de5bd9c00000000, 0x9329b15500000000, 0x5b3e2f9a00000000, + 0x4200fc1100000000, 0x8a1762de00000000, 0x317a2bdd00000000, + 0xf96db51200000000, 0xe053669900000000, 0x2844f85600000000, + 0xddcc0ed000000000, 0x15db901f00000000, 0x0ce5439400000000, + 0xc4f2dd5b00000000, 0x7f9f945800000000, 0xb7880a9700000000, + 0xaeb6d91c00000000, 0x66a147d300000000, 0xd86d4b1a00000000, + 0x107ad5d500000000, 0x0944065e00000000, 0xc153989100000000, + 0x7a3ed19200000000, 0xb2294f5d00000000, 0xab179cd600000000, + 0x6300021900000000, 0x6d1798e400000000, 0xa500062b00000000, + 0xbc3ed5a000000000, 0x74294b6f00000000, 0xcf44026c00000000, + 0x07539ca300000000, 0x1e6d4f2800000000, 0xd67ad1e700000000, + 0x68b6dd2e00000000, 0xa0a143e100000000, 0xb99f906a00000000, + 0x71880ea500000000, 0xcae547a600000000, 0x02f2d96900000000, + 0x1bcc0ae200000000, 0xd3db942d00000000, 0x265362ab00000000, + 0xee44fc6400000000, 0xf77a2fef00000000, 0x3f6db12000000000, + 0x8400f82300000000, 0x4c1766ec00000000, 0x5529b56700000000, + 0x9d3e2ba800000000, 0x23f2276100000000, 0xebe5b9ae00000000, + 0xf2db6a2500000000, 0x3accf4ea00000000, 0x81a1bde900000000, + 0x49b6232600000000, 0x5088f0ad00000000, 0x989f6e6200000000, + 0xfb9f6c7b00000000, 0x3388f2b400000000, 0x2ab6213f00000000, + 0xe2a1bff000000000, 0x59ccf6f300000000, 0x91db683c00000000, + 0x88e5bbb700000000, 0x40f2257800000000, 0xfe3e29b100000000, + 0x3629b77e00000000, 0x2f1764f500000000, 0xe700fa3a00000000, + 0x5c6db33900000000, 0x947a2df600000000, 0x8d44fe7d00000000, + 0x455360b200000000, 0xb0db963400000000, 0x78cc08fb00000000, + 0x61f2db7000000000, 0xa9e545bf00000000, 0x12880cbc00000000, + 0xda9f927300000000, 0xc3a141f800000000, 0x0bb6df3700000000, + 0xb57ad3fe00000000, 0x7d6d4d3100000000, 0x64539eba00000000, + 0xac44007500000000, 0x1729497600000000, 0xdf3ed7b900000000, + 0xc600043200000000, 0x0e179afd00000000, 0x9b28411200000000, + 0x533fdfdd00000000, 0x4a010c5600000000, 0x8216929900000000, + 0x397bdb9a00000000, 0xf16c455500000000, 0xe85296de00000000, + 0x2045081100000000, 0x9e8904d800000000, 0x569e9a1700000000, + 0x4fa0499c00000000, 0x87b7d75300000000, 0x3cda9e5000000000, + 0xf4cd009f00000000, 0xedf3d31400000000, 0x25e44ddb00000000, + 0xd06cbb5d00000000, 0x187b259200000000, 0x0145f61900000000, + 0xc95268d600000000, 0x723f21d500000000, 0xba28bf1a00000000, + 0xa3166c9100000000, 0x6b01f25e00000000, 0xd5cdfe9700000000, + 0x1dda605800000000, 0x04e4b3d300000000, 0xccf32d1c00000000, + 0x779e641f00000000, 0xbf89fad000000000, 0xa6b7295b00000000, + 0x6ea0b79400000000, 0x0da0b58d00000000, 0xc5b72b4200000000, + 0xdc89f8c900000000, 0x149e660600000000, 0xaff32f0500000000, + 0x67e4b1ca00000000, 0x7eda624100000000, 0xb6cdfc8e00000000, + 0x0801f04700000000, 0xc0166e8800000000, 0xd928bd0300000000, + 0x113f23cc00000000, 0xaa526acf00000000, 0x6245f40000000000, + 0x7b7b278b00000000, 0xb36cb94400000000, 0x46e44fc200000000, + 0x8ef3d10d00000000, 0x97cd028600000000, 0x5fda9c4900000000, + 0xe4b7d54a00000000, 0x2ca04b8500000000, 0x359e980e00000000, + 0xfd8906c100000000, 0x43450a0800000000, 0x8b5294c700000000, + 0x926c474c00000000, 0x5a7bd98300000000, 0xe116908000000000, + 0x29010e4f00000000, 0x303fddc400000000, 0xf828430b00000000, + 0xf63fd9f600000000, 0x3e28473900000000, 0x271694b200000000, + 0xef010a7d00000000, 0x546c437e00000000, 0x9c7bddb100000000, + 0x85450e3a00000000, 0x4d5290f500000000, 0xf39e9c3c00000000, + 0x3b8902f300000000, 0x22b7d17800000000, 0xeaa04fb700000000, + 0x51cd06b400000000, 0x99da987b00000000, 0x80e44bf000000000, + 0x48f3d53f00000000, 0xbd7b23b900000000, 0x756cbd7600000000, + 0x6c526efd00000000, 0xa445f03200000000, 0x1f28b93100000000, + 0xd73f27fe00000000, 0xce01f47500000000, 0x06166aba00000000, + 0xb8da667300000000, 0x70cdf8bc00000000, 0x69f32b3700000000, + 0xa1e4b5f800000000, 0x1a89fcfb00000000, 0xd29e623400000000, + 0xcba0b1bf00000000, 0x03b72f7000000000, 0x60b72d6900000000, + 0xa8a0b3a600000000, 0xb19e602d00000000, 0x7989fee200000000, + 0xc2e4b7e100000000, 0x0af3292e00000000, 0x13cdfaa500000000, + 0xdbda646a00000000, 0x651668a300000000, 0xad01f66c00000000, + 0xb43f25e700000000, 0x7c28bb2800000000, 0xc745f22b00000000, + 0x0f526ce400000000, 0x166cbf6f00000000, 0xde7b21a000000000, + 0x2bf3d72600000000, 0xe3e449e900000000, 0xfada9a6200000000, + 0x32cd04ad00000000, 0x89a04dae00000000, 0x41b7d36100000000, + 0x588900ea00000000, 0x909e9e2500000000, 0x2e5292ec00000000, + 0xe6450c2300000000, 0xff7bdfa800000000, 0x376c416700000000, + 0x8c01086400000000, 0x441696ab00000000, 0x5d28452000000000, + 0x953fdbef00000000}, + {0x0000000000000000, 0x95d4709500000000, 0x6baf90f100000000, + 0xfe7be06400000000, 0x9758503800000000, 0x028c20ad00000000, + 0xfcf7c0c900000000, 0x6923b05c00000000, 0x2eb1a07000000000, + 0xbb65d0e500000000, 0x451e308100000000, 0xd0ca401400000000, + 0xb9e9f04800000000, 0x2c3d80dd00000000, 0xd24660b900000000, + 0x4792102c00000000, 0x5c6241e100000000, 0xc9b6317400000000, + 0x37cdd11000000000, 0xa219a18500000000, 0xcb3a11d900000000, + 0x5eee614c00000000, 0xa095812800000000, 0x3541f1bd00000000, + 0x72d3e19100000000, 0xe707910400000000, 0x197c716000000000, + 0x8ca801f500000000, 0xe58bb1a900000000, 0x705fc13c00000000, + 0x8e24215800000000, 0x1bf051cd00000000, 0xf9c2f31900000000, + 0x6c16838c00000000, 0x926d63e800000000, 0x07b9137d00000000, + 0x6e9aa32100000000, 0xfb4ed3b400000000, 0x053533d000000000, + 0x90e1434500000000, 0xd773536900000000, 0x42a723fc00000000, + 0xbcdcc39800000000, 0x2908b30d00000000, 0x402b035100000000, + 0xd5ff73c400000000, 0x2b8493a000000000, 0xbe50e33500000000, + 0xa5a0b2f800000000, 0x3074c26d00000000, 0xce0f220900000000, + 0x5bdb529c00000000, 0x32f8e2c000000000, 0xa72c925500000000, + 0x5957723100000000, 0xcc8302a400000000, 0x8b11128800000000, + 0x1ec5621d00000000, 0xe0be827900000000, 0x756af2ec00000000, + 0x1c4942b000000000, 0x899d322500000000, 0x77e6d24100000000, + 0xe232a2d400000000, 0xf285e73300000000, 0x675197a600000000, + 0x992a77c200000000, 0x0cfe075700000000, 0x65ddb70b00000000, + 0xf009c79e00000000, 0x0e7227fa00000000, 0x9ba6576f00000000, + 0xdc34474300000000, 0x49e037d600000000, 0xb79bd7b200000000, + 0x224fa72700000000, 0x4b6c177b00000000, 0xdeb867ee00000000, + 0x20c3878a00000000, 0xb517f71f00000000, 0xaee7a6d200000000, + 0x3b33d64700000000, 0xc548362300000000, 0x509c46b600000000, + 0x39bff6ea00000000, 0xac6b867f00000000, 0x5210661b00000000, + 0xc7c4168e00000000, 0x805606a200000000, 0x1582763700000000, + 0xebf9965300000000, 0x7e2de6c600000000, 0x170e569a00000000, + 0x82da260f00000000, 0x7ca1c66b00000000, 0xe975b6fe00000000, + 0x0b47142a00000000, 0x9e9364bf00000000, 0x60e884db00000000, + 0xf53cf44e00000000, 0x9c1f441200000000, 0x09cb348700000000, + 0xf7b0d4e300000000, 0x6264a47600000000, 0x25f6b45a00000000, + 0xb022c4cf00000000, 0x4e5924ab00000000, 0xdb8d543e00000000, + 0xb2aee46200000000, 0x277a94f700000000, 0xd901749300000000, + 0x4cd5040600000000, 0x572555cb00000000, 0xc2f1255e00000000, + 0x3c8ac53a00000000, 0xa95eb5af00000000, 0xc07d05f300000000, + 0x55a9756600000000, 0xabd2950200000000, 0x3e06e59700000000, + 0x7994f5bb00000000, 0xec40852e00000000, 0x123b654a00000000, + 0x87ef15df00000000, 0xeecca58300000000, 0x7b18d51600000000, + 0x8563357200000000, 0x10b745e700000000, 0xe40bcf6700000000, + 0x71dfbff200000000, 0x8fa45f9600000000, 0x1a702f0300000000, + 0x73539f5f00000000, 0xe687efca00000000, 0x18fc0fae00000000, + 0x8d287f3b00000000, 0xcaba6f1700000000, 0x5f6e1f8200000000, + 0xa115ffe600000000, 0x34c18f7300000000, 0x5de23f2f00000000, + 0xc8364fba00000000, 0x364dafde00000000, 0xa399df4b00000000, + 0xb8698e8600000000, 0x2dbdfe1300000000, 0xd3c61e7700000000, + 0x46126ee200000000, 0x2f31debe00000000, 0xbae5ae2b00000000, + 0x449e4e4f00000000, 0xd14a3eda00000000, 0x96d82ef600000000, + 0x030c5e6300000000, 0xfd77be0700000000, 0x68a3ce9200000000, + 0x01807ece00000000, 0x94540e5b00000000, 0x6a2fee3f00000000, + 0xfffb9eaa00000000, 0x1dc93c7e00000000, 0x881d4ceb00000000, + 0x7666ac8f00000000, 0xe3b2dc1a00000000, 0x8a916c4600000000, + 0x1f451cd300000000, 0xe13efcb700000000, 0x74ea8c2200000000, + 0x33789c0e00000000, 0xa6acec9b00000000, 0x58d70cff00000000, + 0xcd037c6a00000000, 0xa420cc3600000000, 0x31f4bca300000000, + 0xcf8f5cc700000000, 0x5a5b2c5200000000, 0x41ab7d9f00000000, + 0xd47f0d0a00000000, 0x2a04ed6e00000000, 0xbfd09dfb00000000, + 0xd6f32da700000000, 0x43275d3200000000, 0xbd5cbd5600000000, + 0x2888cdc300000000, 0x6f1addef00000000, 0xfacead7a00000000, + 0x04b54d1e00000000, 0x91613d8b00000000, 0xf8428dd700000000, + 0x6d96fd4200000000, 0x93ed1d2600000000, 0x06396db300000000, + 0x168e285400000000, 0x835a58c100000000, 0x7d21b8a500000000, + 0xe8f5c83000000000, 0x81d6786c00000000, 0x140208f900000000, + 0xea79e89d00000000, 0x7fad980800000000, 0x383f882400000000, + 0xadebf8b100000000, 0x539018d500000000, 0xc644684000000000, + 0xaf67d81c00000000, 0x3ab3a88900000000, 0xc4c848ed00000000, + 0x511c387800000000, 0x4aec69b500000000, 0xdf38192000000000, + 0x2143f94400000000, 0xb49789d100000000, 0xddb4398d00000000, + 0x4860491800000000, 0xb61ba97c00000000, 0x23cfd9e900000000, + 0x645dc9c500000000, 0xf189b95000000000, 0x0ff2593400000000, + 0x9a2629a100000000, 0xf30599fd00000000, 0x66d1e96800000000, + 0x98aa090c00000000, 0x0d7e799900000000, 0xef4cdb4d00000000, + 0x7a98abd800000000, 0x84e34bbc00000000, 0x11373b2900000000, + 0x78148b7500000000, 0xedc0fbe000000000, 0x13bb1b8400000000, + 0x866f6b1100000000, 0xc1fd7b3d00000000, 0x54290ba800000000, + 0xaa52ebcc00000000, 0x3f869b5900000000, 0x56a52b0500000000, + 0xc3715b9000000000, 0x3d0abbf400000000, 0xa8decb6100000000, + 0xb32e9aac00000000, 0x26faea3900000000, 0xd8810a5d00000000, + 0x4d557ac800000000, 0x2476ca9400000000, 0xb1a2ba0100000000, + 0x4fd95a6500000000, 0xda0d2af000000000, 0x9d9f3adc00000000, + 0x084b4a4900000000, 0xf630aa2d00000000, 0x63e4dab800000000, + 0x0ac76ae400000000, 0x9f131a7100000000, 0x6168fa1500000000, + 0xf4bc8a8000000000}, + {0x0000000000000000, 0x1f17f08000000000, 0x7f2891da00000000, + 0x603f615a00000000, 0xbf56536e00000000, 0xa041a3ee00000000, + 0xc07ec2b400000000, 0xdf69323400000000, 0x7eada6dc00000000, + 0x61ba565c00000000, 0x0185370600000000, 0x1e92c78600000000, + 0xc1fbf5b200000000, 0xdeec053200000000, 0xbed3646800000000, + 0xa1c494e800000000, 0xbd5c3c6200000000, 0xa24bcce200000000, + 0xc274adb800000000, 0xdd635d3800000000, 0x020a6f0c00000000, + 0x1d1d9f8c00000000, 0x7d22fed600000000, 0x62350e5600000000, + 0xc3f19abe00000000, 0xdce66a3e00000000, 0xbcd90b6400000000, + 0xa3cefbe400000000, 0x7ca7c9d000000000, 0x63b0395000000000, + 0x038f580a00000000, 0x1c98a88a00000000, 0x7ab978c400000000, + 0x65ae884400000000, 0x0591e91e00000000, 0x1a86199e00000000, + 0xc5ef2baa00000000, 0xdaf8db2a00000000, 0xbac7ba7000000000, + 0xa5d04af000000000, 0x0414de1800000000, 0x1b032e9800000000, + 0x7b3c4fc200000000, 0x642bbf4200000000, 0xbb428d7600000000, + 0xa4557df600000000, 0xc46a1cac00000000, 0xdb7dec2c00000000, + 0xc7e544a600000000, 0xd8f2b42600000000, 0xb8cdd57c00000000, + 0xa7da25fc00000000, 0x78b317c800000000, 0x67a4e74800000000, + 0x079b861200000000, 0x188c769200000000, 0xb948e27a00000000, + 0xa65f12fa00000000, 0xc66073a000000000, 0xd977832000000000, + 0x061eb11400000000, 0x1909419400000000, 0x793620ce00000000, + 0x6621d04e00000000, 0xb574805300000000, 0xaa6370d300000000, + 0xca5c118900000000, 0xd54be10900000000, 0x0a22d33d00000000, + 0x153523bd00000000, 0x750a42e700000000, 0x6a1db26700000000, + 0xcbd9268f00000000, 0xd4ced60f00000000, 0xb4f1b75500000000, + 0xabe647d500000000, 0x748f75e100000000, 0x6b98856100000000, + 0x0ba7e43b00000000, 0x14b014bb00000000, 0x0828bc3100000000, + 0x173f4cb100000000, 0x77002deb00000000, 0x6817dd6b00000000, + 0xb77eef5f00000000, 0xa8691fdf00000000, 0xc8567e8500000000, + 0xd7418e0500000000, 0x76851aed00000000, 0x6992ea6d00000000, + 0x09ad8b3700000000, 0x16ba7bb700000000, 0xc9d3498300000000, + 0xd6c4b90300000000, 0xb6fbd85900000000, 0xa9ec28d900000000, + 0xcfcdf89700000000, 0xd0da081700000000, 0xb0e5694d00000000, + 0xaff299cd00000000, 0x709babf900000000, 0x6f8c5b7900000000, + 0x0fb33a2300000000, 0x10a4caa300000000, 0xb1605e4b00000000, + 0xae77aecb00000000, 0xce48cf9100000000, 0xd15f3f1100000000, + 0x0e360d2500000000, 0x1121fda500000000, 0x711e9cff00000000, + 0x6e096c7f00000000, 0x7291c4f500000000, 0x6d86347500000000, + 0x0db9552f00000000, 0x12aea5af00000000, 0xcdc7979b00000000, + 0xd2d0671b00000000, 0xb2ef064100000000, 0xadf8f6c100000000, + 0x0c3c622900000000, 0x132b92a900000000, 0x7314f3f300000000, + 0x6c03037300000000, 0xb36a314700000000, 0xac7dc1c700000000, + 0xcc42a09d00000000, 0xd355501d00000000, 0x6ae900a700000000, + 0x75fef02700000000, 0x15c1917d00000000, 0x0ad661fd00000000, + 0xd5bf53c900000000, 0xcaa8a34900000000, 0xaa97c21300000000, + 0xb580329300000000, 0x1444a67b00000000, 0x0b5356fb00000000, + 0x6b6c37a100000000, 0x747bc72100000000, 0xab12f51500000000, + 0xb405059500000000, 0xd43a64cf00000000, 0xcb2d944f00000000, + 0xd7b53cc500000000, 0xc8a2cc4500000000, 0xa89dad1f00000000, + 0xb78a5d9f00000000, 0x68e36fab00000000, 0x77f49f2b00000000, + 0x17cbfe7100000000, 0x08dc0ef100000000, 0xa9189a1900000000, + 0xb60f6a9900000000, 0xd6300bc300000000, 0xc927fb4300000000, + 0x164ec97700000000, 0x095939f700000000, 0x696658ad00000000, + 0x7671a82d00000000, 0x1050786300000000, 0x0f4788e300000000, + 0x6f78e9b900000000, 0x706f193900000000, 0xaf062b0d00000000, + 0xb011db8d00000000, 0xd02ebad700000000, 0xcf394a5700000000, + 0x6efddebf00000000, 0x71ea2e3f00000000, 0x11d54f6500000000, + 0x0ec2bfe500000000, 0xd1ab8dd100000000, 0xcebc7d5100000000, + 0xae831c0b00000000, 0xb194ec8b00000000, 0xad0c440100000000, + 0xb21bb48100000000, 0xd224d5db00000000, 0xcd33255b00000000, + 0x125a176f00000000, 0x0d4de7ef00000000, 0x6d7286b500000000, + 0x7265763500000000, 0xd3a1e2dd00000000, 0xccb6125d00000000, + 0xac89730700000000, 0xb39e838700000000, 0x6cf7b1b300000000, + 0x73e0413300000000, 0x13df206900000000, 0x0cc8d0e900000000, + 0xdf9d80f400000000, 0xc08a707400000000, 0xa0b5112e00000000, + 0xbfa2e1ae00000000, 0x60cbd39a00000000, 0x7fdc231a00000000, + 0x1fe3424000000000, 0x00f4b2c000000000, 0xa130262800000000, + 0xbe27d6a800000000, 0xde18b7f200000000, 0xc10f477200000000, + 0x1e66754600000000, 0x017185c600000000, 0x614ee49c00000000, + 0x7e59141c00000000, 0x62c1bc9600000000, 0x7dd64c1600000000, + 0x1de92d4c00000000, 0x02feddcc00000000, 0xdd97eff800000000, + 0xc2801f7800000000, 0xa2bf7e2200000000, 0xbda88ea200000000, + 0x1c6c1a4a00000000, 0x037beaca00000000, 0x63448b9000000000, + 0x7c537b1000000000, 0xa33a492400000000, 0xbc2db9a400000000, + 0xdc12d8fe00000000, 0xc305287e00000000, 0xa524f83000000000, + 0xba3308b000000000, 0xda0c69ea00000000, 0xc51b996a00000000, + 0x1a72ab5e00000000, 0x05655bde00000000, 0x655a3a8400000000, + 0x7a4dca0400000000, 0xdb895eec00000000, 0xc49eae6c00000000, + 0xa4a1cf3600000000, 0xbbb63fb600000000, 0x64df0d8200000000, + 0x7bc8fd0200000000, 0x1bf79c5800000000, 0x04e06cd800000000, + 0x1878c45200000000, 0x076f34d200000000, 0x6750558800000000, + 0x7847a50800000000, 0xa72e973c00000000, 0xb83967bc00000000, + 0xd80606e600000000, 0xc711f66600000000, 0x66d5628e00000000, + 0x79c2920e00000000, 0x19fdf35400000000, 0x06ea03d400000000, + 0xd98331e000000000, 0xc694c16000000000, 0xa6aba03a00000000, + 0xb9bc50ba00000000}, + {0x0000000000000000, 0xe2fd888d00000000, 0x85fd60c000000000, + 0x6700e84d00000000, 0x4bfdb05b00000000, 0xa90038d600000000, + 0xce00d09b00000000, 0x2cfd581600000000, 0x96fa61b700000000, + 0x7407e93a00000000, 0x1307017700000000, 0xf1fa89fa00000000, + 0xdd07d1ec00000000, 0x3ffa596100000000, 0x58fab12c00000000, + 0xba0739a100000000, 0x6df3b2b500000000, 0x8f0e3a3800000000, + 0xe80ed27500000000, 0x0af35af800000000, 0x260e02ee00000000, + 0xc4f38a6300000000, 0xa3f3622e00000000, 0x410eeaa300000000, + 0xfb09d30200000000, 0x19f45b8f00000000, 0x7ef4b3c200000000, + 0x9c093b4f00000000, 0xb0f4635900000000, 0x5209ebd400000000, + 0x3509039900000000, 0xd7f48b1400000000, 0x9be014b000000000, + 0x791d9c3d00000000, 0x1e1d747000000000, 0xfce0fcfd00000000, + 0xd01da4eb00000000, 0x32e02c6600000000, 0x55e0c42b00000000, + 0xb71d4ca600000000, 0x0d1a750700000000, 0xefe7fd8a00000000, + 0x88e715c700000000, 0x6a1a9d4a00000000, 0x46e7c55c00000000, + 0xa41a4dd100000000, 0xc31aa59c00000000, 0x21e72d1100000000, + 0xf613a60500000000, 0x14ee2e8800000000, 0x73eec6c500000000, + 0x91134e4800000000, 0xbdee165e00000000, 0x5f139ed300000000, + 0x3813769e00000000, 0xdaeefe1300000000, 0x60e9c7b200000000, + 0x82144f3f00000000, 0xe514a77200000000, 0x07e92fff00000000, + 0x2b1477e900000000, 0xc9e9ff6400000000, 0xaee9172900000000, + 0x4c149fa400000000, 0x77c758bb00000000, 0x953ad03600000000, + 0xf23a387b00000000, 0x10c7b0f600000000, 0x3c3ae8e000000000, + 0xdec7606d00000000, 0xb9c7882000000000, 0x5b3a00ad00000000, + 0xe13d390c00000000, 0x03c0b18100000000, 0x64c059cc00000000, + 0x863dd14100000000, 0xaac0895700000000, 0x483d01da00000000, + 0x2f3de99700000000, 0xcdc0611a00000000, 0x1a34ea0e00000000, + 0xf8c9628300000000, 0x9fc98ace00000000, 0x7d34024300000000, + 0x51c95a5500000000, 0xb334d2d800000000, 0xd4343a9500000000, + 0x36c9b21800000000, 0x8cce8bb900000000, 0x6e33033400000000, + 0x0933eb7900000000, 0xebce63f400000000, 0xc7333be200000000, + 0x25ceb36f00000000, 0x42ce5b2200000000, 0xa033d3af00000000, + 0xec274c0b00000000, 0x0edac48600000000, 0x69da2ccb00000000, + 0x8b27a44600000000, 0xa7dafc5000000000, 0x452774dd00000000, + 0x22279c9000000000, 0xc0da141d00000000, 0x7add2dbc00000000, + 0x9820a53100000000, 0xff204d7c00000000, 0x1dddc5f100000000, + 0x31209de700000000, 0xd3dd156a00000000, 0xb4ddfd2700000000, + 0x562075aa00000000, 0x81d4febe00000000, 0x6329763300000000, + 0x04299e7e00000000, 0xe6d416f300000000, 0xca294ee500000000, + 0x28d4c66800000000, 0x4fd42e2500000000, 0xad29a6a800000000, + 0x172e9f0900000000, 0xf5d3178400000000, 0x92d3ffc900000000, + 0x702e774400000000, 0x5cd32f5200000000, 0xbe2ea7df00000000, + 0xd92e4f9200000000, 0x3bd3c71f00000000, 0xaf88c0ad00000000, + 0x4d75482000000000, 0x2a75a06d00000000, 0xc88828e000000000, + 0xe47570f600000000, 0x0688f87b00000000, 0x6188103600000000, + 0x837598bb00000000, 0x3972a11a00000000, 0xdb8f299700000000, + 0xbc8fc1da00000000, 0x5e72495700000000, 0x728f114100000000, + 0x907299cc00000000, 0xf772718100000000, 0x158ff90c00000000, + 0xc27b721800000000, 0x2086fa9500000000, 0x478612d800000000, + 0xa57b9a5500000000, 0x8986c24300000000, 0x6b7b4ace00000000, + 0x0c7ba28300000000, 0xee862a0e00000000, 0x548113af00000000, + 0xb67c9b2200000000, 0xd17c736f00000000, 0x3381fbe200000000, + 0x1f7ca3f400000000, 0xfd812b7900000000, 0x9a81c33400000000, + 0x787c4bb900000000, 0x3468d41d00000000, 0xd6955c9000000000, + 0xb195b4dd00000000, 0x53683c5000000000, 0x7f95644600000000, + 0x9d68eccb00000000, 0xfa68048600000000, 0x18958c0b00000000, + 0xa292b5aa00000000, 0x406f3d2700000000, 0x276fd56a00000000, + 0xc5925de700000000, 0xe96f05f100000000, 0x0b928d7c00000000, + 0x6c92653100000000, 0x8e6fedbc00000000, 0x599b66a800000000, + 0xbb66ee2500000000, 0xdc66066800000000, 0x3e9b8ee500000000, + 0x1266d6f300000000, 0xf09b5e7e00000000, 0x979bb63300000000, + 0x75663ebe00000000, 0xcf61071f00000000, 0x2d9c8f9200000000, + 0x4a9c67df00000000, 0xa861ef5200000000, 0x849cb74400000000, + 0x66613fc900000000, 0x0161d78400000000, 0xe39c5f0900000000, + 0xd84f981600000000, 0x3ab2109b00000000, 0x5db2f8d600000000, + 0xbf4f705b00000000, 0x93b2284d00000000, 0x714fa0c000000000, + 0x164f488d00000000, 0xf4b2c00000000000, 0x4eb5f9a100000000, + 0xac48712c00000000, 0xcb48996100000000, 0x29b511ec00000000, + 0x054849fa00000000, 0xe7b5c17700000000, 0x80b5293a00000000, + 0x6248a1b700000000, 0xb5bc2aa300000000, 0x5741a22e00000000, + 0x30414a6300000000, 0xd2bcc2ee00000000, 0xfe419af800000000, + 0x1cbc127500000000, 0x7bbcfa3800000000, 0x994172b500000000, + 0x23464b1400000000, 0xc1bbc39900000000, 0xa6bb2bd400000000, + 0x4446a35900000000, 0x68bbfb4f00000000, 0x8a4673c200000000, + 0xed469b8f00000000, 0x0fbb130200000000, 0x43af8ca600000000, + 0xa152042b00000000, 0xc652ec6600000000, 0x24af64eb00000000, + 0x08523cfd00000000, 0xeaafb47000000000, 0x8daf5c3d00000000, + 0x6f52d4b000000000, 0xd555ed1100000000, 0x37a8659c00000000, + 0x50a88dd100000000, 0xb255055c00000000, 0x9ea85d4a00000000, + 0x7c55d5c700000000, 0x1b553d8a00000000, 0xf9a8b50700000000, + 0x2e5c3e1300000000, 0xcca1b69e00000000, 0xaba15ed300000000, + 0x495cd65e00000000, 0x65a18e4800000000, 0x875c06c500000000, + 0xe05cee8800000000, 0x02a1660500000000, 0xb8a65fa400000000, + 0x5a5bd72900000000, 0x3d5b3f6400000000, 0xdfa6b7e900000000, + 0xf35befff00000000, 0x11a6677200000000, 0x76a68f3f00000000, + 0x945b07b200000000}, + {0x0000000000000000, 0xa90b894e00000000, 0x5217129d00000000, + 0xfb1c9bd300000000, 0xe52855e100000000, 0x4c23dcaf00000000, + 0xb73f477c00000000, 0x1e34ce3200000000, 0x8b57db1900000000, + 0x225c525700000000, 0xd940c98400000000, 0x704b40ca00000000, + 0x6e7f8ef800000000, 0xc77407b600000000, 0x3c689c6500000000, + 0x9563152b00000000, 0x16afb63300000000, 0xbfa43f7d00000000, + 0x44b8a4ae00000000, 0xedb32de000000000, 0xf387e3d200000000, + 0x5a8c6a9c00000000, 0xa190f14f00000000, 0x089b780100000000, + 0x9df86d2a00000000, 0x34f3e46400000000, 0xcfef7fb700000000, + 0x66e4f6f900000000, 0x78d038cb00000000, 0xd1dbb18500000000, + 0x2ac72a5600000000, 0x83cca31800000000, 0x2c5e6d6700000000, + 0x8555e42900000000, 0x7e497ffa00000000, 0xd742f6b400000000, + 0xc976388600000000, 0x607db1c800000000, 0x9b612a1b00000000, + 0x326aa35500000000, 0xa709b67e00000000, 0x0e023f3000000000, + 0xf51ea4e300000000, 0x5c152dad00000000, 0x4221e39f00000000, + 0xeb2a6ad100000000, 0x1036f10200000000, 0xb93d784c00000000, + 0x3af1db5400000000, 0x93fa521a00000000, 0x68e6c9c900000000, + 0xc1ed408700000000, 0xdfd98eb500000000, 0x76d207fb00000000, + 0x8dce9c2800000000, 0x24c5156600000000, 0xb1a6004d00000000, + 0x18ad890300000000, 0xe3b112d000000000, 0x4aba9b9e00000000, + 0x548e55ac00000000, 0xfd85dce200000000, 0x0699473100000000, + 0xaf92ce7f00000000, 0x58bcdace00000000, 0xf1b7538000000000, + 0x0aabc85300000000, 0xa3a0411d00000000, 0xbd948f2f00000000, + 0x149f066100000000, 0xef839db200000000, 0x468814fc00000000, + 0xd3eb01d700000000, 0x7ae0889900000000, 0x81fc134a00000000, + 0x28f79a0400000000, 0x36c3543600000000, 0x9fc8dd7800000000, + 0x64d446ab00000000, 0xcddfcfe500000000, 0x4e136cfd00000000, + 0xe718e5b300000000, 0x1c047e6000000000, 0xb50ff72e00000000, + 0xab3b391c00000000, 0x0230b05200000000, 0xf92c2b8100000000, + 0x5027a2cf00000000, 0xc544b7e400000000, 0x6c4f3eaa00000000, + 0x9753a57900000000, 0x3e582c3700000000, 0x206ce20500000000, + 0x89676b4b00000000, 0x727bf09800000000, 0xdb7079d600000000, + 0x74e2b7a900000000, 0xdde93ee700000000, 0x26f5a53400000000, + 0x8ffe2c7a00000000, 0x91cae24800000000, 0x38c16b0600000000, + 0xc3ddf0d500000000, 0x6ad6799b00000000, 0xffb56cb000000000, + 0x56bee5fe00000000, 0xada27e2d00000000, 0x04a9f76300000000, + 0x1a9d395100000000, 0xb396b01f00000000, 0x488a2bcc00000000, + 0xe181a28200000000, 0x624d019a00000000, 0xcb4688d400000000, + 0x305a130700000000, 0x99519a4900000000, 0x8765547b00000000, + 0x2e6edd3500000000, 0xd57246e600000000, 0x7c79cfa800000000, + 0xe91ada8300000000, 0x401153cd00000000, 0xbb0dc81e00000000, + 0x1206415000000000, 0x0c328f6200000000, 0xa539062c00000000, + 0x5e259dff00000000, 0xf72e14b100000000, 0xf17ec44600000000, + 0x58754d0800000000, 0xa369d6db00000000, 0x0a625f9500000000, + 0x145691a700000000, 0xbd5d18e900000000, 0x4641833a00000000, + 0xef4a0a7400000000, 0x7a291f5f00000000, 0xd322961100000000, + 0x283e0dc200000000, 0x8135848c00000000, 0x9f014abe00000000, + 0x360ac3f000000000, 0xcd16582300000000, 0x641dd16d00000000, + 0xe7d1727500000000, 0x4edafb3b00000000, 0xb5c660e800000000, + 0x1ccde9a600000000, 0x02f9279400000000, 0xabf2aeda00000000, + 0x50ee350900000000, 0xf9e5bc4700000000, 0x6c86a96c00000000, + 0xc58d202200000000, 0x3e91bbf100000000, 0x979a32bf00000000, + 0x89aefc8d00000000, 0x20a575c300000000, 0xdbb9ee1000000000, + 0x72b2675e00000000, 0xdd20a92100000000, 0x742b206f00000000, + 0x8f37bbbc00000000, 0x263c32f200000000, 0x3808fcc000000000, + 0x9103758e00000000, 0x6a1fee5d00000000, 0xc314671300000000, + 0x5677723800000000, 0xff7cfb7600000000, 0x046060a500000000, + 0xad6be9eb00000000, 0xb35f27d900000000, 0x1a54ae9700000000, + 0xe148354400000000, 0x4843bc0a00000000, 0xcb8f1f1200000000, + 0x6284965c00000000, 0x99980d8f00000000, 0x309384c100000000, + 0x2ea74af300000000, 0x87acc3bd00000000, 0x7cb0586e00000000, + 0xd5bbd12000000000, 0x40d8c40b00000000, 0xe9d34d4500000000, + 0x12cfd69600000000, 0xbbc45fd800000000, 0xa5f091ea00000000, + 0x0cfb18a400000000, 0xf7e7837700000000, 0x5eec0a3900000000, + 0xa9c21e8800000000, 0x00c997c600000000, 0xfbd50c1500000000, + 0x52de855b00000000, 0x4cea4b6900000000, 0xe5e1c22700000000, + 0x1efd59f400000000, 0xb7f6d0ba00000000, 0x2295c59100000000, + 0x8b9e4cdf00000000, 0x7082d70c00000000, 0xd9895e4200000000, + 0xc7bd907000000000, 0x6eb6193e00000000, 0x95aa82ed00000000, + 0x3ca10ba300000000, 0xbf6da8bb00000000, 0x166621f500000000, + 0xed7aba2600000000, 0x4471336800000000, 0x5a45fd5a00000000, + 0xf34e741400000000, 0x0852efc700000000, 0xa159668900000000, + 0x343a73a200000000, 0x9d31faec00000000, 0x662d613f00000000, + 0xcf26e87100000000, 0xd112264300000000, 0x7819af0d00000000, + 0x830534de00000000, 0x2a0ebd9000000000, 0x859c73ef00000000, + 0x2c97faa100000000, 0xd78b617200000000, 0x7e80e83c00000000, + 0x60b4260e00000000, 0xc9bfaf4000000000, 0x32a3349300000000, + 0x9ba8bddd00000000, 0x0ecba8f600000000, 0xa7c021b800000000, + 0x5cdcba6b00000000, 0xf5d7332500000000, 0xebe3fd1700000000, + 0x42e8745900000000, 0xb9f4ef8a00000000, 0x10ff66c400000000, + 0x9333c5dc00000000, 0x3a384c9200000000, 0xc124d74100000000, + 0x682f5e0f00000000, 0x761b903d00000000, 0xdf10197300000000, + 0x240c82a000000000, 0x8d070bee00000000, 0x18641ec500000000, + 0xb16f978b00000000, 0x4a730c5800000000, 0xe378851600000000, + 0xfd4c4b2400000000, 0x5447c26a00000000, 0xaf5b59b900000000, + 0x0650d0f700000000}, + {0x0000000000000000, 0x479244af00000000, 0xcf22f88500000000, + 0x88b0bc2a00000000, 0xdf4381d000000000, 0x98d1c57f00000000, + 0x1061795500000000, 0x57f33dfa00000000, 0xff81737a00000000, + 0xb81337d500000000, 0x30a38bff00000000, 0x7731cf5000000000, + 0x20c2f2aa00000000, 0x6750b60500000000, 0xefe00a2f00000000, + 0xa8724e8000000000, 0xfe03e7f400000000, 0xb991a35b00000000, + 0x31211f7100000000, 0x76b35bde00000000, 0x2140662400000000, + 0x66d2228b00000000, 0xee629ea100000000, 0xa9f0da0e00000000, + 0x0182948e00000000, 0x4610d02100000000, 0xcea06c0b00000000, + 0x893228a400000000, 0xdec1155e00000000, 0x995351f100000000, + 0x11e3eddb00000000, 0x5671a97400000000, 0xbd01bf3200000000, + 0xfa93fb9d00000000, 0x722347b700000000, 0x35b1031800000000, + 0x62423ee200000000, 0x25d07a4d00000000, 0xad60c66700000000, + 0xeaf282c800000000, 0x4280cc4800000000, 0x051288e700000000, + 0x8da234cd00000000, 0xca30706200000000, 0x9dc34d9800000000, + 0xda51093700000000, 0x52e1b51d00000000, 0x1573f1b200000000, + 0x430258c600000000, 0x04901c6900000000, 0x8c20a04300000000, + 0xcbb2e4ec00000000, 0x9c41d91600000000, 0xdbd39db900000000, + 0x5363219300000000, 0x14f1653c00000000, 0xbc832bbc00000000, + 0xfb116f1300000000, 0x73a1d33900000000, 0x3433979600000000, + 0x63c0aa6c00000000, 0x2452eec300000000, 0xace252e900000000, + 0xeb70164600000000, 0x7a037e6500000000, 0x3d913aca00000000, + 0xb52186e000000000, 0xf2b3c24f00000000, 0xa540ffb500000000, + 0xe2d2bb1a00000000, 0x6a62073000000000, 0x2df0439f00000000, + 0x85820d1f00000000, 0xc21049b000000000, 0x4aa0f59a00000000, + 0x0d32b13500000000, 0x5ac18ccf00000000, 0x1d53c86000000000, + 0x95e3744a00000000, 0xd27130e500000000, 0x8400999100000000, + 0xc392dd3e00000000, 0x4b22611400000000, 0x0cb025bb00000000, + 0x5b43184100000000, 0x1cd15cee00000000, 0x9461e0c400000000, + 0xd3f3a46b00000000, 0x7b81eaeb00000000, 0x3c13ae4400000000, + 0xb4a3126e00000000, 0xf33156c100000000, 0xa4c26b3b00000000, + 0xe3502f9400000000, 0x6be093be00000000, 0x2c72d71100000000, + 0xc702c15700000000, 0x809085f800000000, 0x082039d200000000, + 0x4fb27d7d00000000, 0x1841408700000000, 0x5fd3042800000000, + 0xd763b80200000000, 0x90f1fcad00000000, 0x3883b22d00000000, + 0x7f11f68200000000, 0xf7a14aa800000000, 0xb0330e0700000000, + 0xe7c033fd00000000, 0xa052775200000000, 0x28e2cb7800000000, + 0x6f708fd700000000, 0x390126a300000000, 0x7e93620c00000000, + 0xf623de2600000000, 0xb1b19a8900000000, 0xe642a77300000000, + 0xa1d0e3dc00000000, 0x29605ff600000000, 0x6ef21b5900000000, + 0xc68055d900000000, 0x8112117600000000, 0x09a2ad5c00000000, + 0x4e30e9f300000000, 0x19c3d40900000000, 0x5e5190a600000000, + 0xd6e12c8c00000000, 0x9173682300000000, 0xf406fcca00000000, + 0xb394b86500000000, 0x3b24044f00000000, 0x7cb640e000000000, + 0x2b457d1a00000000, 0x6cd739b500000000, 0xe467859f00000000, + 0xa3f5c13000000000, 0x0b878fb000000000, 0x4c15cb1f00000000, + 0xc4a5773500000000, 0x8337339a00000000, 0xd4c40e6000000000, + 0x93564acf00000000, 0x1be6f6e500000000, 0x5c74b24a00000000, + 0x0a051b3e00000000, 0x4d975f9100000000, 0xc527e3bb00000000, + 0x82b5a71400000000, 0xd5469aee00000000, 0x92d4de4100000000, + 0x1a64626b00000000, 0x5df626c400000000, 0xf584684400000000, + 0xb2162ceb00000000, 0x3aa690c100000000, 0x7d34d46e00000000, + 0x2ac7e99400000000, 0x6d55ad3b00000000, 0xe5e5111100000000, + 0xa27755be00000000, 0x490743f800000000, 0x0e95075700000000, + 0x8625bb7d00000000, 0xc1b7ffd200000000, 0x9644c22800000000, + 0xd1d6868700000000, 0x59663aad00000000, 0x1ef47e0200000000, + 0xb686308200000000, 0xf114742d00000000, 0x79a4c80700000000, + 0x3e368ca800000000, 0x69c5b15200000000, 0x2e57f5fd00000000, + 0xa6e749d700000000, 0xe1750d7800000000, 0xb704a40c00000000, + 0xf096e0a300000000, 0x78265c8900000000, 0x3fb4182600000000, + 0x684725dc00000000, 0x2fd5617300000000, 0xa765dd5900000000, + 0xe0f799f600000000, 0x4885d77600000000, 0x0f1793d900000000, + 0x87a72ff300000000, 0xc0356b5c00000000, 0x97c656a600000000, + 0xd054120900000000, 0x58e4ae2300000000, 0x1f76ea8c00000000, + 0x8e0582af00000000, 0xc997c60000000000, 0x41277a2a00000000, + 0x06b53e8500000000, 0x5146037f00000000, 0x16d447d000000000, + 0x9e64fbfa00000000, 0xd9f6bf5500000000, 0x7184f1d500000000, + 0x3616b57a00000000, 0xbea6095000000000, 0xf9344dff00000000, + 0xaec7700500000000, 0xe95534aa00000000, 0x61e5888000000000, + 0x2677cc2f00000000, 0x7006655b00000000, 0x379421f400000000, + 0xbf249dde00000000, 0xf8b6d97100000000, 0xaf45e48b00000000, + 0xe8d7a02400000000, 0x60671c0e00000000, 0x27f558a100000000, + 0x8f87162100000000, 0xc815528e00000000, 0x40a5eea400000000, + 0x0737aa0b00000000, 0x50c497f100000000, 0x1756d35e00000000, + 0x9fe66f7400000000, 0xd8742bdb00000000, 0x33043d9d00000000, + 0x7496793200000000, 0xfc26c51800000000, 0xbbb481b700000000, + 0xec47bc4d00000000, 0xabd5f8e200000000, 0x236544c800000000, + 0x64f7006700000000, 0xcc854ee700000000, 0x8b170a4800000000, + 0x03a7b66200000000, 0x4435f2cd00000000, 0x13c6cf3700000000, + 0x54548b9800000000, 0xdce437b200000000, 0x9b76731d00000000, + 0xcd07da6900000000, 0x8a959ec600000000, 0x022522ec00000000, + 0x45b7664300000000, 0x12445bb900000000, 0x55d61f1600000000, + 0xdd66a33c00000000, 0x9af4e79300000000, 0x3286a91300000000, + 0x7514edbc00000000, 0xfda4519600000000, 0xba36153900000000, + 0xedc528c300000000, 0xaa576c6c00000000, 0x22e7d04600000000, + 0x657594e900000000}}; + +#else /* W == 4 */ + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0x65673b46, 0xcace768c, 0xafa94dca, 0x4eedeb59, + 0x2b8ad01f, 0x84239dd5, 0xe144a693, 0x9ddbd6b2, 0xf8bcedf4, + 0x5715a03e, 0x32729b78, 0xd3363deb, 0xb65106ad, 0x19f84b67, + 0x7c9f7021, 0xe0c6ab25, 0x85a19063, 0x2a08dda9, 0x4f6fe6ef, + 0xae2b407c, 0xcb4c7b3a, 0x64e536f0, 0x01820db6, 0x7d1d7d97, + 0x187a46d1, 0xb7d30b1b, 0xd2b4305d, 0x33f096ce, 0x5697ad88, + 0xf93ee042, 0x9c59db04, 0x1afc500b, 0x7f9b6b4d, 0xd0322687, + 0xb5551dc1, 0x5411bb52, 0x31768014, 0x9edfcdde, 0xfbb8f698, + 0x872786b9, 0xe240bdff, 0x4de9f035, 0x288ecb73, 0xc9ca6de0, + 0xacad56a6, 0x03041b6c, 0x6663202a, 0xfa3afb2e, 0x9f5dc068, + 0x30f48da2, 0x5593b6e4, 0xb4d71077, 0xd1b02b31, 0x7e1966fb, + 0x1b7e5dbd, 0x67e12d9c, 0x028616da, 0xad2f5b10, 0xc8486056, + 0x290cc6c5, 0x4c6bfd83, 0xe3c2b049, 0x86a58b0f, 0x35f8a016, + 0x509f9b50, 0xff36d69a, 0x9a51eddc, 0x7b154b4f, 0x1e727009, + 0xb1db3dc3, 0xd4bc0685, 0xa82376a4, 0xcd444de2, 0x62ed0028, + 0x078a3b6e, 0xe6ce9dfd, 0x83a9a6bb, 0x2c00eb71, 0x4967d037, + 0xd53e0b33, 0xb0593075, 0x1ff07dbf, 0x7a9746f9, 0x9bd3e06a, + 0xfeb4db2c, 0x511d96e6, 0x347aada0, 0x48e5dd81, 0x2d82e6c7, + 0x822bab0d, 0xe74c904b, 0x060836d8, 0x636f0d9e, 0xccc64054, + 0xa9a17b12, 0x2f04f01d, 0x4a63cb5b, 0xe5ca8691, 0x80adbdd7, + 0x61e91b44, 0x048e2002, 0xab276dc8, 0xce40568e, 0xb2df26af, + 0xd7b81de9, 0x78115023, 0x1d766b65, 0xfc32cdf6, 0x9955f6b0, + 0x36fcbb7a, 0x539b803c, 0xcfc25b38, 0xaaa5607e, 0x050c2db4, + 0x606b16f2, 0x812fb061, 0xe4488b27, 0x4be1c6ed, 0x2e86fdab, + 0x52198d8a, 0x377eb6cc, 0x98d7fb06, 0xfdb0c040, 0x1cf466d3, + 0x79935d95, 0xd63a105f, 0xb35d2b19, 0x6bf1402c, 0x0e967b6a, + 0xa13f36a0, 0xc4580de6, 0x251cab75, 0x407b9033, 0xefd2ddf9, + 0x8ab5e6bf, 0xf62a969e, 0x934dadd8, 0x3ce4e012, 0x5983db54, + 0xb8c77dc7, 0xdda04681, 0x72090b4b, 0x176e300d, 0x8b37eb09, + 0xee50d04f, 0x41f99d85, 0x249ea6c3, 0xc5da0050, 0xa0bd3b16, + 0x0f1476dc, 0x6a734d9a, 0x16ec3dbb, 0x738b06fd, 0xdc224b37, + 0xb9457071, 0x5801d6e2, 0x3d66eda4, 0x92cfa06e, 0xf7a89b28, + 0x710d1027, 0x146a2b61, 0xbbc366ab, 0xdea45ded, 0x3fe0fb7e, + 0x5a87c038, 0xf52e8df2, 0x9049b6b4, 0xecd6c695, 0x89b1fdd3, + 0x2618b019, 0x437f8b5f, 0xa23b2dcc, 0xc75c168a, 0x68f55b40, + 0x0d926006, 0x91cbbb02, 0xf4ac8044, 0x5b05cd8e, 0x3e62f6c8, + 0xdf26505b, 0xba416b1d, 0x15e826d7, 0x708f1d91, 0x0c106db0, + 0x697756f6, 0xc6de1b3c, 0xa3b9207a, 0x42fd86e9, 0x279abdaf, + 0x8833f065, 0xed54cb23, 0x5e09e03a, 0x3b6edb7c, 0x94c796b6, + 0xf1a0adf0, 0x10e40b63, 0x75833025, 0xda2a7def, 0xbf4d46a9, + 0xc3d23688, 0xa6b50dce, 0x091c4004, 0x6c7b7b42, 0x8d3fddd1, + 0xe858e697, 0x47f1ab5d, 0x2296901b, 0xbecf4b1f, 0xdba87059, + 0x74013d93, 0x116606d5, 0xf022a046, 0x95459b00, 0x3aecd6ca, + 0x5f8bed8c, 0x23149dad, 0x4673a6eb, 0xe9daeb21, 0x8cbdd067, + 0x6df976f4, 0x089e4db2, 0xa7370078, 0xc2503b3e, 0x44f5b031, + 0x21928b77, 0x8e3bc6bd, 0xeb5cfdfb, 0x0a185b68, 0x6f7f602e, + 0xc0d62de4, 0xa5b116a2, 0xd92e6683, 0xbc495dc5, 0x13e0100f, + 0x76872b49, 0x97c38dda, 0xf2a4b69c, 0x5d0dfb56, 0x386ac010, + 0xa4331b14, 0xc1542052, 0x6efd6d98, 0x0b9a56de, 0xeadef04d, + 0x8fb9cb0b, 0x201086c1, 0x4577bd87, 0x39e8cda6, 0x5c8ff6e0, + 0xf326bb2a, 0x9641806c, 0x770526ff, 0x12621db9, 0xbdcb5073, + 0xd8ac6b35}, + {0x00000000, 0xd7e28058, 0x74b406f1, 0xa35686a9, 0xe9680de2, + 0x3e8a8dba, 0x9ddc0b13, 0x4a3e8b4b, 0x09a11d85, 0xde439ddd, + 0x7d151b74, 0xaaf79b2c, 0xe0c91067, 0x372b903f, 0x947d1696, + 0x439f96ce, 0x13423b0a, 0xc4a0bb52, 0x67f63dfb, 0xb014bda3, + 0xfa2a36e8, 0x2dc8b6b0, 0x8e9e3019, 0x597cb041, 0x1ae3268f, + 0xcd01a6d7, 0x6e57207e, 0xb9b5a026, 0xf38b2b6d, 0x2469ab35, + 0x873f2d9c, 0x50ddadc4, 0x26847614, 0xf166f64c, 0x523070e5, + 0x85d2f0bd, 0xcfec7bf6, 0x180efbae, 0xbb587d07, 0x6cbafd5f, + 0x2f256b91, 0xf8c7ebc9, 0x5b916d60, 0x8c73ed38, 0xc64d6673, + 0x11afe62b, 0xb2f96082, 0x651be0da, 0x35c64d1e, 0xe224cd46, + 0x41724bef, 0x9690cbb7, 0xdcae40fc, 0x0b4cc0a4, 0xa81a460d, + 0x7ff8c655, 0x3c67509b, 0xeb85d0c3, 0x48d3566a, 0x9f31d632, + 0xd50f5d79, 0x02eddd21, 0xa1bb5b88, 0x7659dbd0, 0x4d08ec28, + 0x9aea6c70, 0x39bcead9, 0xee5e6a81, 0xa460e1ca, 0x73826192, + 0xd0d4e73b, 0x07366763, 0x44a9f1ad, 0x934b71f5, 0x301df75c, + 0xe7ff7704, 0xadc1fc4f, 0x7a237c17, 0xd975fabe, 0x0e977ae6, + 0x5e4ad722, 0x89a8577a, 0x2afed1d3, 0xfd1c518b, 0xb722dac0, + 0x60c05a98, 0xc396dc31, 0x14745c69, 0x57ebcaa7, 0x80094aff, + 0x235fcc56, 0xf4bd4c0e, 0xbe83c745, 0x6961471d, 0xca37c1b4, + 0x1dd541ec, 0x6b8c9a3c, 0xbc6e1a64, 0x1f389ccd, 0xc8da1c95, + 0x82e497de, 0x55061786, 0xf650912f, 0x21b21177, 0x622d87b9, + 0xb5cf07e1, 0x16998148, 0xc17b0110, 0x8b458a5b, 0x5ca70a03, + 0xfff18caa, 0x28130cf2, 0x78cea136, 0xaf2c216e, 0x0c7aa7c7, + 0xdb98279f, 0x91a6acd4, 0x46442c8c, 0xe512aa25, 0x32f02a7d, + 0x716fbcb3, 0xa68d3ceb, 0x05dbba42, 0xd2393a1a, 0x9807b151, + 0x4fe53109, 0xecb3b7a0, 0x3b5137f8, 0x9a11d850, 0x4df35808, + 0xeea5dea1, 0x39475ef9, 0x7379d5b2, 0xa49b55ea, 0x07cdd343, + 0xd02f531b, 0x93b0c5d5, 0x4452458d, 0xe704c324, 0x30e6437c, + 0x7ad8c837, 0xad3a486f, 0x0e6ccec6, 0xd98e4e9e, 0x8953e35a, + 0x5eb16302, 0xfde7e5ab, 0x2a0565f3, 0x603beeb8, 0xb7d96ee0, + 0x148fe849, 0xc36d6811, 0x80f2fedf, 0x57107e87, 0xf446f82e, + 0x23a47876, 0x699af33d, 0xbe787365, 0x1d2ef5cc, 0xcacc7594, + 0xbc95ae44, 0x6b772e1c, 0xc821a8b5, 0x1fc328ed, 0x55fda3a6, + 0x821f23fe, 0x2149a557, 0xf6ab250f, 0xb534b3c1, 0x62d63399, + 0xc180b530, 0x16623568, 0x5c5cbe23, 0x8bbe3e7b, 0x28e8b8d2, + 0xff0a388a, 0xafd7954e, 0x78351516, 0xdb6393bf, 0x0c8113e7, + 0x46bf98ac, 0x915d18f4, 0x320b9e5d, 0xe5e91e05, 0xa67688cb, + 0x71940893, 0xd2c28e3a, 0x05200e62, 0x4f1e8529, 0x98fc0571, + 0x3baa83d8, 0xec480380, 0xd7193478, 0x00fbb420, 0xa3ad3289, + 0x744fb2d1, 0x3e71399a, 0xe993b9c2, 0x4ac53f6b, 0x9d27bf33, + 0xdeb829fd, 0x095aa9a5, 0xaa0c2f0c, 0x7deeaf54, 0x37d0241f, + 0xe032a447, 0x436422ee, 0x9486a2b6, 0xc45b0f72, 0x13b98f2a, + 0xb0ef0983, 0x670d89db, 0x2d330290, 0xfad182c8, 0x59870461, + 0x8e658439, 0xcdfa12f7, 0x1a1892af, 0xb94e1406, 0x6eac945e, + 0x24921f15, 0xf3709f4d, 0x502619e4, 0x87c499bc, 0xf19d426c, + 0x267fc234, 0x8529449d, 0x52cbc4c5, 0x18f54f8e, 0xcf17cfd6, + 0x6c41497f, 0xbba3c927, 0xf83c5fe9, 0x2fdedfb1, 0x8c885918, + 0x5b6ad940, 0x1154520b, 0xc6b6d253, 0x65e054fa, 0xb202d4a2, + 0xe2df7966, 0x353df93e, 0x966b7f97, 0x4189ffcf, 0x0bb77484, + 0xdc55f4dc, 0x7f037275, 0xa8e1f22d, 0xeb7e64e3, 0x3c9ce4bb, + 0x9fca6212, 0x4828e24a, 0x02166901, 0xd5f4e959, 0x76a26ff0, + 0xa140efa8}, + {0x00000000, 0xef52b6e1, 0x05d46b83, 0xea86dd62, 0x0ba8d706, + 0xe4fa61e7, 0x0e7cbc85, 0xe12e0a64, 0x1751ae0c, 0xf80318ed, + 0x1285c58f, 0xfdd7736e, 0x1cf9790a, 0xf3abcfeb, 0x192d1289, + 0xf67fa468, 0x2ea35c18, 0xc1f1eaf9, 0x2b77379b, 0xc425817a, + 0x250b8b1e, 0xca593dff, 0x20dfe09d, 0xcf8d567c, 0x39f2f214, + 0xd6a044f5, 0x3c269997, 0xd3742f76, 0x325a2512, 0xdd0893f3, + 0x378e4e91, 0xd8dcf870, 0x5d46b830, 0xb2140ed1, 0x5892d3b3, + 0xb7c06552, 0x56ee6f36, 0xb9bcd9d7, 0x533a04b5, 0xbc68b254, + 0x4a17163c, 0xa545a0dd, 0x4fc37dbf, 0xa091cb5e, 0x41bfc13a, + 0xaeed77db, 0x446baab9, 0xab391c58, 0x73e5e428, 0x9cb752c9, + 0x76318fab, 0x9963394a, 0x784d332e, 0x971f85cf, 0x7d9958ad, + 0x92cbee4c, 0x64b44a24, 0x8be6fcc5, 0x616021a7, 0x8e329746, + 0x6f1c9d22, 0x804e2bc3, 0x6ac8f6a1, 0x859a4040, 0xba8d7060, + 0x55dfc681, 0xbf591be3, 0x500bad02, 0xb125a766, 0x5e771187, + 0xb4f1cce5, 0x5ba37a04, 0xaddcde6c, 0x428e688d, 0xa808b5ef, + 0x475a030e, 0xa674096a, 0x4926bf8b, 0xa3a062e9, 0x4cf2d408, + 0x942e2c78, 0x7b7c9a99, 0x91fa47fb, 0x7ea8f11a, 0x9f86fb7e, + 0x70d44d9f, 0x9a5290fd, 0x7500261c, 0x837f8274, 0x6c2d3495, + 0x86abe9f7, 0x69f95f16, 0x88d75572, 0x6785e393, 0x8d033ef1, + 0x62518810, 0xe7cbc850, 0x08997eb1, 0xe21fa3d3, 0x0d4d1532, + 0xec631f56, 0x0331a9b7, 0xe9b774d5, 0x06e5c234, 0xf09a665c, + 0x1fc8d0bd, 0xf54e0ddf, 0x1a1cbb3e, 0xfb32b15a, 0x146007bb, + 0xfee6dad9, 0x11b46c38, 0xc9689448, 0x263a22a9, 0xccbcffcb, + 0x23ee492a, 0xc2c0434e, 0x2d92f5af, 0xc71428cd, 0x28469e2c, + 0xde393a44, 0x316b8ca5, 0xdbed51c7, 0x34bfe726, 0xd591ed42, + 0x3ac35ba3, 0xd04586c1, 0x3f173020, 0xae6be681, 0x41395060, + 0xabbf8d02, 0x44ed3be3, 0xa5c33187, 0x4a918766, 0xa0175a04, + 0x4f45ece5, 0xb93a488d, 0x5668fe6c, 0xbcee230e, 0x53bc95ef, + 0xb2929f8b, 0x5dc0296a, 0xb746f408, 0x581442e9, 0x80c8ba99, + 0x6f9a0c78, 0x851cd11a, 0x6a4e67fb, 0x8b606d9f, 0x6432db7e, + 0x8eb4061c, 0x61e6b0fd, 0x97991495, 0x78cba274, 0x924d7f16, + 0x7d1fc9f7, 0x9c31c393, 0x73637572, 0x99e5a810, 0x76b71ef1, + 0xf32d5eb1, 0x1c7fe850, 0xf6f93532, 0x19ab83d3, 0xf88589b7, + 0x17d73f56, 0xfd51e234, 0x120354d5, 0xe47cf0bd, 0x0b2e465c, + 0xe1a89b3e, 0x0efa2ddf, 0xefd427bb, 0x0086915a, 0xea004c38, + 0x0552fad9, 0xdd8e02a9, 0x32dcb448, 0xd85a692a, 0x3708dfcb, + 0xd626d5af, 0x3974634e, 0xd3f2be2c, 0x3ca008cd, 0xcadfaca5, + 0x258d1a44, 0xcf0bc726, 0x205971c7, 0xc1777ba3, 0x2e25cd42, + 0xc4a31020, 0x2bf1a6c1, 0x14e696e1, 0xfbb42000, 0x1132fd62, + 0xfe604b83, 0x1f4e41e7, 0xf01cf706, 0x1a9a2a64, 0xf5c89c85, + 0x03b738ed, 0xece58e0c, 0x0663536e, 0xe931e58f, 0x081fefeb, + 0xe74d590a, 0x0dcb8468, 0xe2993289, 0x3a45caf9, 0xd5177c18, + 0x3f91a17a, 0xd0c3179b, 0x31ed1dff, 0xdebfab1e, 0x3439767c, + 0xdb6bc09d, 0x2d1464f5, 0xc246d214, 0x28c00f76, 0xc792b997, + 0x26bcb3f3, 0xc9ee0512, 0x2368d870, 0xcc3a6e91, 0x49a02ed1, + 0xa6f29830, 0x4c744552, 0xa326f3b3, 0x4208f9d7, 0xad5a4f36, + 0x47dc9254, 0xa88e24b5, 0x5ef180dd, 0xb1a3363c, 0x5b25eb5e, + 0xb4775dbf, 0x555957db, 0xba0be13a, 0x508d3c58, 0xbfdf8ab9, + 0x670372c9, 0x8851c428, 0x62d7194a, 0x8d85afab, 0x6caba5cf, + 0x83f9132e, 0x697fce4c, 0x862d78ad, 0x7052dcc5, 0x9f006a24, + 0x7586b746, 0x9ad401a7, 0x7bfa0bc3, 0x94a8bd22, 0x7e2e6040, + 0x917cd6a1}, + {0x00000000, 0x87a6cb43, 0xd43c90c7, 0x539a5b84, 0x730827cf, + 0xf4aeec8c, 0xa734b708, 0x20927c4b, 0xe6104f9e, 0x61b684dd, + 0x322cdf59, 0xb58a141a, 0x95186851, 0x12bea312, 0x4124f896, + 0xc68233d5, 0x1751997d, 0x90f7523e, 0xc36d09ba, 0x44cbc2f9, + 0x6459beb2, 0xe3ff75f1, 0xb0652e75, 0x37c3e536, 0xf141d6e3, + 0x76e71da0, 0x257d4624, 0xa2db8d67, 0x8249f12c, 0x05ef3a6f, + 0x567561eb, 0xd1d3aaa8, 0x2ea332fa, 0xa905f9b9, 0xfa9fa23d, + 0x7d39697e, 0x5dab1535, 0xda0dde76, 0x899785f2, 0x0e314eb1, + 0xc8b37d64, 0x4f15b627, 0x1c8feda3, 0x9b2926e0, 0xbbbb5aab, + 0x3c1d91e8, 0x6f87ca6c, 0xe821012f, 0x39f2ab87, 0xbe5460c4, + 0xedce3b40, 0x6a68f003, 0x4afa8c48, 0xcd5c470b, 0x9ec61c8f, + 0x1960d7cc, 0xdfe2e419, 0x58442f5a, 0x0bde74de, 0x8c78bf9d, + 0xaceac3d6, 0x2b4c0895, 0x78d65311, 0xff709852, 0x5d4665f4, + 0xdae0aeb7, 0x897af533, 0x0edc3e70, 0x2e4e423b, 0xa9e88978, + 0xfa72d2fc, 0x7dd419bf, 0xbb562a6a, 0x3cf0e129, 0x6f6abaad, + 0xe8cc71ee, 0xc85e0da5, 0x4ff8c6e6, 0x1c629d62, 0x9bc45621, + 0x4a17fc89, 0xcdb137ca, 0x9e2b6c4e, 0x198da70d, 0x391fdb46, + 0xbeb91005, 0xed234b81, 0x6a8580c2, 0xac07b317, 0x2ba17854, + 0x783b23d0, 0xff9de893, 0xdf0f94d8, 0x58a95f9b, 0x0b33041f, + 0x8c95cf5c, 0x73e5570e, 0xf4439c4d, 0xa7d9c7c9, 0x207f0c8a, + 0x00ed70c1, 0x874bbb82, 0xd4d1e006, 0x53772b45, 0x95f51890, + 0x1253d3d3, 0x41c98857, 0xc66f4314, 0xe6fd3f5f, 0x615bf41c, + 0x32c1af98, 0xb56764db, 0x64b4ce73, 0xe3120530, 0xb0885eb4, + 0x372e95f7, 0x17bce9bc, 0x901a22ff, 0xc380797b, 0x4426b238, + 0x82a481ed, 0x05024aae, 0x5698112a, 0xd13eda69, 0xf1aca622, + 0x760a6d61, 0x259036e5, 0xa236fda6, 0xba8ccbe8, 0x3d2a00ab, + 0x6eb05b2f, 0xe916906c, 0xc984ec27, 0x4e222764, 0x1db87ce0, + 0x9a1eb7a3, 0x5c9c8476, 0xdb3a4f35, 0x88a014b1, 0x0f06dff2, + 0x2f94a3b9, 0xa83268fa, 0xfba8337e, 0x7c0ef83d, 0xaddd5295, + 0x2a7b99d6, 0x79e1c252, 0xfe470911, 0xded5755a, 0x5973be19, + 0x0ae9e59d, 0x8d4f2ede, 0x4bcd1d0b, 0xcc6bd648, 0x9ff18dcc, + 0x1857468f, 0x38c53ac4, 0xbf63f187, 0xecf9aa03, 0x6b5f6140, + 0x942ff912, 0x13893251, 0x401369d5, 0xc7b5a296, 0xe727dedd, + 0x6081159e, 0x331b4e1a, 0xb4bd8559, 0x723fb68c, 0xf5997dcf, + 0xa603264b, 0x21a5ed08, 0x01379143, 0x86915a00, 0xd50b0184, + 0x52adcac7, 0x837e606f, 0x04d8ab2c, 0x5742f0a8, 0xd0e43beb, + 0xf07647a0, 0x77d08ce3, 0x244ad767, 0xa3ec1c24, 0x656e2ff1, + 0xe2c8e4b2, 0xb152bf36, 0x36f47475, 0x1666083e, 0x91c0c37d, + 0xc25a98f9, 0x45fc53ba, 0xe7caae1c, 0x606c655f, 0x33f63edb, + 0xb450f598, 0x94c289d3, 0x13644290, 0x40fe1914, 0xc758d257, + 0x01dae182, 0x867c2ac1, 0xd5e67145, 0x5240ba06, 0x72d2c64d, + 0xf5740d0e, 0xa6ee568a, 0x21489dc9, 0xf09b3761, 0x773dfc22, + 0x24a7a7a6, 0xa3016ce5, 0x839310ae, 0x0435dbed, 0x57af8069, + 0xd0094b2a, 0x168b78ff, 0x912db3bc, 0xc2b7e838, 0x4511237b, + 0x65835f30, 0xe2259473, 0xb1bfcff7, 0x361904b4, 0xc9699ce6, + 0x4ecf57a5, 0x1d550c21, 0x9af3c762, 0xba61bb29, 0x3dc7706a, + 0x6e5d2bee, 0xe9fbe0ad, 0x2f79d378, 0xa8df183b, 0xfb4543bf, + 0x7ce388fc, 0x5c71f4b7, 0xdbd73ff4, 0x884d6470, 0x0febaf33, + 0xde38059b, 0x599eced8, 0x0a04955c, 0x8da25e1f, 0xad302254, + 0x2a96e917, 0x790cb293, 0xfeaa79d0, 0x38284a05, 0xbf8e8146, + 0xec14dac2, 0x6bb21181, 0x4b206dca, 0xcc86a689, 0x9f1cfd0d, + 0x18ba364e}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x00000000, 0x43cba687, 0xc7903cd4, 0x845b9a53, 0xcf270873, + 0x8cecaef4, 0x08b734a7, 0x4b7c9220, 0x9e4f10e6, 0xdd84b661, + 0x59df2c32, 0x1a148ab5, 0x51681895, 0x12a3be12, 0x96f82441, + 0xd53382c6, 0x7d995117, 0x3e52f790, 0xba096dc3, 0xf9c2cb44, + 0xb2be5964, 0xf175ffe3, 0x752e65b0, 0x36e5c337, 0xe3d641f1, + 0xa01de776, 0x24467d25, 0x678ddba2, 0x2cf14982, 0x6f3aef05, + 0xeb617556, 0xa8aad3d1, 0xfa32a32e, 0xb9f905a9, 0x3da29ffa, + 0x7e69397d, 0x3515ab5d, 0x76de0dda, 0xf2859789, 0xb14e310e, + 0x647db3c8, 0x27b6154f, 0xa3ed8f1c, 0xe026299b, 0xab5abbbb, + 0xe8911d3c, 0x6cca876f, 0x2f0121e8, 0x87abf239, 0xc46054be, + 0x403bceed, 0x03f0686a, 0x488cfa4a, 0x0b475ccd, 0x8f1cc69e, + 0xccd76019, 0x19e4e2df, 0x5a2f4458, 0xde74de0b, 0x9dbf788c, + 0xd6c3eaac, 0x95084c2b, 0x1153d678, 0x529870ff, 0xf465465d, + 0xb7aee0da, 0x33f57a89, 0x703edc0e, 0x3b424e2e, 0x7889e8a9, + 0xfcd272fa, 0xbf19d47d, 0x6a2a56bb, 0x29e1f03c, 0xadba6a6f, + 0xee71cce8, 0xa50d5ec8, 0xe6c6f84f, 0x629d621c, 0x2156c49b, + 0x89fc174a, 0xca37b1cd, 0x4e6c2b9e, 0x0da78d19, 0x46db1f39, + 0x0510b9be, 0x814b23ed, 0xc280856a, 0x17b307ac, 0x5478a12b, + 0xd0233b78, 0x93e89dff, 0xd8940fdf, 0x9b5fa958, 0x1f04330b, + 0x5ccf958c, 0x0e57e573, 0x4d9c43f4, 0xc9c7d9a7, 0x8a0c7f20, + 0xc170ed00, 0x82bb4b87, 0x06e0d1d4, 0x452b7753, 0x9018f595, + 0xd3d35312, 0x5788c941, 0x14436fc6, 0x5f3ffde6, 0x1cf45b61, + 0x98afc132, 0xdb6467b5, 0x73ceb464, 0x300512e3, 0xb45e88b0, + 0xf7952e37, 0xbce9bc17, 0xff221a90, 0x7b7980c3, 0x38b22644, + 0xed81a482, 0xae4a0205, 0x2a119856, 0x69da3ed1, 0x22a6acf1, + 0x616d0a76, 0xe5369025, 0xa6fd36a2, 0xe8cb8cba, 0xab002a3d, + 0x2f5bb06e, 0x6c9016e9, 0x27ec84c9, 0x6427224e, 0xe07cb81d, + 0xa3b71e9a, 0x76849c5c, 0x354f3adb, 0xb114a088, 0xf2df060f, + 0xb9a3942f, 0xfa6832a8, 0x7e33a8fb, 0x3df80e7c, 0x9552ddad, + 0xd6997b2a, 0x52c2e179, 0x110947fe, 0x5a75d5de, 0x19be7359, + 0x9de5e90a, 0xde2e4f8d, 0x0b1dcd4b, 0x48d66bcc, 0xcc8df19f, + 0x8f465718, 0xc43ac538, 0x87f163bf, 0x03aaf9ec, 0x40615f6b, + 0x12f92f94, 0x51328913, 0xd5691340, 0x96a2b5c7, 0xddde27e7, + 0x9e158160, 0x1a4e1b33, 0x5985bdb4, 0x8cb63f72, 0xcf7d99f5, + 0x4b2603a6, 0x08eda521, 0x43913701, 0x005a9186, 0x84010bd5, + 0xc7caad52, 0x6f607e83, 0x2cabd804, 0xa8f04257, 0xeb3be4d0, + 0xa04776f0, 0xe38cd077, 0x67d74a24, 0x241ceca3, 0xf12f6e65, + 0xb2e4c8e2, 0x36bf52b1, 0x7574f436, 0x3e086616, 0x7dc3c091, + 0xf9985ac2, 0xba53fc45, 0x1caecae7, 0x5f656c60, 0xdb3ef633, + 0x98f550b4, 0xd389c294, 0x90426413, 0x1419fe40, 0x57d258c7, + 0x82e1da01, 0xc12a7c86, 0x4571e6d5, 0x06ba4052, 0x4dc6d272, + 0x0e0d74f5, 0x8a56eea6, 0xc99d4821, 0x61379bf0, 0x22fc3d77, + 0xa6a7a724, 0xe56c01a3, 0xae109383, 0xeddb3504, 0x6980af57, + 0x2a4b09d0, 0xff788b16, 0xbcb32d91, 0x38e8b7c2, 0x7b231145, + 0x305f8365, 0x739425e2, 0xf7cfbfb1, 0xb4041936, 0xe69c69c9, + 0xa557cf4e, 0x210c551d, 0x62c7f39a, 0x29bb61ba, 0x6a70c73d, + 0xee2b5d6e, 0xade0fbe9, 0x78d3792f, 0x3b18dfa8, 0xbf4345fb, + 0xfc88e37c, 0xb7f4715c, 0xf43fd7db, 0x70644d88, 0x33afeb0f, + 0x9b0538de, 0xd8ce9e59, 0x5c95040a, 0x1f5ea28d, 0x542230ad, + 0x17e9962a, 0x93b20c79, 0xd079aafe, 0x054a2838, 0x46818ebf, + 0xc2da14ec, 0x8111b26b, 0xca6d204b, 0x89a686cc, 0x0dfd1c9f, + 0x4e36ba18}, + {0x00000000, 0xe1b652ef, 0x836bd405, 0x62dd86ea, 0x06d7a80b, + 0xe761fae4, 0x85bc7c0e, 0x640a2ee1, 0x0cae5117, 0xed1803f8, + 0x8fc58512, 0x6e73d7fd, 0x0a79f91c, 0xebcfabf3, 0x89122d19, + 0x68a47ff6, 0x185ca32e, 0xf9eaf1c1, 0x9b37772b, 0x7a8125c4, + 0x1e8b0b25, 0xff3d59ca, 0x9de0df20, 0x7c568dcf, 0x14f2f239, + 0xf544a0d6, 0x9799263c, 0x762f74d3, 0x12255a32, 0xf39308dd, + 0x914e8e37, 0x70f8dcd8, 0x30b8465d, 0xd10e14b2, 0xb3d39258, + 0x5265c0b7, 0x366fee56, 0xd7d9bcb9, 0xb5043a53, 0x54b268bc, + 0x3c16174a, 0xdda045a5, 0xbf7dc34f, 0x5ecb91a0, 0x3ac1bf41, + 0xdb77edae, 0xb9aa6b44, 0x581c39ab, 0x28e4e573, 0xc952b79c, + 0xab8f3176, 0x4a396399, 0x2e334d78, 0xcf851f97, 0xad58997d, + 0x4ceecb92, 0x244ab464, 0xc5fce68b, 0xa7216061, 0x4697328e, + 0x229d1c6f, 0xc32b4e80, 0xa1f6c86a, 0x40409a85, 0x60708dba, + 0x81c6df55, 0xe31b59bf, 0x02ad0b50, 0x66a725b1, 0x8711775e, + 0xe5ccf1b4, 0x047aa35b, 0x6cdedcad, 0x8d688e42, 0xefb508a8, + 0x0e035a47, 0x6a0974a6, 0x8bbf2649, 0xe962a0a3, 0x08d4f24c, + 0x782c2e94, 0x999a7c7b, 0xfb47fa91, 0x1af1a87e, 0x7efb869f, + 0x9f4dd470, 0xfd90529a, 0x1c260075, 0x74827f83, 0x95342d6c, + 0xf7e9ab86, 0x165ff969, 0x7255d788, 0x93e38567, 0xf13e038d, + 0x10885162, 0x50c8cbe7, 0xb17e9908, 0xd3a31fe2, 0x32154d0d, + 0x561f63ec, 0xb7a93103, 0xd574b7e9, 0x34c2e506, 0x5c669af0, + 0xbdd0c81f, 0xdf0d4ef5, 0x3ebb1c1a, 0x5ab132fb, 0xbb076014, + 0xd9dae6fe, 0x386cb411, 0x489468c9, 0xa9223a26, 0xcbffbccc, + 0x2a49ee23, 0x4e43c0c2, 0xaff5922d, 0xcd2814c7, 0x2c9e4628, + 0x443a39de, 0xa58c6b31, 0xc751eddb, 0x26e7bf34, 0x42ed91d5, + 0xa35bc33a, 0xc18645d0, 0x2030173f, 0x81e66bae, 0x60503941, + 0x028dbfab, 0xe33bed44, 0x8731c3a5, 0x6687914a, 0x045a17a0, + 0xe5ec454f, 0x8d483ab9, 0x6cfe6856, 0x0e23eebc, 0xef95bc53, + 0x8b9f92b2, 0x6a29c05d, 0x08f446b7, 0xe9421458, 0x99bac880, + 0x780c9a6f, 0x1ad11c85, 0xfb674e6a, 0x9f6d608b, 0x7edb3264, + 0x1c06b48e, 0xfdb0e661, 0x95149997, 0x74a2cb78, 0x167f4d92, + 0xf7c91f7d, 0x93c3319c, 0x72756373, 0x10a8e599, 0xf11eb776, + 0xb15e2df3, 0x50e87f1c, 0x3235f9f6, 0xd383ab19, 0xb78985f8, + 0x563fd717, 0x34e251fd, 0xd5540312, 0xbdf07ce4, 0x5c462e0b, + 0x3e9ba8e1, 0xdf2dfa0e, 0xbb27d4ef, 0x5a918600, 0x384c00ea, + 0xd9fa5205, 0xa9028edd, 0x48b4dc32, 0x2a695ad8, 0xcbdf0837, + 0xafd526d6, 0x4e637439, 0x2cbef2d3, 0xcd08a03c, 0xa5acdfca, + 0x441a8d25, 0x26c70bcf, 0xc7715920, 0xa37b77c1, 0x42cd252e, + 0x2010a3c4, 0xc1a6f12b, 0xe196e614, 0x0020b4fb, 0x62fd3211, + 0x834b60fe, 0xe7414e1f, 0x06f71cf0, 0x642a9a1a, 0x859cc8f5, + 0xed38b703, 0x0c8ee5ec, 0x6e536306, 0x8fe531e9, 0xebef1f08, + 0x0a594de7, 0x6884cb0d, 0x893299e2, 0xf9ca453a, 0x187c17d5, + 0x7aa1913f, 0x9b17c3d0, 0xff1ded31, 0x1eabbfde, 0x7c763934, + 0x9dc06bdb, 0xf564142d, 0x14d246c2, 0x760fc028, 0x97b992c7, + 0xf3b3bc26, 0x1205eec9, 0x70d86823, 0x916e3acc, 0xd12ea049, + 0x3098f2a6, 0x5245744c, 0xb3f326a3, 0xd7f90842, 0x364f5aad, + 0x5492dc47, 0xb5248ea8, 0xdd80f15e, 0x3c36a3b1, 0x5eeb255b, + 0xbf5d77b4, 0xdb575955, 0x3ae10bba, 0x583c8d50, 0xb98adfbf, + 0xc9720367, 0x28c45188, 0x4a19d762, 0xabaf858d, 0xcfa5ab6c, + 0x2e13f983, 0x4cce7f69, 0xad782d86, 0xc5dc5270, 0x246a009f, + 0x46b78675, 0xa701d49a, 0xc30bfa7b, 0x22bda894, 0x40602e7e, + 0xa1d67c91}, + {0x00000000, 0x5880e2d7, 0xf106b474, 0xa98656a3, 0xe20d68e9, + 0xba8d8a3e, 0x130bdc9d, 0x4b8b3e4a, 0x851da109, 0xdd9d43de, + 0x741b157d, 0x2c9bf7aa, 0x6710c9e0, 0x3f902b37, 0x96167d94, + 0xce969f43, 0x0a3b4213, 0x52bba0c4, 0xfb3df667, 0xa3bd14b0, + 0xe8362afa, 0xb0b6c82d, 0x19309e8e, 0x41b07c59, 0x8f26e31a, + 0xd7a601cd, 0x7e20576e, 0x26a0b5b9, 0x6d2b8bf3, 0x35ab6924, + 0x9c2d3f87, 0xc4addd50, 0x14768426, 0x4cf666f1, 0xe5703052, + 0xbdf0d285, 0xf67beccf, 0xaefb0e18, 0x077d58bb, 0x5ffdba6c, + 0x916b252f, 0xc9ebc7f8, 0x606d915b, 0x38ed738c, 0x73664dc6, + 0x2be6af11, 0x8260f9b2, 0xdae01b65, 0x1e4dc635, 0x46cd24e2, + 0xef4b7241, 0xb7cb9096, 0xfc40aedc, 0xa4c04c0b, 0x0d461aa8, + 0x55c6f87f, 0x9b50673c, 0xc3d085eb, 0x6a56d348, 0x32d6319f, + 0x795d0fd5, 0x21dded02, 0x885bbba1, 0xd0db5976, 0x28ec084d, + 0x706cea9a, 0xd9eabc39, 0x816a5eee, 0xcae160a4, 0x92618273, + 0x3be7d4d0, 0x63673607, 0xadf1a944, 0xf5714b93, 0x5cf71d30, + 0x0477ffe7, 0x4ffcc1ad, 0x177c237a, 0xbefa75d9, 0xe67a970e, + 0x22d74a5e, 0x7a57a889, 0xd3d1fe2a, 0x8b511cfd, 0xc0da22b7, + 0x985ac060, 0x31dc96c3, 0x695c7414, 0xa7caeb57, 0xff4a0980, + 0x56cc5f23, 0x0e4cbdf4, 0x45c783be, 0x1d476169, 0xb4c137ca, + 0xec41d51d, 0x3c9a8c6b, 0x641a6ebc, 0xcd9c381f, 0x951cdac8, + 0xde97e482, 0x86170655, 0x2f9150f6, 0x7711b221, 0xb9872d62, + 0xe107cfb5, 0x48819916, 0x10017bc1, 0x5b8a458b, 0x030aa75c, + 0xaa8cf1ff, 0xf20c1328, 0x36a1ce78, 0x6e212caf, 0xc7a77a0c, + 0x9f2798db, 0xd4aca691, 0x8c2c4446, 0x25aa12e5, 0x7d2af032, + 0xb3bc6f71, 0xeb3c8da6, 0x42badb05, 0x1a3a39d2, 0x51b10798, + 0x0931e54f, 0xa0b7b3ec, 0xf837513b, 0x50d8119a, 0x0858f34d, + 0xa1dea5ee, 0xf95e4739, 0xb2d57973, 0xea559ba4, 0x43d3cd07, + 0x1b532fd0, 0xd5c5b093, 0x8d455244, 0x24c304e7, 0x7c43e630, + 0x37c8d87a, 0x6f483aad, 0xc6ce6c0e, 0x9e4e8ed9, 0x5ae35389, + 0x0263b15e, 0xabe5e7fd, 0xf365052a, 0xb8ee3b60, 0xe06ed9b7, + 0x49e88f14, 0x11686dc3, 0xdffef280, 0x877e1057, 0x2ef846f4, + 0x7678a423, 0x3df39a69, 0x657378be, 0xccf52e1d, 0x9475ccca, + 0x44ae95bc, 0x1c2e776b, 0xb5a821c8, 0xed28c31f, 0xa6a3fd55, + 0xfe231f82, 0x57a54921, 0x0f25abf6, 0xc1b334b5, 0x9933d662, + 0x30b580c1, 0x68356216, 0x23be5c5c, 0x7b3ebe8b, 0xd2b8e828, + 0x8a380aff, 0x4e95d7af, 0x16153578, 0xbf9363db, 0xe713810c, + 0xac98bf46, 0xf4185d91, 0x5d9e0b32, 0x051ee9e5, 0xcb8876a6, + 0x93089471, 0x3a8ec2d2, 0x620e2005, 0x29851e4f, 0x7105fc98, + 0xd883aa3b, 0x800348ec, 0x783419d7, 0x20b4fb00, 0x8932ada3, + 0xd1b24f74, 0x9a39713e, 0xc2b993e9, 0x6b3fc54a, 0x33bf279d, + 0xfd29b8de, 0xa5a95a09, 0x0c2f0caa, 0x54afee7d, 0x1f24d037, + 0x47a432e0, 0xee226443, 0xb6a28694, 0x720f5bc4, 0x2a8fb913, + 0x8309efb0, 0xdb890d67, 0x9002332d, 0xc882d1fa, 0x61048759, + 0x3984658e, 0xf712facd, 0xaf92181a, 0x06144eb9, 0x5e94ac6e, + 0x151f9224, 0x4d9f70f3, 0xe4192650, 0xbc99c487, 0x6c429df1, + 0x34c27f26, 0x9d442985, 0xc5c4cb52, 0x8e4ff518, 0xd6cf17cf, + 0x7f49416c, 0x27c9a3bb, 0xe95f3cf8, 0xb1dfde2f, 0x1859888c, + 0x40d96a5b, 0x0b525411, 0x53d2b6c6, 0xfa54e065, 0xa2d402b2, + 0x6679dfe2, 0x3ef93d35, 0x977f6b96, 0xcfff8941, 0x8474b70b, + 0xdcf455dc, 0x7572037f, 0x2df2e1a8, 0xe3647eeb, 0xbbe49c3c, + 0x1262ca9f, 0x4ae22848, 0x01691602, 0x59e9f4d5, 0xf06fa276, + 0xa8ef40a1}, + {0x00000000, 0x463b6765, 0x8c76ceca, 0xca4da9af, 0x59ebed4e, + 0x1fd08a2b, 0xd59d2384, 0x93a644e1, 0xb2d6db9d, 0xf4edbcf8, + 0x3ea01557, 0x789b7232, 0xeb3d36d3, 0xad0651b6, 0x674bf819, + 0x21709f7c, 0x25abc6e0, 0x6390a185, 0xa9dd082a, 0xefe66f4f, + 0x7c402bae, 0x3a7b4ccb, 0xf036e564, 0xb60d8201, 0x977d1d7d, + 0xd1467a18, 0x1b0bd3b7, 0x5d30b4d2, 0xce96f033, 0x88ad9756, + 0x42e03ef9, 0x04db599c, 0x0b50fc1a, 0x4d6b9b7f, 0x872632d0, + 0xc11d55b5, 0x52bb1154, 0x14807631, 0xdecddf9e, 0x98f6b8fb, + 0xb9862787, 0xffbd40e2, 0x35f0e94d, 0x73cb8e28, 0xe06dcac9, + 0xa656adac, 0x6c1b0403, 0x2a206366, 0x2efb3afa, 0x68c05d9f, + 0xa28df430, 0xe4b69355, 0x7710d7b4, 0x312bb0d1, 0xfb66197e, + 0xbd5d7e1b, 0x9c2de167, 0xda168602, 0x105b2fad, 0x566048c8, + 0xc5c60c29, 0x83fd6b4c, 0x49b0c2e3, 0x0f8ba586, 0x16a0f835, + 0x509b9f50, 0x9ad636ff, 0xdced519a, 0x4f4b157b, 0x0970721e, + 0xc33ddbb1, 0x8506bcd4, 0xa47623a8, 0xe24d44cd, 0x2800ed62, + 0x6e3b8a07, 0xfd9dcee6, 0xbba6a983, 0x71eb002c, 0x37d06749, + 0x330b3ed5, 0x753059b0, 0xbf7df01f, 0xf946977a, 0x6ae0d39b, + 0x2cdbb4fe, 0xe6961d51, 0xa0ad7a34, 0x81dde548, 0xc7e6822d, + 0x0dab2b82, 0x4b904ce7, 0xd8360806, 0x9e0d6f63, 0x5440c6cc, + 0x127ba1a9, 0x1df0042f, 0x5bcb634a, 0x9186cae5, 0xd7bdad80, + 0x441be961, 0x02208e04, 0xc86d27ab, 0x8e5640ce, 0xaf26dfb2, + 0xe91db8d7, 0x23501178, 0x656b761d, 0xf6cd32fc, 0xb0f65599, + 0x7abbfc36, 0x3c809b53, 0x385bc2cf, 0x7e60a5aa, 0xb42d0c05, + 0xf2166b60, 0x61b02f81, 0x278b48e4, 0xedc6e14b, 0xabfd862e, + 0x8a8d1952, 0xccb67e37, 0x06fbd798, 0x40c0b0fd, 0xd366f41c, + 0x955d9379, 0x5f103ad6, 0x192b5db3, 0x2c40f16b, 0x6a7b960e, + 0xa0363fa1, 0xe60d58c4, 0x75ab1c25, 0x33907b40, 0xf9ddd2ef, + 0xbfe6b58a, 0x9e962af6, 0xd8ad4d93, 0x12e0e43c, 0x54db8359, + 0xc77dc7b8, 0x8146a0dd, 0x4b0b0972, 0x0d306e17, 0x09eb378b, + 0x4fd050ee, 0x859df941, 0xc3a69e24, 0x5000dac5, 0x163bbda0, + 0xdc76140f, 0x9a4d736a, 0xbb3dec16, 0xfd068b73, 0x374b22dc, + 0x717045b9, 0xe2d60158, 0xa4ed663d, 0x6ea0cf92, 0x289ba8f7, + 0x27100d71, 0x612b6a14, 0xab66c3bb, 0xed5da4de, 0x7efbe03f, + 0x38c0875a, 0xf28d2ef5, 0xb4b64990, 0x95c6d6ec, 0xd3fdb189, + 0x19b01826, 0x5f8b7f43, 0xcc2d3ba2, 0x8a165cc7, 0x405bf568, + 0x0660920d, 0x02bbcb91, 0x4480acf4, 0x8ecd055b, 0xc8f6623e, + 0x5b5026df, 0x1d6b41ba, 0xd726e815, 0x911d8f70, 0xb06d100c, + 0xf6567769, 0x3c1bdec6, 0x7a20b9a3, 0xe986fd42, 0xafbd9a27, + 0x65f03388, 0x23cb54ed, 0x3ae0095e, 0x7cdb6e3b, 0xb696c794, + 0xf0ada0f1, 0x630be410, 0x25308375, 0xef7d2ada, 0xa9464dbf, + 0x8836d2c3, 0xce0db5a6, 0x04401c09, 0x427b7b6c, 0xd1dd3f8d, + 0x97e658e8, 0x5dabf147, 0x1b909622, 0x1f4bcfbe, 0x5970a8db, + 0x933d0174, 0xd5066611, 0x46a022f0, 0x009b4595, 0xcad6ec3a, + 0x8ced8b5f, 0xad9d1423, 0xeba67346, 0x21ebdae9, 0x67d0bd8c, + 0xf476f96d, 0xb24d9e08, 0x780037a7, 0x3e3b50c2, 0x31b0f544, + 0x778b9221, 0xbdc63b8e, 0xfbfd5ceb, 0x685b180a, 0x2e607f6f, + 0xe42dd6c0, 0xa216b1a5, 0x83662ed9, 0xc55d49bc, 0x0f10e013, + 0x492b8776, 0xda8dc397, 0x9cb6a4f2, 0x56fb0d5d, 0x10c06a38, + 0x141b33a4, 0x522054c1, 0x986dfd6e, 0xde569a0b, 0x4df0deea, + 0x0bcbb98f, 0xc1861020, 0x87bd7745, 0xa6cde839, 0xe0f68f5c, + 0x2abb26f3, 0x6c804196, 0xff260577, 0xb91d6212, 0x7350cbbd, + 0x356bacd8}}; + +#endif + +#endif + +#if N == 6 + +#if W == 8 + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0x3db1ecdc, 0x7b63d9b8, 0x46d23564, 0xf6c7b370, + 0xcb765fac, 0x8da46ac8, 0xb0158614, 0x36fe60a1, 0x0b4f8c7d, + 0x4d9db919, 0x702c55c5, 0xc039d3d1, 0xfd883f0d, 0xbb5a0a69, + 0x86ebe6b5, 0x6dfcc142, 0x504d2d9e, 0x169f18fa, 0x2b2ef426, + 0x9b3b7232, 0xa68a9eee, 0xe058ab8a, 0xdde94756, 0x5b02a1e3, + 0x66b34d3f, 0x2061785b, 0x1dd09487, 0xadc51293, 0x9074fe4f, + 0xd6a6cb2b, 0xeb1727f7, 0xdbf98284, 0xe6486e58, 0xa09a5b3c, + 0x9d2bb7e0, 0x2d3e31f4, 0x108fdd28, 0x565de84c, 0x6bec0490, + 0xed07e225, 0xd0b60ef9, 0x96643b9d, 0xabd5d741, 0x1bc05155, + 0x2671bd89, 0x60a388ed, 0x5d126431, 0xb60543c6, 0x8bb4af1a, + 0xcd669a7e, 0xf0d776a2, 0x40c2f0b6, 0x7d731c6a, 0x3ba1290e, + 0x0610c5d2, 0x80fb2367, 0xbd4acfbb, 0xfb98fadf, 0xc6291603, + 0x763c9017, 0x4b8d7ccb, 0x0d5f49af, 0x30eea573, 0x6c820349, + 0x5133ef95, 0x17e1daf1, 0x2a50362d, 0x9a45b039, 0xa7f45ce5, + 0xe1266981, 0xdc97855d, 0x5a7c63e8, 0x67cd8f34, 0x211fba50, + 0x1cae568c, 0xacbbd098, 0x910a3c44, 0xd7d80920, 0xea69e5fc, + 0x017ec20b, 0x3ccf2ed7, 0x7a1d1bb3, 0x47acf76f, 0xf7b9717b, + 0xca089da7, 0x8cdaa8c3, 0xb16b441f, 0x3780a2aa, 0x0a314e76, + 0x4ce37b12, 0x715297ce, 0xc14711da, 0xfcf6fd06, 0xba24c862, + 0x879524be, 0xb77b81cd, 0x8aca6d11, 0xcc185875, 0xf1a9b4a9, + 0x41bc32bd, 0x7c0dde61, 0x3adfeb05, 0x076e07d9, 0x8185e16c, + 0xbc340db0, 0xfae638d4, 0xc757d408, 0x7742521c, 0x4af3bec0, + 0x0c218ba4, 0x31906778, 0xda87408f, 0xe736ac53, 0xa1e49937, + 0x9c5575eb, 0x2c40f3ff, 0x11f11f23, 0x57232a47, 0x6a92c69b, + 0xec79202e, 0xd1c8ccf2, 0x971af996, 0xaaab154a, 0x1abe935e, + 0x270f7f82, 0x61dd4ae6, 0x5c6ca63a, 0xd9040692, 0xe4b5ea4e, + 0xa267df2a, 0x9fd633f6, 0x2fc3b5e2, 0x1272593e, 0x54a06c5a, + 0x69118086, 0xeffa6633, 0xd24b8aef, 0x9499bf8b, 0xa9285357, + 0x193dd543, 0x248c399f, 0x625e0cfb, 0x5fefe027, 0xb4f8c7d0, + 0x89492b0c, 0xcf9b1e68, 0xf22af2b4, 0x423f74a0, 0x7f8e987c, + 0x395cad18, 0x04ed41c4, 0x8206a771, 0xbfb74bad, 0xf9657ec9, + 0xc4d49215, 0x74c11401, 0x4970f8dd, 0x0fa2cdb9, 0x32132165, + 0x02fd8416, 0x3f4c68ca, 0x799e5dae, 0x442fb172, 0xf43a3766, + 0xc98bdbba, 0x8f59eede, 0xb2e80202, 0x3403e4b7, 0x09b2086b, + 0x4f603d0f, 0x72d1d1d3, 0xc2c457c7, 0xff75bb1b, 0xb9a78e7f, + 0x841662a3, 0x6f014554, 0x52b0a988, 0x14629cec, 0x29d37030, + 0x99c6f624, 0xa4771af8, 0xe2a52f9c, 0xdf14c340, 0x59ff25f5, + 0x644ec929, 0x229cfc4d, 0x1f2d1091, 0xaf389685, 0x92897a59, + 0xd45b4f3d, 0xe9eaa3e1, 0xb58605db, 0x8837e907, 0xcee5dc63, + 0xf35430bf, 0x4341b6ab, 0x7ef05a77, 0x38226f13, 0x059383cf, + 0x8378657a, 0xbec989a6, 0xf81bbcc2, 0xc5aa501e, 0x75bfd60a, + 0x480e3ad6, 0x0edc0fb2, 0x336de36e, 0xd87ac499, 0xe5cb2845, + 0xa3191d21, 0x9ea8f1fd, 0x2ebd77e9, 0x130c9b35, 0x55deae51, + 0x686f428d, 0xee84a438, 0xd33548e4, 0x95e77d80, 0xa856915c, + 0x18431748, 0x25f2fb94, 0x6320cef0, 0x5e91222c, 0x6e7f875f, + 0x53ce6b83, 0x151c5ee7, 0x28adb23b, 0x98b8342f, 0xa509d8f3, + 0xe3dbed97, 0xde6a014b, 0x5881e7fe, 0x65300b22, 0x23e23e46, + 0x1e53d29a, 0xae46548e, 0x93f7b852, 0xd5258d36, 0xe89461ea, + 0x0383461d, 0x3e32aac1, 0x78e09fa5, 0x45517379, 0xf544f56d, + 0xc8f519b1, 0x8e272cd5, 0xb396c009, 0x357d26bc, 0x08ccca60, + 0x4e1eff04, 0x73af13d8, 0xc3ba95cc, 0xfe0b7910, 0xb8d94c74, + 0x8568a0a8}, + {0x00000000, 0x69790b65, 0xd2f216ca, 0xbb8b1daf, 0x7e952bd5, + 0x17ec20b0, 0xac673d1f, 0xc51e367a, 0xfd2a57aa, 0x94535ccf, + 0x2fd84160, 0x46a14a05, 0x83bf7c7f, 0xeac6771a, 0x514d6ab5, + 0x383461d0, 0x2125a915, 0x485ca270, 0xf3d7bfdf, 0x9aaeb4ba, + 0x5fb082c0, 0x36c989a5, 0x8d42940a, 0xe43b9f6f, 0xdc0ffebf, + 0xb576f5da, 0x0efde875, 0x6784e310, 0xa29ad56a, 0xcbe3de0f, + 0x7068c3a0, 0x1911c8c5, 0x424b522a, 0x2b32594f, 0x90b944e0, + 0xf9c04f85, 0x3cde79ff, 0x55a7729a, 0xee2c6f35, 0x87556450, + 0xbf610580, 0xd6180ee5, 0x6d93134a, 0x04ea182f, 0xc1f42e55, + 0xa88d2530, 0x1306389f, 0x7a7f33fa, 0x636efb3f, 0x0a17f05a, + 0xb19cedf5, 0xd8e5e690, 0x1dfbd0ea, 0x7482db8f, 0xcf09c620, + 0xa670cd45, 0x9e44ac95, 0xf73da7f0, 0x4cb6ba5f, 0x25cfb13a, + 0xe0d18740, 0x89a88c25, 0x3223918a, 0x5b5a9aef, 0x8496a454, + 0xedefaf31, 0x5664b29e, 0x3f1db9fb, 0xfa038f81, 0x937a84e4, + 0x28f1994b, 0x4188922e, 0x79bcf3fe, 0x10c5f89b, 0xab4ee534, + 0xc237ee51, 0x0729d82b, 0x6e50d34e, 0xd5dbcee1, 0xbca2c584, + 0xa5b30d41, 0xccca0624, 0x77411b8b, 0x1e3810ee, 0xdb262694, + 0xb25f2df1, 0x09d4305e, 0x60ad3b3b, 0x58995aeb, 0x31e0518e, + 0x8a6b4c21, 0xe3124744, 0x260c713e, 0x4f757a5b, 0xf4fe67f4, + 0x9d876c91, 0xc6ddf67e, 0xafa4fd1b, 0x142fe0b4, 0x7d56ebd1, + 0xb848ddab, 0xd131d6ce, 0x6abacb61, 0x03c3c004, 0x3bf7a1d4, + 0x528eaab1, 0xe905b71e, 0x807cbc7b, 0x45628a01, 0x2c1b8164, + 0x97909ccb, 0xfee997ae, 0xe7f85f6b, 0x8e81540e, 0x350a49a1, + 0x5c7342c4, 0x996d74be, 0xf0147fdb, 0x4b9f6274, 0x22e66911, + 0x1ad208c1, 0x73ab03a4, 0xc8201e0b, 0xa159156e, 0x64472314, + 0x0d3e2871, 0xb6b535de, 0xdfcc3ebb, 0xd25c4ee9, 0xbb25458c, + 0x00ae5823, 0x69d75346, 0xacc9653c, 0xc5b06e59, 0x7e3b73f6, + 0x17427893, 0x2f761943, 0x460f1226, 0xfd840f89, 0x94fd04ec, + 0x51e33296, 0x389a39f3, 0x8311245c, 0xea682f39, 0xf379e7fc, + 0x9a00ec99, 0x218bf136, 0x48f2fa53, 0x8deccc29, 0xe495c74c, + 0x5f1edae3, 0x3667d186, 0x0e53b056, 0x672abb33, 0xdca1a69c, + 0xb5d8adf9, 0x70c69b83, 0x19bf90e6, 0xa2348d49, 0xcb4d862c, + 0x90171cc3, 0xf96e17a6, 0x42e50a09, 0x2b9c016c, 0xee823716, + 0x87fb3c73, 0x3c7021dc, 0x55092ab9, 0x6d3d4b69, 0x0444400c, + 0xbfcf5da3, 0xd6b656c6, 0x13a860bc, 0x7ad16bd9, 0xc15a7676, + 0xa8237d13, 0xb132b5d6, 0xd84bbeb3, 0x63c0a31c, 0x0ab9a879, + 0xcfa79e03, 0xa6de9566, 0x1d5588c9, 0x742c83ac, 0x4c18e27c, + 0x2561e919, 0x9eeaf4b6, 0xf793ffd3, 0x328dc9a9, 0x5bf4c2cc, + 0xe07fdf63, 0x8906d406, 0x56caeabd, 0x3fb3e1d8, 0x8438fc77, + 0xed41f712, 0x285fc168, 0x4126ca0d, 0xfaadd7a2, 0x93d4dcc7, + 0xabe0bd17, 0xc299b672, 0x7912abdd, 0x106ba0b8, 0xd57596c2, + 0xbc0c9da7, 0x07878008, 0x6efe8b6d, 0x77ef43a8, 0x1e9648cd, + 0xa51d5562, 0xcc645e07, 0x097a687d, 0x60036318, 0xdb887eb7, + 0xb2f175d2, 0x8ac51402, 0xe3bc1f67, 0x583702c8, 0x314e09ad, + 0xf4503fd7, 0x9d2934b2, 0x26a2291d, 0x4fdb2278, 0x1481b897, + 0x7df8b3f2, 0xc673ae5d, 0xaf0aa538, 0x6a149342, 0x036d9827, + 0xb8e68588, 0xd19f8eed, 0xe9abef3d, 0x80d2e458, 0x3b59f9f7, + 0x5220f292, 0x973ec4e8, 0xfe47cf8d, 0x45ccd222, 0x2cb5d947, + 0x35a41182, 0x5cdd1ae7, 0xe7560748, 0x8e2f0c2d, 0x4b313a57, + 0x22483132, 0x99c32c9d, 0xf0ba27f8, 0xc88e4628, 0xa1f74d4d, + 0x1a7c50e2, 0x73055b87, 0xb61b6dfd, 0xdf626698, 0x64e97b37, + 0x0d907052}, + {0x00000000, 0x7fc99b93, 0xff933726, 0x805aacb5, 0x2457680d, + 0x5b9ef39e, 0xdbc45f2b, 0xa40dc4b8, 0x48aed01a, 0x37674b89, + 0xb73de73c, 0xc8f47caf, 0x6cf9b817, 0x13302384, 0x936a8f31, + 0xeca314a2, 0x915da034, 0xee943ba7, 0x6ece9712, 0x11070c81, + 0xb50ac839, 0xcac353aa, 0x4a99ff1f, 0x3550648c, 0xd9f3702e, + 0xa63aebbd, 0x26604708, 0x59a9dc9b, 0xfda41823, 0x826d83b0, + 0x02372f05, 0x7dfeb496, 0xf9ca4629, 0x8603ddba, 0x0659710f, + 0x7990ea9c, 0xdd9d2e24, 0xa254b5b7, 0x220e1902, 0x5dc78291, + 0xb1649633, 0xcead0da0, 0x4ef7a115, 0x313e3a86, 0x9533fe3e, + 0xeafa65ad, 0x6aa0c918, 0x1569528b, 0x6897e61d, 0x175e7d8e, + 0x9704d13b, 0xe8cd4aa8, 0x4cc08e10, 0x33091583, 0xb353b936, + 0xcc9a22a5, 0x20393607, 0x5ff0ad94, 0xdfaa0121, 0xa0639ab2, + 0x046e5e0a, 0x7ba7c599, 0xfbfd692c, 0x8434f2bf, 0x28e58a13, + 0x572c1180, 0xd776bd35, 0xa8bf26a6, 0x0cb2e21e, 0x737b798d, + 0xf321d538, 0x8ce84eab, 0x604b5a09, 0x1f82c19a, 0x9fd86d2f, + 0xe011f6bc, 0x441c3204, 0x3bd5a997, 0xbb8f0522, 0xc4469eb1, + 0xb9b82a27, 0xc671b1b4, 0x462b1d01, 0x39e28692, 0x9def422a, + 0xe226d9b9, 0x627c750c, 0x1db5ee9f, 0xf116fa3d, 0x8edf61ae, + 0x0e85cd1b, 0x714c5688, 0xd5419230, 0xaa8809a3, 0x2ad2a516, + 0x551b3e85, 0xd12fcc3a, 0xaee657a9, 0x2ebcfb1c, 0x5175608f, + 0xf578a437, 0x8ab13fa4, 0x0aeb9311, 0x75220882, 0x99811c20, + 0xe64887b3, 0x66122b06, 0x19dbb095, 0xbdd6742d, 0xc21fefbe, + 0x4245430b, 0x3d8cd898, 0x40726c0e, 0x3fbbf79d, 0xbfe15b28, + 0xc028c0bb, 0x64250403, 0x1bec9f90, 0x9bb63325, 0xe47fa8b6, + 0x08dcbc14, 0x77152787, 0xf74f8b32, 0x888610a1, 0x2c8bd419, + 0x53424f8a, 0xd318e33f, 0xacd178ac, 0x51cb1426, 0x2e028fb5, + 0xae582300, 0xd191b893, 0x759c7c2b, 0x0a55e7b8, 0x8a0f4b0d, + 0xf5c6d09e, 0x1965c43c, 0x66ac5faf, 0xe6f6f31a, 0x993f6889, + 0x3d32ac31, 0x42fb37a2, 0xc2a19b17, 0xbd680084, 0xc096b412, + 0xbf5f2f81, 0x3f058334, 0x40cc18a7, 0xe4c1dc1f, 0x9b08478c, + 0x1b52eb39, 0x649b70aa, 0x88386408, 0xf7f1ff9b, 0x77ab532e, + 0x0862c8bd, 0xac6f0c05, 0xd3a69796, 0x53fc3b23, 0x2c35a0b0, + 0xa801520f, 0xd7c8c99c, 0x57926529, 0x285bfeba, 0x8c563a02, + 0xf39fa191, 0x73c50d24, 0x0c0c96b7, 0xe0af8215, 0x9f661986, + 0x1f3cb533, 0x60f52ea0, 0xc4f8ea18, 0xbb31718b, 0x3b6bdd3e, + 0x44a246ad, 0x395cf23b, 0x469569a8, 0xc6cfc51d, 0xb9065e8e, + 0x1d0b9a36, 0x62c201a5, 0xe298ad10, 0x9d513683, 0x71f22221, + 0x0e3bb9b2, 0x8e611507, 0xf1a88e94, 0x55a54a2c, 0x2a6cd1bf, + 0xaa367d0a, 0xd5ffe699, 0x792e9e35, 0x06e705a6, 0x86bda913, + 0xf9743280, 0x5d79f638, 0x22b06dab, 0xa2eac11e, 0xdd235a8d, + 0x31804e2f, 0x4e49d5bc, 0xce137909, 0xb1dae29a, 0x15d72622, + 0x6a1ebdb1, 0xea441104, 0x958d8a97, 0xe8733e01, 0x97baa592, + 0x17e00927, 0x682992b4, 0xcc24560c, 0xb3edcd9f, 0x33b7612a, + 0x4c7efab9, 0xa0ddee1b, 0xdf147588, 0x5f4ed93d, 0x208742ae, + 0x848a8616, 0xfb431d85, 0x7b19b130, 0x04d02aa3, 0x80e4d81c, + 0xff2d438f, 0x7f77ef3a, 0x00be74a9, 0xa4b3b011, 0xdb7a2b82, + 0x5b208737, 0x24e91ca4, 0xc84a0806, 0xb7839395, 0x37d93f20, + 0x4810a4b3, 0xec1d600b, 0x93d4fb98, 0x138e572d, 0x6c47ccbe, + 0x11b97828, 0x6e70e3bb, 0xee2a4f0e, 0x91e3d49d, 0x35ee1025, + 0x4a278bb6, 0xca7d2703, 0xb5b4bc90, 0x5917a832, 0x26de33a1, + 0xa6849f14, 0xd94d0487, 0x7d40c03f, 0x02895bac, 0x82d3f719, + 0xfd1a6c8a}, + {0x00000000, 0xa396284c, 0x9c5d56d9, 0x3fcb7e95, 0xe3cbabf3, + 0x405d83bf, 0x7f96fd2a, 0xdc00d566, 0x1ce651a7, 0xbf7079eb, + 0x80bb077e, 0x232d2f32, 0xff2dfa54, 0x5cbbd218, 0x6370ac8d, + 0xc0e684c1, 0x39cca34e, 0x9a5a8b02, 0xa591f597, 0x0607dddb, + 0xda0708bd, 0x799120f1, 0x465a5e64, 0xe5cc7628, 0x252af2e9, + 0x86bcdaa5, 0xb977a430, 0x1ae18c7c, 0xc6e1591a, 0x65777156, + 0x5abc0fc3, 0xf92a278f, 0x7399469c, 0xd00f6ed0, 0xefc41045, + 0x4c523809, 0x9052ed6f, 0x33c4c523, 0x0c0fbbb6, 0xaf9993fa, + 0x6f7f173b, 0xcce93f77, 0xf32241e2, 0x50b469ae, 0x8cb4bcc8, + 0x2f229484, 0x10e9ea11, 0xb37fc25d, 0x4a55e5d2, 0xe9c3cd9e, + 0xd608b30b, 0x759e9b47, 0xa99e4e21, 0x0a08666d, 0x35c318f8, + 0x965530b4, 0x56b3b475, 0xf5259c39, 0xcaeee2ac, 0x6978cae0, + 0xb5781f86, 0x16ee37ca, 0x2925495f, 0x8ab36113, 0xe7328d38, + 0x44a4a574, 0x7b6fdbe1, 0xd8f9f3ad, 0x04f926cb, 0xa76f0e87, + 0x98a47012, 0x3b32585e, 0xfbd4dc9f, 0x5842f4d3, 0x67898a46, + 0xc41fa20a, 0x181f776c, 0xbb895f20, 0x844221b5, 0x27d409f9, + 0xdefe2e76, 0x7d68063a, 0x42a378af, 0xe13550e3, 0x3d358585, + 0x9ea3adc9, 0xa168d35c, 0x02fefb10, 0xc2187fd1, 0x618e579d, + 0x5e452908, 0xfdd30144, 0x21d3d422, 0x8245fc6e, 0xbd8e82fb, + 0x1e18aab7, 0x94abcba4, 0x373de3e8, 0x08f69d7d, 0xab60b531, + 0x77606057, 0xd4f6481b, 0xeb3d368e, 0x48ab1ec2, 0x884d9a03, + 0x2bdbb24f, 0x1410ccda, 0xb786e496, 0x6b8631f0, 0xc81019bc, + 0xf7db6729, 0x544d4f65, 0xad6768ea, 0x0ef140a6, 0x313a3e33, + 0x92ac167f, 0x4eacc319, 0xed3aeb55, 0xd2f195c0, 0x7167bd8c, + 0xb181394d, 0x12171101, 0x2ddc6f94, 0x8e4a47d8, 0x524a92be, + 0xf1dcbaf2, 0xce17c467, 0x6d81ec2b, 0x15141c31, 0xb682347d, + 0x89494ae8, 0x2adf62a4, 0xf6dfb7c2, 0x55499f8e, 0x6a82e11b, + 0xc914c957, 0x09f24d96, 0xaa6465da, 0x95af1b4f, 0x36393303, + 0xea39e665, 0x49afce29, 0x7664b0bc, 0xd5f298f0, 0x2cd8bf7f, + 0x8f4e9733, 0xb085e9a6, 0x1313c1ea, 0xcf13148c, 0x6c853cc0, + 0x534e4255, 0xf0d86a19, 0x303eeed8, 0x93a8c694, 0xac63b801, + 0x0ff5904d, 0xd3f5452b, 0x70636d67, 0x4fa813f2, 0xec3e3bbe, + 0x668d5aad, 0xc51b72e1, 0xfad00c74, 0x59462438, 0x8546f15e, + 0x26d0d912, 0x191ba787, 0xba8d8fcb, 0x7a6b0b0a, 0xd9fd2346, + 0xe6365dd3, 0x45a0759f, 0x99a0a0f9, 0x3a3688b5, 0x05fdf620, + 0xa66bde6c, 0x5f41f9e3, 0xfcd7d1af, 0xc31caf3a, 0x608a8776, + 0xbc8a5210, 0x1f1c7a5c, 0x20d704c9, 0x83412c85, 0x43a7a844, + 0xe0318008, 0xdffafe9d, 0x7c6cd6d1, 0xa06c03b7, 0x03fa2bfb, + 0x3c31556e, 0x9fa77d22, 0xf2269109, 0x51b0b945, 0x6e7bc7d0, + 0xcdedef9c, 0x11ed3afa, 0xb27b12b6, 0x8db06c23, 0x2e26446f, + 0xeec0c0ae, 0x4d56e8e2, 0x729d9677, 0xd10bbe3b, 0x0d0b6b5d, + 0xae9d4311, 0x91563d84, 0x32c015c8, 0xcbea3247, 0x687c1a0b, + 0x57b7649e, 0xf4214cd2, 0x282199b4, 0x8bb7b1f8, 0xb47ccf6d, + 0x17eae721, 0xd70c63e0, 0x749a4bac, 0x4b513539, 0xe8c71d75, + 0x34c7c813, 0x9751e05f, 0xa89a9eca, 0x0b0cb686, 0x81bfd795, + 0x2229ffd9, 0x1de2814c, 0xbe74a900, 0x62747c66, 0xc1e2542a, + 0xfe292abf, 0x5dbf02f3, 0x9d598632, 0x3ecfae7e, 0x0104d0eb, + 0xa292f8a7, 0x7e922dc1, 0xdd04058d, 0xe2cf7b18, 0x41595354, + 0xb87374db, 0x1be55c97, 0x242e2202, 0x87b80a4e, 0x5bb8df28, + 0xf82ef764, 0xc7e589f1, 0x6473a1bd, 0xa495257c, 0x07030d30, + 0x38c873a5, 0x9b5e5be9, 0x475e8e8f, 0xe4c8a6c3, 0xdb03d856, + 0x7895f01a}, + {0x00000000, 0x2a283862, 0x545070c4, 0x7e7848a6, 0xa8a0e188, + 0x8288d9ea, 0xfcf0914c, 0xd6d8a92e, 0x8a30c551, 0xa018fd33, + 0xde60b595, 0xf4488df7, 0x229024d9, 0x08b81cbb, 0x76c0541d, + 0x5ce86c7f, 0xcf108ce3, 0xe538b481, 0x9b40fc27, 0xb168c445, + 0x67b06d6b, 0x4d985509, 0x33e01daf, 0x19c825cd, 0x452049b2, + 0x6f0871d0, 0x11703976, 0x3b580114, 0xed80a83a, 0xc7a89058, + 0xb9d0d8fe, 0x93f8e09c, 0x45501f87, 0x6f7827e5, 0x11006f43, + 0x3b285721, 0xedf0fe0f, 0xc7d8c66d, 0xb9a08ecb, 0x9388b6a9, + 0xcf60dad6, 0xe548e2b4, 0x9b30aa12, 0xb1189270, 0x67c03b5e, + 0x4de8033c, 0x33904b9a, 0x19b873f8, 0x8a409364, 0xa068ab06, + 0xde10e3a0, 0xf438dbc2, 0x22e072ec, 0x08c84a8e, 0x76b00228, + 0x5c983a4a, 0x00705635, 0x2a586e57, 0x542026f1, 0x7e081e93, + 0xa8d0b7bd, 0x82f88fdf, 0xfc80c779, 0xd6a8ff1b, 0x8aa03f0e, + 0xa088076c, 0xdef04fca, 0xf4d877a8, 0x2200de86, 0x0828e6e4, + 0x7650ae42, 0x5c789620, 0x0090fa5f, 0x2ab8c23d, 0x54c08a9b, + 0x7ee8b2f9, 0xa8301bd7, 0x821823b5, 0xfc606b13, 0xd6485371, + 0x45b0b3ed, 0x6f988b8f, 0x11e0c329, 0x3bc8fb4b, 0xed105265, + 0xc7386a07, 0xb94022a1, 0x93681ac3, 0xcf8076bc, 0xe5a84ede, + 0x9bd00678, 0xb1f83e1a, 0x67209734, 0x4d08af56, 0x3370e7f0, + 0x1958df92, 0xcff02089, 0xe5d818eb, 0x9ba0504d, 0xb188682f, + 0x6750c101, 0x4d78f963, 0x3300b1c5, 0x192889a7, 0x45c0e5d8, + 0x6fe8ddba, 0x1190951c, 0x3bb8ad7e, 0xed600450, 0xc7483c32, + 0xb9307494, 0x93184cf6, 0x00e0ac6a, 0x2ac89408, 0x54b0dcae, + 0x7e98e4cc, 0xa8404de2, 0x82687580, 0xfc103d26, 0xd6380544, + 0x8ad0693b, 0xa0f85159, 0xde8019ff, 0xf4a8219d, 0x227088b3, + 0x0858b0d1, 0x7620f877, 0x5c08c015, 0xce31785d, 0xe419403f, + 0x9a610899, 0xb04930fb, 0x669199d5, 0x4cb9a1b7, 0x32c1e911, + 0x18e9d173, 0x4401bd0c, 0x6e29856e, 0x1051cdc8, 0x3a79f5aa, + 0xeca15c84, 0xc68964e6, 0xb8f12c40, 0x92d91422, 0x0121f4be, + 0x2b09ccdc, 0x5571847a, 0x7f59bc18, 0xa9811536, 0x83a92d54, + 0xfdd165f2, 0xd7f95d90, 0x8b1131ef, 0xa139098d, 0xdf41412b, + 0xf5697949, 0x23b1d067, 0x0999e805, 0x77e1a0a3, 0x5dc998c1, + 0x8b6167da, 0xa1495fb8, 0xdf31171e, 0xf5192f7c, 0x23c18652, + 0x09e9be30, 0x7791f696, 0x5db9cef4, 0x0151a28b, 0x2b799ae9, + 0x5501d24f, 0x7f29ea2d, 0xa9f14303, 0x83d97b61, 0xfda133c7, + 0xd7890ba5, 0x4471eb39, 0x6e59d35b, 0x10219bfd, 0x3a09a39f, + 0xecd10ab1, 0xc6f932d3, 0xb8817a75, 0x92a94217, 0xce412e68, + 0xe469160a, 0x9a115eac, 0xb03966ce, 0x66e1cfe0, 0x4cc9f782, + 0x32b1bf24, 0x18998746, 0x44914753, 0x6eb97f31, 0x10c13797, + 0x3ae90ff5, 0xec31a6db, 0xc6199eb9, 0xb861d61f, 0x9249ee7d, + 0xcea18202, 0xe489ba60, 0x9af1f2c6, 0xb0d9caa4, 0x6601638a, + 0x4c295be8, 0x3251134e, 0x18792b2c, 0x8b81cbb0, 0xa1a9f3d2, + 0xdfd1bb74, 0xf5f98316, 0x23212a38, 0x0909125a, 0x77715afc, + 0x5d59629e, 0x01b10ee1, 0x2b993683, 0x55e17e25, 0x7fc94647, + 0xa911ef69, 0x8339d70b, 0xfd419fad, 0xd769a7cf, 0x01c158d4, + 0x2be960b6, 0x55912810, 0x7fb91072, 0xa961b95c, 0x8349813e, + 0xfd31c998, 0xd719f1fa, 0x8bf19d85, 0xa1d9a5e7, 0xdfa1ed41, + 0xf589d523, 0x23517c0d, 0x0979446f, 0x77010cc9, 0x5d2934ab, + 0xced1d437, 0xe4f9ec55, 0x9a81a4f3, 0xb0a99c91, 0x667135bf, + 0x4c590ddd, 0x3221457b, 0x18097d19, 0x44e11166, 0x6ec92904, + 0x10b161a2, 0x3a9959c0, 0xec41f0ee, 0xc669c88c, 0xb811802a, + 0x9239b848}, + {0x00000000, 0x4713f6fb, 0x8e27edf6, 0xc9341b0d, 0xc73eddad, + 0x802d2b56, 0x4919305b, 0x0e0ac6a0, 0x550cbd1b, 0x121f4be0, + 0xdb2b50ed, 0x9c38a616, 0x923260b6, 0xd521964d, 0x1c158d40, + 0x5b067bbb, 0xaa197a36, 0xed0a8ccd, 0x243e97c0, 0x632d613b, + 0x6d27a79b, 0x2a345160, 0xe3004a6d, 0xa413bc96, 0xff15c72d, + 0xb80631d6, 0x71322adb, 0x3621dc20, 0x382b1a80, 0x7f38ec7b, + 0xb60cf776, 0xf11f018d, 0x8f43f22d, 0xc85004d6, 0x01641fdb, + 0x4677e920, 0x487d2f80, 0x0f6ed97b, 0xc65ac276, 0x8149348d, + 0xda4f4f36, 0x9d5cb9cd, 0x5468a2c0, 0x137b543b, 0x1d71929b, + 0x5a626460, 0x93567f6d, 0xd4458996, 0x255a881b, 0x62497ee0, + 0xab7d65ed, 0xec6e9316, 0xe26455b6, 0xa577a34d, 0x6c43b840, + 0x2b504ebb, 0x70563500, 0x3745c3fb, 0xfe71d8f6, 0xb9622e0d, + 0xb768e8ad, 0xf07b1e56, 0x394f055b, 0x7e5cf3a0, 0xc5f6e21b, + 0x82e514e0, 0x4bd10fed, 0x0cc2f916, 0x02c83fb6, 0x45dbc94d, + 0x8cefd240, 0xcbfc24bb, 0x90fa5f00, 0xd7e9a9fb, 0x1eddb2f6, + 0x59ce440d, 0x57c482ad, 0x10d77456, 0xd9e36f5b, 0x9ef099a0, + 0x6fef982d, 0x28fc6ed6, 0xe1c875db, 0xa6db8320, 0xa8d14580, + 0xefc2b37b, 0x26f6a876, 0x61e55e8d, 0x3ae32536, 0x7df0d3cd, + 0xb4c4c8c0, 0xf3d73e3b, 0xfdddf89b, 0xbace0e60, 0x73fa156d, + 0x34e9e396, 0x4ab51036, 0x0da6e6cd, 0xc492fdc0, 0x83810b3b, + 0x8d8bcd9b, 0xca983b60, 0x03ac206d, 0x44bfd696, 0x1fb9ad2d, + 0x58aa5bd6, 0x919e40db, 0xd68db620, 0xd8877080, 0x9f94867b, + 0x56a09d76, 0x11b36b8d, 0xe0ac6a00, 0xa7bf9cfb, 0x6e8b87f6, + 0x2998710d, 0x2792b7ad, 0x60814156, 0xa9b55a5b, 0xeea6aca0, + 0xb5a0d71b, 0xf2b321e0, 0x3b873aed, 0x7c94cc16, 0x729e0ab6, + 0x358dfc4d, 0xfcb9e740, 0xbbaa11bb, 0x509cc277, 0x178f348c, + 0xdebb2f81, 0x99a8d97a, 0x97a21fda, 0xd0b1e921, 0x1985f22c, + 0x5e9604d7, 0x05907f6c, 0x42838997, 0x8bb7929a, 0xcca46461, + 0xc2aea2c1, 0x85bd543a, 0x4c894f37, 0x0b9ab9cc, 0xfa85b841, + 0xbd964eba, 0x74a255b7, 0x33b1a34c, 0x3dbb65ec, 0x7aa89317, + 0xb39c881a, 0xf48f7ee1, 0xaf89055a, 0xe89af3a1, 0x21aee8ac, + 0x66bd1e57, 0x68b7d8f7, 0x2fa42e0c, 0xe6903501, 0xa183c3fa, + 0xdfdf305a, 0x98ccc6a1, 0x51f8ddac, 0x16eb2b57, 0x18e1edf7, + 0x5ff21b0c, 0x96c60001, 0xd1d5f6fa, 0x8ad38d41, 0xcdc07bba, + 0x04f460b7, 0x43e7964c, 0x4ded50ec, 0x0afea617, 0xc3cabd1a, + 0x84d94be1, 0x75c64a6c, 0x32d5bc97, 0xfbe1a79a, 0xbcf25161, + 0xb2f897c1, 0xf5eb613a, 0x3cdf7a37, 0x7bcc8ccc, 0x20caf777, + 0x67d9018c, 0xaeed1a81, 0xe9feec7a, 0xe7f42ada, 0xa0e7dc21, + 0x69d3c72c, 0x2ec031d7, 0x956a206c, 0xd279d697, 0x1b4dcd9a, + 0x5c5e3b61, 0x5254fdc1, 0x15470b3a, 0xdc731037, 0x9b60e6cc, + 0xc0669d77, 0x87756b8c, 0x4e417081, 0x0952867a, 0x075840da, + 0x404bb621, 0x897fad2c, 0xce6c5bd7, 0x3f735a5a, 0x7860aca1, + 0xb154b7ac, 0xf6474157, 0xf84d87f7, 0xbf5e710c, 0x766a6a01, + 0x31799cfa, 0x6a7fe741, 0x2d6c11ba, 0xe4580ab7, 0xa34bfc4c, + 0xad413aec, 0xea52cc17, 0x2366d71a, 0x647521e1, 0x1a29d241, + 0x5d3a24ba, 0x940e3fb7, 0xd31dc94c, 0xdd170fec, 0x9a04f917, + 0x5330e21a, 0x142314e1, 0x4f256f5a, 0x083699a1, 0xc10282ac, + 0x86117457, 0x881bb2f7, 0xcf08440c, 0x063c5f01, 0x412fa9fa, + 0xb030a877, 0xf7235e8c, 0x3e174581, 0x7904b37a, 0x770e75da, + 0x301d8321, 0xf929982c, 0xbe3a6ed7, 0xe53c156c, 0xa22fe397, + 0x6b1bf89a, 0x2c080e61, 0x2202c8c1, 0x65113e3a, 0xac252537, + 0xeb36d3cc}, + {0x00000000, 0xa13984ee, 0x99020f9d, 0x383b8b73, 0xe975197b, + 0x484c9d95, 0x707716e6, 0xd14e9208, 0x099b34b7, 0xa8a2b059, + 0x90993b2a, 0x31a0bfc4, 0xe0ee2dcc, 0x41d7a922, 0x79ec2251, + 0xd8d5a6bf, 0x1336696e, 0xb20fed80, 0x8a3466f3, 0x2b0de21d, + 0xfa437015, 0x5b7af4fb, 0x63417f88, 0xc278fb66, 0x1aad5dd9, + 0xbb94d937, 0x83af5244, 0x2296d6aa, 0xf3d844a2, 0x52e1c04c, + 0x6ada4b3f, 0xcbe3cfd1, 0x266cd2dc, 0x87555632, 0xbf6edd41, + 0x1e5759af, 0xcf19cba7, 0x6e204f49, 0x561bc43a, 0xf72240d4, + 0x2ff7e66b, 0x8ece6285, 0xb6f5e9f6, 0x17cc6d18, 0xc682ff10, + 0x67bb7bfe, 0x5f80f08d, 0xfeb97463, 0x355abbb2, 0x94633f5c, + 0xac58b42f, 0x0d6130c1, 0xdc2fa2c9, 0x7d162627, 0x452dad54, + 0xe41429ba, 0x3cc18f05, 0x9df80beb, 0xa5c38098, 0x04fa0476, + 0xd5b4967e, 0x748d1290, 0x4cb699e3, 0xed8f1d0d, 0x4cd9a5b8, + 0xede02156, 0xd5dbaa25, 0x74e22ecb, 0xa5acbcc3, 0x0495382d, + 0x3caeb35e, 0x9d9737b0, 0x4542910f, 0xe47b15e1, 0xdc409e92, + 0x7d791a7c, 0xac378874, 0x0d0e0c9a, 0x353587e9, 0x940c0307, + 0x5fefccd6, 0xfed64838, 0xc6edc34b, 0x67d447a5, 0xb69ad5ad, + 0x17a35143, 0x2f98da30, 0x8ea15ede, 0x5674f861, 0xf74d7c8f, + 0xcf76f7fc, 0x6e4f7312, 0xbf01e11a, 0x1e3865f4, 0x2603ee87, + 0x873a6a69, 0x6ab57764, 0xcb8cf38a, 0xf3b778f9, 0x528efc17, + 0x83c06e1f, 0x22f9eaf1, 0x1ac26182, 0xbbfbe56c, 0x632e43d3, + 0xc217c73d, 0xfa2c4c4e, 0x5b15c8a0, 0x8a5b5aa8, 0x2b62de46, + 0x13595535, 0xb260d1db, 0x79831e0a, 0xd8ba9ae4, 0xe0811197, + 0x41b89579, 0x90f60771, 0x31cf839f, 0x09f408ec, 0xa8cd8c02, + 0x70182abd, 0xd121ae53, 0xe91a2520, 0x4823a1ce, 0x996d33c6, + 0x3854b728, 0x006f3c5b, 0xa156b8b5, 0x99b34b70, 0x388acf9e, + 0x00b144ed, 0xa188c003, 0x70c6520b, 0xd1ffd6e5, 0xe9c45d96, + 0x48fdd978, 0x90287fc7, 0x3111fb29, 0x092a705a, 0xa813f4b4, + 0x795d66bc, 0xd864e252, 0xe05f6921, 0x4166edcf, 0x8a85221e, + 0x2bbca6f0, 0x13872d83, 0xb2bea96d, 0x63f03b65, 0xc2c9bf8b, + 0xfaf234f8, 0x5bcbb016, 0x831e16a9, 0x22279247, 0x1a1c1934, + 0xbb259dda, 0x6a6b0fd2, 0xcb528b3c, 0xf369004f, 0x525084a1, + 0xbfdf99ac, 0x1ee61d42, 0x26dd9631, 0x87e412df, 0x56aa80d7, + 0xf7930439, 0xcfa88f4a, 0x6e910ba4, 0xb644ad1b, 0x177d29f5, + 0x2f46a286, 0x8e7f2668, 0x5f31b460, 0xfe08308e, 0xc633bbfd, + 0x670a3f13, 0xace9f0c2, 0x0dd0742c, 0x35ebff5f, 0x94d27bb1, + 0x459ce9b9, 0xe4a56d57, 0xdc9ee624, 0x7da762ca, 0xa572c475, + 0x044b409b, 0x3c70cbe8, 0x9d494f06, 0x4c07dd0e, 0xed3e59e0, + 0xd505d293, 0x743c567d, 0xd56aeec8, 0x74536a26, 0x4c68e155, + 0xed5165bb, 0x3c1ff7b3, 0x9d26735d, 0xa51df82e, 0x04247cc0, + 0xdcf1da7f, 0x7dc85e91, 0x45f3d5e2, 0xe4ca510c, 0x3584c304, + 0x94bd47ea, 0xac86cc99, 0x0dbf4877, 0xc65c87a6, 0x67650348, + 0x5f5e883b, 0xfe670cd5, 0x2f299edd, 0x8e101a33, 0xb62b9140, + 0x171215ae, 0xcfc7b311, 0x6efe37ff, 0x56c5bc8c, 0xf7fc3862, + 0x26b2aa6a, 0x878b2e84, 0xbfb0a5f7, 0x1e892119, 0xf3063c14, + 0x523fb8fa, 0x6a043389, 0xcb3db767, 0x1a73256f, 0xbb4aa181, + 0x83712af2, 0x2248ae1c, 0xfa9d08a3, 0x5ba48c4d, 0x639f073e, + 0xc2a683d0, 0x13e811d8, 0xb2d19536, 0x8aea1e45, 0x2bd39aab, + 0xe030557a, 0x4109d194, 0x79325ae7, 0xd80bde09, 0x09454c01, + 0xa87cc8ef, 0x9047439c, 0x317ec772, 0xe9ab61cd, 0x4892e523, + 0x70a96e50, 0xd190eabe, 0x00de78b6, 0xa1e7fc58, 0x99dc772b, + 0x38e5f3c5}, + {0x00000000, 0xe81790a1, 0x0b5e2703, 0xe349b7a2, 0x16bc4e06, + 0xfeabdea7, 0x1de26905, 0xf5f5f9a4, 0x2d789c0c, 0xc56f0cad, + 0x2626bb0f, 0xce312bae, 0x3bc4d20a, 0xd3d342ab, 0x309af509, + 0xd88d65a8, 0x5af13818, 0xb2e6a8b9, 0x51af1f1b, 0xb9b88fba, + 0x4c4d761e, 0xa45ae6bf, 0x4713511d, 0xaf04c1bc, 0x7789a414, + 0x9f9e34b5, 0x7cd78317, 0x94c013b6, 0x6135ea12, 0x89227ab3, + 0x6a6bcd11, 0x827c5db0, 0xb5e27030, 0x5df5e091, 0xbebc5733, + 0x56abc792, 0xa35e3e36, 0x4b49ae97, 0xa8001935, 0x40178994, + 0x989aec3c, 0x708d7c9d, 0x93c4cb3f, 0x7bd35b9e, 0x8e26a23a, + 0x6631329b, 0x85788539, 0x6d6f1598, 0xef134828, 0x0704d889, + 0xe44d6f2b, 0x0c5aff8a, 0xf9af062e, 0x11b8968f, 0xf2f1212d, + 0x1ae6b18c, 0xc26bd424, 0x2a7c4485, 0xc935f327, 0x21226386, + 0xd4d79a22, 0x3cc00a83, 0xdf89bd21, 0x379e2d80, 0xb0b5e621, + 0x58a27680, 0xbbebc122, 0x53fc5183, 0xa609a827, 0x4e1e3886, + 0xad578f24, 0x45401f85, 0x9dcd7a2d, 0x75daea8c, 0x96935d2e, + 0x7e84cd8f, 0x8b71342b, 0x6366a48a, 0x802f1328, 0x68388389, + 0xea44de39, 0x02534e98, 0xe11af93a, 0x090d699b, 0xfcf8903f, + 0x14ef009e, 0xf7a6b73c, 0x1fb1279d, 0xc73c4235, 0x2f2bd294, + 0xcc626536, 0x2475f597, 0xd1800c33, 0x39979c92, 0xdade2b30, + 0x32c9bb91, 0x05579611, 0xed4006b0, 0x0e09b112, 0xe61e21b3, + 0x13ebd817, 0xfbfc48b6, 0x18b5ff14, 0xf0a26fb5, 0x282f0a1d, + 0xc0389abc, 0x23712d1e, 0xcb66bdbf, 0x3e93441b, 0xd684d4ba, + 0x35cd6318, 0xdddaf3b9, 0x5fa6ae09, 0xb7b13ea8, 0x54f8890a, + 0xbcef19ab, 0x491ae00f, 0xa10d70ae, 0x4244c70c, 0xaa5357ad, + 0x72de3205, 0x9ac9a2a4, 0x79801506, 0x919785a7, 0x64627c03, + 0x8c75eca2, 0x6f3c5b00, 0x872bcba1, 0xba1aca03, 0x520d5aa2, + 0xb144ed00, 0x59537da1, 0xaca68405, 0x44b114a4, 0xa7f8a306, + 0x4fef33a7, 0x9762560f, 0x7f75c6ae, 0x9c3c710c, 0x742be1ad, + 0x81de1809, 0x69c988a8, 0x8a803f0a, 0x6297afab, 0xe0ebf21b, + 0x08fc62ba, 0xebb5d518, 0x03a245b9, 0xf657bc1d, 0x1e402cbc, + 0xfd099b1e, 0x151e0bbf, 0xcd936e17, 0x2584feb6, 0xc6cd4914, + 0x2edad9b5, 0xdb2f2011, 0x3338b0b0, 0xd0710712, 0x386697b3, + 0x0ff8ba33, 0xe7ef2a92, 0x04a69d30, 0xecb10d91, 0x1944f435, + 0xf1536494, 0x121ad336, 0xfa0d4397, 0x2280263f, 0xca97b69e, + 0x29de013c, 0xc1c9919d, 0x343c6839, 0xdc2bf898, 0x3f624f3a, + 0xd775df9b, 0x5509822b, 0xbd1e128a, 0x5e57a528, 0xb6403589, + 0x43b5cc2d, 0xaba25c8c, 0x48ebeb2e, 0xa0fc7b8f, 0x78711e27, + 0x90668e86, 0x732f3924, 0x9b38a985, 0x6ecd5021, 0x86dac080, + 0x65937722, 0x8d84e783, 0x0aaf2c22, 0xe2b8bc83, 0x01f10b21, + 0xe9e69b80, 0x1c136224, 0xf404f285, 0x174d4527, 0xff5ad586, + 0x27d7b02e, 0xcfc0208f, 0x2c89972d, 0xc49e078c, 0x316bfe28, + 0xd97c6e89, 0x3a35d92b, 0xd222498a, 0x505e143a, 0xb849849b, + 0x5b003339, 0xb317a398, 0x46e25a3c, 0xaef5ca9d, 0x4dbc7d3f, + 0xa5abed9e, 0x7d268836, 0x95311897, 0x7678af35, 0x9e6f3f94, + 0x6b9ac630, 0x838d5691, 0x60c4e133, 0x88d37192, 0xbf4d5c12, + 0x575accb3, 0xb4137b11, 0x5c04ebb0, 0xa9f11214, 0x41e682b5, + 0xa2af3517, 0x4ab8a5b6, 0x9235c01e, 0x7a2250bf, 0x996be71d, + 0x717c77bc, 0x84898e18, 0x6c9e1eb9, 0x8fd7a91b, 0x67c039ba, + 0xe5bc640a, 0x0dabf4ab, 0xeee24309, 0x06f5d3a8, 0xf3002a0c, + 0x1b17baad, 0xf85e0d0f, 0x10499dae, 0xc8c4f806, 0x20d368a7, + 0xc39adf05, 0x2b8d4fa4, 0xde78b600, 0x366f26a1, 0xd5269103, + 0x3d3101a2}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x0000000000000000, 0xa19017e800000000, 0x03275e0b00000000, + 0xa2b749e300000000, 0x064ebc1600000000, 0xa7deabfe00000000, + 0x0569e21d00000000, 0xa4f9f5f500000000, 0x0c9c782d00000000, + 0xad0c6fc500000000, 0x0fbb262600000000, 0xae2b31ce00000000, + 0x0ad2c43b00000000, 0xab42d3d300000000, 0x09f59a3000000000, + 0xa8658dd800000000, 0x1838f15a00000000, 0xb9a8e6b200000000, + 0x1b1faf5100000000, 0xba8fb8b900000000, 0x1e764d4c00000000, + 0xbfe65aa400000000, 0x1d51134700000000, 0xbcc104af00000000, + 0x14a4897700000000, 0xb5349e9f00000000, 0x1783d77c00000000, + 0xb613c09400000000, 0x12ea356100000000, 0xb37a228900000000, + 0x11cd6b6a00000000, 0xb05d7c8200000000, 0x3070e2b500000000, + 0x91e0f55d00000000, 0x3357bcbe00000000, 0x92c7ab5600000000, + 0x363e5ea300000000, 0x97ae494b00000000, 0x351900a800000000, + 0x9489174000000000, 0x3cec9a9800000000, 0x9d7c8d7000000000, + 0x3fcbc49300000000, 0x9e5bd37b00000000, 0x3aa2268e00000000, + 0x9b32316600000000, 0x3985788500000000, 0x98156f6d00000000, + 0x284813ef00000000, 0x89d8040700000000, 0x2b6f4de400000000, + 0x8aff5a0c00000000, 0x2e06aff900000000, 0x8f96b81100000000, + 0x2d21f1f200000000, 0x8cb1e61a00000000, 0x24d46bc200000000, + 0x85447c2a00000000, 0x27f335c900000000, 0x8663222100000000, + 0x229ad7d400000000, 0x830ac03c00000000, 0x21bd89df00000000, + 0x802d9e3700000000, 0x21e6b5b000000000, 0x8076a25800000000, + 0x22c1ebbb00000000, 0x8351fc5300000000, 0x27a809a600000000, + 0x86381e4e00000000, 0x248f57ad00000000, 0x851f404500000000, + 0x2d7acd9d00000000, 0x8ceada7500000000, 0x2e5d939600000000, + 0x8fcd847e00000000, 0x2b34718b00000000, 0x8aa4666300000000, + 0x28132f8000000000, 0x8983386800000000, 0x39de44ea00000000, + 0x984e530200000000, 0x3af91ae100000000, 0x9b690d0900000000, + 0x3f90f8fc00000000, 0x9e00ef1400000000, 0x3cb7a6f700000000, + 0x9d27b11f00000000, 0x35423cc700000000, 0x94d22b2f00000000, + 0x366562cc00000000, 0x97f5752400000000, 0x330c80d100000000, + 0x929c973900000000, 0x302bdeda00000000, 0x91bbc93200000000, + 0x1196570500000000, 0xb00640ed00000000, 0x12b1090e00000000, + 0xb3211ee600000000, 0x17d8eb1300000000, 0xb648fcfb00000000, + 0x14ffb51800000000, 0xb56fa2f000000000, 0x1d0a2f2800000000, + 0xbc9a38c000000000, 0x1e2d712300000000, 0xbfbd66cb00000000, + 0x1b44933e00000000, 0xbad484d600000000, 0x1863cd3500000000, + 0xb9f3dadd00000000, 0x09aea65f00000000, 0xa83eb1b700000000, + 0x0a89f85400000000, 0xab19efbc00000000, 0x0fe01a4900000000, + 0xae700da100000000, 0x0cc7444200000000, 0xad5753aa00000000, + 0x0532de7200000000, 0xa4a2c99a00000000, 0x0615807900000000, + 0xa785979100000000, 0x037c626400000000, 0xa2ec758c00000000, + 0x005b3c6f00000000, 0xa1cb2b8700000000, 0x03ca1aba00000000, + 0xa25a0d5200000000, 0x00ed44b100000000, 0xa17d535900000000, + 0x0584a6ac00000000, 0xa414b14400000000, 0x06a3f8a700000000, + 0xa733ef4f00000000, 0x0f56629700000000, 0xaec6757f00000000, + 0x0c713c9c00000000, 0xade12b7400000000, 0x0918de8100000000, + 0xa888c96900000000, 0x0a3f808a00000000, 0xabaf976200000000, + 0x1bf2ebe000000000, 0xba62fc0800000000, 0x18d5b5eb00000000, + 0xb945a20300000000, 0x1dbc57f600000000, 0xbc2c401e00000000, + 0x1e9b09fd00000000, 0xbf0b1e1500000000, 0x176e93cd00000000, + 0xb6fe842500000000, 0x1449cdc600000000, 0xb5d9da2e00000000, + 0x11202fdb00000000, 0xb0b0383300000000, 0x120771d000000000, + 0xb397663800000000, 0x33baf80f00000000, 0x922aefe700000000, + 0x309da60400000000, 0x910db1ec00000000, 0x35f4441900000000, + 0x946453f100000000, 0x36d31a1200000000, 0x97430dfa00000000, + 0x3f26802200000000, 0x9eb697ca00000000, 0x3c01de2900000000, + 0x9d91c9c100000000, 0x39683c3400000000, 0x98f82bdc00000000, + 0x3a4f623f00000000, 0x9bdf75d700000000, 0x2b82095500000000, + 0x8a121ebd00000000, 0x28a5575e00000000, 0x893540b600000000, + 0x2dccb54300000000, 0x8c5ca2ab00000000, 0x2eebeb4800000000, + 0x8f7bfca000000000, 0x271e717800000000, 0x868e669000000000, + 0x24392f7300000000, 0x85a9389b00000000, 0x2150cd6e00000000, + 0x80c0da8600000000, 0x2277936500000000, 0x83e7848d00000000, + 0x222caf0a00000000, 0x83bcb8e200000000, 0x210bf10100000000, + 0x809be6e900000000, 0x2462131c00000000, 0x85f204f400000000, + 0x27454d1700000000, 0x86d55aff00000000, 0x2eb0d72700000000, + 0x8f20c0cf00000000, 0x2d97892c00000000, 0x8c079ec400000000, + 0x28fe6b3100000000, 0x896e7cd900000000, 0x2bd9353a00000000, + 0x8a4922d200000000, 0x3a145e5000000000, 0x9b8449b800000000, + 0x3933005b00000000, 0x98a317b300000000, 0x3c5ae24600000000, + 0x9dcaf5ae00000000, 0x3f7dbc4d00000000, 0x9eedaba500000000, + 0x3688267d00000000, 0x9718319500000000, 0x35af787600000000, + 0x943f6f9e00000000, 0x30c69a6b00000000, 0x91568d8300000000, + 0x33e1c46000000000, 0x9271d38800000000, 0x125c4dbf00000000, + 0xb3cc5a5700000000, 0x117b13b400000000, 0xb0eb045c00000000, + 0x1412f1a900000000, 0xb582e64100000000, 0x1735afa200000000, + 0xb6a5b84a00000000, 0x1ec0359200000000, 0xbf50227a00000000, + 0x1de76b9900000000, 0xbc777c7100000000, 0x188e898400000000, + 0xb91e9e6c00000000, 0x1ba9d78f00000000, 0xba39c06700000000, + 0x0a64bce500000000, 0xabf4ab0d00000000, 0x0943e2ee00000000, + 0xa8d3f50600000000, 0x0c2a00f300000000, 0xadba171b00000000, + 0x0f0d5ef800000000, 0xae9d491000000000, 0x06f8c4c800000000, + 0xa768d32000000000, 0x05df9ac300000000, 0xa44f8d2b00000000, + 0x00b678de00000000, 0xa1266f3600000000, 0x039126d500000000, + 0xa201313d00000000}, + {0x0000000000000000, 0xee8439a100000000, 0x9d0f029900000000, + 0x738b3b3800000000, 0x7b1975e900000000, 0x959d4c4800000000, + 0xe616777000000000, 0x08924ed100000000, 0xb7349b0900000000, + 0x59b0a2a800000000, 0x2a3b999000000000, 0xc4bfa03100000000, + 0xcc2deee000000000, 0x22a9d74100000000, 0x5122ec7900000000, + 0xbfa6d5d800000000, 0x6e69361300000000, 0x80ed0fb200000000, + 0xf366348a00000000, 0x1de20d2b00000000, 0x157043fa00000000, + 0xfbf47a5b00000000, 0x887f416300000000, 0x66fb78c200000000, + 0xd95dad1a00000000, 0x37d994bb00000000, 0x4452af8300000000, + 0xaad6962200000000, 0xa244d8f300000000, 0x4cc0e15200000000, + 0x3f4bda6a00000000, 0xd1cfe3cb00000000, 0xdcd26c2600000000, + 0x3256558700000000, 0x41dd6ebf00000000, 0xaf59571e00000000, + 0xa7cb19cf00000000, 0x494f206e00000000, 0x3ac41b5600000000, + 0xd44022f700000000, 0x6be6f72f00000000, 0x8562ce8e00000000, + 0xf6e9f5b600000000, 0x186dcc1700000000, 0x10ff82c600000000, + 0xfe7bbb6700000000, 0x8df0805f00000000, 0x6374b9fe00000000, + 0xb2bb5a3500000000, 0x5c3f639400000000, 0x2fb458ac00000000, + 0xc130610d00000000, 0xc9a22fdc00000000, 0x2726167d00000000, + 0x54ad2d4500000000, 0xba2914e400000000, 0x058fc13c00000000, + 0xeb0bf89d00000000, 0x9880c3a500000000, 0x7604fa0400000000, + 0x7e96b4d500000000, 0x90128d7400000000, 0xe399b64c00000000, + 0x0d1d8fed00000000, 0xb8a5d94c00000000, 0x5621e0ed00000000, + 0x25aadbd500000000, 0xcb2ee27400000000, 0xc3bcaca500000000, + 0x2d38950400000000, 0x5eb3ae3c00000000, 0xb037979d00000000, + 0x0f91424500000000, 0xe1157be400000000, 0x929e40dc00000000, + 0x7c1a797d00000000, 0x748837ac00000000, 0x9a0c0e0d00000000, + 0xe987353500000000, 0x07030c9400000000, 0xd6ccef5f00000000, + 0x3848d6fe00000000, 0x4bc3edc600000000, 0xa547d46700000000, + 0xadd59ab600000000, 0x4351a31700000000, 0x30da982f00000000, + 0xde5ea18e00000000, 0x61f8745600000000, 0x8f7c4df700000000, + 0xfcf776cf00000000, 0x12734f6e00000000, 0x1ae101bf00000000, + 0xf465381e00000000, 0x87ee032600000000, 0x696a3a8700000000, + 0x6477b56a00000000, 0x8af38ccb00000000, 0xf978b7f300000000, + 0x17fc8e5200000000, 0x1f6ec08300000000, 0xf1eaf92200000000, + 0x8261c21a00000000, 0x6ce5fbbb00000000, 0xd3432e6300000000, + 0x3dc717c200000000, 0x4e4c2cfa00000000, 0xa0c8155b00000000, + 0xa85a5b8a00000000, 0x46de622b00000000, 0x3555591300000000, + 0xdbd160b200000000, 0x0a1e837900000000, 0xe49abad800000000, + 0x971181e000000000, 0x7995b84100000000, 0x7107f69000000000, + 0x9f83cf3100000000, 0xec08f40900000000, 0x028ccda800000000, + 0xbd2a187000000000, 0x53ae21d100000000, 0x20251ae900000000, + 0xcea1234800000000, 0xc6336d9900000000, 0x28b7543800000000, + 0x5b3c6f0000000000, 0xb5b856a100000000, 0x704bb39900000000, + 0x9ecf8a3800000000, 0xed44b10000000000, 0x03c088a100000000, + 0x0b52c67000000000, 0xe5d6ffd100000000, 0x965dc4e900000000, + 0x78d9fd4800000000, 0xc77f289000000000, 0x29fb113100000000, + 0x5a702a0900000000, 0xb4f413a800000000, 0xbc665d7900000000, + 0x52e264d800000000, 0x21695fe000000000, 0xcfed664100000000, + 0x1e22858a00000000, 0xf0a6bc2b00000000, 0x832d871300000000, + 0x6da9beb200000000, 0x653bf06300000000, 0x8bbfc9c200000000, + 0xf834f2fa00000000, 0x16b0cb5b00000000, 0xa9161e8300000000, + 0x4792272200000000, 0x34191c1a00000000, 0xda9d25bb00000000, + 0xd20f6b6a00000000, 0x3c8b52cb00000000, 0x4f0069f300000000, + 0xa184505200000000, 0xac99dfbf00000000, 0x421de61e00000000, + 0x3196dd2600000000, 0xdf12e48700000000, 0xd780aa5600000000, + 0x390493f700000000, 0x4a8fa8cf00000000, 0xa40b916e00000000, + 0x1bad44b600000000, 0xf5297d1700000000, 0x86a2462f00000000, + 0x68267f8e00000000, 0x60b4315f00000000, 0x8e3008fe00000000, + 0xfdbb33c600000000, 0x133f0a6700000000, 0xc2f0e9ac00000000, + 0x2c74d00d00000000, 0x5fffeb3500000000, 0xb17bd29400000000, + 0xb9e99c4500000000, 0x576da5e400000000, 0x24e69edc00000000, + 0xca62a77d00000000, 0x75c472a500000000, 0x9b404b0400000000, + 0xe8cb703c00000000, 0x064f499d00000000, 0x0edd074c00000000, + 0xe0593eed00000000, 0x93d205d500000000, 0x7d563c7400000000, + 0xc8ee6ad500000000, 0x266a537400000000, 0x55e1684c00000000, + 0xbb6551ed00000000, 0xb3f71f3c00000000, 0x5d73269d00000000, + 0x2ef81da500000000, 0xc07c240400000000, 0x7fdaf1dc00000000, + 0x915ec87d00000000, 0xe2d5f34500000000, 0x0c51cae400000000, + 0x04c3843500000000, 0xea47bd9400000000, 0x99cc86ac00000000, + 0x7748bf0d00000000, 0xa6875cc600000000, 0x4803656700000000, + 0x3b885e5f00000000, 0xd50c67fe00000000, 0xdd9e292f00000000, + 0x331a108e00000000, 0x40912bb600000000, 0xae15121700000000, + 0x11b3c7cf00000000, 0xff37fe6e00000000, 0x8cbcc55600000000, + 0x6238fcf700000000, 0x6aaab22600000000, 0x842e8b8700000000, + 0xf7a5b0bf00000000, 0x1921891e00000000, 0x143c06f300000000, + 0xfab83f5200000000, 0x8933046a00000000, 0x67b73dcb00000000, + 0x6f25731a00000000, 0x81a14abb00000000, 0xf22a718300000000, + 0x1cae482200000000, 0xa3089dfa00000000, 0x4d8ca45b00000000, + 0x3e079f6300000000, 0xd083a6c200000000, 0xd811e81300000000, + 0x3695d1b200000000, 0x451eea8a00000000, 0xab9ad32b00000000, + 0x7a5530e000000000, 0x94d1094100000000, 0xe75a327900000000, + 0x09de0bd800000000, 0x014c450900000000, 0xefc87ca800000000, + 0x9c43479000000000, 0x72c77e3100000000, 0xcd61abe900000000, + 0x23e5924800000000, 0x506ea97000000000, 0xbeea90d100000000, + 0xb678de0000000000, 0x58fce7a100000000, 0x2b77dc9900000000, + 0xc5f3e53800000000}, + {0x0000000000000000, 0xfbf6134700000000, 0xf6ed278e00000000, + 0x0d1b34c900000000, 0xaddd3ec700000000, 0x562b2d8000000000, + 0x5b30194900000000, 0xa0c60a0e00000000, 0x1bbd0c5500000000, + 0xe04b1f1200000000, 0xed502bdb00000000, 0x16a6389c00000000, + 0xb660329200000000, 0x4d9621d500000000, 0x408d151c00000000, + 0xbb7b065b00000000, 0x367a19aa00000000, 0xcd8c0aed00000000, + 0xc0973e2400000000, 0x3b612d6300000000, 0x9ba7276d00000000, + 0x6051342a00000000, 0x6d4a00e300000000, 0x96bc13a400000000, + 0x2dc715ff00000000, 0xd63106b800000000, 0xdb2a327100000000, + 0x20dc213600000000, 0x801a2b3800000000, 0x7bec387f00000000, + 0x76f70cb600000000, 0x8d011ff100000000, 0x2df2438f00000000, + 0xd60450c800000000, 0xdb1f640100000000, 0x20e9774600000000, + 0x802f7d4800000000, 0x7bd96e0f00000000, 0x76c25ac600000000, + 0x8d34498100000000, 0x364f4fda00000000, 0xcdb95c9d00000000, + 0xc0a2685400000000, 0x3b547b1300000000, 0x9b92711d00000000, + 0x6064625a00000000, 0x6d7f569300000000, 0x968945d400000000, + 0x1b885a2500000000, 0xe07e496200000000, 0xed657dab00000000, + 0x16936eec00000000, 0xb65564e200000000, 0x4da377a500000000, + 0x40b8436c00000000, 0xbb4e502b00000000, 0x0035567000000000, + 0xfbc3453700000000, 0xf6d871fe00000000, 0x0d2e62b900000000, + 0xade868b700000000, 0x561e7bf000000000, 0x5b054f3900000000, + 0xa0f35c7e00000000, 0x1be2f6c500000000, 0xe014e58200000000, + 0xed0fd14b00000000, 0x16f9c20c00000000, 0xb63fc80200000000, + 0x4dc9db4500000000, 0x40d2ef8c00000000, 0xbb24fccb00000000, + 0x005ffa9000000000, 0xfba9e9d700000000, 0xf6b2dd1e00000000, + 0x0d44ce5900000000, 0xad82c45700000000, 0x5674d71000000000, + 0x5b6fe3d900000000, 0xa099f09e00000000, 0x2d98ef6f00000000, + 0xd66efc2800000000, 0xdb75c8e100000000, 0x2083dba600000000, + 0x8045d1a800000000, 0x7bb3c2ef00000000, 0x76a8f62600000000, + 0x8d5ee56100000000, 0x3625e33a00000000, 0xcdd3f07d00000000, + 0xc0c8c4b400000000, 0x3b3ed7f300000000, 0x9bf8ddfd00000000, + 0x600eceba00000000, 0x6d15fa7300000000, 0x96e3e93400000000, + 0x3610b54a00000000, 0xcde6a60d00000000, 0xc0fd92c400000000, + 0x3b0b818300000000, 0x9bcd8b8d00000000, 0x603b98ca00000000, + 0x6d20ac0300000000, 0x96d6bf4400000000, 0x2dadb91f00000000, + 0xd65baa5800000000, 0xdb409e9100000000, 0x20b68dd600000000, + 0x807087d800000000, 0x7b86949f00000000, 0x769da05600000000, + 0x8d6bb31100000000, 0x006aace000000000, 0xfb9cbfa700000000, + 0xf6878b6e00000000, 0x0d71982900000000, 0xadb7922700000000, + 0x5641816000000000, 0x5b5ab5a900000000, 0xa0aca6ee00000000, + 0x1bd7a0b500000000, 0xe021b3f200000000, 0xed3a873b00000000, + 0x16cc947c00000000, 0xb60a9e7200000000, 0x4dfc8d3500000000, + 0x40e7b9fc00000000, 0xbb11aabb00000000, 0x77c29c5000000000, + 0x8c348f1700000000, 0x812fbbde00000000, 0x7ad9a89900000000, + 0xda1fa29700000000, 0x21e9b1d000000000, 0x2cf2851900000000, + 0xd704965e00000000, 0x6c7f900500000000, 0x9789834200000000, + 0x9a92b78b00000000, 0x6164a4cc00000000, 0xc1a2aec200000000, + 0x3a54bd8500000000, 0x374f894c00000000, 0xccb99a0b00000000, + 0x41b885fa00000000, 0xba4e96bd00000000, 0xb755a27400000000, + 0x4ca3b13300000000, 0xec65bb3d00000000, 0x1793a87a00000000, + 0x1a889cb300000000, 0xe17e8ff400000000, 0x5a0589af00000000, + 0xa1f39ae800000000, 0xace8ae2100000000, 0x571ebd6600000000, + 0xf7d8b76800000000, 0x0c2ea42f00000000, 0x013590e600000000, + 0xfac383a100000000, 0x5a30dfdf00000000, 0xa1c6cc9800000000, + 0xacddf85100000000, 0x572beb1600000000, 0xf7ede11800000000, + 0x0c1bf25f00000000, 0x0100c69600000000, 0xfaf6d5d100000000, + 0x418dd38a00000000, 0xba7bc0cd00000000, 0xb760f40400000000, + 0x4c96e74300000000, 0xec50ed4d00000000, 0x17a6fe0a00000000, + 0x1abdcac300000000, 0xe14bd98400000000, 0x6c4ac67500000000, + 0x97bcd53200000000, 0x9aa7e1fb00000000, 0x6151f2bc00000000, + 0xc197f8b200000000, 0x3a61ebf500000000, 0x377adf3c00000000, + 0xcc8ccc7b00000000, 0x77f7ca2000000000, 0x8c01d96700000000, + 0x811aedae00000000, 0x7aecfee900000000, 0xda2af4e700000000, + 0x21dce7a000000000, 0x2cc7d36900000000, 0xd731c02e00000000, + 0x6c206a9500000000, 0x97d679d200000000, 0x9acd4d1b00000000, + 0x613b5e5c00000000, 0xc1fd545200000000, 0x3a0b471500000000, + 0x371073dc00000000, 0xcce6609b00000000, 0x779d66c000000000, + 0x8c6b758700000000, 0x8170414e00000000, 0x7a86520900000000, + 0xda40580700000000, 0x21b64b4000000000, 0x2cad7f8900000000, + 0xd75b6cce00000000, 0x5a5a733f00000000, 0xa1ac607800000000, + 0xacb754b100000000, 0x574147f600000000, 0xf7874df800000000, + 0x0c715ebf00000000, 0x016a6a7600000000, 0xfa9c793100000000, + 0x41e77f6a00000000, 0xba116c2d00000000, 0xb70a58e400000000, + 0x4cfc4ba300000000, 0xec3a41ad00000000, 0x17cc52ea00000000, + 0x1ad7662300000000, 0xe121756400000000, 0x41d2291a00000000, + 0xba243a5d00000000, 0xb73f0e9400000000, 0x4cc91dd300000000, + 0xec0f17dd00000000, 0x17f9049a00000000, 0x1ae2305300000000, + 0xe114231400000000, 0x5a6f254f00000000, 0xa199360800000000, + 0xac8202c100000000, 0x5774118600000000, 0xf7b21b8800000000, + 0x0c4408cf00000000, 0x015f3c0600000000, 0xfaa92f4100000000, + 0x77a830b000000000, 0x8c5e23f700000000, 0x8145173e00000000, + 0x7ab3047900000000, 0xda750e7700000000, 0x21831d3000000000, + 0x2c9829f900000000, 0xd76e3abe00000000, 0x6c153ce500000000, + 0x97e32fa200000000, 0x9af81b6b00000000, 0x610e082c00000000, + 0xc1c8022200000000, 0x3a3e116500000000, 0x372525ac00000000, + 0xccd336eb00000000}, + {0x0000000000000000, 0x6238282a00000000, 0xc470505400000000, + 0xa648787e00000000, 0x88e1a0a800000000, 0xead9888200000000, + 0x4c91f0fc00000000, 0x2ea9d8d600000000, 0x51c5308a00000000, + 0x33fd18a000000000, 0x95b560de00000000, 0xf78d48f400000000, + 0xd924902200000000, 0xbb1cb80800000000, 0x1d54c07600000000, + 0x7f6ce85c00000000, 0xe38c10cf00000000, 0x81b438e500000000, + 0x27fc409b00000000, 0x45c468b100000000, 0x6b6db06700000000, + 0x0955984d00000000, 0xaf1de03300000000, 0xcd25c81900000000, + 0xb249204500000000, 0xd071086f00000000, 0x7639701100000000, + 0x1401583b00000000, 0x3aa880ed00000000, 0x5890a8c700000000, + 0xfed8d0b900000000, 0x9ce0f89300000000, 0x871f504500000000, + 0xe527786f00000000, 0x436f001100000000, 0x2157283b00000000, + 0x0ffef0ed00000000, 0x6dc6d8c700000000, 0xcb8ea0b900000000, + 0xa9b6889300000000, 0xd6da60cf00000000, 0xb4e248e500000000, + 0x12aa309b00000000, 0x709218b100000000, 0x5e3bc06700000000, + 0x3c03e84d00000000, 0x9a4b903300000000, 0xf873b81900000000, + 0x6493408a00000000, 0x06ab68a000000000, 0xa0e310de00000000, + 0xc2db38f400000000, 0xec72e02200000000, 0x8e4ac80800000000, + 0x2802b07600000000, 0x4a3a985c00000000, 0x3556700000000000, + 0x576e582a00000000, 0xf126205400000000, 0x931e087e00000000, + 0xbdb7d0a800000000, 0xdf8ff88200000000, 0x79c780fc00000000, + 0x1bffa8d600000000, 0x0e3fa08a00000000, 0x6c0788a000000000, + 0xca4ff0de00000000, 0xa877d8f400000000, 0x86de002200000000, + 0xe4e6280800000000, 0x42ae507600000000, 0x2096785c00000000, + 0x5ffa900000000000, 0x3dc2b82a00000000, 0x9b8ac05400000000, + 0xf9b2e87e00000000, 0xd71b30a800000000, 0xb523188200000000, + 0x136b60fc00000000, 0x715348d600000000, 0xedb3b04500000000, + 0x8f8b986f00000000, 0x29c3e01100000000, 0x4bfbc83b00000000, + 0x655210ed00000000, 0x076a38c700000000, 0xa12240b900000000, + 0xc31a689300000000, 0xbc7680cf00000000, 0xde4ea8e500000000, + 0x7806d09b00000000, 0x1a3ef8b100000000, 0x3497206700000000, + 0x56af084d00000000, 0xf0e7703300000000, 0x92df581900000000, + 0x8920f0cf00000000, 0xeb18d8e500000000, 0x4d50a09b00000000, + 0x2f6888b100000000, 0x01c1506700000000, 0x63f9784d00000000, + 0xc5b1003300000000, 0xa789281900000000, 0xd8e5c04500000000, + 0xbadde86f00000000, 0x1c95901100000000, 0x7eadb83b00000000, + 0x500460ed00000000, 0x323c48c700000000, 0x947430b900000000, + 0xf64c189300000000, 0x6aace00000000000, 0x0894c82a00000000, + 0xaedcb05400000000, 0xcce4987e00000000, 0xe24d40a800000000, + 0x8075688200000000, 0x263d10fc00000000, 0x440538d600000000, + 0x3b69d08a00000000, 0x5951f8a000000000, 0xff1980de00000000, + 0x9d21a8f400000000, 0xb388702200000000, 0xd1b0580800000000, + 0x77f8207600000000, 0x15c0085c00000000, 0x5d7831ce00000000, + 0x3f4019e400000000, 0x9908619a00000000, 0xfb3049b000000000, + 0xd599916600000000, 0xb7a1b94c00000000, 0x11e9c13200000000, + 0x73d1e91800000000, 0x0cbd014400000000, 0x6e85296e00000000, + 0xc8cd511000000000, 0xaaf5793a00000000, 0x845ca1ec00000000, + 0xe66489c600000000, 0x402cf1b800000000, 0x2214d99200000000, + 0xbef4210100000000, 0xdccc092b00000000, 0x7a84715500000000, + 0x18bc597f00000000, 0x361581a900000000, 0x542da98300000000, + 0xf265d1fd00000000, 0x905df9d700000000, 0xef31118b00000000, + 0x8d0939a100000000, 0x2b4141df00000000, 0x497969f500000000, + 0x67d0b12300000000, 0x05e8990900000000, 0xa3a0e17700000000, + 0xc198c95d00000000, 0xda67618b00000000, 0xb85f49a100000000, + 0x1e1731df00000000, 0x7c2f19f500000000, 0x5286c12300000000, + 0x30bee90900000000, 0x96f6917700000000, 0xf4ceb95d00000000, + 0x8ba2510100000000, 0xe99a792b00000000, 0x4fd2015500000000, + 0x2dea297f00000000, 0x0343f1a900000000, 0x617bd98300000000, + 0xc733a1fd00000000, 0xa50b89d700000000, 0x39eb714400000000, + 0x5bd3596e00000000, 0xfd9b211000000000, 0x9fa3093a00000000, + 0xb10ad1ec00000000, 0xd332f9c600000000, 0x757a81b800000000, + 0x1742a99200000000, 0x682e41ce00000000, 0x0a1669e400000000, + 0xac5e119a00000000, 0xce6639b000000000, 0xe0cfe16600000000, + 0x82f7c94c00000000, 0x24bfb13200000000, 0x4687991800000000, + 0x5347914400000000, 0x317fb96e00000000, 0x9737c11000000000, + 0xf50fe93a00000000, 0xdba631ec00000000, 0xb99e19c600000000, + 0x1fd661b800000000, 0x7dee499200000000, 0x0282a1ce00000000, + 0x60ba89e400000000, 0xc6f2f19a00000000, 0xa4cad9b000000000, + 0x8a63016600000000, 0xe85b294c00000000, 0x4e13513200000000, + 0x2c2b791800000000, 0xb0cb818b00000000, 0xd2f3a9a100000000, + 0x74bbd1df00000000, 0x1683f9f500000000, 0x382a212300000000, + 0x5a12090900000000, 0xfc5a717700000000, 0x9e62595d00000000, + 0xe10eb10100000000, 0x8336992b00000000, 0x257ee15500000000, + 0x4746c97f00000000, 0x69ef11a900000000, 0x0bd7398300000000, + 0xad9f41fd00000000, 0xcfa769d700000000, 0xd458c10100000000, + 0xb660e92b00000000, 0x1028915500000000, 0x7210b97f00000000, + 0x5cb961a900000000, 0x3e81498300000000, 0x98c931fd00000000, + 0xfaf119d700000000, 0x859df18b00000000, 0xe7a5d9a100000000, + 0x41eda1df00000000, 0x23d589f500000000, 0x0d7c512300000000, + 0x6f44790900000000, 0xc90c017700000000, 0xab34295d00000000, + 0x37d4d1ce00000000, 0x55ecf9e400000000, 0xf3a4819a00000000, + 0x919ca9b000000000, 0xbf35716600000000, 0xdd0d594c00000000, + 0x7b45213200000000, 0x197d091800000000, 0x6611e14400000000, + 0x0429c96e00000000, 0xa261b11000000000, 0xc059993a00000000, + 0xeef041ec00000000, 0x8cc869c600000000, 0x2a8011b800000000, + 0x48b8399200000000}, + {0x0000000000000000, 0x4c2896a300000000, 0xd9565d9c00000000, + 0x957ecb3f00000000, 0xf3abcbe300000000, 0xbf835d4000000000, + 0x2afd967f00000000, 0x66d500dc00000000, 0xa751e61c00000000, + 0xeb7970bf00000000, 0x7e07bb8000000000, 0x322f2d2300000000, + 0x54fa2dff00000000, 0x18d2bb5c00000000, 0x8dac706300000000, + 0xc184e6c000000000, 0x4ea3cc3900000000, 0x028b5a9a00000000, + 0x97f591a500000000, 0xdbdd070600000000, 0xbd0807da00000000, + 0xf120917900000000, 0x645e5a4600000000, 0x2876cce500000000, + 0xe9f22a2500000000, 0xa5dabc8600000000, 0x30a477b900000000, + 0x7c8ce11a00000000, 0x1a59e1c600000000, 0x5671776500000000, + 0xc30fbc5a00000000, 0x8f272af900000000, 0x9c46997300000000, + 0xd06e0fd000000000, 0x4510c4ef00000000, 0x0938524c00000000, + 0x6fed529000000000, 0x23c5c43300000000, 0xb6bb0f0c00000000, + 0xfa9399af00000000, 0x3b177f6f00000000, 0x773fe9cc00000000, + 0xe24122f300000000, 0xae69b45000000000, 0xc8bcb48c00000000, + 0x8494222f00000000, 0x11eae91000000000, 0x5dc27fb300000000, + 0xd2e5554a00000000, 0x9ecdc3e900000000, 0x0bb308d600000000, + 0x479b9e7500000000, 0x214e9ea900000000, 0x6d66080a00000000, + 0xf818c33500000000, 0xb430559600000000, 0x75b4b35600000000, + 0x399c25f500000000, 0xace2eeca00000000, 0xe0ca786900000000, + 0x861f78b500000000, 0xca37ee1600000000, 0x5f49252900000000, + 0x1361b38a00000000, 0x388d32e700000000, 0x74a5a44400000000, + 0xe1db6f7b00000000, 0xadf3f9d800000000, 0xcb26f90400000000, + 0x870e6fa700000000, 0x1270a49800000000, 0x5e58323b00000000, + 0x9fdcd4fb00000000, 0xd3f4425800000000, 0x468a896700000000, + 0x0aa21fc400000000, 0x6c771f1800000000, 0x205f89bb00000000, + 0xb521428400000000, 0xf909d42700000000, 0x762efede00000000, + 0x3a06687d00000000, 0xaf78a34200000000, 0xe35035e100000000, + 0x8585353d00000000, 0xc9ada39e00000000, 0x5cd368a100000000, + 0x10fbfe0200000000, 0xd17f18c200000000, 0x9d578e6100000000, + 0x0829455e00000000, 0x4401d3fd00000000, 0x22d4d32100000000, + 0x6efc458200000000, 0xfb828ebd00000000, 0xb7aa181e00000000, + 0xa4cbab9400000000, 0xe8e33d3700000000, 0x7d9df60800000000, + 0x31b560ab00000000, 0x5760607700000000, 0x1b48f6d400000000, + 0x8e363deb00000000, 0xc21eab4800000000, 0x039a4d8800000000, + 0x4fb2db2b00000000, 0xdacc101400000000, 0x96e486b700000000, + 0xf031866b00000000, 0xbc1910c800000000, 0x2967dbf700000000, + 0x654f4d5400000000, 0xea6867ad00000000, 0xa640f10e00000000, + 0x333e3a3100000000, 0x7f16ac9200000000, 0x19c3ac4e00000000, + 0x55eb3aed00000000, 0xc095f1d200000000, 0x8cbd677100000000, + 0x4d3981b100000000, 0x0111171200000000, 0x946fdc2d00000000, + 0xd8474a8e00000000, 0xbe924a5200000000, 0xf2badcf100000000, + 0x67c417ce00000000, 0x2bec816d00000000, 0x311c141500000000, + 0x7d3482b600000000, 0xe84a498900000000, 0xa462df2a00000000, + 0xc2b7dff600000000, 0x8e9f495500000000, 0x1be1826a00000000, + 0x57c914c900000000, 0x964df20900000000, 0xda6564aa00000000, + 0x4f1baf9500000000, 0x0333393600000000, 0x65e639ea00000000, + 0x29ceaf4900000000, 0xbcb0647600000000, 0xf098f2d500000000, + 0x7fbfd82c00000000, 0x33974e8f00000000, 0xa6e985b000000000, + 0xeac1131300000000, 0x8c1413cf00000000, 0xc03c856c00000000, + 0x55424e5300000000, 0x196ad8f000000000, 0xd8ee3e3000000000, + 0x94c6a89300000000, 0x01b863ac00000000, 0x4d90f50f00000000, + 0x2b45f5d300000000, 0x676d637000000000, 0xf213a84f00000000, + 0xbe3b3eec00000000, 0xad5a8d6600000000, 0xe1721bc500000000, + 0x740cd0fa00000000, 0x3824465900000000, 0x5ef1468500000000, + 0x12d9d02600000000, 0x87a71b1900000000, 0xcb8f8dba00000000, + 0x0a0b6b7a00000000, 0x4623fdd900000000, 0xd35d36e600000000, + 0x9f75a04500000000, 0xf9a0a09900000000, 0xb588363a00000000, + 0x20f6fd0500000000, 0x6cde6ba600000000, 0xe3f9415f00000000, + 0xafd1d7fc00000000, 0x3aaf1cc300000000, 0x76878a6000000000, + 0x10528abc00000000, 0x5c7a1c1f00000000, 0xc904d72000000000, + 0x852c418300000000, 0x44a8a74300000000, 0x088031e000000000, + 0x9dfefadf00000000, 0xd1d66c7c00000000, 0xb7036ca000000000, + 0xfb2bfa0300000000, 0x6e55313c00000000, 0x227da79f00000000, + 0x099126f200000000, 0x45b9b05100000000, 0xd0c77b6e00000000, + 0x9cefedcd00000000, 0xfa3aed1100000000, 0xb6127bb200000000, + 0x236cb08d00000000, 0x6f44262e00000000, 0xaec0c0ee00000000, + 0xe2e8564d00000000, 0x77969d7200000000, 0x3bbe0bd100000000, + 0x5d6b0b0d00000000, 0x11439dae00000000, 0x843d569100000000, + 0xc815c03200000000, 0x4732eacb00000000, 0x0b1a7c6800000000, + 0x9e64b75700000000, 0xd24c21f400000000, 0xb499212800000000, + 0xf8b1b78b00000000, 0x6dcf7cb400000000, 0x21e7ea1700000000, + 0xe0630cd700000000, 0xac4b9a7400000000, 0x3935514b00000000, + 0x751dc7e800000000, 0x13c8c73400000000, 0x5fe0519700000000, + 0xca9e9aa800000000, 0x86b60c0b00000000, 0x95d7bf8100000000, + 0xd9ff292200000000, 0x4c81e21d00000000, 0x00a974be00000000, + 0x667c746200000000, 0x2a54e2c100000000, 0xbf2a29fe00000000, + 0xf302bf5d00000000, 0x3286599d00000000, 0x7eaecf3e00000000, + 0xebd0040100000000, 0xa7f892a200000000, 0xc12d927e00000000, + 0x8d0504dd00000000, 0x187bcfe200000000, 0x5453594100000000, + 0xdb7473b800000000, 0x975ce51b00000000, 0x02222e2400000000, + 0x4e0ab88700000000, 0x28dfb85b00000000, 0x64f72ef800000000, + 0xf189e5c700000000, 0xbda1736400000000, 0x7c2595a400000000, + 0x300d030700000000, 0xa573c83800000000, 0xe95b5e9b00000000, + 0x8f8e5e4700000000, 0xc3a6c8e400000000, 0x56d803db00000000, + 0x1af0957800000000}, + {0x0000000000000000, 0x939bc97f00000000, 0x263793ff00000000, + 0xb5ac5a8000000000, 0x0d68572400000000, 0x9ef39e5b00000000, + 0x2b5fc4db00000000, 0xb8c40da400000000, 0x1ad0ae4800000000, + 0x894b673700000000, 0x3ce73db700000000, 0xaf7cf4c800000000, + 0x17b8f96c00000000, 0x8423301300000000, 0x318f6a9300000000, + 0xa214a3ec00000000, 0x34a05d9100000000, 0xa73b94ee00000000, + 0x1297ce6e00000000, 0x810c071100000000, 0x39c80ab500000000, + 0xaa53c3ca00000000, 0x1fff994a00000000, 0x8c64503500000000, + 0x2e70f3d900000000, 0xbdeb3aa600000000, 0x0847602600000000, + 0x9bdca95900000000, 0x2318a4fd00000000, 0xb0836d8200000000, + 0x052f370200000000, 0x96b4fe7d00000000, 0x2946caf900000000, + 0xbadd038600000000, 0x0f71590600000000, 0x9cea907900000000, + 0x242e9ddd00000000, 0xb7b554a200000000, 0x02190e2200000000, + 0x9182c75d00000000, 0x339664b100000000, 0xa00dadce00000000, + 0x15a1f74e00000000, 0x863a3e3100000000, 0x3efe339500000000, + 0xad65faea00000000, 0x18c9a06a00000000, 0x8b52691500000000, + 0x1de6976800000000, 0x8e7d5e1700000000, 0x3bd1049700000000, + 0xa84acde800000000, 0x108ec04c00000000, 0x8315093300000000, + 0x36b953b300000000, 0xa5229acc00000000, 0x0736392000000000, + 0x94adf05f00000000, 0x2101aadf00000000, 0xb29a63a000000000, + 0x0a5e6e0400000000, 0x99c5a77b00000000, 0x2c69fdfb00000000, + 0xbff2348400000000, 0x138ae52800000000, 0x80112c5700000000, + 0x35bd76d700000000, 0xa626bfa800000000, 0x1ee2b20c00000000, + 0x8d797b7300000000, 0x38d521f300000000, 0xab4ee88c00000000, + 0x095a4b6000000000, 0x9ac1821f00000000, 0x2f6dd89f00000000, + 0xbcf611e000000000, 0x04321c4400000000, 0x97a9d53b00000000, + 0x22058fbb00000000, 0xb19e46c400000000, 0x272ab8b900000000, + 0xb4b171c600000000, 0x011d2b4600000000, 0x9286e23900000000, + 0x2a42ef9d00000000, 0xb9d926e200000000, 0x0c757c6200000000, + 0x9feeb51d00000000, 0x3dfa16f100000000, 0xae61df8e00000000, + 0x1bcd850e00000000, 0x88564c7100000000, 0x309241d500000000, + 0xa30988aa00000000, 0x16a5d22a00000000, 0x853e1b5500000000, + 0x3acc2fd100000000, 0xa957e6ae00000000, 0x1cfbbc2e00000000, + 0x8f60755100000000, 0x37a478f500000000, 0xa43fb18a00000000, + 0x1193eb0a00000000, 0x8208227500000000, 0x201c819900000000, + 0xb38748e600000000, 0x062b126600000000, 0x95b0db1900000000, + 0x2d74d6bd00000000, 0xbeef1fc200000000, 0x0b43454200000000, + 0x98d88c3d00000000, 0x0e6c724000000000, 0x9df7bb3f00000000, + 0x285be1bf00000000, 0xbbc028c000000000, 0x0304256400000000, + 0x909fec1b00000000, 0x2533b69b00000000, 0xb6a87fe400000000, + 0x14bcdc0800000000, 0x8727157700000000, 0x328b4ff700000000, + 0xa110868800000000, 0x19d48b2c00000000, 0x8a4f425300000000, + 0x3fe318d300000000, 0xac78d1ac00000000, 0x2614cb5100000000, + 0xb58f022e00000000, 0x002358ae00000000, 0x93b891d100000000, + 0x2b7c9c7500000000, 0xb8e7550a00000000, 0x0d4b0f8a00000000, + 0x9ed0c6f500000000, 0x3cc4651900000000, 0xaf5fac6600000000, + 0x1af3f6e600000000, 0x89683f9900000000, 0x31ac323d00000000, + 0xa237fb4200000000, 0x179ba1c200000000, 0x840068bd00000000, + 0x12b496c000000000, 0x812f5fbf00000000, 0x3483053f00000000, + 0xa718cc4000000000, 0x1fdcc1e400000000, 0x8c47089b00000000, + 0x39eb521b00000000, 0xaa709b6400000000, 0x0864388800000000, + 0x9bfff1f700000000, 0x2e53ab7700000000, 0xbdc8620800000000, + 0x050c6fac00000000, 0x9697a6d300000000, 0x233bfc5300000000, + 0xb0a0352c00000000, 0x0f5201a800000000, 0x9cc9c8d700000000, + 0x2965925700000000, 0xbafe5b2800000000, 0x023a568c00000000, + 0x91a19ff300000000, 0x240dc57300000000, 0xb7960c0c00000000, + 0x1582afe000000000, 0x8619669f00000000, 0x33b53c1f00000000, + 0xa02ef56000000000, 0x18eaf8c400000000, 0x8b7131bb00000000, + 0x3edd6b3b00000000, 0xad46a24400000000, 0x3bf25c3900000000, + 0xa869954600000000, 0x1dc5cfc600000000, 0x8e5e06b900000000, + 0x369a0b1d00000000, 0xa501c26200000000, 0x10ad98e200000000, + 0x8336519d00000000, 0x2122f27100000000, 0xb2b93b0e00000000, + 0x0715618e00000000, 0x948ea8f100000000, 0x2c4aa55500000000, + 0xbfd16c2a00000000, 0x0a7d36aa00000000, 0x99e6ffd500000000, + 0x359e2e7900000000, 0xa605e70600000000, 0x13a9bd8600000000, + 0x803274f900000000, 0x38f6795d00000000, 0xab6db02200000000, + 0x1ec1eaa200000000, 0x8d5a23dd00000000, 0x2f4e803100000000, + 0xbcd5494e00000000, 0x097913ce00000000, 0x9ae2dab100000000, + 0x2226d71500000000, 0xb1bd1e6a00000000, 0x041144ea00000000, + 0x978a8d9500000000, 0x013e73e800000000, 0x92a5ba9700000000, + 0x2709e01700000000, 0xb492296800000000, 0x0c5624cc00000000, + 0x9fcdedb300000000, 0x2a61b73300000000, 0xb9fa7e4c00000000, + 0x1beedda000000000, 0x887514df00000000, 0x3dd94e5f00000000, + 0xae42872000000000, 0x16868a8400000000, 0x851d43fb00000000, + 0x30b1197b00000000, 0xa32ad00400000000, 0x1cd8e48000000000, + 0x8f432dff00000000, 0x3aef777f00000000, 0xa974be0000000000, + 0x11b0b3a400000000, 0x822b7adb00000000, 0x3787205b00000000, + 0xa41ce92400000000, 0x06084ac800000000, 0x959383b700000000, + 0x203fd93700000000, 0xb3a4104800000000, 0x0b601dec00000000, + 0x98fbd49300000000, 0x2d578e1300000000, 0xbecc476c00000000, + 0x2878b91100000000, 0xbbe3706e00000000, 0x0e4f2aee00000000, + 0x9dd4e39100000000, 0x2510ee3500000000, 0xb68b274a00000000, + 0x03277dca00000000, 0x90bcb4b500000000, 0x32a8175900000000, + 0xa133de2600000000, 0x149f84a600000000, 0x87044dd900000000, + 0x3fc0407d00000000, 0xac5b890200000000, 0x19f7d38200000000, + 0x8a6c1afd00000000}, + {0x0000000000000000, 0x650b796900000000, 0xca16f2d200000000, + 0xaf1d8bbb00000000, 0xd52b957e00000000, 0xb020ec1700000000, + 0x1f3d67ac00000000, 0x7a361ec500000000, 0xaa572afd00000000, + 0xcf5c539400000000, 0x6041d82f00000000, 0x054aa14600000000, + 0x7f7cbf8300000000, 0x1a77c6ea00000000, 0xb56a4d5100000000, + 0xd061343800000000, 0x15a9252100000000, 0x70a25c4800000000, + 0xdfbfd7f300000000, 0xbab4ae9a00000000, 0xc082b05f00000000, + 0xa589c93600000000, 0x0a94428d00000000, 0x6f9f3be400000000, + 0xbffe0fdc00000000, 0xdaf576b500000000, 0x75e8fd0e00000000, + 0x10e3846700000000, 0x6ad59aa200000000, 0x0fdee3cb00000000, + 0xa0c3687000000000, 0xc5c8111900000000, 0x2a524b4200000000, + 0x4f59322b00000000, 0xe044b99000000000, 0x854fc0f900000000, + 0xff79de3c00000000, 0x9a72a75500000000, 0x356f2cee00000000, + 0x5064558700000000, 0x800561bf00000000, 0xe50e18d600000000, + 0x4a13936d00000000, 0x2f18ea0400000000, 0x552ef4c100000000, + 0x30258da800000000, 0x9f38061300000000, 0xfa337f7a00000000, + 0x3ffb6e6300000000, 0x5af0170a00000000, 0xf5ed9cb100000000, + 0x90e6e5d800000000, 0xead0fb1d00000000, 0x8fdb827400000000, + 0x20c609cf00000000, 0x45cd70a600000000, 0x95ac449e00000000, + 0xf0a73df700000000, 0x5fbab64c00000000, 0x3ab1cf2500000000, + 0x4087d1e000000000, 0x258ca88900000000, 0x8a91233200000000, + 0xef9a5a5b00000000, 0x54a4968400000000, 0x31afefed00000000, + 0x9eb2645600000000, 0xfbb91d3f00000000, 0x818f03fa00000000, + 0xe4847a9300000000, 0x4b99f12800000000, 0x2e92884100000000, + 0xfef3bc7900000000, 0x9bf8c51000000000, 0x34e54eab00000000, + 0x51ee37c200000000, 0x2bd8290700000000, 0x4ed3506e00000000, + 0xe1cedbd500000000, 0x84c5a2bc00000000, 0x410db3a500000000, + 0x2406cacc00000000, 0x8b1b417700000000, 0xee10381e00000000, + 0x942626db00000000, 0xf12d5fb200000000, 0x5e30d40900000000, + 0x3b3bad6000000000, 0xeb5a995800000000, 0x8e51e03100000000, + 0x214c6b8a00000000, 0x444712e300000000, 0x3e710c2600000000, + 0x5b7a754f00000000, 0xf467fef400000000, 0x916c879d00000000, + 0x7ef6ddc600000000, 0x1bfda4af00000000, 0xb4e02f1400000000, + 0xd1eb567d00000000, 0xabdd48b800000000, 0xced631d100000000, + 0x61cbba6a00000000, 0x04c0c30300000000, 0xd4a1f73b00000000, + 0xb1aa8e5200000000, 0x1eb705e900000000, 0x7bbc7c8000000000, + 0x018a624500000000, 0x64811b2c00000000, 0xcb9c909700000000, + 0xae97e9fe00000000, 0x6b5ff8e700000000, 0x0e54818e00000000, + 0xa1490a3500000000, 0xc442735c00000000, 0xbe746d9900000000, + 0xdb7f14f000000000, 0x74629f4b00000000, 0x1169e62200000000, + 0xc108d21a00000000, 0xa403ab7300000000, 0x0b1e20c800000000, + 0x6e1559a100000000, 0x1423476400000000, 0x71283e0d00000000, + 0xde35b5b600000000, 0xbb3eccdf00000000, 0xe94e5cd200000000, + 0x8c4525bb00000000, 0x2358ae0000000000, 0x4653d76900000000, + 0x3c65c9ac00000000, 0x596eb0c500000000, 0xf6733b7e00000000, + 0x9378421700000000, 0x4319762f00000000, 0x26120f4600000000, + 0x890f84fd00000000, 0xec04fd9400000000, 0x9632e35100000000, + 0xf3399a3800000000, 0x5c24118300000000, 0x392f68ea00000000, + 0xfce779f300000000, 0x99ec009a00000000, 0x36f18b2100000000, + 0x53faf24800000000, 0x29ccec8d00000000, 0x4cc795e400000000, + 0xe3da1e5f00000000, 0x86d1673600000000, 0x56b0530e00000000, + 0x33bb2a6700000000, 0x9ca6a1dc00000000, 0xf9add8b500000000, + 0x839bc67000000000, 0xe690bf1900000000, 0x498d34a200000000, + 0x2c864dcb00000000, 0xc31c179000000000, 0xa6176ef900000000, + 0x090ae54200000000, 0x6c019c2b00000000, 0x163782ee00000000, + 0x733cfb8700000000, 0xdc21703c00000000, 0xb92a095500000000, + 0x694b3d6d00000000, 0x0c40440400000000, 0xa35dcfbf00000000, + 0xc656b6d600000000, 0xbc60a81300000000, 0xd96bd17a00000000, + 0x76765ac100000000, 0x137d23a800000000, 0xd6b532b100000000, + 0xb3be4bd800000000, 0x1ca3c06300000000, 0x79a8b90a00000000, + 0x039ea7cf00000000, 0x6695dea600000000, 0xc988551d00000000, + 0xac832c7400000000, 0x7ce2184c00000000, 0x19e9612500000000, + 0xb6f4ea9e00000000, 0xd3ff93f700000000, 0xa9c98d3200000000, + 0xccc2f45b00000000, 0x63df7fe000000000, 0x06d4068900000000, + 0xbdeaca5600000000, 0xd8e1b33f00000000, 0x77fc388400000000, + 0x12f741ed00000000, 0x68c15f2800000000, 0x0dca264100000000, + 0xa2d7adfa00000000, 0xc7dcd49300000000, 0x17bde0ab00000000, + 0x72b699c200000000, 0xddab127900000000, 0xb8a06b1000000000, + 0xc29675d500000000, 0xa79d0cbc00000000, 0x0880870700000000, + 0x6d8bfe6e00000000, 0xa843ef7700000000, 0xcd48961e00000000, + 0x62551da500000000, 0x075e64cc00000000, 0x7d687a0900000000, + 0x1863036000000000, 0xb77e88db00000000, 0xd275f1b200000000, + 0x0214c58a00000000, 0x671fbce300000000, 0xc802375800000000, + 0xad094e3100000000, 0xd73f50f400000000, 0xb234299d00000000, + 0x1d29a22600000000, 0x7822db4f00000000, 0x97b8811400000000, + 0xf2b3f87d00000000, 0x5dae73c600000000, 0x38a50aaf00000000, + 0x4293146a00000000, 0x27986d0300000000, 0x8885e6b800000000, + 0xed8e9fd100000000, 0x3defabe900000000, 0x58e4d28000000000, + 0xf7f9593b00000000, 0x92f2205200000000, 0xe8c43e9700000000, + 0x8dcf47fe00000000, 0x22d2cc4500000000, 0x47d9b52c00000000, + 0x8211a43500000000, 0xe71add5c00000000, 0x480756e700000000, + 0x2d0c2f8e00000000, 0x573a314b00000000, 0x3231482200000000, + 0x9d2cc39900000000, 0xf827baf000000000, 0x28468ec800000000, + 0x4d4df7a100000000, 0xe2507c1a00000000, 0x875b057300000000, + 0xfd6d1bb600000000, 0x986662df00000000, 0x377be96400000000, + 0x5270900d00000000}, + {0x0000000000000000, 0xdcecb13d00000000, 0xb8d9637b00000000, + 0x6435d24600000000, 0x70b3c7f600000000, 0xac5f76cb00000000, + 0xc86aa48d00000000, 0x148615b000000000, 0xa160fe3600000000, + 0x7d8c4f0b00000000, 0x19b99d4d00000000, 0xc5552c7000000000, + 0xd1d339c000000000, 0x0d3f88fd00000000, 0x690a5abb00000000, + 0xb5e6eb8600000000, 0x42c1fc6d00000000, 0x9e2d4d5000000000, + 0xfa189f1600000000, 0x26f42e2b00000000, 0x32723b9b00000000, + 0xee9e8aa600000000, 0x8aab58e000000000, 0x5647e9dd00000000, + 0xe3a1025b00000000, 0x3f4db36600000000, 0x5b78612000000000, + 0x8794d01d00000000, 0x9312c5ad00000000, 0x4ffe749000000000, + 0x2bcba6d600000000, 0xf72717eb00000000, 0x8482f9db00000000, + 0x586e48e600000000, 0x3c5b9aa000000000, 0xe0b72b9d00000000, + 0xf4313e2d00000000, 0x28dd8f1000000000, 0x4ce85d5600000000, + 0x9004ec6b00000000, 0x25e207ed00000000, 0xf90eb6d000000000, + 0x9d3b649600000000, 0x41d7d5ab00000000, 0x5551c01b00000000, + 0x89bd712600000000, 0xed88a36000000000, 0x3164125d00000000, + 0xc64305b600000000, 0x1aafb48b00000000, 0x7e9a66cd00000000, + 0xa276d7f000000000, 0xb6f0c24000000000, 0x6a1c737d00000000, + 0x0e29a13b00000000, 0xd2c5100600000000, 0x6723fb8000000000, + 0xbbcf4abd00000000, 0xdffa98fb00000000, 0x031629c600000000, + 0x17903c7600000000, 0xcb7c8d4b00000000, 0xaf495f0d00000000, + 0x73a5ee3000000000, 0x4903826c00000000, 0x95ef335100000000, + 0xf1dae11700000000, 0x2d36502a00000000, 0x39b0459a00000000, + 0xe55cf4a700000000, 0x816926e100000000, 0x5d8597dc00000000, + 0xe8637c5a00000000, 0x348fcd6700000000, 0x50ba1f2100000000, + 0x8c56ae1c00000000, 0x98d0bbac00000000, 0x443c0a9100000000, + 0x2009d8d700000000, 0xfce569ea00000000, 0x0bc27e0100000000, + 0xd72ecf3c00000000, 0xb31b1d7a00000000, 0x6ff7ac4700000000, + 0x7b71b9f700000000, 0xa79d08ca00000000, 0xc3a8da8c00000000, + 0x1f446bb100000000, 0xaaa2803700000000, 0x764e310a00000000, + 0x127be34c00000000, 0xce97527100000000, 0xda1147c100000000, + 0x06fdf6fc00000000, 0x62c824ba00000000, 0xbe24958700000000, + 0xcd817bb700000000, 0x116dca8a00000000, 0x755818cc00000000, + 0xa9b4a9f100000000, 0xbd32bc4100000000, 0x61de0d7c00000000, + 0x05ebdf3a00000000, 0xd9076e0700000000, 0x6ce1858100000000, + 0xb00d34bc00000000, 0xd438e6fa00000000, 0x08d457c700000000, + 0x1c52427700000000, 0xc0bef34a00000000, 0xa48b210c00000000, + 0x7867903100000000, 0x8f4087da00000000, 0x53ac36e700000000, + 0x3799e4a100000000, 0xeb75559c00000000, 0xfff3402c00000000, + 0x231ff11100000000, 0x472a235700000000, 0x9bc6926a00000000, + 0x2e2079ec00000000, 0xf2ccc8d100000000, 0x96f91a9700000000, + 0x4a15abaa00000000, 0x5e93be1a00000000, 0x827f0f2700000000, + 0xe64add6100000000, 0x3aa66c5c00000000, 0x920604d900000000, + 0x4eeab5e400000000, 0x2adf67a200000000, 0xf633d69f00000000, + 0xe2b5c32f00000000, 0x3e59721200000000, 0x5a6ca05400000000, + 0x8680116900000000, 0x3366faef00000000, 0xef8a4bd200000000, + 0x8bbf999400000000, 0x575328a900000000, 0x43d53d1900000000, + 0x9f398c2400000000, 0xfb0c5e6200000000, 0x27e0ef5f00000000, + 0xd0c7f8b400000000, 0x0c2b498900000000, 0x681e9bcf00000000, + 0xb4f22af200000000, 0xa0743f4200000000, 0x7c988e7f00000000, + 0x18ad5c3900000000, 0xc441ed0400000000, 0x71a7068200000000, + 0xad4bb7bf00000000, 0xc97e65f900000000, 0x1592d4c400000000, + 0x0114c17400000000, 0xddf8704900000000, 0xb9cda20f00000000, + 0x6521133200000000, 0x1684fd0200000000, 0xca684c3f00000000, + 0xae5d9e7900000000, 0x72b12f4400000000, 0x66373af400000000, + 0xbadb8bc900000000, 0xdeee598f00000000, 0x0202e8b200000000, + 0xb7e4033400000000, 0x6b08b20900000000, 0x0f3d604f00000000, + 0xd3d1d17200000000, 0xc757c4c200000000, 0x1bbb75ff00000000, + 0x7f8ea7b900000000, 0xa362168400000000, 0x5445016f00000000, + 0x88a9b05200000000, 0xec9c621400000000, 0x3070d32900000000, + 0x24f6c69900000000, 0xf81a77a400000000, 0x9c2fa5e200000000, + 0x40c314df00000000, 0xf525ff5900000000, 0x29c94e6400000000, + 0x4dfc9c2200000000, 0x91102d1f00000000, 0x859638af00000000, + 0x597a899200000000, 0x3d4f5bd400000000, 0xe1a3eae900000000, + 0xdb0586b500000000, 0x07e9378800000000, 0x63dce5ce00000000, + 0xbf3054f300000000, 0xabb6414300000000, 0x775af07e00000000, + 0x136f223800000000, 0xcf83930500000000, 0x7a65788300000000, + 0xa689c9be00000000, 0xc2bc1bf800000000, 0x1e50aac500000000, + 0x0ad6bf7500000000, 0xd63a0e4800000000, 0xb20fdc0e00000000, + 0x6ee36d3300000000, 0x99c47ad800000000, 0x4528cbe500000000, + 0x211d19a300000000, 0xfdf1a89e00000000, 0xe977bd2e00000000, + 0x359b0c1300000000, 0x51aede5500000000, 0x8d426f6800000000, + 0x38a484ee00000000, 0xe44835d300000000, 0x807de79500000000, + 0x5c9156a800000000, 0x4817431800000000, 0x94fbf22500000000, + 0xf0ce206300000000, 0x2c22915e00000000, 0x5f877f6e00000000, + 0x836bce5300000000, 0xe75e1c1500000000, 0x3bb2ad2800000000, + 0x2f34b89800000000, 0xf3d809a500000000, 0x97eddbe300000000, + 0x4b016ade00000000, 0xfee7815800000000, 0x220b306500000000, + 0x463ee22300000000, 0x9ad2531e00000000, 0x8e5446ae00000000, + 0x52b8f79300000000, 0x368d25d500000000, 0xea6194e800000000, + 0x1d46830300000000, 0xc1aa323e00000000, 0xa59fe07800000000, + 0x7973514500000000, 0x6df544f500000000, 0xb119f5c800000000, + 0xd52c278e00000000, 0x09c096b300000000, 0xbc267d3500000000, + 0x60cacc0800000000, 0x04ff1e4e00000000, 0xd813af7300000000, + 0xcc95bac300000000, 0x10790bfe00000000, 0x744cd9b800000000, + 0xa8a0688500000000}}; + +#else /* W == 4 */ + +local const z_crc_t FAR crc_braid_table[][256] = { + {0x00000000, 0x81256527, 0xd93bcc0f, 0x581ea928, 0x69069e5f, + 0xe823fb78, 0xb03d5250, 0x31183777, 0xd20d3cbe, 0x53285999, + 0x0b36f0b1, 0x8a139596, 0xbb0ba2e1, 0x3a2ec7c6, 0x62306eee, + 0xe3150bc9, 0x7f6b7f3d, 0xfe4e1a1a, 0xa650b332, 0x2775d615, + 0x166de162, 0x97488445, 0xcf562d6d, 0x4e73484a, 0xad664383, + 0x2c4326a4, 0x745d8f8c, 0xf578eaab, 0xc460dddc, 0x4545b8fb, + 0x1d5b11d3, 0x9c7e74f4, 0xfed6fe7a, 0x7ff39b5d, 0x27ed3275, + 0xa6c85752, 0x97d06025, 0x16f50502, 0x4eebac2a, 0xcfcec90d, + 0x2cdbc2c4, 0xadfea7e3, 0xf5e00ecb, 0x74c56bec, 0x45dd5c9b, + 0xc4f839bc, 0x9ce69094, 0x1dc3f5b3, 0x81bd8147, 0x0098e460, + 0x58864d48, 0xd9a3286f, 0xe8bb1f18, 0x699e7a3f, 0x3180d317, + 0xb0a5b630, 0x53b0bdf9, 0xd295d8de, 0x8a8b71f6, 0x0bae14d1, + 0x3ab623a6, 0xbb934681, 0xe38defa9, 0x62a88a8e, 0x26dcfab5, + 0xa7f99f92, 0xffe736ba, 0x7ec2539d, 0x4fda64ea, 0xceff01cd, + 0x96e1a8e5, 0x17c4cdc2, 0xf4d1c60b, 0x75f4a32c, 0x2dea0a04, + 0xaccf6f23, 0x9dd75854, 0x1cf23d73, 0x44ec945b, 0xc5c9f17c, + 0x59b78588, 0xd892e0af, 0x808c4987, 0x01a92ca0, 0x30b11bd7, + 0xb1947ef0, 0xe98ad7d8, 0x68afb2ff, 0x8bbab936, 0x0a9fdc11, + 0x52817539, 0xd3a4101e, 0xe2bc2769, 0x6399424e, 0x3b87eb66, + 0xbaa28e41, 0xd80a04cf, 0x592f61e8, 0x0131c8c0, 0x8014ade7, + 0xb10c9a90, 0x3029ffb7, 0x6837569f, 0xe91233b8, 0x0a073871, + 0x8b225d56, 0xd33cf47e, 0x52199159, 0x6301a62e, 0xe224c309, + 0xba3a6a21, 0x3b1f0f06, 0xa7617bf2, 0x26441ed5, 0x7e5ab7fd, + 0xff7fd2da, 0xce67e5ad, 0x4f42808a, 0x175c29a2, 0x96794c85, + 0x756c474c, 0xf449226b, 0xac578b43, 0x2d72ee64, 0x1c6ad913, + 0x9d4fbc34, 0xc551151c, 0x4474703b, 0x4db9f56a, 0xcc9c904d, + 0x94823965, 0x15a75c42, 0x24bf6b35, 0xa59a0e12, 0xfd84a73a, + 0x7ca1c21d, 0x9fb4c9d4, 0x1e91acf3, 0x468f05db, 0xc7aa60fc, + 0xf6b2578b, 0x779732ac, 0x2f899b84, 0xaeacfea3, 0x32d28a57, + 0xb3f7ef70, 0xebe94658, 0x6acc237f, 0x5bd41408, 0xdaf1712f, + 0x82efd807, 0x03cabd20, 0xe0dfb6e9, 0x61fad3ce, 0x39e47ae6, + 0xb8c11fc1, 0x89d928b6, 0x08fc4d91, 0x50e2e4b9, 0xd1c7819e, + 0xb36f0b10, 0x324a6e37, 0x6a54c71f, 0xeb71a238, 0xda69954f, + 0x5b4cf068, 0x03525940, 0x82773c67, 0x616237ae, 0xe0475289, + 0xb859fba1, 0x397c9e86, 0x0864a9f1, 0x8941ccd6, 0xd15f65fe, + 0x507a00d9, 0xcc04742d, 0x4d21110a, 0x153fb822, 0x941add05, + 0xa502ea72, 0x24278f55, 0x7c39267d, 0xfd1c435a, 0x1e094893, + 0x9f2c2db4, 0xc732849c, 0x4617e1bb, 0x770fd6cc, 0xf62ab3eb, + 0xae341ac3, 0x2f117fe4, 0x6b650fdf, 0xea406af8, 0xb25ec3d0, + 0x337ba6f7, 0x02639180, 0x8346f4a7, 0xdb585d8f, 0x5a7d38a8, + 0xb9683361, 0x384d5646, 0x6053ff6e, 0xe1769a49, 0xd06ead3e, + 0x514bc819, 0x09556131, 0x88700416, 0x140e70e2, 0x952b15c5, + 0xcd35bced, 0x4c10d9ca, 0x7d08eebd, 0xfc2d8b9a, 0xa43322b2, + 0x25164795, 0xc6034c5c, 0x4726297b, 0x1f388053, 0x9e1de574, + 0xaf05d203, 0x2e20b724, 0x763e1e0c, 0xf71b7b2b, 0x95b3f1a5, + 0x14969482, 0x4c883daa, 0xcdad588d, 0xfcb56ffa, 0x7d900add, + 0x258ea3f5, 0xa4abc6d2, 0x47becd1b, 0xc69ba83c, 0x9e850114, + 0x1fa06433, 0x2eb85344, 0xaf9d3663, 0xf7839f4b, 0x76a6fa6c, + 0xead88e98, 0x6bfdebbf, 0x33e34297, 0xb2c627b0, 0x83de10c7, + 0x02fb75e0, 0x5ae5dcc8, 0xdbc0b9ef, 0x38d5b226, 0xb9f0d701, + 0xe1ee7e29, 0x60cb1b0e, 0x51d32c79, 0xd0f6495e, 0x88e8e076, + 0x09cd8551}, + {0x00000000, 0x9b73ead4, 0xed96d3e9, 0x76e5393d, 0x005ca193, + 0x9b2f4b47, 0xedca727a, 0x76b998ae, 0x00b94326, 0x9bcaa9f2, + 0xed2f90cf, 0x765c7a1b, 0x00e5e2b5, 0x9b960861, 0xed73315c, + 0x7600db88, 0x0172864c, 0x9a016c98, 0xece455a5, 0x7797bf71, + 0x012e27df, 0x9a5dcd0b, 0xecb8f436, 0x77cb1ee2, 0x01cbc56a, + 0x9ab82fbe, 0xec5d1683, 0x772efc57, 0x019764f9, 0x9ae48e2d, + 0xec01b710, 0x77725dc4, 0x02e50c98, 0x9996e64c, 0xef73df71, + 0x740035a5, 0x02b9ad0b, 0x99ca47df, 0xef2f7ee2, 0x745c9436, + 0x025c4fbe, 0x992fa56a, 0xefca9c57, 0x74b97683, 0x0200ee2d, + 0x997304f9, 0xef963dc4, 0x74e5d710, 0x03978ad4, 0x98e46000, + 0xee01593d, 0x7572b3e9, 0x03cb2b47, 0x98b8c193, 0xee5df8ae, + 0x752e127a, 0x032ec9f2, 0x985d2326, 0xeeb81a1b, 0x75cbf0cf, + 0x03726861, 0x980182b5, 0xeee4bb88, 0x7597515c, 0x05ca1930, + 0x9eb9f3e4, 0xe85ccad9, 0x732f200d, 0x0596b8a3, 0x9ee55277, + 0xe8006b4a, 0x7373819e, 0x05735a16, 0x9e00b0c2, 0xe8e589ff, + 0x7396632b, 0x052ffb85, 0x9e5c1151, 0xe8b9286c, 0x73cac2b8, + 0x04b89f7c, 0x9fcb75a8, 0xe92e4c95, 0x725da641, 0x04e43eef, + 0x9f97d43b, 0xe972ed06, 0x720107d2, 0x0401dc5a, 0x9f72368e, + 0xe9970fb3, 0x72e4e567, 0x045d7dc9, 0x9f2e971d, 0xe9cbae20, + 0x72b844f4, 0x072f15a8, 0x9c5cff7c, 0xeab9c641, 0x71ca2c95, + 0x0773b43b, 0x9c005eef, 0xeae567d2, 0x71968d06, 0x0796568e, + 0x9ce5bc5a, 0xea008567, 0x71736fb3, 0x07caf71d, 0x9cb91dc9, + 0xea5c24f4, 0x712fce20, 0x065d93e4, 0x9d2e7930, 0xebcb400d, + 0x70b8aad9, 0x06013277, 0x9d72d8a3, 0xeb97e19e, 0x70e40b4a, + 0x06e4d0c2, 0x9d973a16, 0xeb72032b, 0x7001e9ff, 0x06b87151, + 0x9dcb9b85, 0xeb2ea2b8, 0x705d486c, 0x0b943260, 0x90e7d8b4, + 0xe602e189, 0x7d710b5d, 0x0bc893f3, 0x90bb7927, 0xe65e401a, + 0x7d2daace, 0x0b2d7146, 0x905e9b92, 0xe6bba2af, 0x7dc8487b, + 0x0b71d0d5, 0x90023a01, 0xe6e7033c, 0x7d94e9e8, 0x0ae6b42c, + 0x91955ef8, 0xe77067c5, 0x7c038d11, 0x0aba15bf, 0x91c9ff6b, + 0xe72cc656, 0x7c5f2c82, 0x0a5ff70a, 0x912c1dde, 0xe7c924e3, + 0x7cbace37, 0x0a035699, 0x9170bc4d, 0xe7958570, 0x7ce66fa4, + 0x09713ef8, 0x9202d42c, 0xe4e7ed11, 0x7f9407c5, 0x092d9f6b, + 0x925e75bf, 0xe4bb4c82, 0x7fc8a656, 0x09c87dde, 0x92bb970a, + 0xe45eae37, 0x7f2d44e3, 0x0994dc4d, 0x92e73699, 0xe4020fa4, + 0x7f71e570, 0x0803b8b4, 0x93705260, 0xe5956b5d, 0x7ee68189, + 0x085f1927, 0x932cf3f3, 0xe5c9cace, 0x7eba201a, 0x08bafb92, + 0x93c91146, 0xe52c287b, 0x7e5fc2af, 0x08e65a01, 0x9395b0d5, + 0xe57089e8, 0x7e03633c, 0x0e5e2b50, 0x952dc184, 0xe3c8f8b9, + 0x78bb126d, 0x0e028ac3, 0x95716017, 0xe394592a, 0x78e7b3fe, + 0x0ee76876, 0x959482a2, 0xe371bb9f, 0x7802514b, 0x0ebbc9e5, + 0x95c82331, 0xe32d1a0c, 0x785ef0d8, 0x0f2cad1c, 0x945f47c8, + 0xe2ba7ef5, 0x79c99421, 0x0f700c8f, 0x9403e65b, 0xe2e6df66, + 0x799535b2, 0x0f95ee3a, 0x94e604ee, 0xe2033dd3, 0x7970d707, + 0x0fc94fa9, 0x94baa57d, 0xe25f9c40, 0x792c7694, 0x0cbb27c8, + 0x97c8cd1c, 0xe12df421, 0x7a5e1ef5, 0x0ce7865b, 0x97946c8f, + 0xe17155b2, 0x7a02bf66, 0x0c0264ee, 0x97718e3a, 0xe194b707, + 0x7ae75dd3, 0x0c5ec57d, 0x972d2fa9, 0xe1c81694, 0x7abbfc40, + 0x0dc9a184, 0x96ba4b50, 0xe05f726d, 0x7b2c98b9, 0x0d950017, + 0x96e6eac3, 0xe003d3fe, 0x7b70392a, 0x0d70e2a2, 0x96030876, + 0xe0e6314b, 0x7b95db9f, 0x0d2c4331, 0x965fa9e5, 0xe0ba90d8, + 0x7bc97a0c}, + {0x00000000, 0x172864c0, 0x2e50c980, 0x3978ad40, 0x5ca19300, + 0x4b89f7c0, 0x72f15a80, 0x65d93e40, 0xb9432600, 0xae6b42c0, + 0x9713ef80, 0x803b8b40, 0xe5e2b500, 0xf2cad1c0, 0xcbb27c80, + 0xdc9a1840, 0xa9f74a41, 0xbedf2e81, 0x87a783c1, 0x908fe701, + 0xf556d941, 0xe27ebd81, 0xdb0610c1, 0xcc2e7401, 0x10b46c41, + 0x079c0881, 0x3ee4a5c1, 0x29ccc101, 0x4c15ff41, 0x5b3d9b81, + 0x624536c1, 0x756d5201, 0x889f92c3, 0x9fb7f603, 0xa6cf5b43, + 0xb1e73f83, 0xd43e01c3, 0xc3166503, 0xfa6ec843, 0xed46ac83, + 0x31dcb4c3, 0x26f4d003, 0x1f8c7d43, 0x08a41983, 0x6d7d27c3, + 0x7a554303, 0x432dee43, 0x54058a83, 0x2168d882, 0x3640bc42, + 0x0f381102, 0x181075c2, 0x7dc94b82, 0x6ae12f42, 0x53998202, + 0x44b1e6c2, 0x982bfe82, 0x8f039a42, 0xb67b3702, 0xa15353c2, + 0xc48a6d82, 0xd3a20942, 0xeadaa402, 0xfdf2c0c2, 0xca4e23c7, + 0xdd664707, 0xe41eea47, 0xf3368e87, 0x96efb0c7, 0x81c7d407, + 0xb8bf7947, 0xaf971d87, 0x730d05c7, 0x64256107, 0x5d5dcc47, + 0x4a75a887, 0x2fac96c7, 0x3884f207, 0x01fc5f47, 0x16d43b87, + 0x63b96986, 0x74910d46, 0x4de9a006, 0x5ac1c4c6, 0x3f18fa86, + 0x28309e46, 0x11483306, 0x066057c6, 0xdafa4f86, 0xcdd22b46, + 0xf4aa8606, 0xe382e2c6, 0x865bdc86, 0x9173b846, 0xa80b1506, + 0xbf2371c6, 0x42d1b104, 0x55f9d5c4, 0x6c817884, 0x7ba91c44, + 0x1e702204, 0x095846c4, 0x3020eb84, 0x27088f44, 0xfb929704, + 0xecbaf3c4, 0xd5c25e84, 0xc2ea3a44, 0xa7330404, 0xb01b60c4, + 0x8963cd84, 0x9e4ba944, 0xeb26fb45, 0xfc0e9f85, 0xc57632c5, + 0xd25e5605, 0xb7876845, 0xa0af0c85, 0x99d7a1c5, 0x8effc505, + 0x5265dd45, 0x454db985, 0x7c3514c5, 0x6b1d7005, 0x0ec44e45, + 0x19ec2a85, 0x209487c5, 0x37bce305, 0x4fed41cf, 0x58c5250f, + 0x61bd884f, 0x7695ec8f, 0x134cd2cf, 0x0464b60f, 0x3d1c1b4f, + 0x2a347f8f, 0xf6ae67cf, 0xe186030f, 0xd8feae4f, 0xcfd6ca8f, + 0xaa0ff4cf, 0xbd27900f, 0x845f3d4f, 0x9377598f, 0xe61a0b8e, + 0xf1326f4e, 0xc84ac20e, 0xdf62a6ce, 0xbabb988e, 0xad93fc4e, + 0x94eb510e, 0x83c335ce, 0x5f592d8e, 0x4871494e, 0x7109e40e, + 0x662180ce, 0x03f8be8e, 0x14d0da4e, 0x2da8770e, 0x3a8013ce, + 0xc772d30c, 0xd05ab7cc, 0xe9221a8c, 0xfe0a7e4c, 0x9bd3400c, + 0x8cfb24cc, 0xb583898c, 0xa2abed4c, 0x7e31f50c, 0x691991cc, + 0x50613c8c, 0x4749584c, 0x2290660c, 0x35b802cc, 0x0cc0af8c, + 0x1be8cb4c, 0x6e85994d, 0x79adfd8d, 0x40d550cd, 0x57fd340d, + 0x32240a4d, 0x250c6e8d, 0x1c74c3cd, 0x0b5ca70d, 0xd7c6bf4d, + 0xc0eedb8d, 0xf99676cd, 0xeebe120d, 0x8b672c4d, 0x9c4f488d, + 0xa537e5cd, 0xb21f810d, 0x85a36208, 0x928b06c8, 0xabf3ab88, + 0xbcdbcf48, 0xd902f108, 0xce2a95c8, 0xf7523888, 0xe07a5c48, + 0x3ce04408, 0x2bc820c8, 0x12b08d88, 0x0598e948, 0x6041d708, + 0x7769b3c8, 0x4e111e88, 0x59397a48, 0x2c542849, 0x3b7c4c89, + 0x0204e1c9, 0x152c8509, 0x70f5bb49, 0x67dddf89, 0x5ea572c9, + 0x498d1609, 0x95170e49, 0x823f6a89, 0xbb47c7c9, 0xac6fa309, + 0xc9b69d49, 0xde9ef989, 0xe7e654c9, 0xf0ce3009, 0x0d3cf0cb, + 0x1a14940b, 0x236c394b, 0x34445d8b, 0x519d63cb, 0x46b5070b, + 0x7fcdaa4b, 0x68e5ce8b, 0xb47fd6cb, 0xa357b20b, 0x9a2f1f4b, + 0x8d077b8b, 0xe8de45cb, 0xfff6210b, 0xc68e8c4b, 0xd1a6e88b, + 0xa4cbba8a, 0xb3e3de4a, 0x8a9b730a, 0x9db317ca, 0xf86a298a, + 0xef424d4a, 0xd63ae00a, 0xc11284ca, 0x1d889c8a, 0x0aa0f84a, + 0x33d8550a, 0x24f031ca, 0x41290f8a, 0x56016b4a, 0x6f79c60a, + 0x7851a2ca}, + {0x00000000, 0x9fda839e, 0xe4c4017d, 0x7b1e82e3, 0x12f904bb, + 0x8d238725, 0xf63d05c6, 0x69e78658, 0x25f20976, 0xba288ae8, + 0xc136080b, 0x5eec8b95, 0x370b0dcd, 0xa8d18e53, 0xd3cf0cb0, + 0x4c158f2e, 0x4be412ec, 0xd43e9172, 0xaf201391, 0x30fa900f, + 0x591d1657, 0xc6c795c9, 0xbdd9172a, 0x220394b4, 0x6e161b9a, + 0xf1cc9804, 0x8ad21ae7, 0x15089979, 0x7cef1f21, 0xe3359cbf, + 0x982b1e5c, 0x07f19dc2, 0x97c825d8, 0x0812a646, 0x730c24a5, + 0xecd6a73b, 0x85312163, 0x1aeba2fd, 0x61f5201e, 0xfe2fa380, + 0xb23a2cae, 0x2de0af30, 0x56fe2dd3, 0xc924ae4d, 0xa0c32815, + 0x3f19ab8b, 0x44072968, 0xdbddaaf6, 0xdc2c3734, 0x43f6b4aa, + 0x38e83649, 0xa732b5d7, 0xced5338f, 0x510fb011, 0x2a1132f2, + 0xb5cbb16c, 0xf9de3e42, 0x6604bddc, 0x1d1a3f3f, 0x82c0bca1, + 0xeb273af9, 0x74fdb967, 0x0fe33b84, 0x9039b81a, 0xf4e14df1, + 0x6b3bce6f, 0x10254c8c, 0x8fffcf12, 0xe618494a, 0x79c2cad4, + 0x02dc4837, 0x9d06cba9, 0xd1134487, 0x4ec9c719, 0x35d745fa, + 0xaa0dc664, 0xc3ea403c, 0x5c30c3a2, 0x272e4141, 0xb8f4c2df, + 0xbf055f1d, 0x20dfdc83, 0x5bc15e60, 0xc41bddfe, 0xadfc5ba6, + 0x3226d838, 0x49385adb, 0xd6e2d945, 0x9af7566b, 0x052dd5f5, + 0x7e335716, 0xe1e9d488, 0x880e52d0, 0x17d4d14e, 0x6cca53ad, + 0xf310d033, 0x63296829, 0xfcf3ebb7, 0x87ed6954, 0x1837eaca, + 0x71d06c92, 0xee0aef0c, 0x95146def, 0x0aceee71, 0x46db615f, + 0xd901e2c1, 0xa21f6022, 0x3dc5e3bc, 0x542265e4, 0xcbf8e67a, + 0xb0e66499, 0x2f3ce707, 0x28cd7ac5, 0xb717f95b, 0xcc097bb8, + 0x53d3f826, 0x3a347e7e, 0xa5eefde0, 0xdef07f03, 0x412afc9d, + 0x0d3f73b3, 0x92e5f02d, 0xe9fb72ce, 0x7621f150, 0x1fc67708, + 0x801cf496, 0xfb027675, 0x64d8f5eb, 0x32b39da3, 0xad691e3d, + 0xd6779cde, 0x49ad1f40, 0x204a9918, 0xbf901a86, 0xc48e9865, + 0x5b541bfb, 0x174194d5, 0x889b174b, 0xf38595a8, 0x6c5f1636, + 0x05b8906e, 0x9a6213f0, 0xe17c9113, 0x7ea6128d, 0x79578f4f, + 0xe68d0cd1, 0x9d938e32, 0x02490dac, 0x6bae8bf4, 0xf474086a, + 0x8f6a8a89, 0x10b00917, 0x5ca58639, 0xc37f05a7, 0xb8618744, + 0x27bb04da, 0x4e5c8282, 0xd186011c, 0xaa9883ff, 0x35420061, + 0xa57bb87b, 0x3aa13be5, 0x41bfb906, 0xde653a98, 0xb782bcc0, + 0x28583f5e, 0x5346bdbd, 0xcc9c3e23, 0x8089b10d, 0x1f533293, + 0x644db070, 0xfb9733ee, 0x9270b5b6, 0x0daa3628, 0x76b4b4cb, + 0xe96e3755, 0xee9faa97, 0x71452909, 0x0a5babea, 0x95812874, + 0xfc66ae2c, 0x63bc2db2, 0x18a2af51, 0x87782ccf, 0xcb6da3e1, + 0x54b7207f, 0x2fa9a29c, 0xb0732102, 0xd994a75a, 0x464e24c4, + 0x3d50a627, 0xa28a25b9, 0xc652d052, 0x598853cc, 0x2296d12f, + 0xbd4c52b1, 0xd4abd4e9, 0x4b715777, 0x306fd594, 0xafb5560a, + 0xe3a0d924, 0x7c7a5aba, 0x0764d859, 0x98be5bc7, 0xf159dd9f, + 0x6e835e01, 0x159ddce2, 0x8a475f7c, 0x8db6c2be, 0x126c4120, + 0x6972c3c3, 0xf6a8405d, 0x9f4fc605, 0x0095459b, 0x7b8bc778, + 0xe45144e6, 0xa844cbc8, 0x379e4856, 0x4c80cab5, 0xd35a492b, + 0xbabdcf73, 0x25674ced, 0x5e79ce0e, 0xc1a34d90, 0x519af58a, + 0xce407614, 0xb55ef4f7, 0x2a847769, 0x4363f131, 0xdcb972af, + 0xa7a7f04c, 0x387d73d2, 0x7468fcfc, 0xebb27f62, 0x90acfd81, + 0x0f767e1f, 0x6691f847, 0xf94b7bd9, 0x8255f93a, 0x1d8f7aa4, + 0x1a7ee766, 0x85a464f8, 0xfebae61b, 0x61606585, 0x0887e3dd, + 0x975d6043, 0xec43e2a0, 0x7399613e, 0x3f8cee10, 0xa0566d8e, + 0xdb48ef6d, 0x44926cf3, 0x2d75eaab, 0xb2af6935, 0xc9b1ebd6, + 0x566b6848}}; + +local const z_word_t FAR crc_braid_big_table[][256] = { + {0x00000000, 0x9e83da9f, 0x7d01c4e4, 0xe3821e7b, 0xbb04f912, + 0x2587238d, 0xc6053df6, 0x5886e769, 0x7609f225, 0xe88a28ba, + 0x0b0836c1, 0x958bec5e, 0xcd0d0b37, 0x538ed1a8, 0xb00ccfd3, + 0x2e8f154c, 0xec12e44b, 0x72913ed4, 0x911320af, 0x0f90fa30, + 0x57161d59, 0xc995c7c6, 0x2a17d9bd, 0xb4940322, 0x9a1b166e, + 0x0498ccf1, 0xe71ad28a, 0x79990815, 0x211fef7c, 0xbf9c35e3, + 0x5c1e2b98, 0xc29df107, 0xd825c897, 0x46a61208, 0xa5240c73, + 0x3ba7d6ec, 0x63213185, 0xfda2eb1a, 0x1e20f561, 0x80a32ffe, + 0xae2c3ab2, 0x30afe02d, 0xd32dfe56, 0x4dae24c9, 0x1528c3a0, + 0x8bab193f, 0x68290744, 0xf6aadddb, 0x34372cdc, 0xaab4f643, + 0x4936e838, 0xd7b532a7, 0x8f33d5ce, 0x11b00f51, 0xf232112a, + 0x6cb1cbb5, 0x423edef9, 0xdcbd0466, 0x3f3f1a1d, 0xa1bcc082, + 0xf93a27eb, 0x67b9fd74, 0x843be30f, 0x1ab83990, 0xf14de1f4, + 0x6fce3b6b, 0x8c4c2510, 0x12cfff8f, 0x4a4918e6, 0xd4cac279, + 0x3748dc02, 0xa9cb069d, 0x874413d1, 0x19c7c94e, 0xfa45d735, + 0x64c60daa, 0x3c40eac3, 0xa2c3305c, 0x41412e27, 0xdfc2f4b8, + 0x1d5f05bf, 0x83dcdf20, 0x605ec15b, 0xfedd1bc4, 0xa65bfcad, + 0x38d82632, 0xdb5a3849, 0x45d9e2d6, 0x6b56f79a, 0xf5d52d05, + 0x1657337e, 0x88d4e9e1, 0xd0520e88, 0x4ed1d417, 0xad53ca6c, + 0x33d010f3, 0x29682963, 0xb7ebf3fc, 0x5469ed87, 0xcaea3718, + 0x926cd071, 0x0cef0aee, 0xef6d1495, 0x71eece0a, 0x5f61db46, + 0xc1e201d9, 0x22601fa2, 0xbce3c53d, 0xe4652254, 0x7ae6f8cb, + 0x9964e6b0, 0x07e73c2f, 0xc57acd28, 0x5bf917b7, 0xb87b09cc, + 0x26f8d353, 0x7e7e343a, 0xe0fdeea5, 0x037ff0de, 0x9dfc2a41, + 0xb3733f0d, 0x2df0e592, 0xce72fbe9, 0x50f12176, 0x0877c61f, + 0x96f41c80, 0x757602fb, 0xebf5d864, 0xa39db332, 0x3d1e69ad, + 0xde9c77d6, 0x401fad49, 0x18994a20, 0x861a90bf, 0x65988ec4, + 0xfb1b545b, 0xd5944117, 0x4b179b88, 0xa89585f3, 0x36165f6c, + 0x6e90b805, 0xf013629a, 0x13917ce1, 0x8d12a67e, 0x4f8f5779, + 0xd10c8de6, 0x328e939d, 0xac0d4902, 0xf48bae6b, 0x6a0874f4, + 0x898a6a8f, 0x1709b010, 0x3986a55c, 0xa7057fc3, 0x448761b8, + 0xda04bb27, 0x82825c4e, 0x1c0186d1, 0xff8398aa, 0x61004235, + 0x7bb87ba5, 0xe53ba13a, 0x06b9bf41, 0x983a65de, 0xc0bc82b7, + 0x5e3f5828, 0xbdbd4653, 0x233e9ccc, 0x0db18980, 0x9332531f, + 0x70b04d64, 0xee3397fb, 0xb6b57092, 0x2836aa0d, 0xcbb4b476, + 0x55376ee9, 0x97aa9fee, 0x09294571, 0xeaab5b0a, 0x74288195, + 0x2cae66fc, 0xb22dbc63, 0x51afa218, 0xcf2c7887, 0xe1a36dcb, + 0x7f20b754, 0x9ca2a92f, 0x022173b0, 0x5aa794d9, 0xc4244e46, + 0x27a6503d, 0xb9258aa2, 0x52d052c6, 0xcc538859, 0x2fd19622, + 0xb1524cbd, 0xe9d4abd4, 0x7757714b, 0x94d56f30, 0x0a56b5af, + 0x24d9a0e3, 0xba5a7a7c, 0x59d86407, 0xc75bbe98, 0x9fdd59f1, + 0x015e836e, 0xe2dc9d15, 0x7c5f478a, 0xbec2b68d, 0x20416c12, + 0xc3c37269, 0x5d40a8f6, 0x05c64f9f, 0x9b459500, 0x78c78b7b, + 0xe64451e4, 0xc8cb44a8, 0x56489e37, 0xb5ca804c, 0x2b495ad3, + 0x73cfbdba, 0xed4c6725, 0x0ece795e, 0x904da3c1, 0x8af59a51, + 0x147640ce, 0xf7f45eb5, 0x6977842a, 0x31f16343, 0xaf72b9dc, + 0x4cf0a7a7, 0xd2737d38, 0xfcfc6874, 0x627fb2eb, 0x81fdac90, + 0x1f7e760f, 0x47f89166, 0xd97b4bf9, 0x3af95582, 0xa47a8f1d, + 0x66e77e1a, 0xf864a485, 0x1be6bafe, 0x85656061, 0xdde38708, + 0x43605d97, 0xa0e243ec, 0x3e619973, 0x10ee8c3f, 0x8e6d56a0, + 0x6def48db, 0xf36c9244, 0xabea752d, 0x3569afb2, 0xd6ebb1c9, + 0x48686b56}, + {0x00000000, 0xc0642817, 0x80c9502e, 0x40ad7839, 0x0093a15c, + 0xc0f7894b, 0x805af172, 0x403ed965, 0x002643b9, 0xc0426bae, + 0x80ef1397, 0x408b3b80, 0x00b5e2e5, 0xc0d1caf2, 0x807cb2cb, + 0x40189adc, 0x414af7a9, 0x812edfbe, 0xc183a787, 0x01e78f90, + 0x41d956f5, 0x81bd7ee2, 0xc11006db, 0x01742ecc, 0x416cb410, + 0x81089c07, 0xc1a5e43e, 0x01c1cc29, 0x41ff154c, 0x819b3d5b, + 0xc1364562, 0x01526d75, 0xc3929f88, 0x03f6b79f, 0x435bcfa6, + 0x833fe7b1, 0xc3013ed4, 0x036516c3, 0x43c86efa, 0x83ac46ed, + 0xc3b4dc31, 0x03d0f426, 0x437d8c1f, 0x8319a408, 0xc3277d6d, + 0x0343557a, 0x43ee2d43, 0x838a0554, 0x82d86821, 0x42bc4036, + 0x0211380f, 0xc2751018, 0x824bc97d, 0x422fe16a, 0x02829953, + 0xc2e6b144, 0x82fe2b98, 0x429a038f, 0x02377bb6, 0xc25353a1, + 0x826d8ac4, 0x4209a2d3, 0x02a4daea, 0xc2c0f2fd, 0xc7234eca, + 0x074766dd, 0x47ea1ee4, 0x878e36f3, 0xc7b0ef96, 0x07d4c781, + 0x4779bfb8, 0x871d97af, 0xc7050d73, 0x07612564, 0x47cc5d5d, + 0x87a8754a, 0xc796ac2f, 0x07f28438, 0x475ffc01, 0x873bd416, + 0x8669b963, 0x460d9174, 0x06a0e94d, 0xc6c4c15a, 0x86fa183f, + 0x469e3028, 0x06334811, 0xc6576006, 0x864ffada, 0x462bd2cd, + 0x0686aaf4, 0xc6e282e3, 0x86dc5b86, 0x46b87391, 0x06150ba8, + 0xc67123bf, 0x04b1d142, 0xc4d5f955, 0x8478816c, 0x441ca97b, + 0x0422701e, 0xc4465809, 0x84eb2030, 0x448f0827, 0x049792fb, + 0xc4f3baec, 0x845ec2d5, 0x443aeac2, 0x040433a7, 0xc4601bb0, + 0x84cd6389, 0x44a94b9e, 0x45fb26eb, 0x859f0efc, 0xc53276c5, + 0x05565ed2, 0x456887b7, 0x850cafa0, 0xc5a1d799, 0x05c5ff8e, + 0x45dd6552, 0x85b94d45, 0xc514357c, 0x05701d6b, 0x454ec40e, + 0x852aec19, 0xc5879420, 0x05e3bc37, 0xcf41ed4f, 0x0f25c558, + 0x4f88bd61, 0x8fec9576, 0xcfd24c13, 0x0fb66404, 0x4f1b1c3d, + 0x8f7f342a, 0xcf67aef6, 0x0f0386e1, 0x4faefed8, 0x8fcad6cf, + 0xcff40faa, 0x0f9027bd, 0x4f3d5f84, 0x8f597793, 0x8e0b1ae6, + 0x4e6f32f1, 0x0ec24ac8, 0xcea662df, 0x8e98bbba, 0x4efc93ad, + 0x0e51eb94, 0xce35c383, 0x8e2d595f, 0x4e497148, 0x0ee40971, + 0xce802166, 0x8ebef803, 0x4edad014, 0x0e77a82d, 0xce13803a, + 0x0cd372c7, 0xccb75ad0, 0x8c1a22e9, 0x4c7e0afe, 0x0c40d39b, + 0xcc24fb8c, 0x8c8983b5, 0x4cedaba2, 0x0cf5317e, 0xcc911969, + 0x8c3c6150, 0x4c584947, 0x0c669022, 0xcc02b835, 0x8cafc00c, + 0x4ccbe81b, 0x4d99856e, 0x8dfdad79, 0xcd50d540, 0x0d34fd57, + 0x4d0a2432, 0x8d6e0c25, 0xcdc3741c, 0x0da75c0b, 0x4dbfc6d7, + 0x8ddbeec0, 0xcd7696f9, 0x0d12beee, 0x4d2c678b, 0x8d484f9c, + 0xcde537a5, 0x0d811fb2, 0x0862a385, 0xc8068b92, 0x88abf3ab, + 0x48cfdbbc, 0x08f102d9, 0xc8952ace, 0x883852f7, 0x485c7ae0, + 0x0844e03c, 0xc820c82b, 0x888db012, 0x48e99805, 0x08d74160, + 0xc8b36977, 0x881e114e, 0x487a3959, 0x4928542c, 0x894c7c3b, + 0xc9e10402, 0x09852c15, 0x49bbf570, 0x89dfdd67, 0xc972a55e, + 0x09168d49, 0x490e1795, 0x896a3f82, 0xc9c747bb, 0x09a36fac, + 0x499db6c9, 0x89f99ede, 0xc954e6e7, 0x0930cef0, 0xcbf03c0d, + 0x0b94141a, 0x4b396c23, 0x8b5d4434, 0xcb639d51, 0x0b07b546, + 0x4baacd7f, 0x8bcee568, 0xcbd67fb4, 0x0bb257a3, 0x4b1f2f9a, + 0x8b7b078d, 0xcb45dee8, 0x0b21f6ff, 0x4b8c8ec6, 0x8be8a6d1, + 0x8abacba4, 0x4adee3b3, 0x0a739b8a, 0xca17b39d, 0x8a296af8, + 0x4a4d42ef, 0x0ae03ad6, 0xca8412c1, 0x8a9c881d, 0x4af8a00a, + 0x0a55d833, 0xca31f024, 0x8a0f2941, 0x4a6b0156, 0x0ac6796f, + 0xcaa25178}, + {0x00000000, 0xd4ea739b, 0xe9d396ed, 0x3d39e576, 0x93a15c00, + 0x474b2f9b, 0x7a72caed, 0xae98b976, 0x2643b900, 0xf2a9ca9b, + 0xcf902fed, 0x1b7a5c76, 0xb5e2e500, 0x6108969b, 0x5c3173ed, + 0x88db0076, 0x4c867201, 0x986c019a, 0xa555e4ec, 0x71bf9777, + 0xdf272e01, 0x0bcd5d9a, 0x36f4b8ec, 0xe21ecb77, 0x6ac5cb01, + 0xbe2fb89a, 0x83165dec, 0x57fc2e77, 0xf9649701, 0x2d8ee49a, + 0x10b701ec, 0xc45d7277, 0x980ce502, 0x4ce69699, 0x71df73ef, + 0xa5350074, 0x0badb902, 0xdf47ca99, 0xe27e2fef, 0x36945c74, + 0xbe4f5c02, 0x6aa52f99, 0x579ccaef, 0x8376b974, 0x2dee0002, + 0xf9047399, 0xc43d96ef, 0x10d7e574, 0xd48a9703, 0x0060e498, + 0x3d5901ee, 0xe9b37275, 0x472bcb03, 0x93c1b898, 0xaef85dee, + 0x7a122e75, 0xf2c92e03, 0x26235d98, 0x1b1ab8ee, 0xcff0cb75, + 0x61687203, 0xb5820198, 0x88bbe4ee, 0x5c519775, 0x3019ca05, + 0xe4f3b99e, 0xd9ca5ce8, 0x0d202f73, 0xa3b89605, 0x7752e59e, + 0x4a6b00e8, 0x9e817373, 0x165a7305, 0xc2b0009e, 0xff89e5e8, + 0x2b639673, 0x85fb2f05, 0x51115c9e, 0x6c28b9e8, 0xb8c2ca73, + 0x7c9fb804, 0xa875cb9f, 0x954c2ee9, 0x41a65d72, 0xef3ee404, + 0x3bd4979f, 0x06ed72e9, 0xd2070172, 0x5adc0104, 0x8e36729f, + 0xb30f97e9, 0x67e5e472, 0xc97d5d04, 0x1d972e9f, 0x20aecbe9, + 0xf444b872, 0xa8152f07, 0x7cff5c9c, 0x41c6b9ea, 0x952cca71, + 0x3bb47307, 0xef5e009c, 0xd267e5ea, 0x068d9671, 0x8e569607, + 0x5abce59c, 0x678500ea, 0xb36f7371, 0x1df7ca07, 0xc91db99c, + 0xf4245cea, 0x20ce2f71, 0xe4935d06, 0x30792e9d, 0x0d40cbeb, + 0xd9aab870, 0x77320106, 0xa3d8729d, 0x9ee197eb, 0x4a0be470, + 0xc2d0e406, 0x163a979d, 0x2b0372eb, 0xffe90170, 0x5171b806, + 0x859bcb9d, 0xb8a22eeb, 0x6c485d70, 0x6032940b, 0xb4d8e790, + 0x89e102e6, 0x5d0b717d, 0xf393c80b, 0x2779bb90, 0x1a405ee6, + 0xceaa2d7d, 0x46712d0b, 0x929b5e90, 0xafa2bbe6, 0x7b48c87d, + 0xd5d0710b, 0x013a0290, 0x3c03e7e6, 0xe8e9947d, 0x2cb4e60a, + 0xf85e9591, 0xc56770e7, 0x118d037c, 0xbf15ba0a, 0x6bffc991, + 0x56c62ce7, 0x822c5f7c, 0x0af75f0a, 0xde1d2c91, 0xe324c9e7, + 0x37ceba7c, 0x9956030a, 0x4dbc7091, 0x708595e7, 0xa46fe67c, + 0xf83e7109, 0x2cd40292, 0x11ede7e4, 0xc507947f, 0x6b9f2d09, + 0xbf755e92, 0x824cbbe4, 0x56a6c87f, 0xde7dc809, 0x0a97bb92, + 0x37ae5ee4, 0xe3442d7f, 0x4ddc9409, 0x9936e792, 0xa40f02e4, + 0x70e5717f, 0xb4b80308, 0x60527093, 0x5d6b95e5, 0x8981e67e, + 0x27195f08, 0xf3f32c93, 0xcecac9e5, 0x1a20ba7e, 0x92fbba08, + 0x4611c993, 0x7b282ce5, 0xafc25f7e, 0x015ae608, 0xd5b09593, + 0xe88970e5, 0x3c63037e, 0x502b5e0e, 0x84c12d95, 0xb9f8c8e3, + 0x6d12bb78, 0xc38a020e, 0x17607195, 0x2a5994e3, 0xfeb3e778, + 0x7668e70e, 0xa2829495, 0x9fbb71e3, 0x4b510278, 0xe5c9bb0e, + 0x3123c895, 0x0c1a2de3, 0xd8f05e78, 0x1cad2c0f, 0xc8475f94, + 0xf57ebae2, 0x2194c979, 0x8f0c700f, 0x5be60394, 0x66dfe6e2, + 0xb2359579, 0x3aee950f, 0xee04e694, 0xd33d03e2, 0x07d77079, + 0xa94fc90f, 0x7da5ba94, 0x409c5fe2, 0x94762c79, 0xc827bb0c, + 0x1ccdc897, 0x21f42de1, 0xf51e5e7a, 0x5b86e70c, 0x8f6c9497, + 0xb25571e1, 0x66bf027a, 0xee64020c, 0x3a8e7197, 0x07b794e1, + 0xd35de77a, 0x7dc55e0c, 0xa92f2d97, 0x9416c8e1, 0x40fcbb7a, + 0x84a1c90d, 0x504bba96, 0x6d725fe0, 0xb9982c7b, 0x1700950d, + 0xc3eae696, 0xfed303e0, 0x2a39707b, 0xa2e2700d, 0x76080396, + 0x4b31e6e0, 0x9fdb957b, 0x31432c0d, 0xe5a95f96, 0xd890bae0, + 0x0c7ac97b}, + {0x00000000, 0x27652581, 0x0fcc3bd9, 0x28a91e58, 0x5f9e0669, + 0x78fb23e8, 0x50523db0, 0x77371831, 0xbe3c0dd2, 0x99592853, + 0xb1f0360b, 0x9695138a, 0xe1a20bbb, 0xc6c72e3a, 0xee6e3062, + 0xc90b15e3, 0x3d7f6b7f, 0x1a1a4efe, 0x32b350a6, 0x15d67527, + 0x62e16d16, 0x45844897, 0x6d2d56cf, 0x4a48734e, 0x834366ad, + 0xa426432c, 0x8c8f5d74, 0xabea78f5, 0xdcdd60c4, 0xfbb84545, + 0xd3115b1d, 0xf4747e9c, 0x7afed6fe, 0x5d9bf37f, 0x7532ed27, + 0x5257c8a6, 0x2560d097, 0x0205f516, 0x2aaceb4e, 0x0dc9cecf, + 0xc4c2db2c, 0xe3a7fead, 0xcb0ee0f5, 0xec6bc574, 0x9b5cdd45, + 0xbc39f8c4, 0x9490e69c, 0xb3f5c31d, 0x4781bd81, 0x60e49800, + 0x484d8658, 0x6f28a3d9, 0x181fbbe8, 0x3f7a9e69, 0x17d38031, + 0x30b6a5b0, 0xf9bdb053, 0xded895d2, 0xf6718b8a, 0xd114ae0b, + 0xa623b63a, 0x814693bb, 0xa9ef8de3, 0x8e8aa862, 0xb5fadc26, + 0x929ff9a7, 0xba36e7ff, 0x9d53c27e, 0xea64da4f, 0xcd01ffce, + 0xe5a8e196, 0xc2cdc417, 0x0bc6d1f4, 0x2ca3f475, 0x040aea2d, + 0x236fcfac, 0x5458d79d, 0x733df21c, 0x5b94ec44, 0x7cf1c9c5, + 0x8885b759, 0xafe092d8, 0x87498c80, 0xa02ca901, 0xd71bb130, + 0xf07e94b1, 0xd8d78ae9, 0xffb2af68, 0x36b9ba8b, 0x11dc9f0a, + 0x39758152, 0x1e10a4d3, 0x6927bce2, 0x4e429963, 0x66eb873b, + 0x418ea2ba, 0xcf040ad8, 0xe8612f59, 0xc0c83101, 0xe7ad1480, + 0x909a0cb1, 0xb7ff2930, 0x9f563768, 0xb83312e9, 0x7138070a, + 0x565d228b, 0x7ef43cd3, 0x59911952, 0x2ea60163, 0x09c324e2, + 0x216a3aba, 0x060f1f3b, 0xf27b61a7, 0xd51e4426, 0xfdb75a7e, + 0xdad27fff, 0xade567ce, 0x8a80424f, 0xa2295c17, 0x854c7996, + 0x4c476c75, 0x6b2249f4, 0x438b57ac, 0x64ee722d, 0x13d96a1c, + 0x34bc4f9d, 0x1c1551c5, 0x3b707444, 0x6af5b94d, 0x4d909ccc, + 0x65398294, 0x425ca715, 0x356bbf24, 0x120e9aa5, 0x3aa784fd, + 0x1dc2a17c, 0xd4c9b49f, 0xf3ac911e, 0xdb058f46, 0xfc60aac7, + 0x8b57b2f6, 0xac329777, 0x849b892f, 0xa3feacae, 0x578ad232, + 0x70eff7b3, 0x5846e9eb, 0x7f23cc6a, 0x0814d45b, 0x2f71f1da, + 0x07d8ef82, 0x20bdca03, 0xe9b6dfe0, 0xced3fa61, 0xe67ae439, + 0xc11fc1b8, 0xb628d989, 0x914dfc08, 0xb9e4e250, 0x9e81c7d1, + 0x100b6fb3, 0x376e4a32, 0x1fc7546a, 0x38a271eb, 0x4f9569da, + 0x68f04c5b, 0x40595203, 0x673c7782, 0xae376261, 0x895247e0, + 0xa1fb59b8, 0x869e7c39, 0xf1a96408, 0xd6cc4189, 0xfe655fd1, + 0xd9007a50, 0x2d7404cc, 0x0a11214d, 0x22b83f15, 0x05dd1a94, + 0x72ea02a5, 0x558f2724, 0x7d26397c, 0x5a431cfd, 0x9348091e, + 0xb42d2c9f, 0x9c8432c7, 0xbbe11746, 0xccd60f77, 0xebb32af6, + 0xc31a34ae, 0xe47f112f, 0xdf0f656b, 0xf86a40ea, 0xd0c35eb2, + 0xf7a67b33, 0x80916302, 0xa7f44683, 0x8f5d58db, 0xa8387d5a, + 0x613368b9, 0x46564d38, 0x6eff5360, 0x499a76e1, 0x3ead6ed0, + 0x19c84b51, 0x31615509, 0x16047088, 0xe2700e14, 0xc5152b95, + 0xedbc35cd, 0xcad9104c, 0xbdee087d, 0x9a8b2dfc, 0xb22233a4, + 0x95471625, 0x5c4c03c6, 0x7b292647, 0x5380381f, 0x74e51d9e, + 0x03d205af, 0x24b7202e, 0x0c1e3e76, 0x2b7b1bf7, 0xa5f1b395, + 0x82949614, 0xaa3d884c, 0x8d58adcd, 0xfa6fb5fc, 0xdd0a907d, + 0xf5a38e25, 0xd2c6aba4, 0x1bcdbe47, 0x3ca89bc6, 0x1401859e, + 0x3364a01f, 0x4453b82e, 0x63369daf, 0x4b9f83f7, 0x6cfaa676, + 0x988ed8ea, 0xbfebfd6b, 0x9742e333, 0xb027c6b2, 0xc710de83, + 0xe075fb02, 0xc8dce55a, 0xefb9c0db, 0x26b2d538, 0x01d7f0b9, + 0x297eeee1, 0x0e1bcb60, 0x792cd351, 0x5e49f6d0, 0x76e0e888, + 0x5185cd09}}; + +#endif + +#endif + +#endif + +local const z_crc_t FAR x2n_table[] = { + 0x40000000, 0x20000000, 0x08000000, 0x00800000, 0x00008000, + 0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467, + 0xd7bbfe6a, 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, + 0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3, 0x31fec169, + 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37, + 0x2e4e5eef, 0x4eaba214, 0xa8a472c0, 0x429a969e, 0x148d302a, + 0xc40ba6d0, 0xc4e22c3c}; diff --git a/lib/zlib/inffast.c b/lib/zlib/inffast.c new file mode 100644 index 0000000..9354676 --- /dev/null +++ b/lib/zlib/inffast.c @@ -0,0 +1,320 @@ +/* inffast.c -- fast decoding + * Copyright (C) 1995-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "inffast.h" + +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") +#else + +/* + Decode literal, length, and distance codes and write out the resulting + literal and match bytes until either not enough input or output is + available, an end-of-block is encountered, or a data error is encountered. + When large enough input and output buffers are supplied to inflate(), for + example, a 16K input buffer and a 64K output buffer, more than 95% of the + inflate execution time is spent in this routine. + + Entry assumptions: + + state->mode == LEN + strm->avail_in >= 6 + strm->avail_out >= 258 + start >= strm->avail_out + state->bits < 8 + + On return, state->mode is one of: + + LEN -- ran out of enough output space or enough available input + TYPE -- reached end of block code, inflate() to interpret next block + BAD -- error in block data + + Notes: + + - The maximum input bits used by a length/distance pair is 15 bits for the + length code, 5 bits for the length extra, 15 bits for the distance code, + and 13 bits for the distance extra. This totals 48 bits, or six bytes. + Therefore if strm->avail_in >= 6, then there is enough input to avoid + checking for available input while decoding. + + - The maximum bytes that a single length/distance pair can output is 258 + bytes, which is the maximum length that can be coded. inflate_fast() + requires strm->avail_out >= 258 for each loop to avoid checking for + output space. + */ +void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { + struct inflate_state FAR *state; + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ + unsigned char FAR *out; /* local strm->next_out */ + unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ + unsigned char FAR *end; /* while out < end, enough space available */ +#ifdef INFLATE_STRICT + unsigned dmax; /* maximum distance from zlib header */ +#endif + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ + unsigned long hold; /* local strm->hold */ + unsigned bits; /* local strm->bits */ + code const FAR *lcode; /* local strm->lencode */ + code const FAR *dcode; /* local strm->distcode */ + unsigned lmask; /* mask for first level of length codes */ + unsigned dmask; /* mask for first level of distance codes */ + code const *here; /* retrieved table entry */ + unsigned op; /* code bits, operation, extra bits, or */ + /* window position, window bytes to copy */ + unsigned len; /* match length, unused bytes */ + unsigned dist; /* match distance */ + unsigned char FAR *from; /* where to copy match from */ + + /* copy state to local variables */ + state = (struct inflate_state FAR *)strm->state; + in = strm->next_in; + last = in + (strm->avail_in - 5); + out = strm->next_out; + beg = out - (start - strm->avail_out); + end = out + (strm->avail_out - 257); +#ifdef INFLATE_STRICT + dmax = state->dmax; +#endif + wsize = state->wsize; + whave = state->whave; + wnext = state->wnext; + window = state->window; + hold = state->hold; + bits = state->bits; + lcode = state->lencode; + dcode = state->distcode; + lmask = (1U << state->lenbits) - 1; + dmask = (1U << state->distbits) - 1; + + /* decode literals and length/distances until end-of-block or not enough + input data or output space */ + do { + if (bits < 15) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + here = lcode + (hold & lmask); + dolen: + op = (unsigned)(here->bits); + hold >>= op; + bits -= op; + op = (unsigned)(here->op); + if (op == 0) { /* literal */ + Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here->val)); + *out++ = (unsigned char)(here->val); + } + else if (op & 16) { /* length base */ + len = (unsigned)(here->val); + op &= 15; /* number of extra bits */ + if (op) { + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + len += (unsigned)hold & ((1U << op) - 1); + hold >>= op; + bits -= op; + } + Tracevv((stderr, "inflate: length %u\n", len)); + if (bits < 15) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + here = dcode + (hold & dmask); + dodist: + op = (unsigned)(here->bits); + hold >>= op; + bits -= op; + op = (unsigned)(here->op); + if (op & 16) { /* distance base */ + dist = (unsigned)(here->val); + op &= 15; /* number of extra bits */ + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + } + dist += (unsigned)hold & ((1U << op) - 1); +#ifdef INFLATE_STRICT + if (dist > dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + hold >>= op; + bits -= op; + Tracevv((stderr, "inflate: distance %u\n", dist)); + op = (unsigned)(out - beg); /* max distance in output */ + if (dist > op) { /* see if copy from window */ + op = dist - op; /* distance back in window */ + if (op > whave) { + if (state->sane) { + strm->msg = + (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + if (len <= op - whave) { + do { + *out++ = 0; + } while (--len); + continue; + } + len -= op - whave; + do { + *out++ = 0; + } while (--op > whave); + if (op == 0) { + from = out - dist; + do { + *out++ = *from++; + } while (--len); + continue; + } +#endif + } + from = window; + if (wnext == 0) { /* very common case */ + from += wsize - op; + if (op < len) { /* some from window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + else if (wnext < op) { /* wrap around window */ + from += wsize + wnext - op; + op -= wnext; + if (op < len) { /* some from end of window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = window; + if (wnext < len) { /* some from start of window */ + op = wnext; + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + } + else { /* contiguous in window */ + from += wnext - op; + if (op < len) { /* some from window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + while (len > 2) { + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; + len -= 3; + } + if (len) { + *out++ = *from++; + if (len > 1) + *out++ = *from++; + } + } + else { + from = out - dist; /* copy direct from output */ + do { /* minimum length is three */ + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; + len -= 3; + } while (len > 2); + if (len) { + *out++ = *from++; + if (len > 1) + *out++ = *from++; + } + } + } + else if ((op & 64) == 0) { /* 2nd level distance code */ + here = dcode + here->val + (hold & ((1U << op) - 1)); + goto dodist; + } + else { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + } + else if ((op & 64) == 0) { /* 2nd level length code */ + here = lcode + here->val + (hold & ((1U << op) - 1)); + goto dolen; + } + else if (op & 32) { /* end-of-block */ + Tracevv((stderr, "inflate: end of block\n")); + state->mode = TYPE; + break; + } + else { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + } while (in < last && out < end); + + /* return unused bytes (on entry, bits < 8, so in won't go too far back) */ + len = bits >> 3; + in -= len; + bits -= len << 3; + hold &= (1U << bits) - 1; + + /* update state and return */ + strm->next_in = in; + strm->next_out = out; + strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); + strm->avail_out = (unsigned)(out < end ? + 257 + (end - out) : 257 - (out - end)); + state->hold = hold; + state->bits = bits; + return; +} + +/* + inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): + - Using bit fields for code structure + - Different op definition to avoid & for extra bits (do & for table bits) + - Three separate decoding do-loops for direct, window, and wnext == 0 + - Special case for distance > 1 copies to do overlapped load and store copy + - Explicit branch predictions (based on measured branch probabilities) + - Deferring match copy and interspersed it with decoding subsequent codes + - Swapping literal/length else + - Swapping window/direct else + - Larger unrolled copy loops (three is about right) + - Moving len -= 3 statement into middle of loop + */ + +#endif /* !ASMINF */ diff --git a/lib/zlib/inffast.h b/lib/zlib/inffast.h new file mode 100644 index 0000000..49c6d15 --- /dev/null +++ b/lib/zlib/inffast.h @@ -0,0 +1,11 @@ +/* inffast.h -- header to use inffast.c + * Copyright (C) 1995-2003, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start); diff --git a/lib/zlib/inffixed.h b/lib/zlib/inffixed.h new file mode 100644 index 0000000..d628327 --- /dev/null +++ b/lib/zlib/inffixed.h @@ -0,0 +1,94 @@ + /* inffixed.h -- table for decoding fixed codes + * Generated automatically by makefixed(). + */ + + /* WARNING: this file should *not* be used by applications. + It is part of the implementation of this library and is + subject to change. Applications should only use zlib.h. + */ + + static const code lenfix[512] = { + {96,7,0},{0,8,80},{0,8,16},{20,8,115},{18,7,31},{0,8,112},{0,8,48}, + {0,9,192},{16,7,10},{0,8,96},{0,8,32},{0,9,160},{0,8,0},{0,8,128}, + {0,8,64},{0,9,224},{16,7,6},{0,8,88},{0,8,24},{0,9,144},{19,7,59}, + {0,8,120},{0,8,56},{0,9,208},{17,7,17},{0,8,104},{0,8,40},{0,9,176}, + {0,8,8},{0,8,136},{0,8,72},{0,9,240},{16,7,4},{0,8,84},{0,8,20}, + {21,8,227},{19,7,43},{0,8,116},{0,8,52},{0,9,200},{17,7,13},{0,8,100}, + {0,8,36},{0,9,168},{0,8,4},{0,8,132},{0,8,68},{0,9,232},{16,7,8}, + {0,8,92},{0,8,28},{0,9,152},{20,7,83},{0,8,124},{0,8,60},{0,9,216}, + {18,7,23},{0,8,108},{0,8,44},{0,9,184},{0,8,12},{0,8,140},{0,8,76}, + {0,9,248},{16,7,3},{0,8,82},{0,8,18},{21,8,163},{19,7,35},{0,8,114}, + {0,8,50},{0,9,196},{17,7,11},{0,8,98},{0,8,34},{0,9,164},{0,8,2}, + {0,8,130},{0,8,66},{0,9,228},{16,7,7},{0,8,90},{0,8,26},{0,9,148}, + {20,7,67},{0,8,122},{0,8,58},{0,9,212},{18,7,19},{0,8,106},{0,8,42}, + {0,9,180},{0,8,10},{0,8,138},{0,8,74},{0,9,244},{16,7,5},{0,8,86}, + {0,8,22},{64,8,0},{19,7,51},{0,8,118},{0,8,54},{0,9,204},{17,7,15}, + {0,8,102},{0,8,38},{0,9,172},{0,8,6},{0,8,134},{0,8,70},{0,9,236}, + {16,7,9},{0,8,94},{0,8,30},{0,9,156},{20,7,99},{0,8,126},{0,8,62}, + {0,9,220},{18,7,27},{0,8,110},{0,8,46},{0,9,188},{0,8,14},{0,8,142}, + {0,8,78},{0,9,252},{96,7,0},{0,8,81},{0,8,17},{21,8,131},{18,7,31}, + {0,8,113},{0,8,49},{0,9,194},{16,7,10},{0,8,97},{0,8,33},{0,9,162}, + {0,8,1},{0,8,129},{0,8,65},{0,9,226},{16,7,6},{0,8,89},{0,8,25}, + {0,9,146},{19,7,59},{0,8,121},{0,8,57},{0,9,210},{17,7,17},{0,8,105}, + {0,8,41},{0,9,178},{0,8,9},{0,8,137},{0,8,73},{0,9,242},{16,7,4}, + {0,8,85},{0,8,21},{16,8,258},{19,7,43},{0,8,117},{0,8,53},{0,9,202}, + {17,7,13},{0,8,101},{0,8,37},{0,9,170},{0,8,5},{0,8,133},{0,8,69}, + {0,9,234},{16,7,8},{0,8,93},{0,8,29},{0,9,154},{20,7,83},{0,8,125}, + {0,8,61},{0,9,218},{18,7,23},{0,8,109},{0,8,45},{0,9,186},{0,8,13}, + {0,8,141},{0,8,77},{0,9,250},{16,7,3},{0,8,83},{0,8,19},{21,8,195}, + {19,7,35},{0,8,115},{0,8,51},{0,9,198},{17,7,11},{0,8,99},{0,8,35}, + {0,9,166},{0,8,3},{0,8,131},{0,8,67},{0,9,230},{16,7,7},{0,8,91}, + {0,8,27},{0,9,150},{20,7,67},{0,8,123},{0,8,59},{0,9,214},{18,7,19}, + {0,8,107},{0,8,43},{0,9,182},{0,8,11},{0,8,139},{0,8,75},{0,9,246}, + {16,7,5},{0,8,87},{0,8,23},{64,8,0},{19,7,51},{0,8,119},{0,8,55}, + {0,9,206},{17,7,15},{0,8,103},{0,8,39},{0,9,174},{0,8,7},{0,8,135}, + {0,8,71},{0,9,238},{16,7,9},{0,8,95},{0,8,31},{0,9,158},{20,7,99}, + {0,8,127},{0,8,63},{0,9,222},{18,7,27},{0,8,111},{0,8,47},{0,9,190}, + {0,8,15},{0,8,143},{0,8,79},{0,9,254},{96,7,0},{0,8,80},{0,8,16}, + {20,8,115},{18,7,31},{0,8,112},{0,8,48},{0,9,193},{16,7,10},{0,8,96}, + {0,8,32},{0,9,161},{0,8,0},{0,8,128},{0,8,64},{0,9,225},{16,7,6}, + {0,8,88},{0,8,24},{0,9,145},{19,7,59},{0,8,120},{0,8,56},{0,9,209}, + {17,7,17},{0,8,104},{0,8,40},{0,9,177},{0,8,8},{0,8,136},{0,8,72}, + {0,9,241},{16,7,4},{0,8,84},{0,8,20},{21,8,227},{19,7,43},{0,8,116}, + {0,8,52},{0,9,201},{17,7,13},{0,8,100},{0,8,36},{0,9,169},{0,8,4}, + {0,8,132},{0,8,68},{0,9,233},{16,7,8},{0,8,92},{0,8,28},{0,9,153}, + {20,7,83},{0,8,124},{0,8,60},{0,9,217},{18,7,23},{0,8,108},{0,8,44}, + {0,9,185},{0,8,12},{0,8,140},{0,8,76},{0,9,249},{16,7,3},{0,8,82}, + {0,8,18},{21,8,163},{19,7,35},{0,8,114},{0,8,50},{0,9,197},{17,7,11}, + {0,8,98},{0,8,34},{0,9,165},{0,8,2},{0,8,130},{0,8,66},{0,9,229}, + {16,7,7},{0,8,90},{0,8,26},{0,9,149},{20,7,67},{0,8,122},{0,8,58}, + {0,9,213},{18,7,19},{0,8,106},{0,8,42},{0,9,181},{0,8,10},{0,8,138}, + {0,8,74},{0,9,245},{16,7,5},{0,8,86},{0,8,22},{64,8,0},{19,7,51}, + {0,8,118},{0,8,54},{0,9,205},{17,7,15},{0,8,102},{0,8,38},{0,9,173}, + {0,8,6},{0,8,134},{0,8,70},{0,9,237},{16,7,9},{0,8,94},{0,8,30}, + {0,9,157},{20,7,99},{0,8,126},{0,8,62},{0,9,221},{18,7,27},{0,8,110}, + {0,8,46},{0,9,189},{0,8,14},{0,8,142},{0,8,78},{0,9,253},{96,7,0}, + {0,8,81},{0,8,17},{21,8,131},{18,7,31},{0,8,113},{0,8,49},{0,9,195}, + {16,7,10},{0,8,97},{0,8,33},{0,9,163},{0,8,1},{0,8,129},{0,8,65}, + {0,9,227},{16,7,6},{0,8,89},{0,8,25},{0,9,147},{19,7,59},{0,8,121}, + {0,8,57},{0,9,211},{17,7,17},{0,8,105},{0,8,41},{0,9,179},{0,8,9}, + {0,8,137},{0,8,73},{0,9,243},{16,7,4},{0,8,85},{0,8,21},{16,8,258}, + {19,7,43},{0,8,117},{0,8,53},{0,9,203},{17,7,13},{0,8,101},{0,8,37}, + {0,9,171},{0,8,5},{0,8,133},{0,8,69},{0,9,235},{16,7,8},{0,8,93}, + {0,8,29},{0,9,155},{20,7,83},{0,8,125},{0,8,61},{0,9,219},{18,7,23}, + {0,8,109},{0,8,45},{0,9,187},{0,8,13},{0,8,141},{0,8,77},{0,9,251}, + {16,7,3},{0,8,83},{0,8,19},{21,8,195},{19,7,35},{0,8,115},{0,8,51}, + {0,9,199},{17,7,11},{0,8,99},{0,8,35},{0,9,167},{0,8,3},{0,8,131}, + {0,8,67},{0,9,231},{16,7,7},{0,8,91},{0,8,27},{0,9,151},{20,7,67}, + {0,8,123},{0,8,59},{0,9,215},{18,7,19},{0,8,107},{0,8,43},{0,9,183}, + {0,8,11},{0,8,139},{0,8,75},{0,9,247},{16,7,5},{0,8,87},{0,8,23}, + {64,8,0},{19,7,51},{0,8,119},{0,8,55},{0,9,207},{17,7,15},{0,8,103}, + {0,8,39},{0,9,175},{0,8,7},{0,8,135},{0,8,71},{0,9,239},{16,7,9}, + {0,8,95},{0,8,31},{0,9,159},{20,7,99},{0,8,127},{0,8,63},{0,9,223}, + {18,7,27},{0,8,111},{0,8,47},{0,9,191},{0,8,15},{0,8,143},{0,8,79}, + {0,9,255} + }; + + static const code distfix[32] = { + {16,5,1},{23,5,257},{19,5,17},{27,5,4097},{17,5,5},{25,5,1025}, + {21,5,65},{29,5,16385},{16,5,3},{24,5,513},{20,5,33},{28,5,8193}, + {18,5,9},{26,5,2049},{22,5,129},{64,5,0},{16,5,2},{23,5,385}, + {19,5,25},{27,5,6145},{17,5,7},{25,5,1537},{21,5,97},{29,5,24577}, + {16,5,4},{24,5,769},{20,5,49},{28,5,12289},{18,5,13},{26,5,3073}, + {22,5,193},{64,5,0} + }; diff --git a/lib/zlib/inflate.c b/lib/zlib/inflate.c new file mode 100644 index 0000000..b0757a9 --- /dev/null +++ b/lib/zlib/inflate.c @@ -0,0 +1,1526 @@ +/* inflate.c -- zlib decompression + * Copyright (C) 1995-2022 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * Change history: + * + * 1.2.beta0 24 Nov 2002 + * - First version -- complete rewrite of inflate to simplify code, avoid + * creation of window when not needed, minimize use of window when it is + * needed, make inffast.c even faster, implement gzip decoding, and to + * improve code readability and style over the previous zlib inflate code + * + * 1.2.beta1 25 Nov 2002 + * - Use pointers for available input and output checking in inffast.c + * - Remove input and output counters in inffast.c + * - Change inffast.c entry and loop from avail_in >= 7 to >= 6 + * - Remove unnecessary second byte pull from length extra in inffast.c + * - Unroll direct copy to three copies per loop in inffast.c + * + * 1.2.beta2 4 Dec 2002 + * - Change external routine names to reduce potential conflicts + * - Correct filename to inffixed.h for fixed tables in inflate.c + * - Make hbuf[] unsigned char to match parameter type in inflate.c + * - Change strm->next_out[-state->offset] to *(strm->next_out - state->offset) + * to avoid negation problem on Alphas (64 bit) in inflate.c + * + * 1.2.beta3 22 Dec 2002 + * - Add comments on state->bits assertion in inffast.c + * - Add comments on op field in inftrees.h + * - Fix bug in reuse of allocated window after inflateReset() + * - Remove bit fields--back to byte structure for speed + * - Remove distance extra == 0 check in inflate_fast()--only helps for lengths + * - Change post-increments to pre-increments in inflate_fast(), PPC biased? + * - Add compile time option, POSTINC, to use post-increments instead (Intel?) + * - Make MATCH copy in inflate() much faster for when inflate_fast() not used + * - Use local copies of stream next and avail values, as well as local bit + * buffer and bit count in inflate()--for speed when inflate_fast() not used + * + * 1.2.beta4 1 Jan 2003 + * - Split ptr - 257 statements in inflate_table() to avoid compiler warnings + * - Move a comment on output buffer sizes from inffast.c to inflate.c + * - Add comments in inffast.c to introduce the inflate_fast() routine + * - Rearrange window copies in inflate_fast() for speed and simplification + * - Unroll last copy for window match in inflate_fast() + * - Use local copies of window variables in inflate_fast() for speed + * - Pull out common wnext == 0 case for speed in inflate_fast() + * - Make op and len in inflate_fast() unsigned for consistency + * - Add FAR to lcode and dcode declarations in inflate_fast() + * - Simplified bad distance check in inflate_fast() + * - Added inflateBackInit(), inflateBack(), and inflateBackEnd() in new + * source file infback.c to provide a call-back interface to inflate for + * programs like gzip and unzip -- uses window as output buffer to avoid + * window copying + * + * 1.2.beta5 1 Jan 2003 + * - Improved inflateBack() interface to allow the caller to provide initial + * input in strm. + * - Fixed stored blocks bug in inflateBack() + * + * 1.2.beta6 4 Jan 2003 + * - Added comments in inffast.c on effectiveness of POSTINC + * - Typecasting all around to reduce compiler warnings + * - Changed loops from while (1) or do {} while (1) to for (;;), again to + * make compilers happy + * - Changed type of window in inflateBackInit() to unsigned char * + * + * 1.2.beta7 27 Jan 2003 + * - Changed many types to unsigned or unsigned short to avoid warnings + * - Added inflateCopy() function + * + * 1.2.0 9 Mar 2003 + * - Changed inflateBack() interface to provide separate opaque descriptors + * for the in() and out() functions + * - Changed inflateBack() argument and in_func typedef to swap the length + * and buffer address return values for the input function + * - Check next_in and next_out for Z_NULL on entry to inflate() + * + * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "inffast.h" + +#ifdef MAKEFIXED +# ifndef BUILDFIXED +# define BUILDFIXED +# endif +#endif + +local int inflateStateCheck(z_streamp strm) { + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + +int ZEXPORT inflateResetKeep(z_streamp strm) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + strm->total_in = strm->total_out = state->total = 0; + strm->msg = Z_NULL; + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; + state->mode = HEAD; + state->last = 0; + state->havedict = 0; + state->flags = -1; + state->dmax = 32768U; + state->head = Z_NULL; + state->hold = 0; + state->bits = 0; + state->lencode = state->distcode = state->next = state->codes; + state->sane = 1; + state->back = -1; + Tracev((stderr, "inflate: reset\n")); + return Z_OK; +} + +int ZEXPORT inflateReset(z_streamp strm) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + state->wsize = 0; + state->whave = 0; + state->wnext = 0; + return inflateResetKeep(strm); +} + +int ZEXPORT inflateReset2(z_streamp strm, int windowBits) { + int wrap; + struct inflate_state FAR *state; + + /* get the state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* extract wrap request from windowBits parameter */ + if (windowBits < 0) { + if (windowBits < -15) + return Z_STREAM_ERROR; + wrap = 0; + windowBits = -windowBits; + } + else { + wrap = (windowBits >> 4) + 5; +#ifdef GUNZIP + if (windowBits < 48) + windowBits &= 15; +#endif + } + + /* set number of window bits, free window if different */ + if (windowBits && (windowBits < 8 || windowBits > 15)) + return Z_STREAM_ERROR; + if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { + ZFREE(strm, state->window); + state->window = Z_NULL; + } + + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; + return inflateReset(strm); +} + +int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, + const char *version, int stream_size) { + int ret; + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL) return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *) + ZALLOC(strm, 1, sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->strm = strm; + state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); + strm->state = Z_NULL; + } + return ret; +} + +int ZEXPORT inflateInit_(z_streamp strm, const char *version, + int stream_size) { + return inflateInit2_(strm, DEF_WBITS, version, stream_size); +} + +int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + if (bits == 0) + return Z_OK; + state = (struct inflate_state FAR *)strm->state; + if (bits < 0) { + state->hold = 0; + state->bits = 0; + return Z_OK; + } + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; + value &= (1L << bits) - 1; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; + return Z_OK; +} + +/* + Return state with length and distance decoding tables and index sizes set to + fixed code decoding. Normally this returns fixed tables from inffixed.h. + If BUILDFIXED is defined, then instead this routine builds the tables the + first time it's called, and returns those tables the first time and + thereafter. This reduces the size of the code by about 2K bytes, in + exchange for a little execution time. However, BUILDFIXED should not be + used for threaded applications, since the rewriting of the tables and virgin + may not be thread-safe. + */ +local void fixedtables(struct inflate_state FAR *state) { +#ifdef BUILDFIXED + static int virgin = 1; + static code *lenfix, *distfix; + static code fixed[544]; + + /* build fixed huffman tables if first call (may not be thread safe) */ + if (virgin) { + unsigned sym, bits; + static code *next; + + /* literal/length table */ + sym = 0; + while (sym < 144) state->lens[sym++] = 8; + while (sym < 256) state->lens[sym++] = 9; + while (sym < 280) state->lens[sym++] = 7; + while (sym < 288) state->lens[sym++] = 8; + next = fixed; + lenfix = next; + bits = 9; + inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); + + /* distance table */ + sym = 0; + while (sym < 32) state->lens[sym++] = 5; + distfix = next; + bits = 5; + inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); + + /* do this just once */ + virgin = 0; + } +#else /* !BUILDFIXED */ +# include "inffixed.h" +#endif /* BUILDFIXED */ + state->lencode = lenfix; + state->lenbits = 9; + state->distcode = distfix; + state->distbits = 5; +} + +#ifdef MAKEFIXED +#include <stdio.h> + +/* + Write out the inffixed.h that is #include'd above. Defining MAKEFIXED also + defines BUILDFIXED, so the tables are built on the fly. makefixed() writes + those tables to stdout, which would be piped to inffixed.h. A small program + can simply call makefixed to do this: + + void makefixed(void); + + int main(void) + { + makefixed(); + return 0; + } + + Then that can be linked with zlib built with MAKEFIXED defined and run: + + a.out > inffixed.h + */ +void makefixed(void) +{ + unsigned low, size; + struct inflate_state state; + + fixedtables(&state); + puts(" /* inffixed.h -- table for decoding fixed codes"); + puts(" * Generated automatically by makefixed()."); + puts(" */"); + puts(""); + puts(" /* WARNING: this file should *not* be used by applications."); + puts(" It is part of the implementation of this library and is"); + puts(" subject to change. Applications should only use zlib.h."); + puts(" */"); + puts(""); + size = 1U << 9; + printf(" static const code lenfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 7) == 0) printf("\n "); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); + size = 1U << 5; + printf("\n static const code distfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 6) == 0) printf("\n "); + printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, + state.distcode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); +} +#endif /* MAKEFIXED */ + +/* + Update the window with the last wsize (normally 32K) bytes written before + returning. If window does not exist yet, create it. This is only called + when a window is already in use, or when output has been written during this + inflate call, but the end of the deflate stream has not been reached yet. + It is also called to create a window for dictionary data when a dictionary + is loaded. + + Providing output buffers larger than 32K to inflate() should provide a speed + advantage, since only the last 32K of output is copied to the sliding window + upon return from inflate(), and since all distances after the first 32K of + output will fall in the output data, making match copies simpler and faster. + The advantage may be dependent on the size of the processor's data caches. + */ +local int updatewindow(z_streamp strm, const Bytef *end, unsigned copy) { + struct inflate_state FAR *state; + unsigned dist; + + state = (struct inflate_state FAR *)strm->state; + + /* if it hasn't been done already, allocate space for the window */ + if (state->window == Z_NULL) { + state->window = (unsigned char FAR *) + ZALLOC(strm, 1U << state->wbits, + sizeof(unsigned char)); + if (state->window == Z_NULL) return 1; + } + + /* if window not in use yet, initialize */ + if (state->wsize == 0) { + state->wsize = 1U << state->wbits; + state->wnext = 0; + state->whave = 0; + } + + /* copy state->wsize or less output bytes into the circular window */ + if (copy >= state->wsize) { + zmemcpy(state->window, end - state->wsize, state->wsize); + state->wnext = 0; + state->whave = state->wsize; + } + else { + dist = state->wsize - state->wnext; + if (dist > copy) dist = copy; + zmemcpy(state->window + state->wnext, end - copy, dist); + copy -= dist; + if (copy) { + zmemcpy(state->window, end - copy, copy); + state->wnext = copy; + state->whave = state->wsize; + } + else { + state->wnext += dist; + if (state->wnext == state->wsize) state->wnext = 0; + if (state->whave < state->wsize) state->whave += dist; + } + } + return 0; +} + +/* Macros for inflate(): */ + +/* check function to use adler32() for zlib or crc32() for gzip */ +#ifdef GUNZIP +# define UPDATE_CHECK(check, buf, len) \ + (state->flags ? crc32(check, buf, len) : adler32(check, buf, len)) +#else +# define UPDATE_CHECK(check, buf, len) adler32(check, buf, len) +#endif + +/* check macros for header crc */ +#ifdef GUNZIP +# define CRC2(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + check = crc32(check, hbuf, 2); \ + } while (0) + +# define CRC4(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + hbuf[2] = (unsigned char)((word) >> 16); \ + hbuf[3] = (unsigned char)((word) >> 24); \ + check = crc32(check, hbuf, 4); \ + } while (0) +#endif + +/* Load registers with state in inflate() for speed */ +#define LOAD() \ + do { \ + put = strm->next_out; \ + left = strm->avail_out; \ + next = strm->next_in; \ + have = strm->avail_in; \ + hold = state->hold; \ + bits = state->bits; \ + } while (0) + +/* Restore state from registers in inflate() */ +#define RESTORE() \ + do { \ + strm->next_out = put; \ + strm->avail_out = left; \ + strm->next_in = next; \ + strm->avail_in = have; \ + state->hold = hold; \ + state->bits = bits; \ + } while (0) + +/* Clear the input bit accumulator */ +#define INITBITS() \ + do { \ + hold = 0; \ + bits = 0; \ + } while (0) + +/* Get a byte of input into the bit accumulator, or return from inflate() + if there is no input available. */ +#define PULLBYTE() \ + do { \ + if (have == 0) goto inf_leave; \ + have--; \ + hold += (unsigned long)(*next++) << bits; \ + bits += 8; \ + } while (0) + +/* Assure that there are at least n bits in the bit accumulator. If there is + not enough available input to do that, then return from inflate(). */ +#define NEEDBITS(n) \ + do { \ + while (bits < (unsigned)(n)) \ + PULLBYTE(); \ + } while (0) + +/* Return the low n bits of the bit accumulator (n < 16) */ +#define BITS(n) \ + ((unsigned)hold & ((1U << (n)) - 1)) + +/* Remove n bits from the bit accumulator */ +#define DROPBITS(n) \ + do { \ + hold >>= (n); \ + bits -= (unsigned)(n); \ + } while (0) + +/* Remove zero to seven bits as needed to go to a byte boundary */ +#define BYTEBITS() \ + do { \ + hold >>= bits & 7; \ + bits -= bits & 7; \ + } while (0) + +/* + inflate() uses a state machine to process as much input data and generate as + much output data as possible before returning. The state machine is + structured roughly as follows: + + for (;;) switch (state) { + ... + case STATEn: + if (not enough input data or output space to make progress) + return; + ... make progress ... + state = STATEm; + break; + ... + } + + so when inflate() is called again, the same case is attempted again, and + if the appropriate resources are provided, the machine proceeds to the + next state. The NEEDBITS() macro is usually the way the state evaluates + whether it can proceed or should return. NEEDBITS() does the return if + the requested bits are not available. The typical use of the BITS macros + is: + + NEEDBITS(n); + ... do something with BITS(n) ... + DROPBITS(n); + + where NEEDBITS(n) either returns from inflate() if there isn't enough + input left to load n bits into the accumulator, or it continues. BITS(n) + gives the low n bits in the accumulator. When done, DROPBITS(n) drops + the low n bits off the accumulator. INITBITS() clears the accumulator + and sets the number of available bits to zero. BYTEBITS() discards just + enough bits to put the accumulator on a byte boundary. After BYTEBITS() + and a NEEDBITS(8), then BITS(8) would return the next byte in the stream. + + NEEDBITS(n) uses PULLBYTE() to get an available byte of input, or to return + if there is no input available. The decoding of variable length codes uses + PULLBYTE() directly in order to pull just enough bytes to decode the next + code, and no more. + + Some states loop until they get enough input, making sure that enough + state information is maintained to continue the loop where it left off + if NEEDBITS() returns in the loop. For example, want, need, and keep + would all have to actually be part of the saved state in case NEEDBITS() + returns: + + case STATEw: + while (want < need) { + NEEDBITS(n); + keep[want++] = BITS(n); + DROPBITS(n); + } + state = STATEx; + case STATEx: + + As shown above, if the next state is also the next case, then the break + is omitted. + + A state may also return if there is not enough output space available to + complete that state. Those states are copying stored data, writing a + literal byte, and copying a matching string. + + When returning, a "goto inf_leave" is used to update the total counters, + update the check value, and determine whether any progress has been made + during that inflate() call in order to return the proper return code. + Progress is defined as a change in either strm->avail_in or strm->avail_out. + When there is a window, goto inf_leave will update the window with the last + output written. If a goto inf_leave occurs in the middle of decompression + and there is no window currently, goto inf_leave will create one and copy + output to the window for the next call of inflate(). + + In this implementation, the flush parameter of inflate() only affects the + return code (per zlib.h). inflate() always writes as much as possible to + strm->next_out, given the space available and the provided input--the effect + documented in zlib.h of Z_SYNC_FLUSH. Furthermore, inflate() always defers + the allocation of and copying into a sliding window until necessary, which + provides the effect documented in zlib.h for Z_FINISH when the entire input + stream available. So the only thing the flush parameter actually does is: + when flush is set to Z_FINISH, inflate() cannot return Z_OK. Instead it + will return Z_BUF_ERROR if it has not reached the end of the stream. + */ + +int ZEXPORT inflate(z_streamp strm, int flush) { + struct inflate_state FAR *state; + z_const unsigned char FAR *next; /* next input */ + unsigned char FAR *put; /* next output */ + unsigned have, left; /* available input and output */ + unsigned long hold; /* bit buffer */ + unsigned bits; /* bits in bit buffer */ + unsigned in, out; /* save starting available input and output */ + unsigned copy; /* number of stored or match bytes to copy */ + unsigned char FAR *from; /* where to copy match bytes from */ + code here; /* current decoding table entry */ + code last; /* parent table entry */ + unsigned len; /* length to copy for repeats, bits to drop */ + int ret; /* return code */ +#ifdef GUNZIP + unsigned char hbuf[4]; /* buffer for gzip header crc calculation */ +#endif + static const unsigned short order[19] = /* permutation of code lengths */ + {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || + (strm->next_in == Z_NULL && strm->avail_in != 0)) + return Z_STREAM_ERROR; + + state = (struct inflate_state FAR *)strm->state; + if (state->mode == TYPE) state->mode = TYPEDO; /* skip check */ + LOAD(); + in = have; + out = left; + ret = Z_OK; + for (;;) + switch (state->mode) { + case HEAD: + if (state->wrap == 0) { + state->mode = TYPEDO; + break; + } + NEEDBITS(16); +#ifdef GUNZIP + if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; + state->check = crc32(0L, Z_NULL, 0); + CRC2(state->check, hold); + INITBITS(); + state->mode = FLAGS; + break; + } + if (state->head != Z_NULL) + state->head->done = -1; + if (!(state->wrap & 1) || /* check if zlib header allowed */ +#else + if ( +#endif + ((BITS(8) << 8) + (hold >> 8)) % 31) { + strm->msg = (char *)"incorrect header check"; + state->mode = BAD; + break; + } + if (BITS(4) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + DROPBITS(4); + len = BITS(4) + 8; + if (state->wbits == 0) + state->wbits = len; + if (len > 15 || len > state->wbits) { + strm->msg = (char *)"invalid window size"; + state->mode = BAD; + break; + } + state->dmax = 1U << len; + state->flags = 0; /* indicate zlib header */ + Tracev((stderr, "inflate: zlib header ok\n")); + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = hold & 0x200 ? DICTID : TYPE; + INITBITS(); + break; +#ifdef GUNZIP + case FLAGS: + NEEDBITS(16); + state->flags = (int)(hold); + if ((state->flags & 0xff) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + if (state->flags & 0xe000) { + strm->msg = (char *)"unknown header flags set"; + state->mode = BAD; + break; + } + if (state->head != Z_NULL) + state->head->text = (int)((hold >> 8) & 1); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = TIME; + /* fallthrough */ + case TIME: + NEEDBITS(32); + if (state->head != Z_NULL) + state->head->time = hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); + INITBITS(); + state->mode = OS; + /* fallthrough */ + case OS: + NEEDBITS(16); + if (state->head != Z_NULL) { + state->head->xflags = (int)(hold & 0xff); + state->head->os = (int)(hold >> 8); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = EXLEN; + /* fallthrough */ + case EXLEN: + if (state->flags & 0x0400) { + NEEDBITS(16); + state->length = (unsigned)(hold); + if (state->head != Z_NULL) + state->head->extra_len = (unsigned)hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + } + else if (state->head != Z_NULL) + state->head->extra = Z_NULL; + state->mode = EXTRA; + /* fallthrough */ + case EXTRA: + if (state->flags & 0x0400) { + copy = state->length; + if (copy > have) copy = have; + if (copy) { + if (state->head != Z_NULL && + state->head->extra != Z_NULL && + (len = state->head->extra_len - state->length) < + state->head->extra_max) { + zmemcpy(state->head->extra + len, next, + len + copy > state->head->extra_max ? + state->head->extra_max - len : copy); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + state->length -= copy; + } + if (state->length) goto inf_leave; + } + state->length = 0; + state->mode = NAME; + /* fallthrough */ + case NAME: + if (state->flags & 0x0800) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->name != Z_NULL && + state->length < state->head->name_max) + state->head->name[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->name = Z_NULL; + state->length = 0; + state->mode = COMMENT; + /* fallthrough */ + case COMMENT: + if (state->flags & 0x1000) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->comment != Z_NULL && + state->length < state->head->comm_max) + state->head->comment[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->comment = Z_NULL; + state->mode = HCRC; + /* fallthrough */ + case HCRC: + if (state->flags & 0x0200) { + NEEDBITS(16); + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { + strm->msg = (char *)"header crc mismatch"; + state->mode = BAD; + break; + } + INITBITS(); + } + if (state->head != Z_NULL) { + state->head->hcrc = (int)((state->flags >> 9) & 1); + state->head->done = 1; + } + strm->adler = state->check = crc32(0L, Z_NULL, 0); + state->mode = TYPE; + break; +#endif + case DICTID: + NEEDBITS(32); + strm->adler = state->check = ZSWAP32(hold); + INITBITS(); + state->mode = DICT; + /* fallthrough */ + case DICT: + if (state->havedict == 0) { + RESTORE(); + return Z_NEED_DICT; + } + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = TYPE; + /* fallthrough */ + case TYPE: + if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case TYPEDO: + if (state->last) { + BYTEBITS(); + state->mode = CHECK; + break; + } + NEEDBITS(3); + state->last = BITS(1); + DROPBITS(1); + switch (BITS(2)) { + case 0: /* stored block */ + Tracev((stderr, "inflate: stored block%s\n", + state->last ? " (last)" : "")); + state->mode = STORED; + break; + case 1: /* fixed block */ + fixedtables(state); + Tracev((stderr, "inflate: fixed codes block%s\n", + state->last ? " (last)" : "")); + state->mode = LEN_; /* decode codes */ + if (flush == Z_TREES) { + DROPBITS(2); + goto inf_leave; + } + break; + case 2: /* dynamic block */ + Tracev((stderr, "inflate: dynamic codes block%s\n", + state->last ? " (last)" : "")); + state->mode = TABLE; + break; + case 3: + strm->msg = (char *)"invalid block type"; + state->mode = BAD; + } + DROPBITS(2); + break; + case STORED: + BYTEBITS(); /* go to byte boundary */ + NEEDBITS(32); + if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { + strm->msg = (char *)"invalid stored block lengths"; + state->mode = BAD; + break; + } + state->length = (unsigned)hold & 0xffff; + Tracev((stderr, "inflate: stored length %u\n", + state->length)); + INITBITS(); + state->mode = COPY_; + if (flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case COPY_: + state->mode = COPY; + /* fallthrough */ + case COPY: + copy = state->length; + if (copy) { + if (copy > have) copy = have; + if (copy > left) copy = left; + if (copy == 0) goto inf_leave; + zmemcpy(put, next, copy); + have -= copy; + next += copy; + left -= copy; + put += copy; + state->length -= copy; + break; + } + Tracev((stderr, "inflate: stored end\n")); + state->mode = TYPE; + break; + case TABLE: + NEEDBITS(14); + state->nlen = BITS(5) + 257; + DROPBITS(5); + state->ndist = BITS(5) + 1; + DROPBITS(5); + state->ncode = BITS(4) + 4; + DROPBITS(4); +#ifndef PKZIP_BUG_WORKAROUND + if (state->nlen > 286 || state->ndist > 30) { + strm->msg = (char *)"too many length or distance symbols"; + state->mode = BAD; + break; + } +#endif + Tracev((stderr, "inflate: table sizes ok\n")); + state->have = 0; + state->mode = LENLENS; + /* fallthrough */ + case LENLENS: + while (state->have < state->ncode) { + NEEDBITS(3); + state->lens[order[state->have++]] = (unsigned short)BITS(3); + DROPBITS(3); + } + while (state->have < 19) + state->lens[order[state->have++]] = 0; + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 7; + ret = inflate_table(CODES, state->lens, 19, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid code lengths set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: code lengths ok\n")); + state->have = 0; + state->mode = CODELENS; + /* fallthrough */ + case CODELENS: + while (state->have < state->nlen + state->ndist) { + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; + } + else { + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); + if (state->have == 0) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + len = state->lens[state->have - 1]; + copy = 3 + BITS(2); + DROPBITS(2); + } + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); + len = 0; + copy = 3 + BITS(3); + DROPBITS(3); + } + else { + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); + len = 0; + copy = 11 + BITS(7); + DROPBITS(7); + } + if (state->have + copy > state->nlen + state->ndist) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + while (copy--) + state->lens[state->have++] = (unsigned short)len; + } + } + + /* handle error breaks in while */ + if (state->mode == BAD) break; + + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 9; + ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid literal/lengths set"; + state->mode = BAD; + break; + } + state->distcode = (const code FAR *)(state->next); + state->distbits = 6; + ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, + &(state->next), &(state->distbits), state->work); + if (ret) { + strm->msg = (char *)"invalid distances set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: codes ok\n")); + state->mode = LEN_; + if (flush == Z_TREES) goto inf_leave; + /* fallthrough */ + case LEN_: + state->mode = LEN; + /* fallthrough */ + case LEN: + if (have >= 6 && left >= 258) { + RESTORE(); + inflate_fast(strm, out); + LOAD(); + if (state->mode == TYPE) + state->back = -1; + break; + } + state->back = 0; + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.op && (here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->lencode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + state->length = (unsigned)here.val; + if ((int)(here.op) == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + state->mode = LIT; + break; + } + if (here.op & 32) { + Tracevv((stderr, "inflate: end of block\n")); + state->back = -1; + state->mode = TYPE; + break; + } + if (here.op & 64) { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + state->extra = (unsigned)(here.op) & 15; + state->mode = LENEXT; + /* fallthrough */ + case LENEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->length += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } + Tracevv((stderr, "inflate: length %u\n", state->length)); + state->was = state->length; + state->mode = DIST; + /* fallthrough */ + case DIST: + for (;;) { + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if ((here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->distcode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + if (here.op & 64) { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + state->offset = (unsigned)here.val; + state->extra = (unsigned)(here.op) & 15; + state->mode = DISTEXT; + /* fallthrough */ + case DISTEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->offset += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } +#ifdef INFLATE_STRICT + if (state->offset > state->dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + Tracevv((stderr, "inflate: distance %u\n", state->offset)); + state->mode = MATCH; + /* fallthrough */ + case MATCH: + if (left == 0) goto inf_leave; + copy = out - left; + if (state->offset > copy) { /* copy from window */ + copy = state->offset - copy; + if (copy > state->whave) { + if (state->sane) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + Trace((stderr, "inflate.c too far\n")); + copy -= state->whave; + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = 0; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; +#endif + } + if (copy > state->wnext) { + copy -= state->wnext; + from = state->window + (state->wsize - copy); + } + else + from = state->window + (state->wnext - copy); + if (copy > state->length) copy = state->length; + } + else { /* copy from output */ + from = put - state->offset; + copy = state->length; + } + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = *from++; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; + case LIT: + if (left == 0) goto inf_leave; + *put++ = (unsigned char)(state->length); + left--; + state->mode = LEN; + break; + case CHECK: + if (state->wrap) { + NEEDBITS(32); + out -= left; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE_CHECK(state->check, put - out, out); + out = left; + if ((state->wrap & 4) && ( +#ifdef GUNZIP + state->flags ? hold : +#endif + ZSWAP32(hold)) != state->check) { + strm->msg = (char *)"incorrect data check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: check matches trailer\n")); + } +#ifdef GUNZIP + state->mode = LENGTH; + /* fallthrough */ + case LENGTH: + if (state->wrap && state->flags) { + NEEDBITS(32); + if ((state->wrap & 4) && hold != (state->total & 0xffffffff)) { + strm->msg = (char *)"incorrect length check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: length matches trailer\n")); + } +#endif + state->mode = DONE; + /* fallthrough */ + case DONE: + ret = Z_STREAM_END; + goto inf_leave; + case BAD: + ret = Z_DATA_ERROR; + goto inf_leave; + case MEM: + return Z_MEM_ERROR; + case SYNC: + /* fallthrough */ + default: + return Z_STREAM_ERROR; + } + + /* + Return from inflate(), updating the total counts and the check value. + If there was no progress during the inflate() call, return a buffer + error. Call updatewindow() to create and/or update the window state. + Note: a memory error from inflate() is non-recoverable. + */ + inf_leave: + RESTORE(); + if (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH))) + if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { + state->mode = MEM; + return Z_MEM_ERROR; + } + in -= strm->avail_in; + out -= strm->avail_out; + strm->total_in += in; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE_CHECK(state->check, strm->next_out - out, out); + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + + (state->mode == TYPE ? 128 : 0) + + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); + if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) + ret = Z_BUF_ERROR; + return ret; +} + +int ZEXPORT inflateEnd(z_streamp strm) { + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->window != Z_NULL) ZFREE(strm, state->window); + ZFREE(strm, strm->state); + strm->state = Z_NULL; + Tracev((stderr, "inflate: end\n")); + return Z_OK; +} + +int ZEXPORT inflateGetDictionary(z_streamp strm, Bytef *dictionary, + uInt *dictLength) { + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* copy dictionary */ + if (state->whave && dictionary != Z_NULL) { + zmemcpy(dictionary, state->window + state->wnext, + state->whave - state->wnext); + zmemcpy(dictionary + state->whave - state->wnext, + state->window, state->wnext); + } + if (dictLength != Z_NULL) + *dictLength = state->whave; + return Z_OK; +} + +int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef *dictionary, + uInt dictLength) { + struct inflate_state FAR *state; + unsigned long dictid; + int ret; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->wrap != 0 && state->mode != DICT) + return Z_STREAM_ERROR; + + /* check for correct dictionary identifier */ + if (state->mode == DICT) { + dictid = adler32(0L, Z_NULL, 0); + dictid = adler32(dictid, dictionary, dictLength); + if (dictid != state->check) + return Z_DATA_ERROR; + } + + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary + dictLength, dictLength); + if (ret) { + state->mode = MEM; + return Z_MEM_ERROR; + } + state->havedict = 1; + Tracev((stderr, "inflate: dictionary set\n")); + return Z_OK; +} + +int ZEXPORT inflateGetHeader(z_streamp strm, gz_headerp head) { + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; + + /* save header structure */ + state->head = head; + head->done = 0; + return Z_OK; +} + +/* + Search buf[0..len-1] for the pattern: 0, 0, 0xff, 0xff. Return when found + or when out of input. When called, *have is the number of pattern bytes + found in order so far, in 0..3. On return *have is updated to the new + state. If on return *have equals four, then the pattern was found and the + return value is how many bytes were read including the last byte of the + pattern. If *have is less than four, then the pattern has not been found + yet and the return value is len. In the latter case, syncsearch() can be + called again with more data and the *have state. *have is initialized to + zero for the first call. + */ +local unsigned syncsearch(unsigned FAR *have, const unsigned char FAR *buf, + unsigned len) { + unsigned got; + unsigned next; + + got = *have; + next = 0; + while (next < len && got < 4) { + if ((int)(buf[next]) == (got < 2 ? 0 : 0xff)) + got++; + else if (buf[next]) + got = 0; + else + got = 4 - got; + next++; + } + *have = got; + return next; +} + +int ZEXPORT inflateSync(z_streamp strm) { + unsigned len; /* number of bytes to look at or looked at */ + int flags; /* temporary to save header status */ + unsigned long in, out; /* temporary to save total_in and total_out */ + unsigned char buf[4]; /* to restore bit buffer to byte string */ + struct inflate_state FAR *state; + + /* check parameters */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; + + /* if first time, start search in bit buffer */ + if (state->mode != SYNC) { + state->mode = SYNC; + state->hold <<= state->bits & 7; + state->bits -= state->bits & 7; + len = 0; + while (state->bits >= 8) { + buf[len++] = (unsigned char)(state->hold); + state->hold >>= 8; + state->bits -= 8; + } + state->have = 0; + syncsearch(&(state->have), buf, len); + } + + /* search available input */ + len = syncsearch(&(state->have), strm->next_in, strm->avail_in); + strm->avail_in -= len; + strm->next_in += len; + strm->total_in += len; + + /* return no joy or set up to restart inflate() on a new block */ + if (state->have != 4) return Z_DATA_ERROR; + if (state->flags == -1) + state->wrap = 0; /* if no header yet, treat as raw */ + else + state->wrap &= ~4; /* no point in computing a check value now */ + flags = state->flags; + in = strm->total_in; out = strm->total_out; + inflateReset(strm); + strm->total_in = in; strm->total_out = out; + state->flags = flags; + state->mode = TYPE; + return Z_OK; +} + +/* + Returns true if inflate is currently at the end of a block generated by + Z_SYNC_FLUSH or Z_FULL_FLUSH. This function is used by one PPP + implementation to provide an additional safety check. PPP uses + Z_SYNC_FLUSH but removes the length bytes of the resulting empty stored + block. When decompressing, PPP checks that at the end of input packet, + inflate is waiting for these length bytes. + */ +int ZEXPORT inflateSyncPoint(z_streamp strm) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + return state->mode == STORED && state->bits == 0; +} + +int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { + struct inflate_state FAR *state; + struct inflate_state FAR *copy; + unsigned char FAR *window; + unsigned wsize; + + /* check input */ + if (inflateStateCheck(source) || dest == Z_NULL) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)source->state; + + /* allocate space */ + copy = (struct inflate_state FAR *) + ZALLOC(source, 1, sizeof(struct inflate_state)); + if (copy == Z_NULL) return Z_MEM_ERROR; + window = Z_NULL; + if (state->window != Z_NULL) { + window = (unsigned char FAR *) + ZALLOC(source, 1U << state->wbits, sizeof(unsigned char)); + if (window == Z_NULL) { + ZFREE(source, copy); + return Z_MEM_ERROR; + } + } + + /* copy state */ + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; + if (state->lencode >= state->codes && + state->lencode <= state->codes + ENOUGH - 1) { + copy->lencode = copy->codes + (state->lencode - state->codes); + copy->distcode = copy->codes + (state->distcode - state->codes); + } + copy->next = copy->codes + (state->next - state->codes); + if (window != Z_NULL) { + wsize = 1U << state->wbits; + zmemcpy(window, state->window, wsize); + } + copy->window = window; + dest->state = (struct internal_state FAR *)copy; + return Z_OK; +} + +int ZEXPORT inflateUndermine(z_streamp strm, int subvert) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; + return Z_OK; +#else + (void)subvert; + state->sane = 1; + return Z_DATA_ERROR; +#endif +} + +int ZEXPORT inflateValidate(z_streamp strm, int check) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check && state->wrap) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + +long ZEXPORT inflateMark(z_streamp strm) { + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) + return -(1L << 16); + state = (struct inflate_state FAR *)strm->state; + return (long)(((unsigned long)((long)state->back)) << 16) + + (state->mode == COPY ? state->length : + (state->mode == MATCH ? state->was - state->length : 0)); +} + +unsigned long ZEXPORT inflateCodesUsed(z_streamp strm) { + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/lib/zlib/inflate.h b/lib/zlib/inflate.h new file mode 100644 index 0000000..f127b6b --- /dev/null +++ b/lib/zlib/inflate.h @@ -0,0 +1,126 @@ +/* inflate.h -- internal inflate state definition + * Copyright (C) 1995-2019 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* define NO_GZIP when compiling if you want to disable gzip header and + trailer decoding by inflate(). NO_GZIP would be used to avoid linking in + the crc code when it is not needed. For shared libraries, gzip decoding + should be left enabled. */ +#ifndef NO_GZIP +# define GUNZIP +#endif + +/* Possible inflate modes between inflate() calls */ +typedef enum { + HEAD = 16180, /* i: waiting for magic header */ + FLAGS, /* i: waiting for method and flags (gzip) */ + TIME, /* i: waiting for modification time (gzip) */ + OS, /* i: waiting for extra flags and operating system (gzip) */ + EXLEN, /* i: waiting for extra length (gzip) */ + EXTRA, /* i: waiting for extra bytes (gzip) */ + NAME, /* i: waiting for end of file name (gzip) */ + COMMENT, /* i: waiting for end of comment (gzip) */ + HCRC, /* i: waiting for header crc (gzip) */ + DICTID, /* i: waiting for dictionary check value */ + DICT, /* waiting for inflateSetDictionary() call */ + TYPE, /* i: waiting for type bits, including last-flag bit */ + TYPEDO, /* i: same, but skip check to exit inflate on new block */ + STORED, /* i: waiting for stored size (length and complement) */ + COPY_, /* i/o: same as COPY below, but only first time in */ + COPY, /* i/o: waiting for input or output to copy stored block */ + TABLE, /* i: waiting for dynamic block table lengths */ + LENLENS, /* i: waiting for code length code lengths */ + CODELENS, /* i: waiting for length/lit and distance code lengths */ + LEN_, /* i: same as LEN below, but only first time in */ + LEN, /* i: waiting for length/lit/eob code */ + LENEXT, /* i: waiting for length extra bits */ + DIST, /* i: waiting for distance code */ + DISTEXT, /* i: waiting for distance extra bits */ + MATCH, /* o: waiting for output space to copy string */ + LIT, /* o: waiting for output space to write literal */ + CHECK, /* i: waiting for 32-bit check value */ + LENGTH, /* i: waiting for 32-bit length (gzip) */ + DONE, /* finished check, done -- remain here until reset */ + BAD, /* got a data error -- remain here until reset */ + MEM, /* got an inflate() memory error -- remain here until reset */ + SYNC /* looking for synchronization bytes to restart inflate() */ +} inflate_mode; + +/* + State transitions between above modes - + + (most modes can go to BAD or MEM on error -- not shown for clarity) + + Process header: + HEAD -> (gzip) or (zlib) or (raw) + (gzip) -> FLAGS -> TIME -> OS -> EXLEN -> EXTRA -> NAME -> COMMENT -> + HCRC -> TYPE + (zlib) -> DICTID or TYPE + DICTID -> DICT -> TYPE + (raw) -> TYPEDO + Read deflate blocks: + TYPE -> TYPEDO -> STORED or TABLE or LEN_ or CHECK + STORED -> COPY_ -> COPY -> TYPE + TABLE -> LENLENS -> CODELENS -> LEN_ + LEN_ -> LEN + Read deflate codes in fixed or dynamic block: + LEN -> LENEXT or LIT or TYPE + LENEXT -> DIST -> DISTEXT -> MATCH -> LEN + LIT -> LEN + Process trailer: + CHECK -> LENGTH -> DONE + */ + +/* State maintained between inflate() calls -- approximately 7K bytes, not + including the allocated sliding window, which is up to 32K bytes. */ +struct inflate_state { + z_streamp strm; /* pointer back to this zlib stream */ + inflate_mode mode; /* current inflate mode */ + int last; /* true if processing last block */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip, + bit 2 true to validate check value */ + int havedict; /* true if dictionary provided */ + int flags; /* gzip header method and flags, 0 if zlib, or + -1 if raw or no header yet */ + unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ + unsigned long check; /* protected copy of check value */ + unsigned long total; /* protected copy of output count */ + gz_headerp head; /* where to save gzip header information */ + /* sliding window */ + unsigned wbits; /* log base 2 of requested window size */ + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if needed */ + /* bit accumulator */ + unsigned long hold; /* input bit accumulator */ + unsigned bits; /* number of bits in "in" */ + /* for string and stored block copying */ + unsigned length; /* literal or length of data to copy */ + unsigned offset; /* distance back to copy string from */ + /* for table and code decoding */ + unsigned extra; /* extra bits needed */ + /* fixed and dynamic code tables */ + code const FAR *lencode; /* starting table for length/literal codes */ + code const FAR *distcode; /* starting table for distance codes */ + unsigned lenbits; /* index bits for lencode */ + unsigned distbits; /* index bits for distcode */ + /* dynamic table building */ + unsigned ncode; /* number of code length code lengths */ + unsigned nlen; /* number of length code lengths */ + unsigned ndist; /* number of distance code lengths */ + unsigned have; /* number of code lengths in lens[] */ + code FAR *next; /* next available space in codes[] */ + unsigned short lens[320]; /* temporary storage for code lengths */ + unsigned short work[288]; /* work area for code table building */ + code codes[ENOUGH]; /* space for code tables */ + int sane; /* if false, allow invalid distance too far */ + int back; /* bits back of last unprocessed length/lit */ + unsigned was; /* initial length of match */ +}; diff --git a/lib/zlib/inftrees.c b/lib/zlib/inftrees.c new file mode 100644 index 0000000..8a208c2 --- /dev/null +++ b/lib/zlib/inftrees.c @@ -0,0 +1,299 @@ +/* inftrees.c -- generate Huffman trees for efficient decoding + * Copyright (C) 1995-2023 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" + +#define MAXBITS 15 + +const char inflate_copyright[] = + " inflate 1.3 Copyright 1995-2023 Mark Adler "; +/* + If you use the zlib library in a product, an acknowledgment is welcome + in the documentation of your product. If for some reason you cannot + include such an acknowledgment, I would appreciate that you keep this + copyright string in the executable of your product. + */ + +/* + Build a set of tables to decode the provided canonical Huffman code. + The code lengths are lens[0..codes-1]. The result starts at *table, + whose indices are 0..2^bits-1. work is a writable array of at least + lens shorts, which is used as a work area. type is the type of code + to be generated, CODES, LENS, or DISTS. On return, zero is success, + -1 is an invalid code, and +1 means that ENOUGH isn't enough. table + on return points to the next available entry's address. bits is the + requested root table index bits, and on return it is the actual root + table index bits. It will differ if the request is greater than the + longest code or if it is less than the shortest code. + */ +int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, + unsigned codes, code FAR * FAR *table, + unsigned FAR *bits, unsigned short FAR *work) { + unsigned len; /* a code's length in bits */ + unsigned sym; /* index of code symbols */ + unsigned min, max; /* minimum and maximum code lengths */ + unsigned root; /* number of index bits for root table */ + unsigned curr; /* number of index bits for current table */ + unsigned drop; /* code bits to drop for sub-table */ + int left; /* number of prefix codes available */ + unsigned used; /* code entries in table used */ + unsigned huff; /* Huffman code */ + unsigned incr; /* for incrementing code, index */ + unsigned fill; /* index for replicating entries */ + unsigned low; /* low bits for current root entry */ + unsigned mask; /* mask for low root bits */ + code here; /* table entry for duplication */ + code FAR *next; /* next available space in table */ + const unsigned short FAR *base; /* base value table to use */ + const unsigned short FAR *extra; /* extra bits table to use */ + unsigned match; /* use base and extra for symbol >= match */ + unsigned short count[MAXBITS+1]; /* number of codes of each length */ + unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ + static const unsigned short lbase[31] = { /* Length codes 257..285 base */ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; + static const unsigned short lext[31] = { /* Length codes 257..285 extra */ + 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 198, 203}; + static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, + 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, + 8193, 12289, 16385, 24577, 0, 0}; + static const unsigned short dext[32] = { /* Distance codes 0..29 extra */ + 16, 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, + 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, + 28, 28, 29, 29, 64, 64}; + + /* + Process a set of code lengths to create a canonical Huffman code. The + code lengths are lens[0..codes-1]. Each length corresponds to the + symbols 0..codes-1. The Huffman code is generated by first sorting the + symbols by length from short to long, and retaining the symbol order + for codes with equal lengths. Then the code starts with all zero bits + for the first code of the shortest length, and the codes are integer + increments for the same length, and zeros are appended as the length + increases. For the deflate format, these bits are stored backwards + from their more natural integer increment ordering, and so when the + decoding tables are built in the large loop below, the integer codes + are incremented backwards. + + This routine assumes, but does not check, that all of the entries in + lens[] are in the range 0..MAXBITS. The caller must assure this. + 1..MAXBITS is interpreted as that code length. zero means that that + symbol does not occur in this code. + + The codes are sorted by computing a count of codes for each length, + creating from that a table of starting indices for each length in the + sorted table, and then entering the symbols in order in the sorted + table. The sorted table is work[], with that space being provided by + the caller. + + The length counts are used for other purposes as well, i.e. finding + the minimum and maximum length codes, determining if there are any + codes at all, checking for a valid set of lengths, and looking ahead + at length counts to determine sub-table sizes when building the + decoding tables. + */ + + /* accumulate lengths for codes (assumes lens[] all in 0..MAXBITS) */ + for (len = 0; len <= MAXBITS; len++) + count[len] = 0; + for (sym = 0; sym < codes; sym++) + count[lens[sym]]++; + + /* bound code lengths, force root to be within code lengths */ + root = *bits; + for (max = MAXBITS; max >= 1; max--) + if (count[max] != 0) break; + if (root > max) root = max; + if (max == 0) { /* no symbols to code at all */ + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)1; + here.val = (unsigned short)0; + *(*table)++ = here; /* make a table to force an error */ + *(*table)++ = here; + *bits = 1; + return 0; /* no symbols, but wait for decoding to report error */ + } + for (min = 1; min < max; min++) + if (count[min] != 0) break; + if (root < min) root = min; + + /* check for an over-subscribed or incomplete set of lengths */ + left = 1; + for (len = 1; len <= MAXBITS; len++) { + left <<= 1; + left -= count[len]; + if (left < 0) return -1; /* over-subscribed */ + } + if (left > 0 && (type == CODES || max != 1)) + return -1; /* incomplete set */ + + /* generate offsets into symbol table for each length for sorting */ + offs[1] = 0; + for (len = 1; len < MAXBITS; len++) + offs[len + 1] = offs[len] + count[len]; + + /* sort symbols by length, by symbol order within each length */ + for (sym = 0; sym < codes; sym++) + if (lens[sym] != 0) work[offs[lens[sym]]++] = (unsigned short)sym; + + /* + Create and fill in decoding tables. In this loop, the table being + filled is at next and has curr index bits. The code being used is huff + with length len. That code is converted to an index by dropping drop + bits off of the bottom. For codes where len is less than drop + curr, + those top drop + curr - len bits are incremented through all values to + fill the table with replicated entries. + + root is the number of index bits for the root table. When len exceeds + root, sub-tables are created pointed to by the root entry with an index + of the low root bits of huff. This is saved in low to check for when a + new sub-table should be started. drop is zero when the root table is + being filled, and drop is root when sub-tables are being filled. + + When a new sub-table is needed, it is necessary to look ahead in the + code lengths to determine what size sub-table is needed. The length + counts are used for this, and so count[] is decremented as codes are + entered in the tables. + + used keeps track of how many table entries have been allocated from the + provided *table space. It is checked for LENS and DIST tables against + the constants ENOUGH_LENS and ENOUGH_DISTS to guard against changes in + the initial root table size constants. See the comments in inftrees.h + for more information. + + sym increments through all symbols, and the loop terminates when + all codes of length max, i.e. all codes, have been processed. This + routine permits incomplete codes, so another loop after this one fills + in the rest of the decoding tables with invalid code markers. + */ + + /* set up for code type */ + switch (type) { + case CODES: + base = extra = work; /* dummy value--not used */ + match = 20; + break; + case LENS: + base = lbase; + extra = lext; + match = 257; + break; + default: /* DISTS */ + base = dbase; + extra = dext; + match = 0; + } + + /* initialize state for loop */ + huff = 0; /* starting code */ + sym = 0; /* starting code symbol */ + len = min; /* starting code length */ + next = *table; /* current table to fill in */ + curr = root; /* current table index bits */ + drop = 0; /* current bits to drop from code for index */ + low = (unsigned)(-1); /* trigger new sub-table when len > root */ + used = 1U << root; /* use root table entries */ + mask = used - 1; /* mask for comparing low */ + + /* check available table space */ + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) + return 1; + + /* process all codes and make table entries */ + for (;;) { + /* create table entry */ + here.bits = (unsigned char)(len - drop); + if (work[sym] + 1U < match) { + here.op = (unsigned char)0; + here.val = work[sym]; + } + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; + } + else { + here.op = (unsigned char)(32 + 64); /* end of block */ + here.val = 0; + } + + /* replicate for those indices with low len bits equal to huff */ + incr = 1U << (len - drop); + fill = 1U << curr; + min = fill; /* save offset to next table */ + do { + fill -= incr; + next[(huff >> drop) + fill] = here; + } while (fill != 0); + + /* backwards increment the len-bit code huff */ + incr = 1U << (len - 1); + while (huff & incr) + incr >>= 1; + if (incr != 0) { + huff &= incr - 1; + huff += incr; + } + else + huff = 0; + + /* go to next symbol, update count, len */ + sym++; + if (--(count[len]) == 0) { + if (len == max) break; + len = lens[work[sym]]; + } + + /* create new sub-table if needed */ + if (len > root && (huff & mask) != low) { + /* if first time, transition to sub-tables */ + if (drop == 0) + drop = root; + + /* increment past last table */ + next += min; /* here min is 1 << curr */ + + /* determine length of next table */ + curr = len - drop; + left = (int)(1 << curr); + while (curr + drop < max) { + left -= count[curr + drop]; + if (left <= 0) break; + curr++; + left <<= 1; + } + + /* check for enough space */ + used += 1U << curr; + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) + return 1; + + /* point entry in root table to sub-table */ + low = huff & mask; + (*table)[low].op = (unsigned char)curr; + (*table)[low].bits = (unsigned char)root; + (*table)[low].val = (unsigned short)(next - *table); + } + } + + /* fill in remaining table entry if code is incomplete (guaranteed to have + at most one remaining entry, since if the code is incomplete, the + maximum code length that was allowed to get this far is one bit) */ + if (huff != 0) { + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)(len - drop); + here.val = (unsigned short)0; + next[huff] = here; + } + + /* set return parameters */ + *table += used; + *bits = root; + return 0; +} diff --git a/lib/zlib/inftrees.h b/lib/zlib/inftrees.h new file mode 100644 index 0000000..a10712d --- /dev/null +++ b/lib/zlib/inftrees.h @@ -0,0 +1,62 @@ +/* inftrees.h -- header to use inftrees.c + * Copyright (C) 1995-2005, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* Structure for decoding tables. Each entry provides either the + information needed to do the operation requested by the code that + indexed that table entry, or it provides a pointer to another + table that indexes more bits of the code. op indicates whether + the entry is a pointer to another table, a literal, a length or + distance, an end-of-block, or an invalid code. For a table + pointer, the low four bits of op is the number of index bits of + that table. For a length or distance, the low four bits of op + is the number of extra bits to get after the code. bits is + the number of bits in this code or part of the code to drop off + of the bit buffer. val is the actual byte to output in the case + of a literal, the base length or distance, or the offset from + the current table to the next table. Each entry is four bytes. */ +typedef struct { + unsigned char op; /* operation, extra bits, table bits */ + unsigned char bits; /* bits in this part of the code */ + unsigned short val; /* offset in table or code value */ +} code; + +/* op values as set by inflate_table(): + 00000000 - literal + 0000tttt - table link, tttt != 0 is the number of table index bits + 0001eeee - length or distance, eeee is the number of extra bits + 01100000 - end of block + 01000000 - invalid code + */ + +/* Maximum size of the dynamic table. The maximum number of code structures is + 1444, which is the sum of 852 for literal/length codes and 592 for distance + codes. These values were found by exhaustive searches using the program + examples/enough.c found in the zlib distribution. The arguments to that + program are the number of symbols, the initial root table size, and the + maximum bit length of a code. "enough 286 9 15" for literal/length codes + returns returns 852, and "enough 30 6 15" for distance codes returns 592. + The initial root table size (9 or 6) is found in the fifth argument of the + inflate_table() calls in inflate.c and infback.c. If the root table size is + changed, then these maximum sizes would be need to be recalculated and + updated. */ +#define ENOUGH_LENS 852 +#define ENOUGH_DISTS 592 +#define ENOUGH (ENOUGH_LENS+ENOUGH_DISTS) + +/* Type of code to build for inflate_table() */ +typedef enum { + CODES, + LENS, + DISTS +} codetype; + +int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, + unsigned codes, code FAR * FAR *table, + unsigned FAR *bits, unsigned short FAR *work); diff --git a/lib/zlib/tf_gunzip.c b/lib/zlib/tf_gunzip.c new file mode 100644 index 0000000..8b8d14d --- /dev/null +++ b/lib/zlib/tf_gunzip.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2018-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include <common/debug.h> +#include <common/tf_crc32.h> +#include <lib/utils.h> +#include <tf_gunzip.h> + +#include "zutil.h" + +/* + * memory allocated by malloc() is supposed to be aligned for any built-in type + */ +#define ZALLOC_ALIGNMENT sizeof(void *) + +static uintptr_t zalloc_start; +static uintptr_t zalloc_end; +static uintptr_t zalloc_current; + +static void * ZLIB_INTERNAL zcalloc(void *opaque, unsigned int items, + unsigned int size) +{ + uintptr_t p, p_end; + + size *= items; + + p = round_up(zalloc_current, ZALLOC_ALIGNMENT); + p_end = p + size; + + if (p_end > zalloc_end) + return NULL; + + memset((void *)p, 0, size); + + zalloc_current = p_end; + + return (void *)p; +} + +static void ZLIB_INTERNAL zfree(void *opaque, void *ptr) +{ +} + +/* + * gunzip - decompress gzip data + * @in_buf: source of compressed input. Upon exit, the end of input. + * @in_len: length of in_buf + * @out_buf: destination of decompressed output. Upon exit, the end of output. + * @out_len: length of out_buf + * @work_buf: workspace + * @work_len: length of workspace + */ +int gunzip(uintptr_t *in_buf, size_t in_len, uintptr_t *out_buf, + size_t out_len, uintptr_t work_buf, size_t work_len) +{ + z_stream stream; + int zret, ret; + + zalloc_start = work_buf; + zalloc_end = work_buf + work_len; + zalloc_current = zalloc_start; + + stream.next_in = (typeof(stream.next_in))*in_buf; + stream.avail_in = in_len; + stream.next_out = (typeof(stream.next_out))*out_buf; + stream.avail_out = out_len; + stream.zalloc = zcalloc; + stream.zfree = zfree; + stream.opaque = (voidpf)0; + + zret = inflateInit(&stream); + if (zret != Z_OK) { + ERROR("zlib: inflate init failed (ret = %d)\n", zret); + return (zret == Z_MEM_ERROR) ? -ENOMEM : -EIO; + } + + zret = inflate(&stream, Z_NO_FLUSH); + if (zret == Z_STREAM_END) { + ret = 0; + } else { + if (stream.msg) + ERROR("%s\n", stream.msg); + ERROR("zlib: inflate failed (ret = %d)\n", zret); + ret = (zret == Z_MEM_ERROR) ? -ENOMEM : -EIO; + } + + VERBOSE("zlib: %lu byte input\n", stream.total_in); + VERBOSE("zlib: %lu byte output\n", stream.total_out); + + *in_buf = (uintptr_t)stream.next_in; + *out_buf = (uintptr_t)stream.next_out; + + inflateEnd(&stream); + + return ret; +} + +/* Wrapper function to calculate CRC + * @crc: previous accumulated CRC + * @buf: buffer base address + * @size: size of the buffer + * + * Return calculated CRC32 value + */ +uint32_t tf_crc32(uint32_t crc, const unsigned char *buf, size_t size) +{ + return (uint32_t)crc32((unsigned long)crc, buf, size); +} diff --git a/lib/zlib/zconf.h b/lib/zlib/zconf.h new file mode 100644 index 0000000..fb76ffe --- /dev/null +++ b/lib/zlib/zconf.h @@ -0,0 +1,551 @@ +/* zconf.h -- configuration of the zlib compression library + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#ifndef ZCONF_H +#define ZCONF_H + +/* + * If you *really* need a unique prefix for all types and library functions, + * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. + * Even better than compiling with -DZ_PREFIX would be to use configure to set + * this permanently in zconf.h using "./configure --zprefix". + */ +#ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ +# define Z_PREFIX_SET + +/* all linked symbols and init macros */ +# define _dist_code z__dist_code +# define _length_code z__length_code +# define _tr_align z__tr_align +# define _tr_flush_bits z__tr_flush_bits +# define _tr_flush_block z__tr_flush_block +# define _tr_init z__tr_init +# define _tr_stored_block z__tr_stored_block +# define _tr_tally z__tr_tally +# define adler32 z_adler32 +# define adler32_combine z_adler32_combine +# define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z +# ifndef Z_SOLO +# define compress z_compress +# define compress2 z_compress2 +# define compressBound z_compressBound +# endif +# define crc32 z_crc32 +# define crc32_combine z_crc32_combine +# define crc32_combine64 z_crc32_combine64 +# define crc32_combine_gen z_crc32_combine_gen +# define crc32_combine_gen64 z_crc32_combine_gen64 +# define crc32_combine_op z_crc32_combine_op +# define crc32_z z_crc32_z +# define deflate z_deflate +# define deflateBound z_deflateBound +# define deflateCopy z_deflateCopy +# define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 +# define deflateInit2_ z_deflateInit2_ +# define deflateInit_ z_deflateInit_ +# define deflateParams z_deflateParams +# define deflatePending z_deflatePending +# define deflatePrime z_deflatePrime +# define deflateReset z_deflateReset +# define deflateResetKeep z_deflateResetKeep +# define deflateSetDictionary z_deflateSetDictionary +# define deflateSetHeader z_deflateSetHeader +# define deflateTune z_deflateTune +# define deflate_copyright z_deflate_copyright +# define get_crc_table z_get_crc_table +# ifndef Z_SOLO +# define gz_error z_gz_error +# define gz_intmax z_gz_intmax +# define gz_strwinerror z_gz_strwinerror +# define gzbuffer z_gzbuffer +# define gzclearerr z_gzclearerr +# define gzclose z_gzclose +# define gzclose_r z_gzclose_r +# define gzclose_w z_gzclose_w +# define gzdirect z_gzdirect +# define gzdopen z_gzdopen +# define gzeof z_gzeof +# define gzerror z_gzerror +# define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite +# define gzgetc z_gzgetc +# define gzgetc_ z_gzgetc_ +# define gzgets z_gzgets +# define gzoffset z_gzoffset +# define gzoffset64 z_gzoffset64 +# define gzopen z_gzopen +# define gzopen64 z_gzopen64 +# ifdef _WIN32 +# define gzopen_w z_gzopen_w +# endif +# define gzprintf z_gzprintf +# define gzputc z_gzputc +# define gzputs z_gzputs +# define gzread z_gzread +# define gzrewind z_gzrewind +# define gzseek z_gzseek +# define gzseek64 z_gzseek64 +# define gzsetparams z_gzsetparams +# define gztell z_gztell +# define gztell64 z_gztell64 +# define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf +# define gzwrite z_gzwrite +# endif +# define inflate z_inflate +# define inflateBack z_inflateBack +# define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit +# define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed +# define inflateCopy z_inflateCopy +# define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary +# define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 +# define inflateInit2_ z_inflateInit2_ +# define inflateInit_ z_inflateInit_ +# define inflateMark z_inflateMark +# define inflatePrime z_inflatePrime +# define inflateReset z_inflateReset +# define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep +# define inflateSetDictionary z_inflateSetDictionary +# define inflateSync z_inflateSync +# define inflateSyncPoint z_inflateSyncPoint +# define inflateUndermine z_inflateUndermine +# define inflateValidate z_inflateValidate +# define inflate_copyright z_inflate_copyright +# define inflate_fast z_inflate_fast +# define inflate_table z_inflate_table +# ifndef Z_SOLO +# define uncompress z_uncompress +# define uncompress2 z_uncompress2 +# endif +# define zError z_zError +# ifndef Z_SOLO +# define zcalloc z_zcalloc +# define zcfree z_zcfree +# endif +# define zlibCompileFlags z_zlibCompileFlags +# define zlibVersion z_zlibVersion + +/* all zlib typedefs in zlib.h and zconf.h */ +# define Byte z_Byte +# define Bytef z_Bytef +# define alloc_func z_alloc_func +# define charf z_charf +# define free_func z_free_func +# ifndef Z_SOLO +# define gzFile z_gzFile +# endif +# define gz_header z_gz_header +# define gz_headerp z_gz_headerp +# define in_func z_in_func +# define intf z_intf +# define out_func z_out_func +# define uInt z_uInt +# define uIntf z_uIntf +# define uLong z_uLong +# define uLongf z_uLongf +# define voidp z_voidp +# define voidpc z_voidpc +# define voidpf z_voidpf + +/* all zlib structs in zlib.h and zconf.h */ +# define gz_header_s z_gz_header_s +# define internal_state z_internal_state + +#endif + +#if defined(__MSDOS__) && !defined(MSDOS) +# define MSDOS +#endif +#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2) +# define OS2 +#endif +#if defined(_WINDOWS) && !defined(WINDOWS) +# define WINDOWS +#endif +#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__) +# ifndef WIN32 +# define WIN32 +# endif +#endif +#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32) +# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__) +# ifndef SYS16BIT +# define SYS16BIT +# endif +# endif +#endif + +/* + * Compile with -DMAXSEG_64K if the alloc function cannot allocate more + * than 64k bytes at a time (needed on systems with 16-bit int). + */ +#ifdef SYS16BIT +# define MAXSEG_64K +#endif +#ifdef MSDOS +# define UNALIGNED_OK +#endif + +#ifdef __STDC_VERSION__ +# ifndef STDC +# define STDC +# endif +# if __STDC_VERSION__ >= 199901L +# ifndef STDC99 +# define STDC99 +# endif +# endif +#endif +#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus)) +# define STDC +#endif +#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__)) +# define STDC +#endif +#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32)) +# define STDC +#endif +#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__)) +# define STDC +#endif + +#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */ +# define STDC +#endif + +#ifndef STDC +# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */ +# define const /* note: need a more gentle solution here */ +# endif +#endif + +#if defined(ZLIB_CONST) && !defined(z_const) +# define z_const const +#else +# define z_const +#endif + +#ifdef Z_SOLO +# ifdef _WIN64 + typedef unsigned long long z_size_t; +# else + typedef unsigned long z_size_t; +# endif +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include <stddef.h> + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong +#endif + +/* Maximum value for memLevel in deflateInit2 */ +#ifndef MAX_MEM_LEVEL +# ifdef MAXSEG_64K +# define MAX_MEM_LEVEL 8 +# else +# define MAX_MEM_LEVEL 9 +# endif +#endif + +/* Maximum value for windowBits in deflateInit2 and inflateInit2. + * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files + * created by gzip. (Files created by minigzip can still be extracted by + * gzip.) + */ +#ifndef MAX_WBITS +# define MAX_WBITS 15 /* 32K LZ77 window */ +#endif + +/* The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) + plus a few kilobytes for small objects. For example, if you want to reduce + the default memory requirements from 256K to 128K, compile with + make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" + Of course this will generally degrade compression (there's no free lunch). + + The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes + for small objects. +*/ + + /* Type declarations */ + +#ifndef OF /* function prototypes */ +# ifdef STDC +# define OF(args) args +# else +# define OF(args) () +# endif +#endif + +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + +/* The following definitions for FAR are needed only for MSDOS mixed + * model programming (small or medium model with some far allocations). + * This was tested only with MSC; for other MSDOS compilers you may have + * to define NO_MEMCPY in zutil.h. If you don't need the mixed model, + * just define FAR to be empty. + */ +#ifdef SYS16BIT +# if defined(M_I86SM) || defined(M_I86MM) + /* MSC small or medium model */ +# define SMALL_MEDIUM +# ifdef _MSC_VER +# define FAR _far +# else +# define FAR far +# endif +# endif +# if (defined(__SMALL__) || defined(__MEDIUM__)) + /* Turbo C small or medium model */ +# define SMALL_MEDIUM +# ifdef __BORLANDC__ +# define FAR _far +# else +# define FAR far +# endif +# endif +#endif + +#if defined(WINDOWS) || defined(WIN32) + /* If building or using zlib as a DLL, define ZLIB_DLL. + * This is not mandatory, but it offers a little performance increase. + */ +# ifdef ZLIB_DLL +# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500)) +# ifdef ZLIB_INTERNAL +# define ZEXTERN extern __declspec(dllexport) +# else +# define ZEXTERN extern __declspec(dllimport) +# endif +# endif +# endif /* ZLIB_DLL */ + /* If building or using zlib with the WINAPI/WINAPIV calling convention, + * define ZLIB_WINAPI. + * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. + */ +# ifdef ZLIB_WINAPI +# ifdef FAR +# undef FAR +# endif +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include <windows.h> + /* No need for _export, use ZLIB.DEF instead. */ + /* For complete Windows compatibility, use WINAPI, not __stdcall. */ +# define ZEXPORT WINAPI +# ifdef WIN32 +# define ZEXPORTVA WINAPIV +# else +# define ZEXPORTVA FAR CDECL +# endif +# endif +#endif + +#if defined (__BEOS__) +# ifdef ZLIB_DLL +# ifdef ZLIB_INTERNAL +# define ZEXPORT __declspec(dllexport) +# define ZEXPORTVA __declspec(dllexport) +# else +# define ZEXPORT __declspec(dllimport) +# define ZEXPORTVA __declspec(dllimport) +# endif +# endif +#endif + +#ifndef ZEXTERN +# define ZEXTERN extern +#endif +#ifndef ZEXPORT +# define ZEXPORT +#endif +#ifndef ZEXPORTVA +# define ZEXPORTVA +#endif + +#ifndef FAR +# define FAR +#endif + +#if !defined(__MACTYPES__) +typedef unsigned char Byte; /* 8 bits */ +#endif +typedef unsigned int uInt; /* 16 bits or more */ +typedef unsigned long uLong; /* 32 bits or more */ + +#ifdef SMALL_MEDIUM + /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ +# define Bytef Byte FAR +#else + typedef Byte FAR Bytef; +#endif +typedef char FAR charf; +typedef int FAR intf; +typedef uInt FAR uIntf; +typedef uLong FAR uLongf; + +#ifdef STDC + typedef void const *voidpc; + typedef void FAR *voidpf; + typedef void *voidp; +#else + typedef Byte const *voidpc; + typedef Byte FAR *voidpf; + typedef Byte *voidp; +#endif + +#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) +# include <limits.h> +# if (UINT_MAX == 0xffffffffUL) +# define Z_U4 unsigned +# elif (ULONG_MAX == 0xffffffffUL) +# define Z_U4 unsigned long +# elif (USHRT_MAX == 0xffffffffUL) +# define Z_U4 unsigned short +# endif +#endif + +#ifdef Z_U4 + typedef Z_U4 z_crc_t; +#else + typedef unsigned long z_crc_t; +#endif + +#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_UNISTD_H +#endif + +#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_STDARG_H +#endif + +#ifdef STDC +# ifndef Z_SOLO +# include <sys/types.h> /* for off_t */ +# endif +#endif + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +# include <stdarg.h> /* for va_list */ +# endif +#endif + +#ifdef _WIN32 +# ifndef Z_SOLO +# include <stddef.h> /* for wchar_t */ +# endif +#endif + +/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and + * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even + * though the former does not conform to the LFS document), but considering + * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as + * equivalently requesting no 64-bit operations + */ +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 +# undef _LARGEFILE64_SOURCE +#endif + +#ifndef Z_HAVE_UNISTD_H +# ifdef __WATCOMC__ +# define Z_HAVE_UNISTD_H +# endif +#endif +#ifndef Z_HAVE_UNISTD_H +# if defined(_LARGEFILE64_SOURCE) && !defined(_WIN32) +# define Z_HAVE_UNISTD_H +# endif +#endif +#ifndef Z_SOLO +# if defined(Z_HAVE_UNISTD_H) +# include <unistd.h> /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifdef VMS +# include <unixio.h> /* for off_t */ +# endif +# ifndef z_off_t +# define z_off_t off_t +# endif +# endif +#endif + +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) && !defined(Z_SOLO) +# define SEEK_SET 0 /* Seek from beginning of file. */ +# define SEEK_CUR 1 /* Seek from current position. */ +# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ +#endif + +#ifndef z_off_t +# define z_off_t long +#endif + +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(_WIN32) && !defined(__GNUC__) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t +# endif +#endif + +/* MVS linker does not support external names larger than 8 bytes */ +#if defined(__MVS__) + #pragma map(deflateInit_,"DEIN") + #pragma map(deflateInit2_,"DEIN2") + #pragma map(deflateEnd,"DEEND") + #pragma map(deflateBound,"DEBND") + #pragma map(inflateInit_,"ININ") + #pragma map(inflateInit2_,"ININ2") + #pragma map(inflateEnd,"INEND") + #pragma map(inflateSync,"INSY") + #pragma map(inflateSetDictionary,"INSEDI") + #pragma map(compressBound,"CMBND") + #pragma map(inflate_table,"INTABL") + #pragma map(inflate_fast,"INFA") + #pragma map(inflate_copyright,"INCOPY") +#endif + +#endif /* ZCONF_H */ diff --git a/lib/zlib/zlib.h b/lib/zlib/zlib.h new file mode 100644 index 0000000..6b7244f --- /dev/null +++ b/lib/zlib/zlib.h @@ -0,0 +1,1938 @@ +/* zlib.h -- interface of the 'zlib' general purpose compression library + version 1.3, August 18th, 2023 + + Copyright (C) 1995-2023 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + + The data format used by the zlib library is described by RFCs (Request for + Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 + (zlib format), rfc1951 (deflate format) and rfc1952 (gzip format). +*/ + +#ifndef ZLIB_H +#define ZLIB_H + +#include "zconf.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ZLIB_VERSION "1.3" +#define ZLIB_VERNUM 0x1300 +#define ZLIB_VER_MAJOR 1 +#define ZLIB_VER_MINOR 3 +#define ZLIB_VER_REVISION 0 +#define ZLIB_VER_SUBREVISION 0 + +/* + The 'zlib' compression library provides in-memory compression and + decompression functions, including integrity checks of the uncompressed data. + This version of the library supports only one compression method (deflation) + but other algorithms will be added later and will have the same stream + interface. + + Compression can be done in a single step if the buffers are large enough, + or can be done by repeated calls of the compression function. In the latter + case, the application must provide more input and/or consume the output + (providing more output space) before each call. + + The compressed data format used by default by the in-memory functions is + the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped + around a deflate stream, which is itself documented in RFC 1951. + + The library also supports reading and writing files in gzip (.gz) format + with an interface similar to that of stdio using the functions that start + with "gz". The gzip format is different from the zlib format. gzip is a + gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. + + This library can optionally read and write gzip and raw deflate streams in + memory as well. + + The zlib format was designed to be compact and fast for use in memory + and on communications channels. The gzip format was designed for single- + file compression on file systems, has a larger header than zlib to maintain + directory information, and uses a different, slower check method than zlib. + + The library does not install any signal handler. The decoder checks + the consistency of the compressed data, so the library should never crash + even in the case of corrupted input. +*/ + +typedef voidpf (*alloc_func)(voidpf opaque, uInt items, uInt size); +typedef void (*free_func)(voidpf opaque, voidpf address); + +struct internal_state; + +typedef struct z_stream_s { + z_const Bytef *next_in; /* next input byte */ + uInt avail_in; /* number of bytes available at next_in */ + uLong total_in; /* total number of input bytes read so far */ + + Bytef *next_out; /* next output byte will go here */ + uInt avail_out; /* remaining free space at next_out */ + uLong total_out; /* total number of bytes output so far */ + + z_const char *msg; /* last error message, NULL if no error */ + struct internal_state FAR *state; /* not visible by applications */ + + alloc_func zalloc; /* used to allocate the internal state */ + free_func zfree; /* used to free the internal state */ + voidpf opaque; /* private data object passed to zalloc and zfree */ + + int data_type; /* best guess about the data type: binary or text + for deflate, or the decoding state for inflate */ + uLong adler; /* Adler-32 or CRC-32 value of the uncompressed data */ + uLong reserved; /* reserved for future use */ +} z_stream; + +typedef z_stream FAR *z_streamp; + +/* + gzip header information passed to and from zlib routines. See RFC 1952 + for more details on the meanings of these fields. +*/ +typedef struct gz_header_s { + int text; /* true if compressed data believed to be text */ + uLong time; /* modification time */ + int xflags; /* extra flags (not used when writing a gzip file) */ + int os; /* operating system */ + Bytef *extra; /* pointer to extra field or Z_NULL if none */ + uInt extra_len; /* extra field length (valid if extra != Z_NULL) */ + uInt extra_max; /* space at extra (only when reading header) */ + Bytef *name; /* pointer to zero-terminated file name or Z_NULL */ + uInt name_max; /* space at name (only when reading header) */ + Bytef *comment; /* pointer to zero-terminated comment or Z_NULL */ + uInt comm_max; /* space at comment (only when reading header) */ + int hcrc; /* true if there was or will be a header crc */ + int done; /* true when done reading gzip header (not used + when writing a gzip file) */ +} gz_header; + +typedef gz_header FAR *gz_headerp; + +/* + The application must update next_in and avail_in when avail_in has dropped + to zero. It must update next_out and avail_out when avail_out has dropped + to zero. The application must initialize zalloc, zfree and opaque before + calling the init function. All other fields are set by the compression + library and must not be updated by the application. + + The opaque value provided by the application will be passed as the first + parameter for calls of zalloc and zfree. This can be useful for custom + memory management. The compression library attaches no meaning to the + opaque value. + + zalloc must return Z_NULL if there is not enough memory for the object. + If zlib is used in a multi-threaded application, zalloc and zfree must be + thread safe. In that case, zlib is thread-safe. When zalloc and zfree are + Z_NULL on entry to the initialization function, they are set to internal + routines that use the standard library functions malloc() and free(). + + On 16-bit systems, the functions zalloc and zfree must be able to allocate + exactly 65536 bytes, but will not be required to allocate more than this if + the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS, pointers + returned by zalloc for objects of exactly 65536 bytes *must* have their + offset normalized to zero. The default allocation function provided by this + library ensures this (see zutil.c). To reduce memory requirements and avoid + any allocation of 64K objects, at the expense of compression ratio, compile + the library with -DMAX_WBITS=14 (see zconf.h). + + The fields total_in and total_out can be used for statistics or progress + reports. After compression, total_in holds the total size of the + uncompressed data and may be saved for use by the decompressor (particularly + if the decompressor wants to decompress everything in a single step). +*/ + + /* constants */ + +#define Z_NO_FLUSH 0 +#define Z_PARTIAL_FLUSH 1 +#define Z_SYNC_FLUSH 2 +#define Z_FULL_FLUSH 3 +#define Z_FINISH 4 +#define Z_BLOCK 5 +#define Z_TREES 6 +/* Allowed flush values; see deflate() and inflate() below for details */ + +#define Z_OK 0 +#define Z_STREAM_END 1 +#define Z_NEED_DICT 2 +#define Z_ERRNO (-1) +#define Z_STREAM_ERROR (-2) +#define Z_DATA_ERROR (-3) +#define Z_MEM_ERROR (-4) +#define Z_BUF_ERROR (-5) +#define Z_VERSION_ERROR (-6) +/* Return codes for the compression/decompression functions. Negative values + * are errors, positive values are used for special but normal events. + */ + +#define Z_NO_COMPRESSION 0 +#define Z_BEST_SPEED 1 +#define Z_BEST_COMPRESSION 9 +#define Z_DEFAULT_COMPRESSION (-1) +/* compression levels */ + +#define Z_FILTERED 1 +#define Z_HUFFMAN_ONLY 2 +#define Z_RLE 3 +#define Z_FIXED 4 +#define Z_DEFAULT_STRATEGY 0 +/* compression strategy; see deflateInit2() below for details */ + +#define Z_BINARY 0 +#define Z_TEXT 1 +#define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ +#define Z_UNKNOWN 2 +/* Possible values of the data_type field for deflate() */ + +#define Z_DEFLATED 8 +/* The deflate compression method (the only one supported in this version) */ + +#define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ + +#define zlib_version zlibVersion() +/* for compatibility with versions < 1.0.2 */ + + + /* basic functions */ + +ZEXTERN const char * ZEXPORT zlibVersion(void); +/* The application can compare zlibVersion and ZLIB_VERSION for consistency. + If the first character differs, the library code actually used is not + compatible with the zlib.h header file used by the application. This check + is automatically made by deflateInit and inflateInit. + */ + +/* +ZEXTERN int ZEXPORT deflateInit(z_streamp strm, int level); + + Initializes the internal stream state for compression. The fields + zalloc, zfree and opaque must be initialized before by the caller. If + zalloc and zfree are set to Z_NULL, deflateInit updates them to use default + allocation functions. total_in, total_out, adler, and msg are initialized. + + The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: + 1 gives best speed, 9 gives best compression, 0 gives no compression at all + (the input data is simply copied a block at a time). Z_DEFAULT_COMPRESSION + requests a default compromise between speed and compression (currently + equivalent to level 6). + + deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if level is not a valid compression level, or + Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible + with the version assumed by the caller (ZLIB_VERSION). msg is set to null + if there is no error message. deflateInit does not perform any compression: + this will be done by deflate(). +*/ + + +ZEXTERN int ZEXPORT deflate(z_streamp strm, int flush); +/* + deflate compresses as much data as possible, and stops when the input + buffer becomes empty or the output buffer becomes full. It may introduce + some output latency (reading input without producing any output) except when + forced to flush. + + The detailed semantics are as follows. deflate performs one or both of the + following actions: + + - Compress more input starting at next_in and update next_in and avail_in + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), next_in and avail_in are updated and + processing will resume at this point for the next call of deflate(). + + - Generate more output starting at next_out and update next_out and avail_out + accordingly. This action is forced if the parameter flush is non zero. + Forcing flush frequently degrades the compression ratio, so this parameter + should be set only when necessary. Some output may be provided even if + flush is zero. + + Before the call of deflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming more + output, and updating avail_in or avail_out accordingly; avail_out should + never be zero before the call. The application can consume the compressed + output when it wants, for example when the output buffer is full (avail_out + == 0), or after each call of deflate(). If deflate returns Z_OK and with + zero avail_out, it must be called again after making room in the output + buffer because there might be more output pending. See deflatePending(), + which can be used if desired to determine whether or not there is more output + in that case. + + Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to + decide how much data to accumulate before producing output, in order to + maximize compression. + + If the parameter flush is set to Z_SYNC_FLUSH, all pending output is + flushed to the output buffer and the output is aligned on a byte boundary, so + that the decompressor can get all input data available so far. (In + particular avail_in is zero after the call if enough output space has been + provided before the call.) Flushing may degrade compression for some + compression algorithms and so it should be used only when necessary. This + completes the current deflate block and follows it with an empty stored block + that is three bits plus filler bits to the next byte, followed by four bytes + (00 00 ff ff). + + If flush is set to Z_PARTIAL_FLUSH, all pending output is flushed to the + output buffer, but the output is not aligned to a byte boundary. All of the + input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. + This completes the current deflate block and follows it with an empty fixed + codes block that is 10 bits long. This assures that enough bytes are output + in order for the decompressor to finish the block before the empty fixed + codes block. + + If flush is set to Z_BLOCK, a deflate block is completed and emitted, as + for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to + seven bits of the current block are held to be written as the next byte after + the next deflate block is completed. In this case, the decompressor may not + be provided enough bits at this point in order to complete decompression of + the data provided so far to the compressor. It may need to wait for the next + block to be emitted. This is for advanced applications that need to control + the emission of deflate blocks. + + If flush is set to Z_FULL_FLUSH, all output is flushed as with + Z_SYNC_FLUSH, and the compression state is reset so that decompression can + restart from this point if previous compressed data has been damaged or if + random access is desired. Using Z_FULL_FLUSH too often can seriously degrade + compression. + + If deflate returns with avail_out == 0, this function must be called again + with the same value of the flush parameter and more output space (updated + avail_out), until the flush is complete (deflate returns with non-zero + avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that + avail_out is greater than six when the flush marker begins, in order to avoid + repeated flush markers upon calling deflate() again when avail_out == 0. + + If the parameter flush is set to Z_FINISH, pending input is processed, + pending output is flushed and deflate returns with Z_STREAM_END if there was + enough output space. If deflate returns with Z_OK or Z_BUF_ERROR, this + function must be called again with Z_FINISH and more output space (updated + avail_out) but no more input data, until it returns with Z_STREAM_END or an + error. After deflate has returned Z_STREAM_END, the only possible operations + on the stream are deflateReset or deflateEnd. + + Z_FINISH can be used in the first deflate call after deflateInit if all the + compression is to be done in a single step. In order to complete in one + call, avail_out must be at least the value returned by deflateBound (see + below). Then deflate is guaranteed to return Z_STREAM_END. If not enough + output space is provided, deflate will not return Z_STREAM_END, and it must + be called again as described above. + + deflate() sets strm->adler to the Adler-32 checksum of all input read + so far (that is, total_in bytes). If a gzip stream is being generated, then + strm->adler will be the CRC-32 checksum of the input read so far. (See + deflateInit2 below.) + + deflate() may update strm->data_type if it can make a good guess about + the input data type (Z_BINARY or Z_TEXT). If in doubt, the data is + considered binary. This field is only for information purposes and does not + affect the compression algorithm in any manner. + + deflate() returns Z_OK if some progress has been made (more input + processed or more output produced), Z_STREAM_END if all input has been + consumed and all output has been produced (only when flush is set to + Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example + if next_in or next_out was Z_NULL or the state was inadvertently written over + by the application), or Z_BUF_ERROR if no progress is possible (for example + avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and + deflate() can be called again with more input and more output space to + continue compressing. +*/ + + +ZEXTERN int ZEXPORT deflateEnd(z_streamp strm); +/* + All dynamically allocated data structures for this stream are freed. + This function discards any unprocessed input and does not flush any pending + output. + + deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the + stream state was inconsistent, Z_DATA_ERROR if the stream was freed + prematurely (some input or output was discarded). In the error case, msg + may be set but then points to a static string (which must not be + deallocated). +*/ + + +/* +ZEXTERN int ZEXPORT inflateInit(z_streamp strm); + + Initializes the internal stream state for decompression. The fields + next_in, avail_in, zalloc, zfree and opaque must be initialized before by + the caller. In the current version of inflate, the provided input is not + read or consumed. The allocation of a sliding window will be deferred to + the first call of inflate (if the decompression does not complete on the + first call). If zalloc and zfree are set to Z_NULL, inflateInit updates + them to use default allocation functions. total_in, total_out, adler, and + msg are initialized. + + inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_VERSION_ERROR if the zlib library version is incompatible with the + version assumed by the caller, or Z_STREAM_ERROR if the parameters are + invalid, such as a null pointer to the structure. msg is set to null if + there is no error message. inflateInit does not perform any decompression. + Actual decompression will be done by inflate(). So next_in, and avail_in, + next_out, and avail_out are unused and unchanged. The current + implementation of inflateInit() does not process any header information -- + that is deferred until inflate() is called. +*/ + + +ZEXTERN int ZEXPORT inflate(z_streamp strm, int flush); +/* + inflate decompresses as much data as possible, and stops when the input + buffer becomes empty or the output buffer becomes full. It may introduce + some output latency (reading input without producing any output) except when + forced to flush. + + The detailed semantics are as follows. inflate performs one or both of the + following actions: + + - Decompress more input starting at next_in and update next_in and avail_in + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), then next_in and avail_in are updated + accordingly, and processing will resume at this point for the next call of + inflate(). + + - Generate more output starting at next_out and update next_out and avail_out + accordingly. inflate() provides as much output as possible, until there is + no more input data or no more space in the output buffer (see below about + the flush parameter). + + Before the call of inflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming more + output, and updating the next_* and avail_* values accordingly. If the + caller of inflate() does not provide both available input and available + output space, it is possible that there will be no progress made. The + application can consume the uncompressed output when it wants, for example + when the output buffer is full (avail_out == 0), or after each call of + inflate(). If inflate returns Z_OK and with zero avail_out, it must be + called again after making room in the output buffer because there might be + more output pending. + + The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FINISH, + Z_BLOCK, or Z_TREES. Z_SYNC_FLUSH requests that inflate() flush as much + output as possible to the output buffer. Z_BLOCK requests that inflate() + stop if and when it gets to the next deflate block boundary. When decoding + the zlib or gzip format, this will cause inflate() to return immediately + after the header and before the first block. When doing a raw inflate, + inflate() will go ahead and process the first block, and will return when it + gets to the end of that block, or when it runs out of data. + + The Z_BLOCK option assists in appending to or combining deflate streams. + To assist in this, on return inflate() always sets strm->data_type to the + number of unused bits in the last byte taken from strm->next_in, plus 64 if + inflate() is currently decoding the last block in the deflate stream, plus + 128 if inflate() returned immediately after decoding an end-of-block code or + decoding the complete header up to just before the first byte of the deflate + stream. The end-of-block will not be indicated until all of the uncompressed + data from that block has been written to strm->next_out. The number of + unused bits may in general be greater than seven, except when bit 7 of + data_type is set, in which case the number of unused bits will be less than + eight. data_type is set as noted here every time inflate() returns for all + flush options, and so can be used to determine the amount of currently + consumed input in bits. + + The Z_TREES option behaves as Z_BLOCK does, but it also returns when the + end of each deflate block header is reached, before any actual data in that + block is decoded. This allows the caller to determine the length of the + deflate block header for later use in random access within a deflate block. + 256 is added to the value of strm->data_type when inflate() returns + immediately after reaching the end of the deflate block header. + + inflate() should normally be called until it returns Z_STREAM_END or an + error. However if all decompression is to be performed in a single step (a + single call of inflate), the parameter flush should be set to Z_FINISH. In + this case all pending input is processed and all pending output is flushed; + avail_out must be large enough to hold all of the uncompressed data for the + operation to complete. (The size of the uncompressed data may have been + saved by the compressor for this purpose.) The use of Z_FINISH is not + required to perform an inflation in one step. However it may be used to + inform inflate that a faster approach can be used for the single inflate() + call. Z_FINISH also informs inflate to not maintain a sliding window if the + stream completes, which reduces inflate's memory footprint. If the stream + does not complete, either because not all of the stream is provided or not + enough output space is provided, then a sliding window will be allocated and + inflate() can be called again to continue the operation as if Z_NO_FLUSH had + been used. + + In this implementation, inflate() always flushes as much output as + possible to the output buffer, and always uses the faster approach on the + first call. So the effects of the flush parameter in this implementation are + on the return value of inflate() as noted below, when inflate() returns early + when Z_BLOCK or Z_TREES is used, and when inflate() avoids the allocation of + memory for a sliding window when Z_FINISH is used. + + If a preset dictionary is needed after this call (see inflateSetDictionary + below), inflate sets strm->adler to the Adler-32 checksum of the dictionary + chosen by the compressor and returns Z_NEED_DICT; otherwise it sets + strm->adler to the Adler-32 checksum of all output produced so far (that is, + total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described + below. At the end of the stream, inflate() checks that its computed Adler-32 + checksum is equal to that saved by the compressor and returns Z_STREAM_END + only if the checksum is correct. + + inflate() can decompress and check either zlib-wrapped or gzip-wrapped + deflate data. The header type is detected automatically, if requested when + initializing with inflateInit2(). Any information contained in the gzip + header is not retained unless inflateGetHeader() is used. When processing + gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output + produced so far. The CRC-32 is checked against the gzip trailer, as is the + uncompressed length, modulo 2^32. + + inflate() returns Z_OK if some progress has been made (more input processed + or more output produced), Z_STREAM_END if the end of the compressed data has + been reached and all uncompressed output has been produced, Z_NEED_DICT if a + preset dictionary is needed at this point, Z_DATA_ERROR if the input data was + corrupted (input stream not conforming to the zlib format or incorrect check + value, in which case strm->msg points to a string with a more specific + error), Z_STREAM_ERROR if the stream structure was inconsistent (for example + next_in or next_out was Z_NULL, or the state was inadvertently written over + by the application), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR + if no progress was possible or if there was not enough room in the output + buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + inflate() can be called again with more input and more output space to + continue decompressing. If Z_DATA_ERROR is returned, the application may + then call inflateSync() to look for a good compression block if a partial + recovery of the data is to be attempted. +*/ + + +ZEXTERN int ZEXPORT inflateEnd(z_streamp strm); +/* + All dynamically allocated data structures for this stream are freed. + This function discards any unprocessed input and does not flush any pending + output. + + inflateEnd returns Z_OK if success, or Z_STREAM_ERROR if the stream state + was inconsistent. +*/ + + + /* Advanced functions */ + +/* + The following functions are needed only in some special applications. +*/ + +/* +ZEXTERN int ZEXPORT deflateInit2(z_streamp strm, + int level, + int method, + int windowBits, + int memLevel, + int strategy); + + This is another version of deflateInit with more compression options. The + fields zalloc, zfree and opaque must be initialized before by the caller. + + The method parameter is the compression method. It must be Z_DEFLATED in + this version of the library. + + The windowBits parameter is the base two logarithm of the window size + (the size of the history buffer). It should be in the range 8..15 for this + version of the library. Larger values of this parameter result in better + compression at the expense of memory usage. The default value is 15 if + deflateInit is used instead. + + For the current implementation of deflate(), a windowBits value of 8 (a + window size of 256 bytes) is not supported. As a result, a request for 8 + will result in 9 (a 512-byte window). In that case, providing 8 to + inflateInit2() will result in an error when the zlib header with 9 is + checked against the initialization of inflate(). The remedy is to not use 8 + with deflateInit2() with this initialization, or at least in that case use 9 + with inflateInit2(). + + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits + determines the window size. deflate() will then generate raw deflate data + with no zlib header or trailer, and will not compute a check value. + + windowBits can also be greater than 15 for optional gzip encoding. Add + 16 to windowBits to write a simple gzip header and trailer around the + compressed data instead of a zlib wrapper. The gzip header will have no + file name, no extra data, no comment, no modification time (set to zero), no + header crc, and the operating system will be set to the appropriate value, + if the operating system was determined at compile time. If a gzip stream is + being written, strm->adler is a CRC-32 instead of an Adler-32. + + For raw deflate or gzip encoding, a request for a 256-byte window is + rejected as invalid, since only the zlib header provides a means of + transmitting the window size to the decompressor. + + The memLevel parameter specifies how much memory should be allocated + for the internal compression state. memLevel=1 uses minimum memory but is + slow and reduces compression ratio; memLevel=9 uses maximum memory for + optimal speed. The default value is 8. See zconf.h for total memory usage + as a function of windowBits and memLevel. + + The strategy parameter is used to tune the compression algorithm. Use the + value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a + filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no + string match), or Z_RLE to limit match distances to one (run-length + encoding). Filtered data consists mostly of small values with a somewhat + random distribution. In this case, the compression algorithm is tuned to + compress them better. The effect of Z_FILTERED is to force more Huffman + coding and less string matching; it is somewhat intermediate between + Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as + fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data. The + strategy parameter only affects the compression ratio but not the + correctness of the compressed output even if it is not set appropriately. + Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler + decoder for special applications. + + deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid + method), or Z_VERSION_ERROR if the zlib library version (zlib_version) is + incompatible with the version assumed by the caller (ZLIB_VERSION). msg is + set to null if there is no error message. deflateInit2 does not perform any + compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateSetDictionary(z_streamp strm, + const Bytef *dictionary, + uInt dictLength); +/* + Initializes the compression dictionary from the given byte sequence + without producing any compressed output. When using the zlib format, this + function must be called immediately after deflateInit, deflateInit2 or + deflateReset, and before any call of deflate. When doing raw deflate, this + function must be called either before any call of deflate, or immediately + after the completion of a deflate block, i.e. after all input has been + consumed and all output has been delivered when using any of the flush + options Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, or Z_FULL_FLUSH. The + compressor and decompressor must use exactly the same dictionary (see + inflateSetDictionary). + + The dictionary should consist of strings (byte sequences) that are likely + to be encountered later in the data to be compressed, with the most commonly + used strings preferably put towards the end of the dictionary. Using a + dictionary is most useful when the data to be compressed is short and can be + predicted with good accuracy; the data can then be compressed better than + with the default empty dictionary. + + Depending on the size of the compression data structures selected by + deflateInit or deflateInit2, a part of the dictionary may in effect be + discarded, for example if the dictionary is larger than the window size + provided in deflateInit or deflateInit2. Thus the strings most likely to be + useful should be put at the end of the dictionary, not at the front. In + addition, the current implementation of deflate will use at most the window + size minus 262 bytes of the provided dictionary. + + Upon return of this function, strm->adler is set to the Adler-32 value + of the dictionary; the decompressor may later use this value to determine + which dictionary has been used by the compressor. (The Adler-32 value + applies to the whole dictionary even if only a subset of the dictionary is + actually used by the compressor.) If a raw deflate was requested, then the + Adler-32 value is not computed and strm->adler is not set. + + deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a + parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is + inconsistent (for example if deflate has already been called for this stream + or if not at a block boundary for raw deflate). deflateSetDictionary does + not perform any compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateGetDictionary(z_streamp strm, + Bytef *dictionary, + uInt *dictLength); +/* + Returns the sliding dictionary being maintained by deflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If deflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similarly, if dictLength is Z_NULL, then it is not set. + + deflateGetDictionary() may return a length less than the window size, even + when more than the window size in input has been provided. It may return up + to 258 bytes less in that case, due to how zlib's implementation of deflate + manages the sliding window and lookahead for matches, where matches can be + up to 258 bytes long. If the application needs the last window-size bytes of + input, then that would need to be saved by the application outside of zlib. + + deflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + +ZEXTERN int ZEXPORT deflateCopy(z_streamp dest, + z_streamp source); +/* + Sets the destination stream as a complete copy of the source stream. + + This function can be useful when several compression strategies will be + tried, for example when there are several ways of pre-processing the input + data with a filter. The streams that will be discarded should then be freed + by calling deflateEnd. Note that deflateCopy duplicates the internal + compression state which can be quite large, so this strategy is slow and can + consume lots of memory. + + deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if the source stream state was inconsistent + (such as zalloc being Z_NULL). msg is left unchanged in both source and + destination. +*/ + +ZEXTERN int ZEXPORT deflateReset(z_streamp strm); +/* + This function is equivalent to deflateEnd followed by deflateInit, but + does not free and reallocate the internal compression state. The stream + will leave the compression level and any other attributes that may have been + set unchanged. total_in, total_out, adler, and msg are initialized. + + deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL). +*/ + +ZEXTERN int ZEXPORT deflateParams(z_streamp strm, + int level, + int strategy); +/* + Dynamically update the compression level and compression strategy. The + interpretation of level and strategy is as in deflateInit2(). This can be + used to switch between compression and straight copy of the input data, or + to switch to a different kind of input data requiring a different strategy. + If the compression approach (which is a function of the level) or the + strategy is changed, and if there have been any deflate() calls since the + state was initialized or reset, then the input available so far is + compressed with the old level and strategy using deflate(strm, Z_BLOCK). + There are three approaches for the compression levels 0, 1..3, and 4..9 + respectively. The new level and strategy will take effect at the next call + of deflate(). + + If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does + not have enough output space to complete, then the parameter change will not + take effect. In this case, deflateParams() can be called again with the + same parameters and more output space to try again. + + In order to assure a change in the parameters on the first try, the + deflate stream should be flushed using deflate() with Z_BLOCK or other flush + request until strm.avail_out is not zero, before calling deflateParams(). + Then no more input data should be provided before the deflateParams() call. + If this is done, the old level and strategy will be applied to the data + compressed before deflateParams(), and the new level and strategy will be + applied to the data compressed after deflateParams(). + + deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream + state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if + there was not enough output space to complete the compression of the + available input data before a change in the strategy or approach. Note that + in the case of a Z_BUF_ERROR, the parameters are not changed. A return + value of Z_BUF_ERROR is not fatal, in which case deflateParams() can be + retried with more output space. +*/ + +ZEXTERN int ZEXPORT deflateTune(z_streamp strm, + int good_length, + int max_lazy, + int nice_length, + int max_chain); +/* + Fine tune deflate's internal compression parameters. This should only be + used by someone who understands the algorithm used by zlib's deflate for + searching for the best matching string, and even then only by the most + fanatic optimizer trying to squeeze out the last compressed bit for their + specific input data. Read the deflate.c source code for the meaning of the + max_lazy, good_length, nice_length, and max_chain parameters. + + deflateTune() can be called after deflateInit() or deflateInit2(), and + returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream. + */ + +ZEXTERN uLong ZEXPORT deflateBound(z_streamp strm, + uLong sourceLen); +/* + deflateBound() returns an upper bound on the compressed size after + deflation of sourceLen bytes. It must be called after deflateInit() or + deflateInit2(), and after deflateSetHeader(), if used. This would be used + to allocate an output buffer for deflation in a single pass, and so would be + called before deflate(). If that first deflate() call is provided the + sourceLen input bytes, an output buffer allocated to the size returned by + deflateBound(), and the flush value Z_FINISH, then deflate() is guaranteed + to return Z_STREAM_END. Note that it is possible for the compressed size to + be larger than the value returned by deflateBound() if flush options other + than Z_FINISH or Z_NO_FLUSH are used. +*/ + +ZEXTERN int ZEXPORT deflatePending(z_streamp strm, + unsigned *pending, + int *bits); +/* + deflatePending() returns the number of bytes and bits of output that have + been generated, but not yet provided in the available output. The bytes not + provided would be due to the available output space having being consumed. + The number of bits of output not provided are between 0 and 7, where they + await more bits to join them in order to fill out a full byte. If pending + or bits are Z_NULL, then those values are not set. + + deflatePending returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. + */ + +ZEXTERN int ZEXPORT deflatePrime(z_streamp strm, + int bits, + int value); +/* + deflatePrime() inserts bits in the deflate output stream. The intent + is that this function is used to start off the deflate output with the bits + leftover from a previous deflate stream when appending to it. As such, this + function can only be used for raw deflate, and must be used before the first + deflate() call after a deflateInit2() or deflateReset(). bits must be less + than or equal to 16, and that many of the least significant bits of value + will be inserted in the output. + + deflatePrime returns Z_OK if success, Z_BUF_ERROR if there was not enough + room in the internal buffer to insert the bits, or Z_STREAM_ERROR if the + source stream state was inconsistent. +*/ + +ZEXTERN int ZEXPORT deflateSetHeader(z_streamp strm, + gz_headerp head); +/* + deflateSetHeader() provides gzip header information for when a gzip + stream is requested by deflateInit2(). deflateSetHeader() may be called + after deflateInit2() or deflateReset() and before the first call of + deflate(). The text, time, os, extra field, name, and comment information + in the provided gz_header structure are written to the gzip header (xflag is + ignored -- the extra flags are set according to the compression level). The + caller must assure that, if not Z_NULL, name and comment are terminated with + a zero byte, and that if extra is not Z_NULL, that extra_len bytes are + available there. If hcrc is true, a gzip header crc is included. Note that + the current versions of the command-line version of gzip (up through version + 1.3.x) do not support header crc's, and will report that it is a "multi-part + gzip file" and give up. + + If deflateSetHeader is not used, the default gzip header has text false, + the time set to zero, and os set to the current operating system, with no + extra, name, or comment fields. The gzip header is returned to the default + state by deflateReset(). + + deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +/* +ZEXTERN int ZEXPORT inflateInit2(z_streamp strm, + int windowBits); + + This is another version of inflateInit with an extra parameter. The + fields next_in, avail_in, zalloc, zfree and opaque must be initialized + before by the caller. + + The windowBits parameter is the base two logarithm of the maximum window + size (the size of the history buffer). It should be in the range 8..15 for + this version of the library. The default value is 15 if inflateInit is used + instead. windowBits must be greater than or equal to the windowBits value + provided to deflateInit2() while compressing, or it must be equal to 15 if + deflateInit2() was not used. If a compressed stream with a larger window + size is given as input, inflate() will return with the error code + Z_DATA_ERROR instead of trying to allocate a larger window. + + windowBits can also be zero to request that inflate use the window size in + the zlib header of the compressed stream. + + windowBits can also be -8..-15 for raw inflate. In this case, -windowBits + determines the window size. inflate() will then process raw deflate data, + not looking for a zlib or gzip header, not generating a check value, and not + looking for any check values for comparison at the end of the stream. This + is for use with other formats that use the deflate compressed data format + such as zip. Those formats provide their own check values. If a custom + format is developed using the raw deflate format for compressed data, it is + recommended that a check value such as an Adler-32 or a CRC-32 be applied to + the uncompressed data as is done in the zlib, gzip, and zip formats. For + most applications, the zlib format should be used as is. Note that comments + above on the use in deflateInit2() applies to the magnitude of windowBits. + + windowBits can also be greater than 15 for optional gzip decoding. Add + 32 to windowBits to enable zlib and gzip decoding with automatic header + detection, or add 16 to decode only the gzip format (the zlib format will + return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + below), inflate() will *not* automatically decode concatenated gzip members. + inflate() will return Z_STREAM_END at the end of the gzip member. The state + would need to be reset to continue decoding a subsequent gzip member. This + *must* be done if there is more data after a gzip member, in order for the + decompression to be compliant with the gzip standard (RFC 1952). + + inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_VERSION_ERROR if the zlib library version is incompatible with the + version assumed by the caller, or Z_STREAM_ERROR if the parameters are + invalid, such as a null pointer to the structure. msg is set to null if + there is no error message. inflateInit2 does not perform any decompression + apart from possibly reading the zlib header if present: actual decompression + will be done by inflate(). (So next_in and avail_in may be modified, but + next_out and avail_out are unused and unchanged.) The current implementation + of inflateInit2() does not process any header information -- that is + deferred until inflate() is called. +*/ + +ZEXTERN int ZEXPORT inflateSetDictionary(z_streamp strm, + const Bytef *dictionary, + uInt dictLength); +/* + Initializes the decompression dictionary from the given uncompressed byte + sequence. This function must be called immediately after a call of inflate, + if that call returned Z_NEED_DICT. The dictionary chosen by the compressor + can be determined from the Adler-32 value returned by that call of inflate. + The compressor and decompressor must use exactly the same dictionary (see + deflateSetDictionary). For raw inflate, this function can be called at any + time to set the dictionary. If the provided dictionary is smaller than the + window and there is already data in the window, then the provided dictionary + will amend what's there. The application must insure that the dictionary + that was used for compression is provided. + + inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a + parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is + inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the + expected one (incorrect Adler-32 value). inflateSetDictionary does not + perform any decompression: this will be done by subsequent calls of + inflate(). +*/ + +ZEXTERN int ZEXPORT inflateGetDictionary(z_streamp strm, + Bytef *dictionary, + uInt *dictLength); +/* + Returns the sliding dictionary being maintained by inflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If inflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similarly, if dictLength is Z_NULL, then it is not set. + + inflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + +ZEXTERN int ZEXPORT inflateSync(z_streamp strm); +/* + Skips invalid compressed data until a possible full flush point (see above + for the description of deflate with Z_FULL_FLUSH) can be found, or until all + available input is skipped. No output is provided. + + inflateSync searches for a 00 00 FF FF pattern in the compressed data. + All full flush points have this pattern, but not all occurrences of this + pattern are full flush points. + + inflateSync returns Z_OK if a possible full flush point has been found, + Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point + has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. + In the success case, the application may save the current current value of + total_in which indicates where valid compressed data was found. In the + error case, the application may repeatedly call inflateSync, providing more + input each time, until success or end of the input data. +*/ + +ZEXTERN int ZEXPORT inflateCopy(z_streamp dest, + z_streamp source); +/* + Sets the destination stream as a complete copy of the source stream. + + This function can be useful when randomly accessing a large stream. The + first pass through the stream can periodically record the inflate state, + allowing restarting inflate at those points when randomly accessing the + stream. + + inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if the source stream state was inconsistent + (such as zalloc being Z_NULL). msg is left unchanged in both source and + destination. +*/ + +ZEXTERN int ZEXPORT inflateReset(z_streamp strm); +/* + This function is equivalent to inflateEnd followed by inflateInit, + but does not free and reallocate the internal decompression state. The + stream will keep attributes that may have been set by inflateInit2. + total_in, total_out, adler, and msg are initialized. + + inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL). +*/ + +ZEXTERN int ZEXPORT inflateReset2(z_streamp strm, + int windowBits); +/* + This function is the same as inflateReset, but it also permits changing + the wrap and window size requests. The windowBits parameter is interpreted + the same as it is for inflateInit2. If the window size is changed, then the + memory allocated for the window is freed, and the window will be reallocated + by inflate() if needed. + + inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL), or if + the windowBits parameter is invalid. +*/ + +ZEXTERN int ZEXPORT inflatePrime(z_streamp strm, + int bits, + int value); +/* + This function inserts bits in the inflate input stream. The intent is + that this function is used to start inflating at a bit position in the + middle of a byte. The provided bits will be used before any bytes are used + from next_in. This function should only be used with raw inflate, and + should be used before the first inflate() call after inflateInit2() or + inflateReset(). bits must be less than or equal to 16, and that many of the + least significant bits of value will be inserted in the input. + + If bits is negative, then the input stream bit buffer is emptied. Then + inflatePrime() can be called again to put bits in the buffer. This is used + to clear out bits leftover after feeding inflate a block description prior + to feeding inflate codes. + + inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +ZEXTERN long ZEXPORT inflateMark(z_streamp strm); +/* + This function returns two values, one in the lower 16 bits of the return + value, and the other in the remaining upper bits, obtained by shifting the + return value down 16 bits. If the upper value is -1 and the lower value is + zero, then inflate() is currently decoding information outside of a block. + If the upper value is -1 and the lower value is non-zero, then inflate is in + the middle of a stored block, with the lower value equaling the number of + bytes from the input remaining to copy. If the upper value is not -1, then + it is the number of bits back from the current bit position in the input of + the code (literal or length/distance pair) currently being processed. In + that case the lower value is the number of bytes already emitted for that + code. + + A code is being processed if inflate is waiting for more input to complete + decoding of the code, or if it has completed decoding but is waiting for + more output space to write the literal or match data. + + inflateMark() is used to mark locations in the input data for random + access, which may be at bit positions, and to note those cases where the + output of a code may span boundaries of random access blocks. The current + location in the input stream can be determined from avail_in and data_type + as noted in the description for the Z_BLOCK flush parameter for inflate. + + inflateMark returns the value noted above, or -65536 if the provided + source stream state was inconsistent. +*/ + +ZEXTERN int ZEXPORT inflateGetHeader(z_streamp strm, + gz_headerp head); +/* + inflateGetHeader() requests that gzip header information be stored in the + provided gz_header structure. inflateGetHeader() may be called after + inflateInit2() or inflateReset(), and before the first call of inflate(). + As inflate() processes the gzip stream, head->done is zero until the header + is completed, at which time head->done is set to one. If a zlib stream is + being decoded, then head->done is set to -1 to indicate that there will be + no gzip header information forthcoming. Note that Z_BLOCK or Z_TREES can be + used to force inflate() to return immediately after header processing is + complete and before any actual data is decompressed. + + The text, time, xflags, and os fields are filled in with the gzip header + contents. hcrc is set to true if there is a header CRC. (The header CRC + was valid if done is set to one.) If extra is not Z_NULL, then extra_max + contains the maximum number of bytes to write to extra. Once done is true, + extra_len contains the actual extra field length, and extra contains the + extra field, or that field truncated if extra_max is less than extra_len. + If name is not Z_NULL, then up to name_max characters are written there, + terminated with a zero unless the length is greater than name_max. If + comment is not Z_NULL, then up to comm_max characters are written there, + terminated with a zero unless the length is greater than comm_max. When any + of extra, name, or comment are not Z_NULL and the respective field is not + present in the header, then that field is set to Z_NULL to signal its + absence. This allows the use of deflateSetHeader() with the returned + structure to duplicate the header. However if those fields are set to + allocated memory, then the application will need to save those pointers + elsewhere so that they can be eventually freed. + + If inflateGetHeader is not used, then the header information is simply + discarded. The header is always checked for validity, including the header + CRC if present. inflateReset() will reset the process to discard the header + information. The application would need to call inflateGetHeader() again to + retrieve the header from the next gzip stream. + + inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +/* +ZEXTERN int ZEXPORT inflateBackInit(z_streamp strm, int windowBits, + unsigned char FAR *window); + + Initialize the internal stream state for decompression using inflateBack() + calls. The fields zalloc, zfree and opaque in strm must be initialized + before the call. If zalloc and zfree are Z_NULL, then the default library- + derived memory allocation routines are used. windowBits is the base two + logarithm of the window size, in the range 8..15. window is a caller + supplied buffer of that size. Except for special applications where it is + assured that deflate was used with small window sizes, windowBits must be 15 + and a 32K byte window must be supplied to be able to decompress general + deflate streams. + + See inflateBack() for the usage of these routines. + + inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of + the parameters are invalid, Z_MEM_ERROR if the internal state could not be + allocated, or Z_VERSION_ERROR if the version of the library does not match + the version of the header file. +*/ + +typedef unsigned (*in_func)(void FAR *, + z_const unsigned char FAR * FAR *); +typedef int (*out_func)(void FAR *, unsigned char FAR *, unsigned); + +ZEXTERN int ZEXPORT inflateBack(z_streamp strm, + in_func in, void FAR *in_desc, + out_func out, void FAR *out_desc); +/* + inflateBack() does a raw inflate with a single call using a call-back + interface for input and output. This is potentially more efficient than + inflate() for file i/o applications, in that it avoids copying between the + output and the sliding window by simply making the window itself the output + buffer. inflate() can be faster on modern CPUs when used with large + buffers. inflateBack() trusts the application to not change the output + buffer passed by the output function, at least until inflateBack() returns. + + inflateBackInit() must be called first to allocate the internal state + and to initialize the state with the user-provided window buffer. + inflateBack() may then be used multiple times to inflate a complete, raw + deflate stream with each call. inflateBackEnd() is then called to free the + allocated state. + + A raw deflate stream is one with no zlib or gzip header or trailer. + This routine would normally be used in a utility that reads zip or gzip + files and writes out uncompressed files. The utility would decode the + header and process the trailer on its own, hence this routine expects only + the raw deflate stream to decompress. This is different from the default + behavior of inflate(), which expects a zlib header and trailer around the + deflate stream. + + inflateBack() uses two subroutines supplied by the caller that are then + called by inflateBack() for input and output. inflateBack() calls those + routines until it reads a complete deflate stream and writes out all of the + uncompressed data, or until it encounters an error. The function's + parameters and return types are defined above in the in_func and out_func + typedefs. inflateBack() will call in(in_desc, &buf) which should return the + number of bytes of provided input, and a pointer to that input in buf. If + there is no input available, in() must return zero -- buf is ignored in that + case -- and inflateBack() will return a buffer error. inflateBack() will + call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. + out() should return zero on success, or non-zero on failure. If out() + returns non-zero, inflateBack() will return with an error. Neither in() nor + out() are permitted to change the contents of the window provided to + inflateBackInit(), which is also the buffer that out() uses to write from. + The length written by out() will be at most the window size. Any non-zero + amount of input may be provided by in(). + + For convenience, inflateBack() can be provided input on the first call by + setting strm->next_in and strm->avail_in. If that input is exhausted, then + in() will be called. Therefore strm->next_in must be initialized before + calling inflateBack(). If strm->next_in is Z_NULL, then in() will be called + immediately for input. If strm->next_in is not Z_NULL, then strm->avail_in + must also be initialized, and then if strm->avail_in is not zero, input will + initially be taken from strm->next_in[0 .. strm->avail_in - 1]. + + The in_desc and out_desc parameters of inflateBack() is passed as the + first parameter of in() and out() respectively when they are called. These + descriptors can be optionally used to pass any information that the caller- + supplied in() and out() functions need to do their job. + + On return, inflateBack() will set strm->next_in and strm->avail_in to + pass back any unused input that was provided by the last in() call. The + return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR + if in() or out() returned an error, Z_DATA_ERROR if there was a format error + in the deflate stream (in which case strm->msg is set to indicate the nature + of the error), or Z_STREAM_ERROR if the stream was not properly initialized. + In the case of Z_BUF_ERROR, an input or output error can be distinguished + using strm->next_in which will be Z_NULL only if in() returned an error. If + strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning + non-zero. (in() will always be called before out(), so strm->next_in is + assured to be defined if out() returns non-zero.) Note that inflateBack() + cannot return Z_OK. +*/ + +ZEXTERN int ZEXPORT inflateBackEnd(z_streamp strm); +/* + All memory allocated by inflateBackInit() is freed. + + inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream + state was inconsistent. +*/ + +ZEXTERN uLong ZEXPORT zlibCompileFlags(void); +/* Return flags indicating compile-time options. + + Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other: + 1.0: size of uInt + 3.2: size of uLong + 5.4: size of voidpf (pointer) + 7.6: size of z_off_t + + Compiler, assembler, and debug options: + 8: ZLIB_DEBUG + 9: ASMV or ASMINF -- use ASM code + 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention + 11: 0 (reserved) + + One-time table building (smaller code, but not thread-safe if true): + 12: BUILDFIXED -- build static block decoding tables when needed + 13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed + 14,15: 0 (reserved) + + Library content (indicates missing functionality): + 16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking + deflate code when not needed) + 17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect + and decode gzip streams (to avoid linking crc code) + 18-19: 0 (reserved) + + Operation variations (changes in library functionality): + 20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate + 21: FASTEST -- deflate algorithm with only one, lowest compression level + 22,23: 0 (reserved) + + The sprintf variant used by gzprintf (zero is best): + 24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format + 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure! + 26: 0 = returns value, 1 = void -- 1 means inferred string length returned + + Remainder: + 27-31: 0 (reserved) + */ + +#ifndef Z_SOLO + + /* utility functions */ + +/* + The following utility functions are implemented on top of the basic + stream-oriented functions. To simplify the interface, some default options + are assumed (compression level and memory usage, standard memory allocation + functions). The source code of these utility functions can be modified if + you need special options. +*/ + +ZEXTERN int ZEXPORT compress(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen); +/* + Compresses the source buffer into the destination buffer. sourceLen is + the byte length of the source buffer. Upon entry, destLen is the total size + of the destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed data. compress() is equivalent to compress2() with a level + parameter of Z_DEFAULT_COMPRESSION. + + compress returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_BUF_ERROR if there was not enough room in the output + buffer. +*/ + +ZEXTERN int ZEXPORT compress2(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int level); +/* + Compresses the source buffer into the destination buffer. The level + parameter has the same meaning as in deflateInit. sourceLen is the byte + length of the source buffer. Upon entry, destLen is the total size of the + destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed data. + + compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, + Z_STREAM_ERROR if the level parameter is invalid. +*/ + +ZEXTERN uLong ZEXPORT compressBound(uLong sourceLen); +/* + compressBound() returns an upper bound on the compressed size after + compress() or compress2() on sourceLen bytes. It would be used before a + compress() or compress2() call to allocate the destination buffer. +*/ + +ZEXTERN int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen); +/* + Decompresses the source buffer into the destination buffer. sourceLen is + the byte length of the source buffer. Upon entry, destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, destLen + is the actual size of the uncompressed data. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_BUF_ERROR if there was not enough room in the output + buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In + the case where there is not enough room, uncompress() will fill the output + buffer with the uncompressed data up to that point. +*/ + +ZEXTERN int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen); +/* + Same as uncompress, except that sourceLen is a pointer, where the + length of the source is *sourceLen. On return, *sourceLen is the number of + source bytes consumed. +*/ + + /* gzip file access functions */ + +/* + This library supports reading and writing files in gzip (.gz) format with + an interface similar to that of stdio, using the functions that start with + "gz". The gzip format is different from the zlib format. gzip is a gzip + wrapper, documented in RFC 1952, wrapped around a deflate stream. +*/ + +typedef struct gzFile_s *gzFile; /* semi-opaque gzip file descriptor */ + +/* +ZEXTERN gzFile ZEXPORT gzopen(const char *path, const char *mode); + + Open the gzip (.gz) file at path for reading and decompressing, or + compressing and writing. The mode parameter is as in fopen ("rb" or "wb") + but can also include a compression level ("wb9") or a strategy: 'f' for + filtered data as in "wb6f", 'h' for Huffman-only compression as in "wb1h", + 'R' for run-length encoding as in "wb1R", or 'F' for fixed code compression + as in "wb9F". (See the description of deflateInit2 for more information + about the strategy parameter.) 'T' will request transparent writing or + appending with no compression and not using the gzip format. + + "a" can be used instead of "w" to request that the gzip stream that will + be written be appended to the file. "+" will result in an error, since + reading and writing to the same gzip file is not supported. The addition of + "x" when writing will create the file exclusively, which fails if the file + already exists. On systems that support it, the addition of "e" when + reading or writing will set the flag to close the file on an execve() call. + + These functions, as well as gzip, will read and decode a sequence of gzip + streams in a file. The append function of gzopen() can be used to create + such a file. (Also see gzflush() for another way to do this.) When + appending, gzopen does not test whether the file begins with a gzip stream, + nor does it look for the end of the gzip streams to begin appending. gzopen + will simply append a gzip stream to the existing file. + + gzopen can be used to read a file which is not in gzip format; in this + case gzread will directly read from the file without decompression. When + reading, this will be detected automatically by looking for the magic two- + byte gzip header. + + gzopen returns NULL if the file could not be opened, if there was + insufficient memory to allocate the gzFile state, or if an invalid mode was + specified (an 'r', 'w', or 'a' was not provided, or '+' was provided). + errno can be checked to determine if the reason gzopen failed was that the + file could not be opened. +*/ + +ZEXTERN gzFile ZEXPORT gzdopen(int fd, const char *mode); +/* + Associate a gzFile with the file descriptor fd. File descriptors are + obtained from calls like open, dup, creat, pipe or fileno (if the file has + been previously opened with fopen). The mode parameter is as in gzopen. + + The next call of gzclose on the returned gzFile will also close the file + descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor + fd. If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd, + mode);. The duplicated descriptor should be saved to avoid a leak, since + gzdopen does not close fd if it fails. If you are using fileno() to get the + file descriptor from a FILE *, then you will have to use dup() to avoid + double-close()ing the file descriptor. Both gzclose() and fclose() will + close the associated file descriptor, so they need to have different file + descriptors. + + gzdopen returns NULL if there was insufficient memory to allocate the + gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not + provided, or '+' was provided), or if fd is -1. The file descriptor is not + used until the next gz* read, write, seek, or close operation, so gzdopen + will not detect if fd is invalid (unless fd is -1). +*/ + +ZEXTERN int ZEXPORT gzbuffer(gzFile file, unsigned size); +/* + Set the internal buffer size used by this library's functions for file to + size. The default buffer size is 8192 bytes. This function must be called + after gzopen() or gzdopen(), and before any other calls that read or write + the file. The buffer memory allocation is always deferred to the first read + or write. Three times that size in buffer space is allocated. A larger + buffer size of, for example, 64K or 128K bytes will noticeably increase the + speed of decompression (reading). + + The new buffer size also affects the maximum length for gzprintf(). + + gzbuffer() returns 0 on success, or -1 on failure, such as being called + too late. +*/ + +ZEXTERN int ZEXPORT gzsetparams(gzFile file, int level, int strategy); +/* + Dynamically update the compression level and strategy for file. See the + description of deflateInit2 for the meaning of these parameters. Previously + provided data is flushed before applying the parameter changes. + + gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not + opened for writing, Z_ERRNO if there is an error writing the flushed data, + or Z_MEM_ERROR if there is a memory allocation error. +*/ + +ZEXTERN int ZEXPORT gzread(gzFile file, voidp buf, unsigned len); +/* + Read and decompress up to len uncompressed bytes from file into buf. If + the input file is not in gzip format, gzread copies the given number of + bytes into the buffer directly from the file. + + After reaching the end of a gzip stream in the input, gzread will continue + to read, looking for another gzip stream. Any number of gzip streams may be + concatenated in the input file, and will all be decompressed by gzread(). + If something other than a gzip stream is encountered after a gzip stream, + that remaining trailing garbage is ignored (and no error is returned). + + gzread can be used to read a gzip file that is being concurrently written. + Upon reaching the end of the input, gzread will return with the available + data. If the error code returned by gzerror is Z_OK or Z_BUF_ERROR, then + gzclearerr can be used to clear the end of file indicator in order to permit + gzread to be tried again. Z_OK indicates that a gzip stream was completed + on the last gzread. Z_BUF_ERROR indicates that the input file ended in the + middle of a gzip stream. Note that gzread does not return -1 in the event + of an incomplete gzip stream. This error is deferred until gzclose(), which + will return Z_BUF_ERROR if the last gzread ended in the middle of a gzip + stream. Alternatively, gzerror can be used before gzclose to detect this + case. + + gzread returns the number of uncompressed bytes actually read, less than + len for end of file, or -1 for error. If len is too large to fit in an int, + then nothing is read, -1 is returned, and the error state is set to + Z_STREAM_ERROR. +*/ + +ZEXTERN z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, + gzFile file); +/* + Read and decompress up to nitems items of size size from file into buf, + otherwise operating as gzread() does. This duplicates the interface of + stdio's fread(), with size_t request and return types. If the library + defines size_t, then z_size_t is identical to size_t. If not, then z_size_t + is an unsigned integer type that can contain a pointer. + + gzfread() returns the number of full items read of size size, or zero if + the end of the file was reached and a full item could not be read, or if + there was an error. gzerror() must be consulted if zero is returned in + order to determine if there was an error. If the multiplication of size and + nitems overflows, i.e. the product does not fit in a z_size_t, then nothing + is read, zero is returned, and the error state is set to Z_STREAM_ERROR. + + In the event that the end of file is reached and only a partial item is + available at the end, i.e. the remaining uncompressed data length is not a + multiple of size, then the final partial item is nevertheless read into buf + and the end-of-file flag is set. The length of the partial item read is not + provided, but could be inferred from the result of gztell(). This behavior + is the same as the behavior of fread() implementations in common libraries, + but it prevents the direct use of gzfread() to read a concurrently written + file, resetting and retrying on end-of-file, when size is not 1. +*/ + +ZEXTERN int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len); +/* + Compress and write the len uncompressed bytes at buf to file. gzwrite + returns the number of uncompressed bytes written or 0 in case of error. +*/ + +ZEXTERN z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, + z_size_t nitems, gzFile file); +/* + Compress and write nitems items of size size from buf to file, duplicating + the interface of stdio's fwrite(), with size_t request and return types. If + the library defines size_t, then z_size_t is identical to size_t. If not, + then z_size_t is an unsigned integer type that can contain a pointer. + + gzfwrite() returns the number of full items written of size size, or zero + if there was an error. If the multiplication of size and nitems overflows, + i.e. the product does not fit in a z_size_t, then nothing is written, zero + is returned, and the error state is set to Z_STREAM_ERROR. +*/ + +ZEXTERN int ZEXPORTVA gzprintf(gzFile file, const char *format, ...); +/* + Convert, format, compress, and write the arguments (...) to file under + control of the string format, as in fprintf. gzprintf returns the number of + uncompressed bytes actually written, or a negative zlib error code in case + of error. The number of uncompressed bytes written is limited to 8191, or + one less than the buffer size given to gzbuffer(). The caller should assure + that this limit is not exceeded. If it is exceeded, then gzprintf() will + return an error (0) with nothing written. In this case, there may also be a + buffer overflow with unpredictable consequences, which is possible only if + zlib was compiled with the insecure functions sprintf() or vsprintf(), + because the secure snprintf() or vsnprintf() functions were not available. + This can be determined using zlibCompileFlags(). +*/ + +ZEXTERN int ZEXPORT gzputs(gzFile file, const char *s); +/* + Compress and write the given null-terminated string s to file, excluding + the terminating null character. + + gzputs returns the number of characters written, or -1 in case of error. +*/ + +ZEXTERN char * ZEXPORT gzgets(gzFile file, char *buf, int len); +/* + Read and decompress bytes from file into buf, until len-1 characters are + read, or until a newline character is read and transferred to buf, or an + end-of-file condition is encountered. If any characters are read or if len + is one, the string is terminated with a null character. If no characters + are read due to an end-of-file or len is less than one, then the buffer is + left untouched. + + gzgets returns buf which is a null-terminated string, or it returns NULL + for end-of-file or in case of error. If there was an error, the contents at + buf are indeterminate. +*/ + +ZEXTERN int ZEXPORT gzputc(gzFile file, int c); +/* + Compress and write c, converted to an unsigned char, into file. gzputc + returns the value that was written, or -1 in case of error. +*/ + +ZEXTERN int ZEXPORT gzgetc(gzFile file); +/* + Read and decompress one byte from file. gzgetc returns this byte or -1 + in case of end of file or error. This is implemented as a macro for speed. + As such, it does not do all of the checking the other functions do. I.e. + it does not check to see if file is NULL, nor whether the structure file + points to has been clobbered or not. +*/ + +ZEXTERN int ZEXPORT gzungetc(int c, gzFile file); +/* + Push c back onto the stream for file to be read as the first character on + the next read. At least one character of push-back is always allowed. + gzungetc() returns the character pushed, or -1 on failure. gzungetc() will + fail if c is -1, and may fail if a character has been pushed but not read + yet. If gzungetc is used immediately after gzopen or gzdopen, at least the + output buffer size of pushed characters is allowed. (See gzbuffer above.) + The pushed character will be discarded if the stream is repositioned with + gzseek() or gzrewind(). +*/ + +ZEXTERN int ZEXPORT gzflush(gzFile file, int flush); +/* + Flush all pending output to file. The parameter flush is as in the + deflate() function. The return value is the zlib error number (see function + gzerror below). gzflush is only permitted when writing. + + If the flush parameter is Z_FINISH, the remaining data is written and the + gzip stream is completed in the output. If gzwrite() is called again, a new + gzip stream will be started in the output. gzread() is able to read such + concatenated gzip streams. + + gzflush should be called only when strictly necessary because it will + degrade compression if called too often. +*/ + +/* +ZEXTERN z_off_t ZEXPORT gzseek(gzFile file, + z_off_t offset, int whence); + + Set the starting position to offset relative to whence for the next gzread + or gzwrite on file. The offset represents a number of bytes in the + uncompressed data stream. The whence parameter is defined as in lseek(2); + the value SEEK_END is not supported. + + If the file is opened for reading, this function is emulated but can be + extremely slow. If the file is opened for writing, only forward seeks are + supported; gzseek then compresses a sequence of zeroes up to the new + starting position. + + gzseek returns the resulting offset location as measured in bytes from + the beginning of the uncompressed stream, or -1 in case of error, in + particular if the file is opened for writing and the new starting position + would be before the current position. +*/ + +ZEXTERN int ZEXPORT gzrewind(gzFile file); +/* + Rewind file. This function is supported only for reading. + + gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET). +*/ + +/* +ZEXTERN z_off_t ZEXPORT gztell(gzFile file); + + Return the starting position for the next gzread or gzwrite on file. + This position represents a number of bytes in the uncompressed data stream, + and is zero when starting, even if appending or reading a gzip stream from + the middle of a file using gzdopen(). + + gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR) +*/ + +/* +ZEXTERN z_off_t ZEXPORT gzoffset(gzFile file); + + Return the current compressed (actual) read or write offset of file. This + offset includes the count of bytes that precede the gzip stream, for example + when appending or when using gzdopen() for reading. When reading, the + offset does not include as yet unused buffered input. This information can + be used for a progress indicator. On error, gzoffset() returns -1. +*/ + +ZEXTERN int ZEXPORT gzeof(gzFile file); +/* + Return true (1) if the end-of-file indicator for file has been set while + reading, false (0) otherwise. Note that the end-of-file indicator is set + only if the read tried to go past the end of the input, but came up short. + Therefore, just like feof(), gzeof() may return false even if there is no + more data to read, in the event that the last read request was for the exact + number of bytes remaining in the input file. This will happen if the input + file size is an exact multiple of the buffer size. + + If gzeof() returns true, then the read functions will return no more data, + unless the end-of-file indicator is reset by gzclearerr() and the input file + has grown since the previous end of file was detected. +*/ + +ZEXTERN int ZEXPORT gzdirect(gzFile file); +/* + Return true (1) if file is being copied directly while reading, or false + (0) if file is a gzip stream being decompressed. + + If the input file is empty, gzdirect() will return true, since the input + does not contain a gzip stream. + + If gzdirect() is used immediately after gzopen() or gzdopen() it will + cause buffers to be allocated to allow reading the file to determine if it + is a gzip file. Therefore if gzbuffer() is used, it should be called before + gzdirect(). + + When writing, gzdirect() returns true (1) if transparent writing was + requested ("wT" for the gzopen() mode), or false (0) otherwise. (Note: + gzdirect() is not needed when writing. Transparent writing must be + explicitly requested, so the application already knows the answer. When + linking statically, using gzdirect() will include all of the zlib code for + gzip file reading and decompression, which may not be desired.) +*/ + +ZEXTERN int ZEXPORT gzclose(gzFile file); +/* + Flush all pending output for file, if necessary, close file and + deallocate the (de)compression state. Note that once file is closed, you + cannot call gzerror with file, since its structures have been deallocated. + gzclose must not be called more than once on the same file, just as free + must not be called more than once on the same allocation. + + gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a + file operation error, Z_MEM_ERROR if out of memory, Z_BUF_ERROR if the + last read ended in the middle of a gzip stream, or Z_OK on success. +*/ + +ZEXTERN int ZEXPORT gzclose_r(gzFile file); +ZEXTERN int ZEXPORT gzclose_w(gzFile file); +/* + Same as gzclose(), but gzclose_r() is only for use when reading, and + gzclose_w() is only for use when writing or appending. The advantage to + using these instead of gzclose() is that they avoid linking in zlib + compression or decompression code that is not used when only reading or only + writing respectively. If gzclose() is used, then both compression and + decompression code will be included the application when linking to a static + zlib library. +*/ + +ZEXTERN const char * ZEXPORT gzerror(gzFile file, int *errnum); +/* + Return the error message for the last error which occurred on file. + errnum is set to zlib error number. If an error occurred in the file system + and not in the compression library, errnum is set to Z_ERRNO and the + application may consult errno to get the exact error code. + + The application must not modify the returned string. Future calls to + this function may invalidate the previously returned string. If file is + closed, then the string previously returned by gzerror will no longer be + available. + + gzerror() should be used to distinguish errors from end-of-file for those + functions above that do not distinguish those cases in their return values. +*/ + +ZEXTERN void ZEXPORT gzclearerr(gzFile file); +/* + Clear the error and end-of-file flags for file. This is analogous to the + clearerr() function in stdio. This is useful for continuing to read a gzip + file that is being written concurrently. +*/ + +#endif /* !Z_SOLO */ + + /* checksum functions */ + +/* + These functions are not related to compression but are exported + anyway because they might be useful in applications using the compression + library. +*/ + +ZEXTERN uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len); +/* + Update a running Adler-32 checksum with the bytes buf[0..len-1] and + return the updated checksum. An Adler-32 value is in the range of a 32-bit + unsigned integer. If buf is Z_NULL, this function returns the required + initial value for the checksum. + + An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed + much faster. + + Usage example: + + uLong adler = adler32(0L, Z_NULL, 0); + + while (read_buffer(buffer, length) != EOF) { + adler = adler32(adler, buffer, length); + } + if (adler != original_adler) error(); +*/ + +ZEXTERN uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, + z_size_t len); +/* + Same as adler32(), but with a size_t length. +*/ + +/* +ZEXTERN uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2, + z_off_t len2); + + Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 + and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for + each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of + seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. Note + that the z_off_t type (like off_t) is a signed integer. If len2 is + negative, the result has no meaning or utility. +*/ + +ZEXTERN uLong ZEXPORT crc32(uLong crc, const Bytef *buf, uInt len); +/* + Update a running CRC-32 with the bytes buf[0..len-1] and return the + updated CRC-32. A CRC-32 value is in the range of a 32-bit unsigned integer. + If buf is Z_NULL, this function returns the required initial value for the + crc. Pre- and post-conditioning (one's complement) is performed within this + function so it shouldn't be done by the application. + + Usage example: + + uLong crc = crc32(0L, Z_NULL, 0); + + while (read_buffer(buffer, length) != EOF) { + crc = crc32(crc, buffer, length); + } + if (crc != original_crc) error(); +*/ + +ZEXTERN uLong ZEXPORT crc32_z(uLong crc, const Bytef *buf, + z_size_t len); +/* + Same as crc32(), but with a size_t length. +*/ + +/* +ZEXTERN uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2); + + Combine two CRC-32 check values into one. For two sequences of bytes, + seq1 and seq2 with lengths len1 and len2, CRC-32 check values were + calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 + check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and + len2. +*/ + +/* +ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t len2); + + Return the operator corresponding to length len2, to be used with + crc32_combine_op(). +*/ + +ZEXTERN uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op); +/* + Give the same result as crc32_combine(), using op in place of len2. op is + is generated from len2 by crc32_combine_gen(). This will be faster than + crc32_combine() if the generated op is used more than once. +*/ + + + /* various hacks, don't look :) */ + +/* deflateInit and inflateInit are macros to allow checking the zlib version + * and the compiler's view of z_stream: + */ +ZEXTERN int ZEXPORT deflateInit_(z_streamp strm, int level, + const char *version, int stream_size); +ZEXTERN int ZEXPORT inflateInit_(z_streamp strm, + const char *version, int stream_size); +ZEXTERN int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, + int windowBits, int memLevel, + int strategy, const char *version, + int stream_size); +ZEXTERN int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, + const char *version, int stream_size); +ZEXTERN int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits, + unsigned char FAR *window, + const char *version, + int stream_size); +#ifdef Z_PREFIX_SET +# define z_deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define z_inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#else +# define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#endif + +#ifndef Z_SOLO + +/* gzgetc() macro and its supporting function and exposed data structure. Note + * that the real internal state is much larger than the exposed structure. + * This abbreviated structure exposes just enough for the gzgetc() macro. The + * user should not mess with these exposed elements, since their names or + * behavior could change in the future, perhaps even capriciously. They can + * only be used by the gzgetc() macro. You have been warned. + */ +struct gzFile_s { + unsigned have; + unsigned char *next; + z_off64_t pos; +}; +ZEXTERN int ZEXPORT gzgetc_(gzFile file); /* backward compatibility */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +# define z_gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#else +# define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#endif + +/* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or + * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if + * both are true, the application gets the *64 functions, and the regular + * functions are changed to 64 bits) -- in case these are set on systems + * without large file support, _LFS64_LARGEFILE must also be true + */ +#ifdef Z_LARGE64 + ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); + ZEXTERN z_off64_t ZEXPORT gzseek64(gzFile, z_off64_t, int); + ZEXTERN z_off64_t ZEXPORT gztell64(gzFile); + ZEXTERN z_off64_t ZEXPORT gzoffset64(gzFile); + ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off64_t); + ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off64_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off64_t); +#endif + +#if !defined(ZLIB_INTERNAL) && defined(Z_WANT64) +# ifdef Z_PREFIX_SET +# define z_gzopen z_gzopen64 +# define z_gzseek z_gzseek64 +# define z_gztell z_gztell64 +# define z_gzoffset z_gzoffset64 +# define z_adler32_combine z_adler32_combine64 +# define z_crc32_combine z_crc32_combine64 +# define z_crc32_combine_gen z_crc32_combine_gen64 +# else +# define gzopen gzopen64 +# define gzseek gzseek64 +# define gztell gztell64 +# define gzoffset gzoffset64 +# define adler32_combine adler32_combine64 +# define crc32_combine crc32_combine64 +# define crc32_combine_gen crc32_combine_gen64 +# endif +# ifndef Z_LARGE64 + ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); + ZEXTERN z_off_t ZEXPORT gzseek64(gzFile, z_off_t, int); + ZEXTERN z_off_t ZEXPORT gztell64(gzFile); + ZEXTERN z_off_t ZEXPORT gzoffset64(gzFile); + ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t); +# endif +#else + ZEXTERN gzFile ZEXPORT gzopen(const char *, const char *); + ZEXTERN z_off_t ZEXPORT gzseek(gzFile, z_off_t, int); + ZEXTERN z_off_t ZEXPORT gztell(gzFile); + ZEXTERN z_off_t ZEXPORT gzoffset(gzFile); + ZEXTERN uLong ZEXPORT adler32_combine(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t); +#endif + +#else /* Z_SOLO */ + + ZEXTERN uLong ZEXPORT adler32_combine(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t); + +#endif /* !Z_SOLO */ + +/* undocumented functions */ +ZEXTERN const char * ZEXPORT zError(int); +ZEXTERN int ZEXPORT inflateSyncPoint(z_streamp); +ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table(void); +ZEXTERN int ZEXPORT inflateUndermine(z_streamp, int); +ZEXTERN int ZEXPORT inflateValidate(z_streamp, int); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed(z_streamp); +ZEXTERN int ZEXPORT inflateResetKeep(z_streamp); +ZEXTERN int ZEXPORT deflateResetKeep(z_streamp); +#if defined(_WIN32) && !defined(Z_SOLO) +ZEXTERN gzFile ZEXPORT gzopen_w(const wchar_t *path, + const char *mode); +#endif +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +ZEXTERN int ZEXPORTVA gzvprintf(gzFile file, + const char *format, + va_list va); +# endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* ZLIB_H */ diff --git a/lib/zlib/zlib.mk b/lib/zlib/zlib.mk new file mode 100644 index 0000000..a8937a7 --- /dev/null +++ b/lib/zlib/zlib.mk @@ -0,0 +1,25 @@ +# +# Copyright (c) 2018, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +ZLIB_PATH := lib/zlib + +# Imported from zlib 1.2.11 (do not modify them) +ZLIB_SOURCES := $(addprefix $(ZLIB_PATH)/, \ + adler32.c \ + crc32.c \ + inffast.c \ + inflate.c \ + inftrees.c \ + zutil.c) + +# Implemented for TF +ZLIB_SOURCES += $(addprefix $(ZLIB_PATH)/, \ + tf_gunzip.c) + +INCLUDES += -Iinclude/lib/zlib + +# REVISIT: the following flags need not be given globally +TF_CFLAGS += -DZ_SOLO -DDEF_WBITS=31 diff --git a/lib/zlib/zutil.c b/lib/zlib/zutil.c new file mode 100644 index 0000000..b1c5d2d --- /dev/null +++ b/lib/zlib/zutil.c @@ -0,0 +1,299 @@ +/* zutil.c -- target dependent utility functions for the compression library + * Copyright (C) 1995-2017 Jean-loup Gailly + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#include "zutil.h" +#ifndef Z_SOLO +# include "gzguts.h" +#endif + +z_const char * const z_errmsg[10] = { + (z_const char *)"need dictionary", /* Z_NEED_DICT 2 */ + (z_const char *)"stream end", /* Z_STREAM_END 1 */ + (z_const char *)"", /* Z_OK 0 */ + (z_const char *)"file error", /* Z_ERRNO (-1) */ + (z_const char *)"stream error", /* Z_STREAM_ERROR (-2) */ + (z_const char *)"data error", /* Z_DATA_ERROR (-3) */ + (z_const char *)"insufficient memory", /* Z_MEM_ERROR (-4) */ + (z_const char *)"buffer error", /* Z_BUF_ERROR (-5) */ + (z_const char *)"incompatible version",/* Z_VERSION_ERROR (-6) */ + (z_const char *)"" +}; + + +const char * ZEXPORT zlibVersion(void) { + return ZLIB_VERSION; +} + +uLong ZEXPORT zlibCompileFlags(void) { + uLong flags; + + flags = 0; + switch ((int)(sizeof(uInt))) { + case 2: break; + case 4: flags += 1; break; + case 8: flags += 2; break; + default: flags += 3; + } + switch ((int)(sizeof(uLong))) { + case 2: break; + case 4: flags += 1 << 2; break; + case 8: flags += 2 << 2; break; + default: flags += 3 << 2; + } + switch ((int)(sizeof(voidpf))) { + case 2: break; + case 4: flags += 1 << 4; break; + case 8: flags += 2 << 4; break; + default: flags += 3 << 4; + } + switch ((int)(sizeof(z_off_t))) { + case 2: break; + case 4: flags += 1 << 6; break; + case 8: flags += 2 << 6; break; + default: flags += 3 << 6; + } +#ifdef ZLIB_DEBUG + flags += 1 << 8; +#endif + /* +#if defined(ASMV) || defined(ASMINF) + flags += 1 << 9; +#endif + */ +#ifdef ZLIB_WINAPI + flags += 1 << 10; +#endif +#ifdef BUILDFIXED + flags += 1 << 12; +#endif +#ifdef DYNAMIC_CRC_TABLE + flags += 1 << 13; +#endif +#ifdef NO_GZCOMPRESS + flags += 1L << 16; +#endif +#ifdef NO_GZIP + flags += 1L << 17; +#endif +#ifdef PKZIP_BUG_WORKAROUND + flags += 1L << 20; +#endif +#ifdef FASTEST + flags += 1L << 21; +#endif +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifdef NO_vsnprintf + flags += 1L << 25; +# ifdef HAS_vsprintf_void + flags += 1L << 26; +# endif +# else +# ifdef HAS_vsnprintf_void + flags += 1L << 26; +# endif +# endif +#else + flags += 1L << 24; +# ifdef NO_snprintf + flags += 1L << 25; +# ifdef HAS_sprintf_void + flags += 1L << 26; +# endif +# else +# ifdef HAS_snprintf_void + flags += 1L << 26; +# endif +# endif +#endif + return flags; +} + +#ifdef ZLIB_DEBUG +#include <stdlib.h> +# ifndef verbose +# define verbose 0 +# endif +int ZLIB_INTERNAL z_verbose = verbose; + +void ZLIB_INTERNAL z_error(char *m) { + fprintf(stderr, "%s\n", m); + exit(1); +} +#endif + +/* exported to allow conversion of error code to string for compress() and + * uncompress() + */ +const char * ZEXPORT zError(int err) { + return ERR_MSG(err); +} + +#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 + /* The older Microsoft C Run-Time Library for Windows CE doesn't have + * errno. We define it as a global variable to simplify porting. + * Its value is always 0 and should not be used. + */ + int errno = 0; +#endif + +#ifndef HAVE_MEMCPY + +void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len) { + if (len == 0) return; + do { + *dest++ = *source++; /* ??? to be unrolled */ + } while (--len != 0); +} + +int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len) { + uInt j; + + for (j = 0; j < len; j++) { + if (s1[j] != s2[j]) return 2*(s1[j] > s2[j])-1; + } + return 0; +} + +void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len) { + if (len == 0) return; + do { + *dest++ = 0; /* ??? to be unrolled */ + } while (--len != 0); +} +#endif + +#ifndef Z_SOLO + +#ifdef SYS16BIT + +#ifdef __TURBOC__ +/* Turbo C in 16-bit mode */ + +# define MY_ZCALLOC + +/* Turbo C malloc() does not allow dynamic allocation of 64K bytes + * and farmalloc(64K) returns a pointer with an offset of 8, so we + * must fix the pointer. Warning: the pointer must be put back to its + * original form in order to free it, use zcfree(). + */ + +#define MAX_PTR 10 +/* 10*64K = 640K */ + +local int next_ptr = 0; + +typedef struct ptr_table_s { + voidpf org_ptr; + voidpf new_ptr; +} ptr_table; + +local ptr_table table[MAX_PTR]; +/* This table is used to remember the original form of pointers + * to large buffers (64K). Such pointers are normalized with a zero offset. + * Since MSDOS is not a preemptive multitasking OS, this table is not + * protected from concurrent access. This hack doesn't work anyway on + * a protected system like OS/2. Use Microsoft C instead. + */ + +voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) { + voidpf buf; + ulg bsize = (ulg)items*size; + + (void)opaque; + + /* If we allocate less than 65520 bytes, we assume that farmalloc + * will return a usable pointer which doesn't have to be normalized. + */ + if (bsize < 65520L) { + buf = farmalloc(bsize); + if (*(ush*)&buf != 0) return buf; + } else { + buf = farmalloc(bsize + 16L); + } + if (buf == NULL || next_ptr >= MAX_PTR) return NULL; + table[next_ptr].org_ptr = buf; + + /* Normalize the pointer to seg:0 */ + *((ush*)&buf+1) += ((ush)((uch*)buf-0) + 15) >> 4; + *(ush*)&buf = 0; + table[next_ptr++].new_ptr = buf; + return buf; +} + +void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { + int n; + + (void)opaque; + + if (*(ush*)&ptr != 0) { /* object < 64K */ + farfree(ptr); + return; + } + /* Find the original pointer */ + for (n = 0; n < next_ptr; n++) { + if (ptr != table[n].new_ptr) continue; + + farfree(table[n].org_ptr); + while (++n < next_ptr) { + table[n-1] = table[n]; + } + next_ptr--; + return; + } + Assert(0, "zcfree: ptr not found"); +} + +#endif /* __TURBOC__ */ + + +#ifdef M_I86 +/* Microsoft C in 16-bit mode */ + +# define MY_ZCALLOC + +#if (!defined(_MSC_VER) || (_MSC_VER <= 600)) +# define _halloc halloc +# define _hfree hfree +#endif + +voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, uInt items, uInt size) { + (void)opaque; + return _halloc((long)items, size); +} + +void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { + (void)opaque; + _hfree(ptr); +} + +#endif /* M_I86 */ + +#endif /* SYS16BIT */ + + +#ifndef MY_ZCALLOC /* Any system without a special alloc function */ + +#ifndef STDC +extern voidp malloc(uInt size); +extern voidp calloc(uInt items, uInt size); +extern void free(voidpf ptr); +#endif + +voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) { + (void)opaque; + return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : + (voidpf)calloc(items, size); +} + +void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { + (void)opaque; + free(ptr); +} + +#endif /* MY_ZCALLOC */ + +#endif /* !Z_SOLO */ diff --git a/lib/zlib/zutil.h b/lib/zlib/zutil.h new file mode 100644 index 0000000..902a304 --- /dev/null +++ b/lib/zlib/zutil.h @@ -0,0 +1,275 @@ +/* zutil.h -- internal interface and configuration of the compression library + * Copyright (C) 1995-2022 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* @(#) $Id$ */ + +#ifndef ZUTIL_H +#define ZUTIL_H + +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + +#include "zlib.h" + +#if defined(STDC) && !defined(Z_SOLO) +# if !(defined(_WIN32_WCE) && defined(_MSC_VER)) +# include <stddef.h> +# endif +# include <string.h> +# include <stdlib.h> +#endif + +#ifndef local +# define local static +#endif +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ + +typedef unsigned char uch; +typedef uch FAR uchf; +typedef unsigned short ush; +typedef ush FAR ushf; +typedef unsigned long ulg; + +#if !defined(Z_U8) && !defined(Z_SOLO) && defined(STDC) +# include <limits.h> +# if (ULONG_MAX == 0xffffffffffffffff) +# define Z_U8 unsigned long +# elif (ULLONG_MAX == 0xffffffffffffffff) +# define Z_U8 unsigned long long +# elif (UINT_MAX == 0xffffffffffffffff) +# define Z_U8 unsigned +# endif +#endif + +extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ +/* (size given to avoid silly warnings with Visual C++) */ + +#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] + +#define ERR_RETURN(strm,err) \ + return (strm->msg = ERR_MSG(err), (err)) +/* To be used only when the state is known to be valid */ + + /* common constants */ + +#ifndef DEF_WBITS +# define DEF_WBITS MAX_WBITS +#endif +/* default windowBits for decompression. MAX_WBITS is for compression only */ + +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +/* default memLevel */ + +#define STORED_BLOCK 0 +#define STATIC_TREES 1 +#define DYN_TREES 2 +/* The three kinds of block type */ + +#define MIN_MATCH 3 +#define MAX_MATCH 258 +/* The minimum and maximum match lengths */ + +#define PRESET_DICT 0x20 /* preset dictionary flag in zlib header */ + + /* target dependencies */ + +#if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32)) +# define OS_CODE 0x00 +# ifndef Z_SOLO +# if defined(__TURBOC__) || defined(__BORLANDC__) +# if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) + /* Allow compilation with ANSI keywords only enabled */ + void _Cdecl farfree( void *block ); + void *_Cdecl farmalloc( unsigned long nbytes ); +# else +# include <alloc.h> +# endif +# else /* MSC or DJGPP */ +# include <malloc.h> +# endif +# endif +#endif + +#ifdef AMIGA +# define OS_CODE 1 +#endif + +#if defined(VAXC) || defined(VMS) +# define OS_CODE 2 +# define F_OPEN(name, mode) \ + fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") +#endif + +#ifdef __370__ +# if __TARGET_LIB__ < 0x20000000 +# define OS_CODE 4 +# elif __TARGET_LIB__ < 0x40000000 +# define OS_CODE 11 +# else +# define OS_CODE 8 +# endif +#endif + +#if defined(ATARI) || defined(atarist) +# define OS_CODE 5 +#endif + +#ifdef OS2 +# define OS_CODE 6 +# if defined(M_I86) && !defined(Z_SOLO) +# include <malloc.h> +# endif +#endif + +#if defined(MACOS) || defined(TARGET_OS_MAC) +# define OS_CODE 7 +# ifndef Z_SOLO +# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include <unix.h> /* for fdopen */ +# else +# ifndef fdopen +# define fdopen(fd,mode) NULL /* No fdopen() */ +# endif +# endif +# endif +#endif + +#ifdef __acorn +# define OS_CODE 13 +#endif + +#if defined(WIN32) && !defined(__CYGWIN__) +# define OS_CODE 10 +#endif + +#ifdef _BEOS_ +# define OS_CODE 16 +#endif + +#ifdef __TOS_OS400__ +# define OS_CODE 18 +#endif + +#ifdef __APPLE__ +# define OS_CODE 19 +#endif + +#if defined(_BEOS_) || defined(RISCOS) +# define fdopen(fd,mode) NULL /* No fdopen() */ +#endif + +#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX +# if defined(_WIN32_WCE) +# define fdopen(fd,mode) NULL /* No fdopen() */ +# else +# define fdopen(fd,type) _fdopen(fd,type) +# endif +#endif + +#if defined(__BORLANDC__) && !defined(MSDOS) + #pragma warn -8004 + #pragma warn -8008 + #pragma warn -8066 +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_WIN32) && \ + (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) + ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t); +#endif + + /* common defaults */ + +#ifndef OS_CODE +# define OS_CODE 3 /* assume Unix */ +#endif + +#ifndef F_OPEN +# define F_OPEN(name, mode) fopen((name), (mode)) +#endif + + /* functions */ + +#if defined(pyr) || defined(Z_SOLO) +# define NO_MEMCPY +#endif +#if defined(SMALL_MEDIUM) && !defined(_MSC_VER) && !defined(__SC__) + /* Use our own functions for small and medium model with MSC <= 5.0. + * You may have to use the same strategy for Borland C (untested). + * The __SC__ check is for Symantec. + */ +# define NO_MEMCPY +#endif +#if defined(STDC) && !defined(HAVE_MEMCPY) && !defined(NO_MEMCPY) +# define HAVE_MEMCPY +#endif +#ifdef HAVE_MEMCPY +# ifdef SMALL_MEDIUM /* MSDOS small or medium model */ +# define zmemcpy _fmemcpy +# define zmemcmp _fmemcmp +# define zmemzero(dest, len) _fmemset(dest, 0, len) +# else +# define zmemcpy memcpy +# define zmemcmp memcmp +# define zmemzero(dest, len) memset(dest, 0, len) +# endif +#else + void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len); + int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len); + void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len); +#endif + +/* Diagnostic functions */ +#ifdef ZLIB_DEBUG +# include <stdio.h> + extern int ZLIB_INTERNAL z_verbose; + extern void ZLIB_INTERNAL z_error(char *m); +# define Assert(cond,msg) {if(!(cond)) z_error(msg);} +# define Trace(x) {if (z_verbose>=0) fprintf x ;} +# define Tracev(x) {if (z_verbose>0) fprintf x ;} +# define Tracevv(x) {if (z_verbose>1) fprintf x ;} +# define Tracec(c,x) {if (z_verbose>0 && (c)) fprintf x ;} +# define Tracecv(c,x) {if (z_verbose>1 && (c)) fprintf x ;} +#else +# define Assert(cond,msg) +# define Trace(x) +# define Tracev(x) +# define Tracevv(x) +# define Tracec(c,x) +# define Tracecv(c,x) +#endif + +#ifndef Z_SOLO + voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, + unsigned size); + void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr); +#endif + +#define ZALLOC(strm, items, size) \ + (*((strm)->zalloc))((strm)->opaque, (items), (size)) +#define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr)) +#define TRY_FREE(s, p) {if (p) ZFREE(s, p);} + +/* Reverse the bytes in a 32-bit value */ +#define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) + +#endif /* ZUTIL_H */ |