From ace9429bb58fd418f0c81d4c2835699bddf6bde6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 11 Apr 2024 10:27:49 +0200 Subject: Adding upstream version 6.6.15. Signed-off-by: Daniel Baumann --- arch/mips/lib/Makefile | 18 ++ arch/mips/lib/bitops.c | 148 +++++++++ arch/mips/lib/bswapdi.c | 13 + arch/mips/lib/bswapsi.c | 13 + arch/mips/lib/csum_partial.S | 754 +++++++++++++++++++++++++++++++++++++++++++ arch/mips/lib/delay.c | 68 ++++ arch/mips/lib/dump_tlb.c | 191 +++++++++++ arch/mips/lib/iomap-pci.c | 46 +++ arch/mips/lib/iomap_copy.c | 29 ++ arch/mips/lib/libgcc.h | 43 +++ arch/mips/lib/memcpy.S | 704 ++++++++++++++++++++++++++++++++++++++++ arch/mips/lib/memset.S | 325 +++++++++++++++++++ arch/mips/lib/mips-atomic.c | 113 +++++++ arch/mips/lib/multi3.c | 54 ++++ arch/mips/lib/r3k_dump_tlb.c | 71 ++++ arch/mips/lib/strncpy_user.S | 65 ++++ arch/mips/lib/strnlen_user.S | 64 ++++ arch/mips/lib/uncached.c | 82 +++++ 18 files changed, 2801 insertions(+) create mode 100644 arch/mips/lib/Makefile create mode 100644 arch/mips/lib/bitops.c create mode 100644 arch/mips/lib/bswapdi.c create mode 100644 arch/mips/lib/bswapsi.c create mode 100644 arch/mips/lib/csum_partial.S create mode 100644 arch/mips/lib/delay.c create mode 100644 arch/mips/lib/dump_tlb.c create mode 100644 arch/mips/lib/iomap-pci.c create mode 100644 arch/mips/lib/iomap_copy.c create mode 100644 arch/mips/lib/libgcc.h create mode 100644 arch/mips/lib/memcpy.S create mode 100644 arch/mips/lib/memset.S create mode 100644 arch/mips/lib/mips-atomic.c create mode 100644 arch/mips/lib/multi3.c create mode 100644 arch/mips/lib/r3k_dump_tlb.c create mode 100644 arch/mips/lib/strncpy_user.S create mode 100644 arch/mips/lib/strnlen_user.S create mode 100644 arch/mips/lib/uncached.c (limited to 'arch/mips/lib') diff --git a/arch/mips/lib/Makefile b/arch/mips/lib/Makefile new file mode 100644 index 0000000000..5d5b993cbc --- /dev/null +++ b/arch/mips/lib/Makefile @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for MIPS-specific library files.. +# + +lib-y += bitops.o csum_partial.o delay.o memcpy.o memset.o \ + mips-atomic.o strncpy_user.o \ + strnlen_user.o uncached.o + +obj-y += iomap_copy.o +obj-$(CONFIG_PCI) += iomap-pci.o +lib-$(CONFIG_GENERIC_CSUM) := $(filter-out csum_partial.o, $(lib-y)) + +obj-$(CONFIG_CPU_GENERIC_DUMP_TLB) += dump_tlb.o +obj-$(CONFIG_CPU_R3000) += r3k_dump_tlb.o + +# libgcc-style stuff needed in the kernel +obj-y += bswapsi.o bswapdi.o multi3.o diff --git a/arch/mips/lib/bitops.c b/arch/mips/lib/bitops.c new file mode 100644 index 0000000000..116d0bd8b2 --- /dev/null +++ b/arch/mips/lib/bitops.c @@ -0,0 +1,148 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (c) 1994-1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org) + * Copyright (c) 1999, 2000 Silicon Graphics, Inc. + */ +#include +#include +#include +#include + + +/** + * __mips_set_bit - Atomically set a bit in memory. This is called by + * set_bit() if it cannot find a faster solution. + * @nr: the bit to set + * @addr: the address to start counting from + */ +void __mips_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + volatile unsigned long *a = &addr[BIT_WORD(nr)]; + unsigned int bit = nr % BITS_PER_LONG; + unsigned long mask; + unsigned long flags; + + mask = 1UL << bit; + raw_local_irq_save(flags); + *a |= mask; + raw_local_irq_restore(flags); +} +EXPORT_SYMBOL(__mips_set_bit); + + +/** + * __mips_clear_bit - Clears a bit in memory. This is called by clear_bit() if + * it cannot find a faster solution. + * @nr: Bit to clear + * @addr: Address to start counting from + */ +void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + volatile unsigned long *a = &addr[BIT_WORD(nr)]; + unsigned int bit = nr % BITS_PER_LONG; + unsigned long mask; + unsigned long flags; + + mask = 1UL << bit; + raw_local_irq_save(flags); + *a &= ~mask; + raw_local_irq_restore(flags); +} +EXPORT_SYMBOL(__mips_clear_bit); + + +/** + * __mips_change_bit - Toggle a bit in memory. This is called by change_bit() + * if it cannot find a faster solution. + * @nr: Bit to change + * @addr: Address to start counting from + */ +void __mips_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + volatile unsigned long *a = &addr[BIT_WORD(nr)]; + unsigned int bit = nr % BITS_PER_LONG; + unsigned long mask; + unsigned long flags; + + mask = 1UL << bit; + raw_local_irq_save(flags); + *a ^= mask; + raw_local_irq_restore(flags); +} +EXPORT_SYMBOL(__mips_change_bit); + + +/** + * __mips_test_and_set_bit_lock - Set a bit and return its old value. This is + * called by test_and_set_bit_lock() if it cannot find a faster solution. + * @nr: Bit to set + * @addr: Address to count from + */ +int __mips_test_and_set_bit_lock(unsigned long nr, + volatile unsigned long *addr) +{ + volatile unsigned long *a = &addr[BIT_WORD(nr)]; + unsigned int bit = nr % BITS_PER_LONG; + unsigned long mask; + unsigned long flags; + int res; + + mask = 1UL << bit; + raw_local_irq_save(flags); + res = (mask & *a) != 0; + *a |= mask; + raw_local_irq_restore(flags); + return res; +} +EXPORT_SYMBOL(__mips_test_and_set_bit_lock); + + +/** + * __mips_test_and_clear_bit - Clear a bit and return its old value. This is + * called by test_and_clear_bit() if it cannot find a faster solution. + * @nr: Bit to clear + * @addr: Address to count from + */ +int __mips_test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + volatile unsigned long *a = &addr[BIT_WORD(nr)]; + unsigned int bit = nr % BITS_PER_LONG; + unsigned long mask; + unsigned long flags; + int res; + + mask = 1UL << bit; + raw_local_irq_save(flags); + res = (mask & *a) != 0; + *a &= ~mask; + raw_local_irq_restore(flags); + return res; +} +EXPORT_SYMBOL(__mips_test_and_clear_bit); + + +/** + * __mips_test_and_change_bit - Change a bit and return its old value. This is + * called by test_and_change_bit() if it cannot find a faster solution. + * @nr: Bit to change + * @addr: Address to count from + */ +int __mips_test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + volatile unsigned long *a = &addr[BIT_WORD(nr)]; + unsigned int bit = nr % BITS_PER_LONG; + unsigned long mask; + unsigned long flags; + int res; + + mask = 1UL << bit; + raw_local_irq_save(flags); + res = (mask & *a) != 0; + *a ^= mask; + raw_local_irq_restore(flags); + return res; +} +EXPORT_SYMBOL(__mips_test_and_change_bit); diff --git a/arch/mips/lib/bswapdi.c b/arch/mips/lib/bswapdi.c new file mode 100644 index 0000000000..88242dc7de --- /dev/null +++ b/arch/mips/lib/bswapdi.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +/* To silence -Wmissing-prototypes. */ +unsigned long long __bswapdi2(unsigned long long u); + +unsigned long long notrace __bswapdi2(unsigned long long u) +{ + return ___constant_swab64(u); +} +EXPORT_SYMBOL(__bswapdi2); diff --git a/arch/mips/lib/bswapsi.c b/arch/mips/lib/bswapsi.c new file mode 100644 index 0000000000..2ed655497d --- /dev/null +++ b/arch/mips/lib/bswapsi.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +/* To silence -Wmissing-prototypes. */ +unsigned int __bswapsi2(unsigned int u); + +unsigned int notrace __bswapsi2(unsigned int u) +{ + return ___constant_swab32(u); +} +EXPORT_SYMBOL(__bswapsi2); diff --git a/arch/mips/lib/csum_partial.S b/arch/mips/lib/csum_partial.S new file mode 100644 index 0000000000..3d2ff4118d --- /dev/null +++ b/arch/mips/lib/csum_partial.S @@ -0,0 +1,754 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Quick'n'dirty IP checksum ... + * + * Copyright (C) 1998, 1999 Ralf Baechle + * Copyright (C) 1999 Silicon Graphics, Inc. + * Copyright (C) 2007 Maciej W. Rozycki + * Copyright (C) 2014 Imagination Technologies Ltd. + */ +#include +#include +#include +#include +#include + +#ifdef CONFIG_64BIT +/* + * As we are sharing code base with the mips32 tree (which use the o32 ABI + * register definitions). We need to redefine the register definitions from + * the n64 ABI register naming to the o32 ABI register naming. + */ +#undef t0 +#undef t1 +#undef t2 +#undef t3 +#define t0 $8 +#define t1 $9 +#define t2 $10 +#define t3 $11 +#define t4 $12 +#define t5 $13 +#define t6 $14 +#define t7 $15 + +#define USE_DOUBLE +#endif + +#ifdef USE_DOUBLE + +#define LOAD ld +#define LOAD32 lwu +#define ADD daddu +#define NBYTES 8 + +#else + +#define LOAD lw +#define LOAD32 lw +#define ADD addu +#define NBYTES 4 + +#endif /* USE_DOUBLE */ + +#define UNIT(unit) ((unit)*NBYTES) + +#define ADDC(sum,reg) \ + .set push; \ + .set noat; \ + ADD sum, reg; \ + sltu v1, sum, reg; \ + ADD sum, v1; \ + .set pop + +#define ADDC32(sum,reg) \ + .set push; \ + .set noat; \ + addu sum, reg; \ + sltu v1, sum, reg; \ + addu sum, v1; \ + .set pop + +#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \ + LOAD _t0, (offset + UNIT(0))(src); \ + LOAD _t1, (offset + UNIT(1))(src); \ + LOAD _t2, (offset + UNIT(2))(src); \ + LOAD _t3, (offset + UNIT(3))(src); \ + ADDC(_t0, _t1); \ + ADDC(_t2, _t3); \ + ADDC(sum, _t0); \ + ADDC(sum, _t2) + +#ifdef USE_DOUBLE +#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \ + CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) +#else +#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \ + CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \ + CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3) +#endif + +/* + * a0: source address + * a1: length of the area to checksum + * a2: partial checksum + */ + +#define src a0 +#define sum v0 + + .text + .set noreorder + .align 5 +LEAF(csum_partial) +EXPORT_SYMBOL(csum_partial) + move sum, zero + move t7, zero + + sltiu t8, a1, 0x8 + bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */ + move t2, a1 + + andi t7, src, 0x1 /* odd buffer? */ + +.Lhword_align: + beqz t7, .Lword_align + andi t8, src, 0x2 + + lbu t0, (src) + LONG_SUBU a1, a1, 0x1 +#ifdef __MIPSEL__ + sll t0, t0, 8 +#endif + ADDC(sum, t0) + PTR_ADDU src, src, 0x1 + andi t8, src, 0x2 + +.Lword_align: + beqz t8, .Ldword_align + sltiu t8, a1, 56 + + lhu t0, (src) + LONG_SUBU a1, a1, 0x2 + ADDC(sum, t0) + sltiu t8, a1, 56 + PTR_ADDU src, src, 0x2 + +.Ldword_align: + bnez t8, .Ldo_end_words + move t8, a1 + + andi t8, src, 0x4 + beqz t8, .Lqword_align + andi t8, src, 0x8 + + LOAD32 t0, 0x00(src) + LONG_SUBU a1, a1, 0x4 + ADDC(sum, t0) + PTR_ADDU src, src, 0x4 + andi t8, src, 0x8 + +.Lqword_align: + beqz t8, .Loword_align + andi t8, src, 0x10 + +#ifdef USE_DOUBLE + ld t0, 0x00(src) + LONG_SUBU a1, a1, 0x8 + ADDC(sum, t0) +#else + lw t0, 0x00(src) + lw t1, 0x04(src) + LONG_SUBU a1, a1, 0x8 + ADDC(sum, t0) + ADDC(sum, t1) +#endif + PTR_ADDU src, src, 0x8 + andi t8, src, 0x10 + +.Loword_align: + beqz t8, .Lbegin_movement + LONG_SRL t8, a1, 0x7 + +#ifdef USE_DOUBLE + ld t0, 0x00(src) + ld t1, 0x08(src) + ADDC(sum, t0) + ADDC(sum, t1) +#else + CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4) +#endif + LONG_SUBU a1, a1, 0x10 + PTR_ADDU src, src, 0x10 + LONG_SRL t8, a1, 0x7 + +.Lbegin_movement: + beqz t8, 1f + andi t2, a1, 0x40 + +.Lmove_128bytes: + CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) + CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4) + CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4) + CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4) + LONG_SUBU t8, t8, 0x01 + .set reorder /* DADDI_WAR */ + PTR_ADDU src, src, 0x80 + bnez t8, .Lmove_128bytes + .set noreorder + +1: + beqz t2, 1f + andi t2, a1, 0x20 + +.Lmove_64bytes: + CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) + CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4) + PTR_ADDU src, src, 0x40 + +1: + beqz t2, .Ldo_end_words + andi t8, a1, 0x1c + +.Lmove_32bytes: + CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4) + andi t8, a1, 0x1c + PTR_ADDU src, src, 0x20 + +.Ldo_end_words: + beqz t8, .Lsmall_csumcpy + andi t2, a1, 0x3 + LONG_SRL t8, t8, 0x2 + +.Lend_words: + LOAD32 t0, (src) + LONG_SUBU t8, t8, 0x1 + ADDC(sum, t0) + .set reorder /* DADDI_WAR */ + PTR_ADDU src, src, 0x4 + bnez t8, .Lend_words + .set noreorder + +/* unknown src alignment and < 8 bytes to go */ +.Lsmall_csumcpy: + move a1, t2 + + andi t0, a1, 4 + beqz t0, 1f + andi t0, a1, 2 + + /* Still a full word to go */ + ulw t1, (src) + PTR_ADDIU src, 4 +#ifdef USE_DOUBLE + dsll t1, t1, 32 /* clear lower 32bit */ +#endif + ADDC(sum, t1) + +1: move t1, zero + beqz t0, 1f + andi t0, a1, 1 + + /* Still a halfword to go */ + ulhu t1, (src) + PTR_ADDIU src, 2 + +1: beqz t0, 1f + sll t1, t1, 16 + + lbu t2, (src) + nop + +#ifdef __MIPSEB__ + sll t2, t2, 8 +#endif + or t1, t2 + +1: ADDC(sum, t1) + + /* fold checksum */ +#ifdef USE_DOUBLE + dsll32 v1, sum, 0 + daddu sum, v1 + sltu v1, sum, v1 + dsra32 sum, sum, 0 + addu sum, v1 +#endif + + /* odd buffer alignment? */ +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_LOONGSON64) + .set push + .set arch=mips32r2 + wsbh v1, sum + movn sum, v1, t7 + .set pop +#else + beqz t7, 1f /* odd buffer alignment? */ + lui v1, 0x00ff + addu v1, 0x00ff + and t0, sum, v1 + sll t0, t0, 8 + srl sum, sum, 8 + and sum, sum, v1 + or sum, sum, t0 +1: +#endif + .set reorder + /* Add the passed partial csum. */ + ADDC32(sum, a2) + jr ra + .set noreorder + END(csum_partial) + + +/* + * checksum and copy routines based on memcpy.S + * + * csum_partial_copy_nocheck(src, dst, len) + * __csum_partial_copy_kernel(src, dst, len) + * + * See "Spec" in memcpy.S for details. Unlike __copy_user, all + * function in this file use the standard calling convention. + */ + +#define src a0 +#define dst a1 +#define len a2 +#define sum v0 +#define odd t8 + +/* + * All exception handlers simply return 0. + */ + +/* Instruction type */ +#define LD_INSN 1 +#define ST_INSN 2 +#define LEGACY_MODE 1 +#define EVA_MODE 2 +#define USEROP 1 +#define KERNELOP 2 + +/* + * Wrapper to add an entry in the exception table + * in case the insn causes a memory exception. + * Arguments: + * insn : Load/store instruction + * type : Instruction type + * reg : Register + * addr : Address + * handler : Exception handler + */ +#define EXC(insn, type, reg, addr) \ + .if \mode == LEGACY_MODE; \ +9: insn reg, addr; \ + .section __ex_table,"a"; \ + PTR_WD 9b, .L_exc; \ + .previous; \ + /* This is enabled in EVA mode */ \ + .else; \ + /* If loading from user or storing to user */ \ + .if ((\from == USEROP) && (type == LD_INSN)) || \ + ((\to == USEROP) && (type == ST_INSN)); \ +9: __BUILD_EVA_INSN(insn##e, reg, addr); \ + .section __ex_table,"a"; \ + PTR_WD 9b, .L_exc; \ + .previous; \ + .else; \ + /* EVA without exception */ \ + insn reg, addr; \ + .endif; \ + .endif + +#undef LOAD + +#ifdef USE_DOUBLE + +#define LOADK ld /* No exception */ +#define LOAD(reg, addr) EXC(ld, LD_INSN, reg, addr) +#define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr) +#define LOADL(reg, addr) EXC(ldl, LD_INSN, reg, addr) +#define LOADR(reg, addr) EXC(ldr, LD_INSN, reg, addr) +#define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr) +#define STOREL(reg, addr) EXC(sdl, ST_INSN, reg, addr) +#define STORER(reg, addr) EXC(sdr, ST_INSN, reg, addr) +#define STORE(reg, addr) EXC(sd, ST_INSN, reg, addr) +#define ADD daddu +#define SUB dsubu +#define SRL dsrl +#define SLL dsll +#define SLLV dsllv +#define SRLV dsrlv +#define NBYTES 8 +#define LOG_NBYTES 3 + +#else + +#define LOADK lw /* No exception */ +#define LOAD(reg, addr) EXC(lw, LD_INSN, reg, addr) +#define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr) +#define LOADL(reg, addr) EXC(lwl, LD_INSN, reg, addr) +#define LOADR(reg, addr) EXC(lwr, LD_INSN, reg, addr) +#define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr) +#define STOREL(reg, addr) EXC(swl, ST_INSN, reg, addr) +#define STORER(reg, addr) EXC(swr, ST_INSN, reg, addr) +#define STORE(reg, addr) EXC(sw, ST_INSN, reg, addr) +#define ADD addu +#define SUB subu +#define SRL srl +#define SLL sll +#define SLLV sllv +#define SRLV srlv +#define NBYTES 4 +#define LOG_NBYTES 2 + +#endif /* USE_DOUBLE */ + +#ifdef CONFIG_CPU_LITTLE_ENDIAN +#define LDFIRST LOADR +#define LDREST LOADL +#define STFIRST STORER +#define STREST STOREL +#define SHIFT_DISCARD SLLV +#define SHIFT_DISCARD_REVERT SRLV +#else +#define LDFIRST LOADL +#define LDREST LOADR +#define STFIRST STOREL +#define STREST STORER +#define SHIFT_DISCARD SRLV +#define SHIFT_DISCARD_REVERT SLLV +#endif + +#define FIRST(unit) ((unit)*NBYTES) +#define REST(unit) (FIRST(unit)+NBYTES-1) + +#define ADDRMASK (NBYTES-1) + +#ifndef CONFIG_CPU_DADDI_WORKAROUNDS + .set noat +#else + .set at=v1 +#endif + + .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to + + li sum, -1 + move odd, zero + /* + * Note: dst & src may be unaligned, len may be 0 + * Temps + */ + /* + * The "issue break"s below are very approximate. + * Issue delays for dcache fills will perturb the schedule, as will + * load queue full replay traps, etc. + * + * If len < NBYTES use byte operations. + */ + sltu t2, len, NBYTES + and t1, dst, ADDRMASK + bnez t2, .Lcopy_bytes_checklen\@ + and t0, src, ADDRMASK + andi odd, dst, 0x1 /* odd buffer? */ + bnez t1, .Ldst_unaligned\@ + nop + bnez t0, .Lsrc_unaligned_dst_aligned\@ + /* + * use delay slot for fall-through + * src and dst are aligned; need to compute rem + */ +.Lboth_aligned\@: + SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter + beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES + nop + SUB len, 8*NBYTES # subtract here for bgez loop + .align 4 +1: + LOAD(t0, UNIT(0)(src)) + LOAD(t1, UNIT(1)(src)) + LOAD(t2, UNIT(2)(src)) + LOAD(t3, UNIT(3)(src)) + LOAD(t4, UNIT(4)(src)) + LOAD(t5, UNIT(5)(src)) + LOAD(t6, UNIT(6)(src)) + LOAD(t7, UNIT(7)(src)) + SUB len, len, 8*NBYTES + ADD src, src, 8*NBYTES + STORE(t0, UNIT(0)(dst)) + ADDC(t0, t1) + STORE(t1, UNIT(1)(dst)) + ADDC(sum, t0) + STORE(t2, UNIT(2)(dst)) + ADDC(t2, t3) + STORE(t3, UNIT(3)(dst)) + ADDC(sum, t2) + STORE(t4, UNIT(4)(dst)) + ADDC(t4, t5) + STORE(t5, UNIT(5)(dst)) + ADDC(sum, t4) + STORE(t6, UNIT(6)(dst)) + ADDC(t6, t7) + STORE(t7, UNIT(7)(dst)) + ADDC(sum, t6) + .set reorder /* DADDI_WAR */ + ADD dst, dst, 8*NBYTES + bgez len, 1b + .set noreorder + ADD len, 8*NBYTES # revert len (see above) + + /* + * len == the number of bytes left to copy < 8*NBYTES + */ +.Lcleanup_both_aligned\@: +#define rem t7 + beqz len, .Ldone\@ + sltu t0, len, 4*NBYTES + bnez t0, .Lless_than_4units\@ + and rem, len, (NBYTES-1) # rem = len % NBYTES + /* + * len >= 4*NBYTES + */ + LOAD(t0, UNIT(0)(src)) + LOAD(t1, UNIT(1)(src)) + LOAD(t2, UNIT(2)(src)) + LOAD(t3, UNIT(3)(src)) + SUB len, len, 4*NBYTES + ADD src, src, 4*NBYTES + STORE(t0, UNIT(0)(dst)) + ADDC(t0, t1) + STORE(t1, UNIT(1)(dst)) + ADDC(sum, t0) + STORE(t2, UNIT(2)(dst)) + ADDC(t2, t3) + STORE(t3, UNIT(3)(dst)) + ADDC(sum, t2) + .set reorder /* DADDI_WAR */ + ADD dst, dst, 4*NBYTES + beqz len, .Ldone\@ + .set noreorder +.Lless_than_4units\@: + /* + * rem = len % NBYTES + */ + beq rem, len, .Lcopy_bytes\@ + nop +1: + LOAD(t0, 0(src)) + ADD src, src, NBYTES + SUB len, len, NBYTES + STORE(t0, 0(dst)) + ADDC(sum, t0) + .set reorder /* DADDI_WAR */ + ADD dst, dst, NBYTES + bne rem, len, 1b + .set noreorder + + /* + * src and dst are aligned, need to copy rem bytes (rem < NBYTES) + * A loop would do only a byte at a time with possible branch + * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE + * because can't assume read-access to dst. Instead, use + * STREST dst, which doesn't require read access to dst. + * + * This code should perform better than a simple loop on modern, + * wide-issue mips processors because the code has fewer branches and + * more instruction-level parallelism. + */ +#define bits t2 + beqz len, .Ldone\@ + ADD t1, dst, len # t1 is just past last byte of dst + li bits, 8*NBYTES + SLL rem, len, 3 # rem = number of bits to keep + LOAD(t0, 0(src)) + SUB bits, bits, rem # bits = number of bits to discard + SHIFT_DISCARD t0, t0, bits + STREST(t0, -1(t1)) + SHIFT_DISCARD_REVERT t0, t0, bits + .set reorder + ADDC(sum, t0) + b .Ldone\@ + .set noreorder +.Ldst_unaligned\@: + /* + * dst is unaligned + * t0 = src & ADDRMASK + * t1 = dst & ADDRMASK; T1 > 0 + * len >= NBYTES + * + * Copy enough bytes to align dst + * Set match = (src and dst have same alignment) + */ +#define match rem + LDFIRST(t3, FIRST(0)(src)) + ADD t2, zero, NBYTES + LDREST(t3, REST(0)(src)) + SUB t2, t2, t1 # t2 = number of bytes copied + xor match, t0, t1 + STFIRST(t3, FIRST(0)(dst)) + SLL t4, t1, 3 # t4 = number of bits to discard + SHIFT_DISCARD t3, t3, t4 + /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */ + ADDC(sum, t3) + beq len, t2, .Ldone\@ + SUB len, len, t2 + ADD dst, dst, t2 + beqz match, .Lboth_aligned\@ + ADD src, src, t2 + +.Lsrc_unaligned_dst_aligned\@: + SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter + beqz t0, .Lcleanup_src_unaligned\@ + and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES +1: +/* + * Avoid consecutive LD*'s to the same register since some mips + * implementations can't issue them in the same cycle. + * It's OK to load FIRST(N+1) before REST(N) because the two addresses + * are to the same unit (unless src is aligned, but it's not). + */ + LDFIRST(t0, FIRST(0)(src)) + LDFIRST(t1, FIRST(1)(src)) + SUB len, len, 4*NBYTES + LDREST(t0, REST(0)(src)) + LDREST(t1, REST(1)(src)) + LDFIRST(t2, FIRST(2)(src)) + LDFIRST(t3, FIRST(3)(src)) + LDREST(t2, REST(2)(src)) + LDREST(t3, REST(3)(src)) + ADD src, src, 4*NBYTES +#ifdef CONFIG_CPU_SB1 + nop # improves slotting +#endif + STORE(t0, UNIT(0)(dst)) + ADDC(t0, t1) + STORE(t1, UNIT(1)(dst)) + ADDC(sum, t0) + STORE(t2, UNIT(2)(dst)) + ADDC(t2, t3) + STORE(t3, UNIT(3)(dst)) + ADDC(sum, t2) + .set reorder /* DADDI_WAR */ + ADD dst, dst, 4*NBYTES + bne len, rem, 1b + .set noreorder + +.Lcleanup_src_unaligned\@: + beqz len, .Ldone\@ + and rem, len, NBYTES-1 # rem = len % NBYTES + beq rem, len, .Lcopy_bytes\@ + nop +1: + LDFIRST(t0, FIRST(0)(src)) + LDREST(t0, REST(0)(src)) + ADD src, src, NBYTES + SUB len, len, NBYTES + STORE(t0, 0(dst)) + ADDC(sum, t0) + .set reorder /* DADDI_WAR */ + ADD dst, dst, NBYTES + bne len, rem, 1b + .set noreorder + +.Lcopy_bytes_checklen\@: + beqz len, .Ldone\@ + nop +.Lcopy_bytes\@: + /* 0 < len < NBYTES */ +#ifdef CONFIG_CPU_LITTLE_ENDIAN +#define SHIFT_START 0 +#define SHIFT_INC 8 +#else +#define SHIFT_START 8*(NBYTES-1) +#define SHIFT_INC -8 +#endif + move t2, zero # partial word + li t3, SHIFT_START # shift +#define COPY_BYTE(N) \ + LOADBU(t0, N(src)); \ + SUB len, len, 1; \ + STOREB(t0, N(dst)); \ + SLLV t0, t0, t3; \ + addu t3, SHIFT_INC; \ + beqz len, .Lcopy_bytes_done\@; \ + or t2, t0 + + COPY_BYTE(0) + COPY_BYTE(1) +#ifdef USE_DOUBLE + COPY_BYTE(2) + COPY_BYTE(3) + COPY_BYTE(4) + COPY_BYTE(5) +#endif + LOADBU(t0, NBYTES-2(src)) + SUB len, len, 1 + STOREB(t0, NBYTES-2(dst)) + SLLV t0, t0, t3 + or t2, t0 +.Lcopy_bytes_done\@: + ADDC(sum, t2) +.Ldone\@: + /* fold checksum */ + .set push + .set noat +#ifdef USE_DOUBLE + dsll32 v1, sum, 0 + daddu sum, v1 + sltu v1, sum, v1 + dsra32 sum, sum, 0 + addu sum, v1 +#endif + +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_LOONGSON64) + .set push + .set arch=mips32r2 + wsbh v1, sum + movn sum, v1, odd + .set pop +#else + beqz odd, 1f /* odd buffer alignment? */ + lui v1, 0x00ff + addu v1, 0x00ff + and t0, sum, v1 + sll t0, t0, 8 + srl sum, sum, 8 + and sum, sum, v1 + or sum, sum, t0 +1: +#endif + .set pop + .set reorder + jr ra + .set noreorder + .endm + + .set noreorder +.L_exc: + jr ra + li v0, 0 + +FEXPORT(__csum_partial_copy_nocheck) +EXPORT_SYMBOL(__csum_partial_copy_nocheck) +#ifndef CONFIG_EVA +FEXPORT(__csum_partial_copy_to_user) +EXPORT_SYMBOL(__csum_partial_copy_to_user) +FEXPORT(__csum_partial_copy_from_user) +EXPORT_SYMBOL(__csum_partial_copy_from_user) +#endif +__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP + +#ifdef CONFIG_EVA +LEAF(__csum_partial_copy_to_user) +__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP +END(__csum_partial_copy_to_user) + +LEAF(__csum_partial_copy_from_user) +__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP +END(__csum_partial_copy_from_user) +#endif diff --git a/arch/mips/lib/delay.c b/arch/mips/lib/delay.c new file mode 100644 index 0000000000..ccdb1fc1e4 --- /dev/null +++ b/arch/mips/lib/delay.c @@ -0,0 +1,68 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994 by Waldorf Electronics + * Copyright (C) 1995 - 2000, 01, 03 by Ralf Baechle + * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2007, 2014 Maciej W. Rozycki + */ +#include +#include +#include +#include +#include + +#include +#include + +#ifndef CONFIG_CPU_DADDI_WORKAROUNDS +#define GCC_DADDI_IMM_ASM() "I" +#else +#define GCC_DADDI_IMM_ASM() "r" +#endif + +#ifndef CONFIG_HAVE_PLAT_DELAY + +void __delay(unsigned long loops) +{ + __asm__ __volatile__ ( + " .set noreorder \n" + " .align 3 \n" + "1: bnez %0, 1b \n" + " " __stringify(LONG_SUBU) " %0, %1 \n" + " .set reorder \n" + : "=r" (loops) + : GCC_DADDI_IMM_ASM() (1), "0" (loops)); +} +EXPORT_SYMBOL(__delay); + +/* + * Division by multiplication: you don't have to worry about + * loss of precision. + * + * Use only for very small delays ( < 1 msec). Should probably use a + * lookup table, really, as the multiplications take much too long with + * short delays. This is a "reasonable" implementation, though (and the + * first constant multiplications gets optimized away if the delay is + * a constant) + */ + +void __udelay(unsigned long us) +{ + unsigned int lpj = raw_current_cpu_data.udelay_val; + + __delay((us * 0x000010c7ull * HZ * lpj) >> 32); +} +EXPORT_SYMBOL(__udelay); + +void __ndelay(unsigned long ns) +{ + unsigned int lpj = raw_current_cpu_data.udelay_val; + + __delay((ns * 0x00000005ull * HZ * lpj) >> 32); +} +EXPORT_SYMBOL(__ndelay); + +#endif diff --git a/arch/mips/lib/dump_tlb.c b/arch/mips/lib/dump_tlb.c new file mode 100644 index 0000000000..8ce97b8741 --- /dev/null +++ b/arch/mips/lib/dump_tlb.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Dump R4x00 TLB for debugging purposes. + * + * Copyright (C) 1994, 1995 by Waldorf Electronics, written by Ralf Baechle. + * Copyright (C) 1999 by Silicon Graphics, Inc. + */ +#include +#include + +#include +#include +#include +#include +#include + +void dump_tlb_regs(void) +{ + const int field = 2 * sizeof(unsigned long); + + pr_info("Index : %0x\n", read_c0_index()); + pr_info("PageMask : %0x\n", read_c0_pagemask()); + if (cpu_has_guestid) + pr_info("GuestCtl1: %0x\n", read_c0_guestctl1()); + pr_info("EntryHi : %0*lx\n", field, read_c0_entryhi()); + pr_info("EntryLo0 : %0*lx\n", field, read_c0_entrylo0()); + pr_info("EntryLo1 : %0*lx\n", field, read_c0_entrylo1()); + pr_info("Wired : %0x\n", read_c0_wired()); + switch (current_cpu_type()) { + case CPU_R10000: + case CPU_R12000: + case CPU_R14000: + case CPU_R16000: + pr_info("FrameMask: %0x\n", read_c0_framemask()); + break; + } + if (cpu_has_small_pages || cpu_has_rixi || cpu_has_xpa) + pr_info("PageGrain: %0x\n", read_c0_pagegrain()); + if (cpu_has_htw) { + pr_info("PWField : %0*lx\n", field, read_c0_pwfield()); + pr_info("PWSize : %0*lx\n", field, read_c0_pwsize()); + pr_info("PWCtl : %0x\n", read_c0_pwctl()); + } +} + +static inline const char *msk2str(unsigned int mask) +{ + switch (mask) { + case PM_4K: return "4kb"; + case PM_16K: return "16kb"; + case PM_64K: return "64kb"; + case PM_256K: return "256kb"; +#ifdef CONFIG_CPU_CAVIUM_OCTEON + case PM_8K: return "8kb"; + case PM_32K: return "32kb"; + case PM_128K: return "128kb"; + case PM_512K: return "512kb"; + case PM_2M: return "2Mb"; + case PM_8M: return "8Mb"; + case PM_32M: return "32Mb"; +#endif + } + return ""; +} + +static void dump_tlb(int first, int last) +{ + unsigned long s_entryhi, entryhi, asid, mmid; + unsigned long long entrylo0, entrylo1, pa; + unsigned int s_index, s_pagemask, s_guestctl1 = 0; + unsigned int pagemask, guestctl1 = 0, c0, c1, i; + unsigned long asidmask = cpu_asid_mask(¤t_cpu_data); + int asidwidth = DIV_ROUND_UP(ilog2(asidmask) + 1, 4); + unsigned long s_mmid; +#ifdef CONFIG_32BIT + bool xpa = cpu_has_xpa && (read_c0_pagegrain() & PG_ELPA); + int pwidth = xpa ? 11 : 8; + int vwidth = 8; +#else + bool xpa = false; + int pwidth = 11; + int vwidth = 11; +#endif + + s_pagemask = read_c0_pagemask(); + s_entryhi = read_c0_entryhi(); + s_index = read_c0_index(); + + if (cpu_has_mmid) + asid = s_mmid = read_c0_memorymapid(); + else + asid = s_entryhi & asidmask; + + if (cpu_has_guestid) + s_guestctl1 = read_c0_guestctl1(); + + for (i = first; i <= last; i++) { + write_c0_index(i); + mtc0_tlbr_hazard(); + tlb_read(); + tlb_read_hazard(); + pagemask = read_c0_pagemask(); + entryhi = read_c0_entryhi(); + entrylo0 = read_c0_entrylo0(); + entrylo1 = read_c0_entrylo1(); + + if (cpu_has_mmid) + mmid = read_c0_memorymapid(); + else + mmid = entryhi & asidmask; + + if (cpu_has_guestid) + guestctl1 = read_c0_guestctl1(); + + /* EHINV bit marks entire entry as invalid */ + if (cpu_has_tlbinv && entryhi & MIPS_ENTRYHI_EHINV) + continue; + /* + * Prior to tlbinv, unused entries have a virtual address of + * CKSEG0. + */ + if ((entryhi & ~0x1ffffUL) == CKSEG0) + continue; + /* + * ASID takes effect in absence of G (global) bit. + * We check both G bits, even though architecturally they should + * match one another, because some revisions of the SB1 core may + * leave only a single G bit set after a machine check exception + * due to duplicate TLB entry. + */ + if (!((entrylo0 | entrylo1) & ENTRYLO_G) && (mmid != asid)) + continue; + + /* + * Only print entries in use + */ + printk("Index: %2d pgmask=%s ", i, msk2str(pagemask)); + + c0 = (entrylo0 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; + c1 = (entrylo1 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; + + pr_cont("va=%0*lx asid=%0*lx", + vwidth, (entryhi & ~0x1fffUL), + asidwidth, mmid); + if (cpu_has_guestid) + pr_cont(" gid=%02lx", + (guestctl1 & MIPS_GCTL1_RID) + >> MIPS_GCTL1_RID_SHIFT); + /* RI/XI are in awkward places, so mask them off separately */ + pa = entrylo0 & ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI); + if (xpa) + pa |= (unsigned long long)readx_c0_entrylo0() << 30; + pa = (pa << 6) & PAGE_MASK; + pr_cont("\n\t["); + if (cpu_has_rixi) + pr_cont("ri=%d xi=%d ", + (entrylo0 & MIPS_ENTRYLO_RI) ? 1 : 0, + (entrylo0 & MIPS_ENTRYLO_XI) ? 1 : 0); + pr_cont("pa=%0*llx c=%d d=%d v=%d g=%d] [", + pwidth, pa, c0, + (entrylo0 & ENTRYLO_D) ? 1 : 0, + (entrylo0 & ENTRYLO_V) ? 1 : 0, + (entrylo0 & ENTRYLO_G) ? 1 : 0); + /* RI/XI are in awkward places, so mask them off separately */ + pa = entrylo1 & ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI); + if (xpa) + pa |= (unsigned long long)readx_c0_entrylo1() << 30; + pa = (pa << 6) & PAGE_MASK; + if (cpu_has_rixi) + pr_cont("ri=%d xi=%d ", + (entrylo1 & MIPS_ENTRYLO_RI) ? 1 : 0, + (entrylo1 & MIPS_ENTRYLO_XI) ? 1 : 0); + pr_cont("pa=%0*llx c=%d d=%d v=%d g=%d]\n", + pwidth, pa, c1, + (entrylo1 & ENTRYLO_D) ? 1 : 0, + (entrylo1 & ENTRYLO_V) ? 1 : 0, + (entrylo1 & ENTRYLO_G) ? 1 : 0); + } + printk("\n"); + + write_c0_entryhi(s_entryhi); + write_c0_index(s_index); + write_c0_pagemask(s_pagemask); + if (cpu_has_guestid) + write_c0_guestctl1(s_guestctl1); +} + +void dump_tlb_all(void) +{ + dump_tlb(0, current_cpu_data.tlbsize - 1); +} diff --git a/arch/mips/lib/iomap-pci.c b/arch/mips/lib/iomap-pci.c new file mode 100644 index 0000000000..a9cb28813f --- /dev/null +++ b/arch/mips/lib/iomap-pci.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implement the default iomap interfaces + * + * (C) Copyright 2004 Linus Torvalds + * (C) Copyright 2006 Ralf Baechle + * (C) Copyright 2007 MIPS Technologies, Inc. + * written by Ralf Baechle + */ +#include +#include +#include + +#ifdef CONFIG_PCI_DRIVERS_LEGACY + +void __iomem *__pci_ioport_map(struct pci_dev *dev, + unsigned long port, unsigned int nr) +{ + struct pci_controller *ctrl = dev->bus->sysdata; + unsigned long base = ctrl->io_map_base; + + /* This will eventually become a BUG_ON but for now be gentle */ + if (unlikely(!ctrl->io_map_base)) { + struct pci_bus *bus = dev->bus; + char name[8]; + + while (bus->parent) + bus = bus->parent; + + ctrl->io_map_base = base = mips_io_port_base; + + sprintf(name, "%04x:%02x", pci_domain_nr(bus), bus->number); + printk(KERN_WARNING "io_map_base of root PCI bus %s unset. " + "Trying to continue but you better\nfix this issue or " + "report it to linux-mips@vger.kernel.org or your " + "vendor.\n", name); +#ifdef CONFIG_PCI_DOMAINS + panic("To avoid data corruption io_map_base MUST be set with " + "multiple PCI domains."); +#endif + } + + return (void __iomem *) (ctrl->io_map_base + port); +} + +#endif /* CONFIG_PCI_DRIVERS_LEGACY */ diff --git a/arch/mips/lib/iomap_copy.c b/arch/mips/lib/iomap_copy.c new file mode 100644 index 0000000000..157500a09a --- /dev/null +++ b/arch/mips/lib/iomap_copy.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +/** + * __ioread64_copy - copy data from MMIO space, in 64-bit units + * @to: destination (must be 64-bit aligned) + * @from: source, in MMIO space (must be 64-bit aligned) + * @count: number of 64-bit quantities to copy + * + * Copy data from MMIO space to kernel space, in units of 32 or 64 bits at a + * time. Order of access is not guaranteed, nor is a memory barrier + * performed afterwards. + */ +void __ioread64_copy(void *to, const void __iomem *from, size_t count) +{ +#ifdef CONFIG_64BIT + u64 *dst = to; + const u64 __iomem *src = from; + const u64 __iomem *end = src + count; + + while (src < end) + *dst++ = __raw_readq(src++); +#else + __ioread32_copy(to, from, count * 2); +#endif +} +EXPORT_SYMBOL_GPL(__ioread64_copy); diff --git a/arch/mips/lib/libgcc.h b/arch/mips/lib/libgcc.h new file mode 100644 index 0000000000..199a7f9628 --- /dev/null +++ b/arch/mips/lib/libgcc.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_LIBGCC_H +#define __ASM_LIBGCC_H + +#include + +typedef int word_type __attribute__ ((mode (__word__))); + +#ifdef __BIG_ENDIAN +struct DWstruct { + int high, low; +}; + +struct TWstruct { + long long high, low; +}; +#elif defined(__LITTLE_ENDIAN) +struct DWstruct { + int low, high; +}; + +struct TWstruct { + long long low, high; +}; +#else +#error I feel sick. +#endif + +typedef union { + struct DWstruct s; + long long ll; +} DWunion; + +#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6) +typedef int ti_type __attribute__((mode(TI))); + +typedef union { + struct TWstruct s; + ti_type ti; +} TWunion; +#endif + +#endif /* __ASM_LIBGCC_H */ diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S new file mode 100644 index 0000000000..a4b4e805ff --- /dev/null +++ b/arch/mips/lib/memcpy.S @@ -0,0 +1,704 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Unified implementation of memcpy, memmove and the __copy_user backend. + * + * Copyright (C) 1998, 99, 2000, 01, 2002 Ralf Baechle (ralf@gnu.org) + * Copyright (C) 1999, 2000, 01, 2002 Silicon Graphics, Inc. + * Copyright (C) 2002 Broadcom, Inc. + * memcpy/copy_user author: Mark Vandevoorde + * Copyright (C) 2007 Maciej W. Rozycki + * Copyright (C) 2014 Imagination Technologies Ltd. + * + * Mnemonic names for arguments to memcpy/__copy_user + */ + +/* + * Hack to resolve longstanding prefetch issue + * + * Prefetching may be fatal on some systems if we're prefetching beyond the + * end of memory on some systems. It's also a seriously bad idea on non + * dma-coherent systems. + */ +#ifdef CONFIG_DMA_NONCOHERENT +#undef CONFIG_CPU_HAS_PREFETCH +#endif +#ifdef CONFIG_MIPS_MALTA +#undef CONFIG_CPU_HAS_PREFETCH +#endif +#ifdef CONFIG_CPU_MIPSR6 +#undef CONFIG_CPU_HAS_PREFETCH +#endif + +#include +#include +#include +#include + +#define dst a0 +#define src a1 +#define len a2 + +/* + * Spec + * + * memcpy copies len bytes from src to dst and sets v0 to dst. + * It assumes that + * - src and dst don't overlap + * - src is readable + * - dst is writable + * memcpy uses the standard calling convention + * + * __copy_user copies up to len bytes from src to dst and sets a2 (len) to + * the number of uncopied bytes due to an exception caused by a read or write. + * __copy_user assumes that src and dst don't overlap, and that the call is + * implementing one of the following: + * copy_to_user + * - src is readable (no exceptions when reading src) + * copy_from_user + * - dst is writable (no exceptions when writing dst) + * __copy_user uses a non-standard calling convention; see + * include/asm-mips/uaccess.h + * + * When an exception happens on a load, the handler must + # ensure that all of the destination buffer is overwritten to prevent + * leaking information to user mode programs. + */ + +/* + * Implementation + */ + +/* + * The exception handler for loads requires that: + * 1- AT contain the address of the byte just past the end of the source + * of the copy, + * 2- src_entry <= src < AT, and + * 3- (dst - src) == (dst_entry - src_entry), + * The _entry suffix denotes values when __copy_user was called. + * + * (1) is set up up by uaccess.h and maintained by not writing AT in copy_user + * (2) is met by incrementing src by the number of bytes copied + * (3) is met by not doing loads between a pair of increments of dst and src + * + * The exception handlers for stores adjust len (if necessary) and return. + * These handlers do not need to overwrite any data. + * + * For __rmemcpy and memmove an exception is always a kernel bug, therefore + * they're not protected. + */ + +/* Instruction type */ +#define LD_INSN 1 +#define ST_INSN 2 +/* Pretech type */ +#define SRC_PREFETCH 1 +#define DST_PREFETCH 2 +#define LEGACY_MODE 1 +#define EVA_MODE 2 +#define USEROP 1 +#define KERNELOP 2 + +/* + * Wrapper to add an entry in the exception table + * in case the insn causes a memory exception. + * Arguments: + * insn : Load/store instruction + * type : Instruction type + * reg : Register + * addr : Address + * handler : Exception handler + */ + +#define EXC(insn, type, reg, addr, handler) \ + .if \mode == LEGACY_MODE; \ +9: insn reg, addr; \ + .section __ex_table,"a"; \ + PTR_WD 9b, handler; \ + .previous; \ + /* This is assembled in EVA mode */ \ + .else; \ + /* If loading from user or storing to user */ \ + .if ((\from == USEROP) && (type == LD_INSN)) || \ + ((\to == USEROP) && (type == ST_INSN)); \ +9: __BUILD_EVA_INSN(insn##e, reg, addr); \ + .section __ex_table,"a"; \ + PTR_WD 9b, handler; \ + .previous; \ + .else; \ + /* \ + * Still in EVA, but no need for \ + * exception handler or EVA insn \ + */ \ + insn reg, addr; \ + .endif; \ + .endif + +/* + * Only on the 64-bit kernel we can made use of 64-bit registers. + */ +#ifdef CONFIG_64BIT +#define USE_DOUBLE +#endif + +#ifdef USE_DOUBLE + +#define LOADK ld /* No exception */ +#define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler) +#define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler) +#define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler) +#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler) +#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler) +#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler) +#define ADD daddu +#define SUB dsubu +#define SRL dsrl +#define SRA dsra +#define SLL dsll +#define SLLV dsllv +#define SRLV dsrlv +#define NBYTES 8 +#define LOG_NBYTES 3 + +/* + * As we are sharing code base with the mips32 tree (which use the o32 ABI + * register definitions). We need to redefine the register definitions from + * the n64 ABI register naming to the o32 ABI register naming. + */ +#undef t0 +#undef t1 +#undef t2 +#undef t3 +#define t0 $8 +#define t1 $9 +#define t2 $10 +#define t3 $11 +#define t4 $12 +#define t5 $13 +#define t6 $14 +#define t7 $15 + +#else + +#define LOADK lw /* No exception */ +#define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler) +#define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler) +#define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler) +#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler) +#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler) +#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler) +#define ADD addu +#define SUB subu +#define SRL srl +#define SLL sll +#define SRA sra +#define SLLV sllv +#define SRLV srlv +#define NBYTES 4 +#define LOG_NBYTES 2 + +#endif /* USE_DOUBLE */ + +#define LOADB(reg, addr, handler) EXC(lb, LD_INSN, reg, addr, handler) +#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler) + +#ifdef CONFIG_CPU_HAS_PREFETCH +# define _PREF(hint, addr, type) \ + .if \mode == LEGACY_MODE; \ + kernel_pref(hint, addr); \ + .else; \ + .if ((\from == USEROP) && (type == SRC_PREFETCH)) || \ + ((\to == USEROP) && (type == DST_PREFETCH)); \ + /* \ + * PREFE has only 9 bits for the offset \ + * compared to PREF which has 16, so it may \ + * need to use the $at register but this \ + * register should remain intact because it's \ + * used later on. Therefore use $v1. \ + */ \ + .set at=v1; \ + user_pref(hint, addr); \ + .set noat; \ + .else; \ + kernel_pref(hint, addr); \ + .endif; \ + .endif +#else +# define _PREF(hint, addr, type) +#endif + +#define PREFS(hint, addr) _PREF(hint, addr, SRC_PREFETCH) +#define PREFD(hint, addr) _PREF(hint, addr, DST_PREFETCH) + +#ifdef CONFIG_CPU_LITTLE_ENDIAN +#define LDFIRST LOADR +#define LDREST LOADL +#define STFIRST STORER +#define STREST STOREL +#define SHIFT_DISCARD SLLV +#else +#define LDFIRST LOADL +#define LDREST LOADR +#define STFIRST STOREL +#define STREST STORER +#define SHIFT_DISCARD SRLV +#endif + +#define FIRST(unit) ((unit)*NBYTES) +#define REST(unit) (FIRST(unit)+NBYTES-1) +#define UNIT(unit) FIRST(unit) + +#define ADDRMASK (NBYTES-1) + + .text + .set noreorder +#ifndef CONFIG_CPU_DADDI_WORKAROUNDS + .set noat +#else + .set at=v1 +#endif + + .align 5 + + /* + * Macro to build the __copy_user common code + * Arguments: + * mode : LEGACY_MODE or EVA_MODE + * from : Source operand. USEROP or KERNELOP + * to : Destination operand. USEROP or KERNELOP + */ + .macro __BUILD_COPY_USER mode, from, to + + /* initialize __memcpy if this the first time we execute this macro */ + .ifnotdef __memcpy + .set __memcpy, 1 + .hidden __memcpy /* make sure it does not leak */ + .endif + + /* + * Note: dst & src may be unaligned, len may be 0 + * Temps + */ +#define rem t8 + + R10KCBARRIER(0(ra)) + /* + * The "issue break"s below are very approximate. + * Issue delays for dcache fills will perturb the schedule, as will + * load queue full replay traps, etc. + * + * If len < NBYTES use byte operations. + */ + PREFS( 0, 0(src) ) + PREFD( 1, 0(dst) ) + sltu t2, len, NBYTES + and t1, dst, ADDRMASK + PREFS( 0, 1*32(src) ) + PREFD( 1, 1*32(dst) ) + bnez t2, .Lcopy_bytes_checklen\@ + and t0, src, ADDRMASK + PREFS( 0, 2*32(src) ) + PREFD( 1, 2*32(dst) ) +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR + bnez t1, .Ldst_unaligned\@ + nop + bnez t0, .Lsrc_unaligned_dst_aligned\@ +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ + or t0, t0, t1 + bnez t0, .Lcopy_unaligned_bytes\@ +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + /* + * use delay slot for fall-through + * src and dst are aligned; need to compute rem + */ +.Lboth_aligned\@: + SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter + beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES + and rem, len, (8*NBYTES-1) # rem = len % (8*NBYTES) + PREFS( 0, 3*32(src) ) + PREFD( 1, 3*32(dst) ) + .align 4 +1: + R10KCBARRIER(0(ra)) + LOAD(t0, UNIT(0)(src), .Ll_exc\@) + LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@) + LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@) + LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@) + SUB len, len, 8*NBYTES + LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@) + LOAD(t7, UNIT(5)(src), .Ll_exc_copy\@) + STORE(t0, UNIT(0)(dst), .Ls_exc_p8u\@) + STORE(t1, UNIT(1)(dst), .Ls_exc_p7u\@) + LOAD(t0, UNIT(6)(src), .Ll_exc_copy\@) + LOAD(t1, UNIT(7)(src), .Ll_exc_copy\@) + ADD src, src, 8*NBYTES + ADD dst, dst, 8*NBYTES + STORE(t2, UNIT(-6)(dst), .Ls_exc_p6u\@) + STORE(t3, UNIT(-5)(dst), .Ls_exc_p5u\@) + STORE(t4, UNIT(-4)(dst), .Ls_exc_p4u\@) + STORE(t7, UNIT(-3)(dst), .Ls_exc_p3u\@) + STORE(t0, UNIT(-2)(dst), .Ls_exc_p2u\@) + STORE(t1, UNIT(-1)(dst), .Ls_exc_p1u\@) + PREFS( 0, 8*32(src) ) + PREFD( 1, 8*32(dst) ) + bne len, rem, 1b + nop + + /* + * len == rem == the number of bytes left to copy < 8*NBYTES + */ +.Lcleanup_both_aligned\@: + beqz len, .Ldone\@ + sltu t0, len, 4*NBYTES + bnez t0, .Lless_than_4units\@ + and rem, len, (NBYTES-1) # rem = len % NBYTES + /* + * len >= 4*NBYTES + */ + LOAD( t0, UNIT(0)(src), .Ll_exc\@) + LOAD( t1, UNIT(1)(src), .Ll_exc_copy\@) + LOAD( t2, UNIT(2)(src), .Ll_exc_copy\@) + LOAD( t3, UNIT(3)(src), .Ll_exc_copy\@) + SUB len, len, 4*NBYTES + ADD src, src, 4*NBYTES + R10KCBARRIER(0(ra)) + STORE(t0, UNIT(0)(dst), .Ls_exc_p4u\@) + STORE(t1, UNIT(1)(dst), .Ls_exc_p3u\@) + STORE(t2, UNIT(2)(dst), .Ls_exc_p2u\@) + STORE(t3, UNIT(3)(dst), .Ls_exc_p1u\@) + .set reorder /* DADDI_WAR */ + ADD dst, dst, 4*NBYTES + beqz len, .Ldone\@ + .set noreorder +.Lless_than_4units\@: + /* + * rem = len % NBYTES + */ + beq rem, len, .Lcopy_bytes\@ + nop +1: + R10KCBARRIER(0(ra)) + LOAD(t0, 0(src), .Ll_exc\@) + ADD src, src, NBYTES + SUB len, len, NBYTES + STORE(t0, 0(dst), .Ls_exc_p1u\@) + .set reorder /* DADDI_WAR */ + ADD dst, dst, NBYTES + bne rem, len, 1b + .set noreorder + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR + /* + * src and dst are aligned, need to copy rem bytes (rem < NBYTES) + * A loop would do only a byte at a time with possible branch + * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE + * because can't assume read-access to dst. Instead, use + * STREST dst, which doesn't require read access to dst. + * + * This code should perform better than a simple loop on modern, + * wide-issue mips processors because the code has fewer branches and + * more instruction-level parallelism. + */ +#define bits t2 + beqz len, .Ldone\@ + ADD t1, dst, len # t1 is just past last byte of dst + li bits, 8*NBYTES + SLL rem, len, 3 # rem = number of bits to keep + LOAD(t0, 0(src), .Ll_exc\@) + SUB bits, bits, rem # bits = number of bits to discard + SHIFT_DISCARD t0, t0, bits + STREST(t0, -1(t1), .Ls_exc\@) + jr ra + move len, zero +.Ldst_unaligned\@: + /* + * dst is unaligned + * t0 = src & ADDRMASK + * t1 = dst & ADDRMASK; T1 > 0 + * len >= NBYTES + * + * Copy enough bytes to align dst + * Set match = (src and dst have same alignment) + */ +#define match rem + LDFIRST(t3, FIRST(0)(src), .Ll_exc\@) + ADD t2, zero, NBYTES + LDREST(t3, REST(0)(src), .Ll_exc_copy\@) + SUB t2, t2, t1 # t2 = number of bytes copied + xor match, t0, t1 + R10KCBARRIER(0(ra)) + STFIRST(t3, FIRST(0)(dst), .Ls_exc\@) + beq len, t2, .Ldone\@ + SUB len, len, t2 + ADD dst, dst, t2 + beqz match, .Lboth_aligned\@ + ADD src, src, t2 + +.Lsrc_unaligned_dst_aligned\@: + SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter + PREFS( 0, 3*32(src) ) + beqz t0, .Lcleanup_src_unaligned\@ + and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES + PREFD( 1, 3*32(dst) ) +1: +/* + * Avoid consecutive LD*'s to the same register since some mips + * implementations can't issue them in the same cycle. + * It's OK to load FIRST(N+1) before REST(N) because the two addresses + * are to the same unit (unless src is aligned, but it's not). + */ + R10KCBARRIER(0(ra)) + LDFIRST(t0, FIRST(0)(src), .Ll_exc\@) + LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@) + SUB len, len, 4*NBYTES + LDREST(t0, REST(0)(src), .Ll_exc_copy\@) + LDREST(t1, REST(1)(src), .Ll_exc_copy\@) + LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@) + LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@) + LDREST(t2, REST(2)(src), .Ll_exc_copy\@) + LDREST(t3, REST(3)(src), .Ll_exc_copy\@) + PREFS( 0, 9*32(src) ) # 0 is PREF_LOAD (not streamed) + ADD src, src, 4*NBYTES +#ifdef CONFIG_CPU_SB1 + nop # improves slotting +#endif + STORE(t0, UNIT(0)(dst), .Ls_exc_p4u\@) + STORE(t1, UNIT(1)(dst), .Ls_exc_p3u\@) + STORE(t2, UNIT(2)(dst), .Ls_exc_p2u\@) + STORE(t3, UNIT(3)(dst), .Ls_exc_p1u\@) + PREFD( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed) + .set reorder /* DADDI_WAR */ + ADD dst, dst, 4*NBYTES + bne len, rem, 1b + .set noreorder + +.Lcleanup_src_unaligned\@: + beqz len, .Ldone\@ + and rem, len, NBYTES-1 # rem = len % NBYTES + beq rem, len, .Lcopy_bytes\@ + nop +1: + R10KCBARRIER(0(ra)) + LDFIRST(t0, FIRST(0)(src), .Ll_exc\@) + LDREST(t0, REST(0)(src), .Ll_exc_copy\@) + ADD src, src, NBYTES + SUB len, len, NBYTES + STORE(t0, 0(dst), .Ls_exc_p1u\@) + .set reorder /* DADDI_WAR */ + ADD dst, dst, NBYTES + bne len, rem, 1b + .set noreorder + +#endif /* !CONFIG_CPU_NO_LOAD_STORE_LR */ +.Lcopy_bytes_checklen\@: + beqz len, .Ldone\@ + nop +.Lcopy_bytes\@: + /* 0 < len < NBYTES */ + R10KCBARRIER(0(ra)) +#define COPY_BYTE(N) \ + LOADB(t0, N(src), .Ll_exc\@); \ + SUB len, len, 1; \ + beqz len, .Ldone\@; \ + STOREB(t0, N(dst), .Ls_exc_p1\@) + + COPY_BYTE(0) + COPY_BYTE(1) +#ifdef USE_DOUBLE + COPY_BYTE(2) + COPY_BYTE(3) + COPY_BYTE(4) + COPY_BYTE(5) +#endif + LOADB(t0, NBYTES-2(src), .Ll_exc\@) + SUB len, len, 1 + jr ra + STOREB(t0, NBYTES-2(dst), .Ls_exc_p1\@) +.Ldone\@: + jr ra + nop + +#ifdef CONFIG_CPU_NO_LOAD_STORE_LR +.Lcopy_unaligned_bytes\@: +1: + COPY_BYTE(0) + COPY_BYTE(1) + COPY_BYTE(2) + COPY_BYTE(3) + COPY_BYTE(4) + COPY_BYTE(5) + COPY_BYTE(6) + COPY_BYTE(7) + ADD src, src, 8 + b 1b + ADD dst, dst, 8 +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + .if __memcpy == 1 + END(memcpy) + .set __memcpy, 0 + .hidden __memcpy + .endif + +.Ll_exc_copy\@: + /* + * Copy bytes from src until faulting load address (or until a + * lb faults) + * + * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28) + * may be more than a byte beyond the last address. + * Hence, the lb below may get an exception. + * + * Assumes src < THREAD_BUADDR($28) + */ + LOADK t0, TI_TASK($28) + nop + LOADK t0, THREAD_BUADDR(t0) +1: + LOADB(t1, 0(src), .Ll_exc\@) + ADD src, src, 1 + sb t1, 0(dst) # can't fault -- we're copy_from_user + .set reorder /* DADDI_WAR */ + ADD dst, dst, 1 + bne src, t0, 1b + .set noreorder +.Ll_exc\@: + LOADK t0, TI_TASK($28) + nop + LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address + nop + SUB len, AT, t0 # len number of uncopied bytes + jr ra + nop + +#define SEXC(n) \ + .set reorder; /* DADDI_WAR */ \ +.Ls_exc_p ## n ## u\@: \ + ADD len, len, n*NBYTES; \ + jr ra; \ + .set noreorder + +SEXC(8) +SEXC(7) +SEXC(6) +SEXC(5) +SEXC(4) +SEXC(3) +SEXC(2) +SEXC(1) + +.Ls_exc_p1\@: + .set reorder /* DADDI_WAR */ + ADD len, len, 1 + jr ra + .set noreorder +.Ls_exc\@: + jr ra + nop + .endm + +#ifndef CONFIG_HAVE_PLAT_MEMCPY + .align 5 +LEAF(memmove) +EXPORT_SYMBOL(memmove) + ADD t0, a0, a2 + ADD t1, a1, a2 + sltu t0, a1, t0 # dst + len <= src -> memcpy + sltu t1, a0, t1 # dst >= src + len -> memcpy + and t0, t1 + beqz t0, .L__memcpy + move v0, a0 /* return value */ + beqz a2, .Lr_out + END(memmove) + + /* fall through to __rmemcpy */ +LEAF(__rmemcpy) /* a0=dst a1=src a2=len */ + sltu t0, a1, a0 + beqz t0, .Lr_end_bytes_up # src >= dst + nop + ADD a0, a2 # dst = dst + len + ADD a1, a2 # src = src + len + +.Lr_end_bytes: + R10KCBARRIER(0(ra)) + lb t0, -1(a1) + SUB a2, a2, 0x1 + sb t0, -1(a0) + SUB a1, a1, 0x1 + .set reorder /* DADDI_WAR */ + SUB a0, a0, 0x1 + bnez a2, .Lr_end_bytes + .set noreorder + +.Lr_out: + jr ra + move a2, zero + +.Lr_end_bytes_up: + R10KCBARRIER(0(ra)) + lb t0, (a1) + SUB a2, a2, 0x1 + sb t0, (a0) + ADD a1, a1, 0x1 + .set reorder /* DADDI_WAR */ + ADD a0, a0, 0x1 + bnez a2, .Lr_end_bytes_up + .set noreorder + + jr ra + move a2, zero + END(__rmemcpy) + +/* + * A combined memcpy/__copy_user + * __copy_user sets len to 0 for success; else to an upper bound of + * the number of uncopied bytes. + * memcpy sets v0 to dst. + */ + .align 5 +LEAF(memcpy) /* a0=dst a1=src a2=len */ +EXPORT_SYMBOL(memcpy) + move v0, dst /* return value */ +.L__memcpy: +#ifndef CONFIG_EVA +FEXPORT(__raw_copy_from_user) +EXPORT_SYMBOL(__raw_copy_from_user) +FEXPORT(__raw_copy_to_user) +EXPORT_SYMBOL(__raw_copy_to_user) +#endif + /* Legacy Mode, user <-> user */ + __BUILD_COPY_USER LEGACY_MODE USEROP USEROP + +#endif + +#ifdef CONFIG_EVA + +/* + * For EVA we need distinct symbols for reading and writing to user space. + * This is because we need to use specific EVA instructions to perform the + * virtual <-> physical translation when a virtual address is actually in user + * space + */ + +/* + * __copy_from_user (EVA) + */ + +LEAF(__raw_copy_from_user) +EXPORT_SYMBOL(__raw_copy_from_user) + __BUILD_COPY_USER EVA_MODE USEROP KERNELOP +END(__raw_copy_from_user) + + + +/* + * __copy_to_user (EVA) + */ + +LEAF(__raw_copy_to_user) +EXPORT_SYMBOL(__raw_copy_to_user) +__BUILD_COPY_USER EVA_MODE KERNELOP USEROP +END(__raw_copy_to_user) + +#endif diff --git a/arch/mips/lib/memset.S b/arch/mips/lib/memset.S new file mode 100644 index 0000000000..79405c32cc --- /dev/null +++ b/arch/mips/lib/memset.S @@ -0,0 +1,325 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1998, 1999, 2000 by Ralf Baechle + * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2007 by Maciej W. Rozycki + * Copyright (C) 2011, 2012 MIPS Technologies, Inc. + */ +#include +#include +#include +#include + +#if LONGSIZE == 4 +#define LONG_S_L swl +#define LONG_S_R swr +#else +#define LONG_S_L sdl +#define LONG_S_R sdr +#endif + +#ifdef CONFIG_CPU_MICROMIPS +#define STORSIZE (LONGSIZE * 2) +#define STORMASK (STORSIZE - 1) +#define FILL64RG t8 +#define FILLPTRG t7 +#undef LONG_S +#define LONG_S LONG_SP +#else +#define STORSIZE LONGSIZE +#define STORMASK LONGMASK +#define FILL64RG a1 +#define FILLPTRG t0 +#endif + +#define LEGACY_MODE 1 +#define EVA_MODE 2 + +/* + * No need to protect it with EVA #ifdefery. The generated block of code + * will never be assembled if EVA is not enabled. + */ +#define __EVAFY(insn, reg, addr) __BUILD_EVA_INSN(insn##e, reg, addr) +#define ___BUILD_EVA_INSN(insn, reg, addr) __EVAFY(insn, reg, addr) + +#define EX(insn,reg,addr,handler) \ + .if \mode == LEGACY_MODE; \ +9: insn reg, addr; \ + .else; \ +9: ___BUILD_EVA_INSN(insn, reg, addr); \ + .endif; \ + .section __ex_table,"a"; \ + PTR_WD 9b, handler; \ + .previous + + .macro f_fill64 dst, offset, val, fixup, mode + EX(LONG_S, \val, (\offset + 0 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 1 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 2 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 3 * STORSIZE)(\dst), \fixup) +#if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) || !defined(CONFIG_CPU_MICROMIPS)) + EX(LONG_S, \val, (\offset + 4 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 5 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 6 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 7 * STORSIZE)(\dst), \fixup) +#endif +#if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) + EX(LONG_S, \val, (\offset + 8 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 9 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 10 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 11 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 12 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 13 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 14 * STORSIZE)(\dst), \fixup) + EX(LONG_S, \val, (\offset + 15 * STORSIZE)(\dst), \fixup) +#endif + .endm + + .align 5 + + /* + * Macro to generate the __bzero{,_user} symbol + * Arguments: + * mode: LEGACY_MODE or EVA_MODE + */ + .macro __BUILD_BZERO mode + /* Initialize __memset if this is the first time we call this macro */ + .ifnotdef __memset + .set __memset, 1 + .hidden __memset /* Make sure it does not leak */ + .endif + + sltiu t0, a2, STORSIZE /* very small region? */ + .set noreorder + bnez t0, .Lsmall_memset\@ + andi t0, a0, STORMASK /* aligned? */ + .set reorder + +#ifdef CONFIG_CPU_MICROMIPS + move t8, a1 /* used by 'swp' instruction */ + move t9, a1 +#endif + .set noreorder +#ifndef CONFIG_CPU_DADDI_WORKAROUNDS + beqz t0, 1f + PTR_SUBU t0, STORSIZE /* alignment in bytes */ +#else + .set noat + li AT, STORSIZE + beqz t0, 1f + PTR_SUBU t0, AT /* alignment in bytes */ + .set at +#endif + .set reorder + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR + R10KCBARRIER(0(ra)) +#ifdef __MIPSEB__ + EX(LONG_S_L, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */ +#else + EX(LONG_S_R, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */ +#endif + PTR_SUBU a0, t0 /* long align ptr */ + PTR_ADDU a2, t0 /* correct size */ + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +#define STORE_BYTE(N) \ + EX(sb, a1, N(a0), .Lbyte_fixup\@); \ + .set noreorder; \ + beqz t0, 0f; \ + PTR_ADDU t0, 1; \ + .set reorder; + + PTR_ADDU a2, t0 /* correct size */ + PTR_ADDU t0, 1 + STORE_BYTE(0) + STORE_BYTE(1) +#if LONGSIZE == 4 + EX(sb, a1, 2(a0), .Lbyte_fixup\@) +#else + STORE_BYTE(2) + STORE_BYTE(3) + STORE_BYTE(4) + STORE_BYTE(5) + EX(sb, a1, 6(a0), .Lbyte_fixup\@) +#endif +0: + ori a0, STORMASK + xori a0, STORMASK + PTR_ADDIU a0, STORSIZE +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ +1: ori t1, a2, 0x3f /* # of full blocks */ + xori t1, 0x3f + andi t0, a2, 0x40-STORSIZE + beqz t1, .Lmemset_partial\@ /* no block to fill */ + + PTR_ADDU t1, a0 /* end address */ +1: PTR_ADDIU a0, 64 + R10KCBARRIER(0(ra)) + f_fill64 a0, -64, FILL64RG, .Lfwd_fixup\@, \mode + bne t1, a0, 1b + +.Lmemset_partial\@: + R10KCBARRIER(0(ra)) + PTR_LA t1, 2f /* where to start */ +#ifdef CONFIG_CPU_MICROMIPS + LONG_SRL t7, t0, 1 +#endif +#if LONGSIZE == 4 + PTR_SUBU t1, FILLPTRG +#else + .set noat + LONG_SRL AT, FILLPTRG, 1 + PTR_SUBU t1, AT + .set at +#endif + PTR_ADDU a0, t0 /* dest ptr */ + jr t1 + + /* ... but first do longs ... */ + f_fill64 a0, -64, FILL64RG, .Lpartial_fixup\@, \mode +2: andi a2, STORMASK /* At most one long to go */ + + .set noreorder + beqz a2, 1f +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR + PTR_ADDU a0, a2 /* What's left */ + .set reorder + R10KCBARRIER(0(ra)) +#ifdef __MIPSEB__ + EX(LONG_S_R, a1, -1(a0), .Llast_fixup\@) +#else + EX(LONG_S_L, a1, -1(a0), .Llast_fixup\@) +#endif +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ + PTR_SUBU t0, $0, a2 + .set reorder + move a2, zero /* No remaining longs */ + PTR_ADDIU t0, 1 + STORE_BYTE(0) + STORE_BYTE(1) +#if LONGSIZE == 4 + EX(sb, a1, 2(a0), .Lbyte_fixup\@) +#else + STORE_BYTE(2) + STORE_BYTE(3) + STORE_BYTE(4) + STORE_BYTE(5) + EX(sb, a1, 6(a0), .Lbyte_fixup\@) +#endif +0: +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ +1: move a2, zero + jr ra + +.Lsmall_memset\@: + PTR_ADDU t1, a0, a2 + beqz a2, 2f + +1: PTR_ADDIU a0, 1 /* fill bytewise */ + R10KCBARRIER(0(ra)) + .set noreorder + bne t1, a0, 1b + EX(sb, a1, -1(a0), .Lsmall_fixup\@) + .set reorder + +2: move a2, zero + jr ra /* done */ + .if __memset == 1 + END(memset) + .set __memset, 0 + .hidden __memset + .endif + +#ifdef CONFIG_CPU_NO_LOAD_STORE_LR +.Lbyte_fixup\@: + /* + * unset_bytes = (#bytes - (#unaligned bytes)) - (-#unaligned bytes remaining + 1) + 1 + * a2 = a2 - t0 + 1 + */ + PTR_SUBU a2, t0 + PTR_ADDIU a2, 1 + jr ra +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + +.Lfirst_fixup\@: + /* unset_bytes already in a2 */ + jr ra + +.Lfwd_fixup\@: + /* + * unset_bytes = partial_start_addr + #bytes - fault_addr + * a2 = t1 + (a2 & 3f) - $28->task->BUADDR + */ + PTR_L t0, TI_TASK($28) + andi a2, 0x3f + LONG_L t0, THREAD_BUADDR(t0) + LONG_ADDU a2, t1 + LONG_SUBU a2, t0 + jr ra + +.Lpartial_fixup\@: + /* + * unset_bytes = partial_end_addr + #bytes - fault_addr + * a2 = a0 + (a2 & STORMASK) - $28->task->BUADDR + */ + PTR_L t0, TI_TASK($28) + andi a2, STORMASK + LONG_L t0, THREAD_BUADDR(t0) + LONG_ADDU a2, a0 + LONG_SUBU a2, t0 + jr ra + +.Llast_fixup\@: + /* unset_bytes already in a2 */ + jr ra + +.Lsmall_fixup\@: + /* + * unset_bytes = end_addr - current_addr + 1 + * a2 = t1 - a0 + 1 + */ + PTR_SUBU a2, t1, a0 + PTR_ADDIU a2, 1 + jr ra + + .endm + +/* + * memset(void *s, int c, size_t n) + * + * a0: start of area to clear + * a1: char to fill with + * a2: size of area to clear + */ + +LEAF(memset) +EXPORT_SYMBOL(memset) + move v0, a0 /* result */ + beqz a1, 1f + + andi a1, 0xff /* spread fillword */ + LONG_SLL t1, a1, 8 + or a1, t1 + LONG_SLL t1, a1, 16 +#if LONGSIZE == 8 + or a1, t1 + LONG_SLL t1, a1, 32 +#endif + or a1, t1 +1: +#ifndef CONFIG_EVA +FEXPORT(__bzero) +EXPORT_SYMBOL(__bzero) +#endif + __BUILD_BZERO LEGACY_MODE + +#ifdef CONFIG_EVA +LEAF(__bzero) +EXPORT_SYMBOL(__bzero) + __BUILD_BZERO EVA_MODE +END(__bzero) +#endif diff --git a/arch/mips/lib/mips-atomic.c b/arch/mips/lib/mips-atomic.c new file mode 100644 index 0000000000..a9b72eacfc --- /dev/null +++ b/arch/mips/lib/mips-atomic.c @@ -0,0 +1,113 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994, 95, 96, 97, 98, 99, 2003 by Ralf Baechle + * Copyright (C) 1996 by Paul M. Antoine + * Copyright (C) 1999 Silicon Graphics + * Copyright (C) 2000 MIPS Technologies, Inc. + */ +#include +#include +#include +#include +#include +#include + +#if !defined(CONFIG_CPU_HAS_DIEI) + +/* + * For cli() we have to insert nops to make sure that the new value + * has actually arrived in the status register before the end of this + * macro. + * R4000/R4400 need three nops, the R4600 two nops and the R10000 needs + * no nops at all. + */ +/* + * For TX49, operating only IE bit is not enough. + * + * If mfc0 $12 follows store and the mfc0 is last instruction of a + * page and fetching the next instruction causes TLB miss, the result + * of the mfc0 might wrongly contain EXL bit. + * + * ERT-TX49H2-027, ERT-TX49H3-012, ERT-TX49HL3-006, ERT-TX49H4-008 + * + * Workaround: mask EXL bit of the result or place a nop before mfc0. + */ +notrace void arch_local_irq_disable(void) +{ + preempt_disable_notrace(); + + __asm__ __volatile__( + " .set push \n" + " .set noat \n" + " mfc0 $1,$12 \n" + " ori $1,0x1f \n" + " xori $1,0x1f \n" + " .set noreorder \n" + " mtc0 $1,$12 \n" + " " __stringify(__irq_disable_hazard) " \n" + " .set pop \n" + : /* no outputs */ + : /* no inputs */ + : "memory"); + + preempt_enable_notrace(); +} +EXPORT_SYMBOL(arch_local_irq_disable); + +notrace unsigned long arch_local_irq_save(void) +{ + unsigned long flags; + + preempt_disable_notrace(); + + __asm__ __volatile__( + " .set push \n" + " .set reorder \n" + " .set noat \n" + " mfc0 %[flags], $12 \n" + " ori $1, %[flags], 0x1f \n" + " xori $1, 0x1f \n" + " .set noreorder \n" + " mtc0 $1, $12 \n" + " " __stringify(__irq_disable_hazard) " \n" + " .set pop \n" + : [flags] "=r" (flags) + : /* no inputs */ + : "memory"); + + preempt_enable_notrace(); + + return flags; +} +EXPORT_SYMBOL(arch_local_irq_save); + +notrace void arch_local_irq_restore(unsigned long flags) +{ + unsigned long __tmp1; + + preempt_disable_notrace(); + + __asm__ __volatile__( + " .set push \n" + " .set noreorder \n" + " .set noat \n" + " mfc0 $1, $12 \n" + " andi %[flags], 1 \n" + " ori $1, 0x1f \n" + " xori $1, 0x1f \n" + " or %[flags], $1 \n" + " mtc0 %[flags], $12 \n" + " " __stringify(__irq_disable_hazard) " \n" + " .set pop \n" + : [flags] "=r" (__tmp1) + : "0" (flags) + : "memory"); + + preempt_enable_notrace(); +} +EXPORT_SYMBOL(arch_local_irq_restore); + +#endif /* !CONFIG_CPU_HAS_DIEI */ diff --git a/arch/mips/lib/multi3.c b/arch/mips/lib/multi3.c new file mode 100644 index 0000000000..4c2483f410 --- /dev/null +++ b/arch/mips/lib/multi3.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include "libgcc.h" + +/* + * GCC 7 & older can suboptimally generate __multi3 calls for mips64r6, so for + * that specific case only we implement that intrinsic here. + * + * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82981 + */ +#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6) && (__GNUC__ < 8) + +/* multiply 64-bit values, low 64-bits returned */ +static inline long long notrace dmulu(long long a, long long b) +{ + long long res; + + asm ("dmulu %0,%1,%2" : "=r" (res) : "r" (a), "r" (b)); + return res; +} + +/* multiply 64-bit unsigned values, high 64-bits of 128-bit result returned */ +static inline long long notrace dmuhu(long long a, long long b) +{ + long long res; + + asm ("dmuhu %0,%1,%2" : "=r" (res) : "r" (a), "r" (b)); + return res; +} + +/* multiply 128-bit values, low 128-bits returned */ +ti_type notrace __multi3(ti_type a, ti_type b) +{ + TWunion res, aa, bb; + + aa.ti = a; + bb.ti = b; + + /* + * a * b = (a.lo * b.lo) + * + 2^64 * (a.hi * b.lo + a.lo * b.hi) + * [+ 2^128 * (a.hi * b.hi)] + */ + res.s.low = dmulu(aa.s.low, bb.s.low); + res.s.high = dmuhu(aa.s.low, bb.s.low); + res.s.high += dmulu(aa.s.high, bb.s.low); + res.s.high += dmulu(aa.s.low, bb.s.high); + + return res.ti; +} +EXPORT_SYMBOL(__multi3); + +#endif /* 64BIT && CPU_MIPSR6 && GCC7 */ diff --git a/arch/mips/lib/r3k_dump_tlb.c b/arch/mips/lib/r3k_dump_tlb.c new file mode 100644 index 0000000000..fcf594af00 --- /dev/null +++ b/arch/mips/lib/r3k_dump_tlb.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Dump R3000 TLB for debugging purposes. + * + * Copyright (C) 1994, 1995 by Waldorf Electronics, written by Ralf Baechle. + * Copyright (C) 1999 by Silicon Graphics, Inc. + * Copyright (C) 1999 by Harald Koerfgen + */ +#include +#include + +#include +#include +#include +#include + +void dump_tlb_regs(void) +{ + pr_info("Index : %0x\n", read_c0_index()); + pr_info("EntryHi : %0lx\n", read_c0_entryhi()); + pr_info("EntryLo : %0lx\n", read_c0_entrylo0()); +} + +static void dump_tlb(int first, int last) +{ + int i; + unsigned int asid; + unsigned long entryhi, entrylo0, asid_mask; + + asid_mask = cpu_asid_mask(¤t_cpu_data); + asid = read_c0_entryhi() & asid_mask; + + for (i = first; i <= last; i++) { + write_c0_index(i<<8); + __asm__ __volatile__( + ".set\tnoreorder\n\t" + "tlbr\n\t" + "nop\n\t" + ".set\treorder"); + entryhi = read_c0_entryhi(); + entrylo0 = read_c0_entrylo0(); + + /* Unused entries have a virtual address of KSEG0. */ + if ((entryhi & PAGE_MASK) != KSEG0 && + (entrylo0 & R3K_ENTRYLO_G || + (entryhi & asid_mask) == asid)) { + /* + * Only print entries in use + */ + printk("Index: %2d ", i); + + pr_cont("va=%08lx asid=%08lx" + " [pa=%06lx n=%d d=%d v=%d g=%d]", + entryhi & PAGE_MASK, + entryhi & asid_mask, + entrylo0 & PAGE_MASK, + (entrylo0 & R3K_ENTRYLO_N) ? 1 : 0, + (entrylo0 & R3K_ENTRYLO_D) ? 1 : 0, + (entrylo0 & R3K_ENTRYLO_V) ? 1 : 0, + (entrylo0 & R3K_ENTRYLO_G) ? 1 : 0); + } + } + printk("\n"); + + write_c0_entryhi(asid); +} + +void dump_tlb_all(void) +{ + dump_tlb(0, current_cpu_data.tlbsize - 1); +} diff --git a/arch/mips/lib/strncpy_user.S b/arch/mips/lib/strncpy_user.S new file mode 100644 index 0000000000..94f4203563 --- /dev/null +++ b/arch/mips/lib/strncpy_user.S @@ -0,0 +1,65 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1996, 1999 by Ralf Baechle + * Copyright (C) 2011 MIPS Technologies, Inc. + */ +#include +#include +#include +#include +#include + +#define EX(insn,reg,addr,handler) \ +9: insn reg, addr; \ + .section __ex_table,"a"; \ + PTR_WD 9b, handler; \ + .previous + +/* + * Returns: -EFAULT if exception before terminator, N if the entire + * buffer filled, else strlen. + */ + +/* + * Ugly special case have to check: we might get passed a user space + * pointer which wraps into the kernel space. We don't deal with that. If + * it happens at most some bytes of the exceptions handlers will be copied. + */ + +LEAF(__strncpy_from_user_asm) + move t0, zero + move v1, a1 +#ifdef CONFIG_EVA + .set push + .set eva +1: EX(lbue, v0, (v1), .Lfault) + .set pop +#else +1: EX(lbu, v0, (v1), .Lfault) +#endif + PTR_ADDIU v1, 1 + R10KCBARRIER(0(ra)) + sb v0, (a0) + beqz v0, 2f + PTR_ADDIU t0, 1 + PTR_ADDIU a0, 1 + bne t0, a2, 1b +2: PTR_ADDU v0, a1, t0 + xor v0, a1 + bltz v0, .Lfault + move v0, t0 + jr ra # return n + END(__strncpy_from_user_asm) + +.Lfault: + li v0, -EFAULT + jr ra + + .section __ex_table,"a" + PTR_WD 1b, .Lfault + .previous + + EXPORT_SYMBOL(__strncpy_from_user_asm) diff --git a/arch/mips/lib/strnlen_user.S b/arch/mips/lib/strnlen_user.S new file mode 100644 index 0000000000..c192a6f6cd --- /dev/null +++ b/arch/mips/lib/strnlen_user.S @@ -0,0 +1,64 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (c) 1996, 1998, 1999, 2004 by Ralf Baechle + * Copyright (c) 1999 Silicon Graphics, Inc. + */ +#include +#include +#include +#include + +#define EX(insn,reg,addr,handler) \ +9: insn reg, addr; \ + .section __ex_table,"a"; \ + PTR_WD 9b, handler; \ + .previous + +/* + * Return the size of a string including the ending NUL character up to a + * maximum of a1 or 0 in case of error. + * + * Note: for performance reasons we deliberately accept that a user may + * make strlen_user and strnlen_user access the first few KSEG0 + * bytes. There's nothing secret there. On 64-bit accessing beyond + * the maximum is a tad hairier ... + */ +LEAF(__strnlen_user_asm) + move v0, a0 + PTR_ADDU a1, a0 # stop pointer +1: +#ifdef CONFIG_CPU_DADDI_WORKAROUNDS + .set noat + li AT, 1 +#endif + beq v0, a1, 1f # limit reached? +#ifdef CONFIG_EVA + .set push + .set eva + EX(lbe, t0, (v0), .Lfault) + .set pop +#else + EX(lb, t0, (v0), .Lfault) +#endif + .set noreorder + bnez t0, 1b +1: +#ifndef CONFIG_CPU_DADDI_WORKAROUNDS + PTR_ADDIU v0, 1 +#else + PTR_ADDU v0, AT + .set at +#endif + .set reorder + PTR_SUBU v0, a0 + jr ra + END(__strnlen_user_asm) + +.Lfault: + move v0, zero + jr ra + + EXPORT_SYMBOL(__strnlen_user_asm) diff --git a/arch/mips/lib/uncached.c b/arch/mips/lib/uncached.c new file mode 100644 index 0000000000..f80a67c092 --- /dev/null +++ b/arch/mips/lib/uncached.c @@ -0,0 +1,82 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2005 Thiemo Seufer + * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. + * Author: Maciej W. Rozycki + */ + + +#include +#include +#include + +#ifndef CKSEG2 +#define CKSEG2 CKSSEG +#endif +#ifndef TO_PHYS_MASK +#define TO_PHYS_MASK -1 +#endif + +/* + * FUNC is executed in one of the uncached segments, depending on its + * original address as follows: + * + * 1. If the original address is in CKSEG0 or CKSEG1, then the uncached + * segment used is CKSEG1. + * 2. If the original address is in XKPHYS, then the uncached segment + * used is XKPHYS(2). + * 3. Otherwise it's a bug. + * + * The same remapping is done with the stack pointer. Stack handling + * works because we don't handle stack arguments or more complex return + * values, so we can avoid sharing the same stack area between a cached + * and the uncached mode. + */ +unsigned long run_uncached(void *func) +{ + register long ret __asm__("$2"); + long lfunc = (long)func, ufunc; + long usp; + long sp; + + __asm__("move %0, $sp" : "=r" (sp)); + + if (sp >= (long)CKSEG0 && sp < (long)CKSEG2) + usp = CKSEG1ADDR(sp); +#ifdef CONFIG_64BIT + else if ((long long)sp >= (long long)PHYS_TO_XKPHYS(0, 0) && + (long long)sp < (long long)PHYS_TO_XKPHYS(8, 0)) + usp = PHYS_TO_XKPHYS(K_CALG_UNCACHED, + XKPHYS_TO_PHYS((long long)sp)); +#endif + else { + BUG(); + usp = sp; + } + if (lfunc >= (long)CKSEG0 && lfunc < (long)CKSEG2) + ufunc = CKSEG1ADDR(lfunc); +#ifdef CONFIG_64BIT + else if ((long long)lfunc >= (long long)PHYS_TO_XKPHYS(0, 0) && + (long long)lfunc < (long long)PHYS_TO_XKPHYS(8, 0)) + ufunc = PHYS_TO_XKPHYS(K_CALG_UNCACHED, + XKPHYS_TO_PHYS((long long)lfunc)); +#endif + else { + BUG(); + ufunc = lfunc; + } + + __asm__ __volatile__ ( + " move $16, $sp\n" + " move $sp, %1\n" + " jalr %2\n" + " move $sp, $16" + : "=r" (ret) + : "r" (usp), "r" (ufunc) + : "$16", "$31"); + + return ret; +} -- cgit v1.2.3