diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 10:05:51 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 10:05:51 +0000 |
| commit | 5d1646d90e1f2cceb9f0828f4b28318cd0ec7744 (patch) | |
| tree | a94efe259b9009378be6d90eb30d2b019d95c194 /arch/riscv/include/asm | |
| parent | Initial commit. (diff) | |
| download | linux-5d1646d90e1f2cceb9f0828f4b28318cd0ec7744.tar.xz linux-5d1646d90e1f2cceb9f0828f4b28318cd0ec7744.zip | |
Adding upstream version 5.10.209.upstream/5.10.209upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'arch/riscv/include/asm')
84 files changed, 6345 insertions, 0 deletions
diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild new file mode 100644 index 000000000..445ccc973 --- /dev/null +++ b/arch/riscv/include/asm/Kbuild @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +generic-y += early_ioremap.h +generic-y += extable.h +generic-y += flat.h +generic-y += kvm_para.h +generic-y += user.h +generic-y += vmlinux.lds.h diff --git a/arch/riscv/include/asm/asm-offsets.h b/arch/riscv/include/asm/asm-offsets.h new file mode 100644 index 000000000..d370ee36a --- /dev/null +++ b/arch/riscv/include/asm/asm-offsets.h @@ -0,0 +1 @@ +#include <generated/asm-offsets.h> diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h new file mode 100644 index 000000000..27e005fca --- /dev/null +++ b/arch/riscv/include/asm/asm-prototypes.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_PROTOTYPES_H +#define _ASM_RISCV_PROTOTYPES_H + +#include <linux/ftrace.h> +#include <asm-generic/asm-prototypes.h> + +long long __lshrti3(long long a, int b); +long long __ashrti3(long long a, int b); +long long __ashlti3(long long a, int b); + +#endif /* _ASM_RISCV_PROTOTYPES_H */ diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h new file mode 100644 index 000000000..9c992a88d --- /dev/null +++ b/arch/riscv/include/asm/asm.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2015 Regents of the University of California + */ + +#ifndef _ASM_RISCV_ASM_H +#define _ASM_RISCV_ASM_H + +#ifdef __ASSEMBLY__ +#define __ASM_STR(x) x +#else +#define __ASM_STR(x) #x +#endif + +#if __riscv_xlen == 64 +#define __REG_SEL(a, b) __ASM_STR(a) +#elif __riscv_xlen == 32 +#define __REG_SEL(a, b) __ASM_STR(b) +#else +#error "Unexpected __riscv_xlen" +#endif + +#define REG_L __REG_SEL(ld, lw) +#define REG_S __REG_SEL(sd, sw) +#define REG_SC __REG_SEL(sc.d, sc.w) +#define SZREG __REG_SEL(8, 4) +#define LGREG __REG_SEL(3, 2) + +#if __SIZEOF_POINTER__ == 8 +#ifdef __ASSEMBLY__ +#define RISCV_PTR .dword +#define RISCV_SZPTR 8 +#define RISCV_LGPTR 3 +#else +#define RISCV_PTR ".dword" +#define RISCV_SZPTR "8" +#define RISCV_LGPTR "3" +#endif +#elif __SIZEOF_POINTER__ == 4 +#ifdef __ASSEMBLY__ +#define RISCV_PTR .word +#define RISCV_SZPTR 4 +#define RISCV_LGPTR 2 +#else +#define RISCV_PTR ".word" +#define RISCV_SZPTR "4" +#define RISCV_LGPTR "2" +#endif +#else +#error "Unexpected __SIZEOF_POINTER__" +#endif + +#if (__SIZEOF_INT__ == 4) +#define RISCV_INT __ASM_STR(.word) +#define RISCV_SZINT __ASM_STR(4) +#define RISCV_LGINT __ASM_STR(2) +#else +#error "Unexpected __SIZEOF_INT__" +#endif + +#if (__SIZEOF_SHORT__ == 2) +#define RISCV_SHORT __ASM_STR(.half) +#define RISCV_SZSHORT __ASM_STR(2) +#define RISCV_LGSHORT __ASM_STR(1) +#else +#error "Unexpected __SIZEOF_SHORT__" +#endif + +#endif /* _ASM_RISCV_ASM_H */ diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h new file mode 100644 index 000000000..400a8c8b6 --- /dev/null +++ b/arch/riscv/include/asm/atomic.h @@ -0,0 +1,356 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#ifndef _ASM_RISCV_ATOMIC_H +#define _ASM_RISCV_ATOMIC_H + +#ifdef CONFIG_GENERIC_ATOMIC64 +# include <asm-generic/atomic64.h> +#else +# if (__riscv_xlen < 64) +# error "64-bit atomics require XLEN to be at least 64" +# endif +#endif + +#include <asm/cmpxchg.h> +#include <asm/barrier.h> + +#define __atomic_acquire_fence() \ + __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory") + +#define __atomic_release_fence() \ + __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); + +static __always_inline int atomic_read(const atomic_t *v) +{ + return READ_ONCE(v->counter); +} +static __always_inline void atomic_set(atomic_t *v, int i) +{ + WRITE_ONCE(v->counter, i); +} + +#ifndef CONFIG_GENERIC_ATOMIC64 +#define ATOMIC64_INIT(i) { (i) } +static __always_inline s64 atomic64_read(const atomic64_t *v) +{ + return READ_ONCE(v->counter); +} +static __always_inline void atomic64_set(atomic64_t *v, s64 i) +{ + WRITE_ONCE(v->counter, i); +} +#endif + +/* + * First, the atomic ops that have no ordering constraints and therefor don't + * have the AQ or RL bits set. These don't return anything, so there's only + * one version to worry about. + */ +#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \ +static __always_inline \ +void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ +{ \ + __asm__ __volatile__ ( \ + " amo" #asm_op "." #asm_type " zero, %1, %0" \ + : "+A" (v->counter) \ + : "r" (I) \ + : "memory"); \ +} \ + +#ifdef CONFIG_GENERIC_ATOMIC64 +#define ATOMIC_OPS(op, asm_op, I) \ + ATOMIC_OP (op, asm_op, I, w, int, ) +#else +#define ATOMIC_OPS(op, asm_op, I) \ + ATOMIC_OP (op, asm_op, I, w, int, ) \ + ATOMIC_OP (op, asm_op, I, d, s64, 64) +#endif + +ATOMIC_OPS(add, add, i) +ATOMIC_OPS(sub, add, -i) +ATOMIC_OPS(and, and, i) +ATOMIC_OPS( or, or, i) +ATOMIC_OPS(xor, xor, i) + +#undef ATOMIC_OP +#undef ATOMIC_OPS + +/* + * Atomic ops that have ordered, relaxed, acquire, and release variants. + * There's two flavors of these: the arithmatic ops have both fetch and return + * versions, while the logical ops only have fetch versions. + */ +#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \ +static __always_inline \ +c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \ + atomic##prefix##_t *v) \ +{ \ + register c_type ret; \ + __asm__ __volatile__ ( \ + " amo" #asm_op "." #asm_type " %1, %2, %0" \ + : "+A" (v->counter), "=r" (ret) \ + : "r" (I) \ + : "memory"); \ + return ret; \ +} \ +static __always_inline \ +c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ +{ \ + register c_type ret; \ + __asm__ __volatile__ ( \ + " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \ + : "+A" (v->counter), "=r" (ret) \ + : "r" (I) \ + : "memory"); \ + return ret; \ +} + +#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \ +static __always_inline \ +c_type atomic##prefix##_##op##_return_relaxed(c_type i, \ + atomic##prefix##_t *v) \ +{ \ + return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ +} \ +static __always_inline \ +c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ +{ \ + return atomic##prefix##_fetch_##op(i, v) c_op I; \ +} + +#ifdef CONFIG_GENERIC_ATOMIC64 +#define ATOMIC_OPS(op, asm_op, c_op, I) \ + ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \ + ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) +#else +#define ATOMIC_OPS(op, asm_op, c_op, I) \ + ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \ + ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \ + ATOMIC_FETCH_OP( op, asm_op, I, d, s64, 64) \ + ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64) +#endif + +ATOMIC_OPS(add, add, +, i) +ATOMIC_OPS(sub, add, +, -i) + +#define atomic_add_return_relaxed atomic_add_return_relaxed +#define atomic_sub_return_relaxed atomic_sub_return_relaxed +#define atomic_add_return atomic_add_return +#define atomic_sub_return atomic_sub_return + +#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed +#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed +#define atomic_fetch_add atomic_fetch_add +#define atomic_fetch_sub atomic_fetch_sub + +#ifndef CONFIG_GENERIC_ATOMIC64 +#define atomic64_add_return_relaxed atomic64_add_return_relaxed +#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed +#define atomic64_add_return atomic64_add_return +#define atomic64_sub_return atomic64_sub_return + +#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed +#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed +#define atomic64_fetch_add atomic64_fetch_add +#define atomic64_fetch_sub atomic64_fetch_sub +#endif + +#undef ATOMIC_OPS + +#ifdef CONFIG_GENERIC_ATOMIC64 +#define ATOMIC_OPS(op, asm_op, I) \ + ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) +#else +#define ATOMIC_OPS(op, asm_op, I) \ + ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \ + ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64) +#endif + +ATOMIC_OPS(and, and, i) +ATOMIC_OPS( or, or, i) +ATOMIC_OPS(xor, xor, i) + +#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed +#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed +#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed +#define atomic_fetch_and atomic_fetch_and +#define atomic_fetch_or atomic_fetch_or +#define atomic_fetch_xor atomic_fetch_xor + +#ifndef CONFIG_GENERIC_ATOMIC64 +#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed +#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed +#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed +#define atomic64_fetch_and atomic64_fetch_and +#define atomic64_fetch_or atomic64_fetch_or +#define atomic64_fetch_xor atomic64_fetch_xor +#endif + +#undef ATOMIC_OPS + +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN + +/* This is required to provide a full barrier on success. */ +static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int prev, rc; + + __asm__ __volatile__ ( + "0: lr.w %[p], %[c]\n" + " beq %[p], %[u], 1f\n" + " add %[rc], %[p], %[a]\n" + " sc.w.rl %[rc], %[rc], %[c]\n" + " bnez %[rc], 0b\n" + " fence rw, rw\n" + "1:\n" + : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) + : [a]"r" (a), [u]"r" (u) + : "memory"); + return prev; +} +#define atomic_fetch_add_unless atomic_fetch_add_unless + +#ifndef CONFIG_GENERIC_ATOMIC64 +static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +{ + s64 prev; + long rc; + + __asm__ __volatile__ ( + "0: lr.d %[p], %[c]\n" + " beq %[p], %[u], 1f\n" + " add %[rc], %[p], %[a]\n" + " sc.d.rl %[rc], %[rc], %[c]\n" + " bnez %[rc], 0b\n" + " fence rw, rw\n" + "1:\n" + : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) + : [a]"r" (a), [u]"r" (u) + : "memory"); + return prev; +} +#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#endif + +/* + * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as + * {cmp,}xchg and the operations that return, so they need a full barrier. + */ +#define ATOMIC_OP(c_t, prefix, size) \ +static __always_inline \ +c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \ +{ \ + return __xchg_relaxed(&(v->counter), n, size); \ +} \ +static __always_inline \ +c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \ +{ \ + return __xchg_acquire(&(v->counter), n, size); \ +} \ +static __always_inline \ +c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \ +{ \ + return __xchg_release(&(v->counter), n, size); \ +} \ +static __always_inline \ +c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \ +{ \ + return __xchg(&(v->counter), n, size); \ +} \ +static __always_inline \ +c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \ + c_t o, c_t n) \ +{ \ + return __cmpxchg_relaxed(&(v->counter), o, n, size); \ +} \ +static __always_inline \ +c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \ + c_t o, c_t n) \ +{ \ + return __cmpxchg_acquire(&(v->counter), o, n, size); \ +} \ +static __always_inline \ +c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \ + c_t o, c_t n) \ +{ \ + return __cmpxchg_release(&(v->counter), o, n, size); \ +} \ +static __always_inline \ +c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \ +{ \ + return __cmpxchg(&(v->counter), o, n, size); \ +} + +#ifdef CONFIG_GENERIC_ATOMIC64 +#define ATOMIC_OPS() \ + ATOMIC_OP(int, , 4) +#else +#define ATOMIC_OPS() \ + ATOMIC_OP(int, , 4) \ + ATOMIC_OP(s64, 64, 8) +#endif + +ATOMIC_OPS() + +#define atomic_xchg_relaxed atomic_xchg_relaxed +#define atomic_xchg_acquire atomic_xchg_acquire +#define atomic_xchg_release atomic_xchg_release +#define atomic_xchg atomic_xchg +#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed +#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire +#define atomic_cmpxchg_release atomic_cmpxchg_release +#define atomic_cmpxchg atomic_cmpxchg + +#undef ATOMIC_OPS +#undef ATOMIC_OP + +static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) +{ + int prev, rc; + + __asm__ __volatile__ ( + "0: lr.w %[p], %[c]\n" + " sub %[rc], %[p], %[o]\n" + " bltz %[rc], 1f\n" + " sc.w.rl %[rc], %[rc], %[c]\n" + " bnez %[rc], 0b\n" + " fence rw, rw\n" + "1:\n" + : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) + : [o]"r" (offset) + : "memory"); + return prev - offset; +} + +#define atomic_dec_if_positive(v) atomic_sub_if_positive(v, 1) + +#ifndef CONFIG_GENERIC_ATOMIC64 +static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) +{ + s64 prev; + long rc; + + __asm__ __volatile__ ( + "0: lr.d %[p], %[c]\n" + " sub %[rc], %[p], %[o]\n" + " bltz %[rc], 1f\n" + " sc.d.rl %[rc], %[rc], %[c]\n" + " bnez %[rc], 0b\n" + " fence rw, rw\n" + "1:\n" + : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) + : [o]"r" (offset) + : "memory"); + return prev - offset; +} + +#define atomic64_dec_if_positive(v) atomic64_sub_if_positive(v, 1) +#endif + +#endif /* _ASM_RISCV_ATOMIC_H */ diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h new file mode 100644 index 000000000..d0e24aaa2 --- /dev/null +++ b/arch/riscv/include/asm/barrier.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Based on arch/arm/include/asm/barrier.h + * + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2013 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#ifndef _ASM_RISCV_BARRIER_H +#define _ASM_RISCV_BARRIER_H + +#ifndef __ASSEMBLY__ + +#define nop() __asm__ __volatile__ ("nop") + +#define RISCV_FENCE(p, s) \ + __asm__ __volatile__ ("fence " #p "," #s : : : "memory") + +/* These barriers need to enforce ordering on both devices or memory. */ +#define mb() RISCV_FENCE(iorw,iorw) +#define rmb() RISCV_FENCE(ir,ir) +#define wmb() RISCV_FENCE(ow,ow) + +/* These barriers do not need to enforce ordering on devices, just memory. */ +#define __smp_mb() RISCV_FENCE(rw,rw) +#define __smp_rmb() RISCV_FENCE(r,r) +#define __smp_wmb() RISCV_FENCE(w,w) + +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + RISCV_FENCE(rw,w); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + RISCV_FENCE(r,rw); \ + ___p1; \ +}) + +/* + * This is a very specific barrier: it's currently only used in two places in + * the kernel, both in the scheduler. See include/linux/spinlock.h for the two + * orderings it guarantees, but the "critical section is RCsc" guarantee + * mandates a barrier on RISC-V. The sequence looks like: + * + * lr.aq lock + * sc lock <= LOCKED + * smp_mb__after_spinlock() + * // critical section + * lr lock + * sc.rl lock <= UNLOCKED + * + * The AQ/RL pair provides a RCpc critical section, but there's not really any + * way we can take advantage of that here because the ordering is only enforced + * on that one lock. Thus, we're just doing a full fence. + * + * Since we allow writeX to be called from preemptive regions we need at least + * an "o" in the predecessor set to ensure device writes are visible before the + * task is marked as available for scheduling on a new hart. While I don't see + * any concrete reason we need a full IO fence, it seems safer to just upgrade + * this in order to avoid any IO crossing a scheduling boundary. In both + * instances the scheduler pairs this with an mb(), so nothing is necessary on + * the new hart. + */ +#define smp_mb__after_spinlock() RISCV_FENCE(iorw,iorw) + +#include <asm-generic/barrier.h> + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_RISCV_BARRIER_H */ diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h new file mode 100644 index 000000000..396a3303c --- /dev/null +++ b/arch/riscv/include/asm/bitops.h @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_BITOPS_H +#define _ASM_RISCV_BITOPS_H + +#ifndef _LINUX_BITOPS_H +#error "Only <linux/bitops.h> can be included directly" +#endif /* _LINUX_BITOPS_H */ + +#include <linux/compiler.h> +#include <linux/irqflags.h> +#include <asm/barrier.h> +#include <asm/bitsperlong.h> + +#include <asm-generic/bitops/__ffs.h> +#include <asm-generic/bitops/ffz.h> +#include <asm-generic/bitops/fls.h> +#include <asm-generic/bitops/__fls.h> +#include <asm-generic/bitops/fls64.h> +#include <asm-generic/bitops/find.h> +#include <asm-generic/bitops/sched.h> +#include <asm-generic/bitops/ffs.h> + +#include <asm-generic/bitops/hweight.h> + +#if (BITS_PER_LONG == 64) +#define __AMO(op) "amo" #op ".d" +#elif (BITS_PER_LONG == 32) +#define __AMO(op) "amo" #op ".w" +#else +#error "Unexpected BITS_PER_LONG" +#endif + +#define __test_and_op_bit_ord(op, mod, nr, addr, ord) \ +({ \ + unsigned long __res, __mask; \ + __mask = BIT_MASK(nr); \ + __asm__ __volatile__ ( \ + __AMO(op) #ord " %0, %2, %1" \ + : "=r" (__res), "+A" (addr[BIT_WORD(nr)]) \ + : "r" (mod(__mask)) \ + : "memory"); \ + ((__res & __mask) != 0); \ +}) + +#define __op_bit_ord(op, mod, nr, addr, ord) \ + __asm__ __volatile__ ( \ + __AMO(op) #ord " zero, %1, %0" \ + : "+A" (addr[BIT_WORD(nr)]) \ + : "r" (mod(BIT_MASK(nr))) \ + : "memory"); + +#define __test_and_op_bit(op, mod, nr, addr) \ + __test_and_op_bit_ord(op, mod, nr, addr, .aqrl) +#define __op_bit(op, mod, nr, addr) \ + __op_bit_ord(op, mod, nr, addr, ) + +/* Bitmask modifiers */ +#define __NOP(x) (x) +#define __NOT(x) (~(x)) + +/** + * test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation may be reordered on other architectures than x86. + */ +static inline int test_and_set_bit(int nr, volatile unsigned long *addr) +{ + return __test_and_op_bit(or, __NOP, nr, addr); +} + +/** + * test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation can be reordered on other architectures other than x86. + */ +static inline int test_and_clear_bit(int nr, volatile unsigned long *addr) +{ + return __test_and_op_bit(and, __NOT, nr, addr); +} + +/** + * test_and_change_bit - Change a bit and return its old value + * @nr: Bit to change + * @addr: Address to count from + * + * This operation is atomic and cannot be reordered. + * It also implies a memory barrier. + */ +static inline int test_and_change_bit(int nr, volatile unsigned long *addr) +{ + return __test_and_op_bit(xor, __NOP, nr, addr); +} + +/** + * set_bit - Atomically set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Note: there are no guarantees that this function will not be reordered + * on non x86 architectures, so if you are writing portable code, + * make sure not to rely on its reordering guarantees. + * + * Note that @nr may be almost arbitrarily large; this function is not + * restricted to acting on a single-word quantity. + */ +static inline void set_bit(int nr, volatile unsigned long *addr) +{ + __op_bit(or, __NOP, nr, addr); +} + +/** + * clear_bit - Clears a bit in memory + * @nr: Bit to clear + * @addr: Address to start counting from + * + * Note: there are no guarantees that this function will not be reordered + * on non x86 architectures, so if you are writing portable code, + * make sure not to rely on its reordering guarantees. + */ +static inline void clear_bit(int nr, volatile unsigned long *addr) +{ + __op_bit(and, __NOT, nr, addr); +} + +/** + * change_bit - Toggle a bit in memory + * @nr: Bit to change + * @addr: Address to start counting from + * + * change_bit() may be reordered on other architectures than x86. + * Note that @nr may be almost arbitrarily large; this function is not + * restricted to acting on a single-word quantity. + */ +static inline void change_bit(int nr, volatile unsigned long *addr) +{ + __op_bit(xor, __NOP, nr, addr); +} + +/** + * test_and_set_bit_lock - Set a bit and return its old value, for lock + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is atomic and provides acquire barrier semantics. + * It can be used to implement bit locks. + */ +static inline int test_and_set_bit_lock( + unsigned long nr, volatile unsigned long *addr) +{ + return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq); +} + +/** + * clear_bit_unlock - Clear a bit in memory, for unlock + * @nr: the bit to set + * @addr: the address to start counting from + * + * This operation is atomic and provides release barrier semantics. + */ +static inline void clear_bit_unlock( + unsigned long nr, volatile unsigned long *addr) +{ + __op_bit_ord(and, __NOT, nr, addr, .rl); +} + +/** + * __clear_bit_unlock - Clear a bit in memory, for unlock + * @nr: the bit to set + * @addr: the address to start counting from + * + * This operation is like clear_bit_unlock, however it is not atomic. + * It does provide release barrier semantics so it can be used to unlock + * a bit lock, however it would only be used if no other CPU can modify + * any bits in the memory until the lock is released (a good example is + * if the bit lock itself protects access to the other bits in the word). + * + * On RISC-V systems there seems to be no benefit to taking advantage of the + * non-atomic property here: it's a lot more instructions and we still have to + * provide release semantics anyway. + */ +static inline void __clear_bit_unlock( + unsigned long nr, volatile unsigned long *addr) +{ + clear_bit_unlock(nr, addr); +} + +#undef __test_and_op_bit +#undef __op_bit +#undef __NOP +#undef __NOT +#undef __AMO + +#include <asm-generic/bitops/non-atomic.h> +#include <asm-generic/bitops/le.h> +#include <asm-generic/bitops/ext2-atomic.h> + +#endif /* _ASM_RISCV_BITOPS_H */ diff --git a/arch/riscv/include/asm/bug.h b/arch/riscv/include/asm/bug.h new file mode 100644 index 000000000..d6f1ec08d --- /dev/null +++ b/arch/riscv/include/asm/bug.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_BUG_H +#define _ASM_RISCV_BUG_H + +#include <linux/compiler.h> +#include <linux/const.h> +#include <linux/types.h> + +#include <asm/asm.h> + +#define __INSN_LENGTH_MASK _UL(0x3) +#define __INSN_LENGTH_32 _UL(0x3) +#define __COMPRESSED_INSN_MASK _UL(0xffff) + +#define __BUG_INSN_32 _UL(0x00100073) /* ebreak */ +#define __BUG_INSN_16 _UL(0x9002) /* c.ebreak */ + +#define GET_INSN_LENGTH(insn) \ +({ \ + unsigned long __len; \ + __len = ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) ? \ + 4UL : 2UL; \ + __len; \ +}) + +typedef u32 bug_insn_t; + +#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS +#define __BUG_ENTRY_ADDR RISCV_INT " 1b - 2b" +#define __BUG_ENTRY_FILE RISCV_INT " %0 - 2b" +#else +#define __BUG_ENTRY_ADDR RISCV_PTR " 1b" +#define __BUG_ENTRY_FILE RISCV_PTR " %0" +#endif + +#ifdef CONFIG_DEBUG_BUGVERBOSE +#define __BUG_ENTRY \ + __BUG_ENTRY_ADDR "\n\t" \ + __BUG_ENTRY_FILE "\n\t" \ + RISCV_SHORT " %1\n\t" \ + RISCV_SHORT " %2" +#else +#define __BUG_ENTRY \ + __BUG_ENTRY_ADDR "\n\t" \ + RISCV_SHORT " %2" +#endif + +#ifdef CONFIG_GENERIC_BUG +#define __BUG_FLAGS(flags) \ +do { \ + __asm__ __volatile__ ( \ + "1:\n\t" \ + "ebreak\n" \ + ".pushsection __bug_table,\"aw\"\n\t" \ + "2:\n\t" \ + __BUG_ENTRY "\n\t" \ + ".org 2b + %3\n\t" \ + ".popsection" \ + : \ + : "i" (__FILE__), "i" (__LINE__), \ + "i" (flags), \ + "i" (sizeof(struct bug_entry))); \ +} while (0) +#else /* CONFIG_GENERIC_BUG */ +#define __BUG_FLAGS(flags) do { \ + __asm__ __volatile__ ("ebreak\n"); \ +} while (0) +#endif /* CONFIG_GENERIC_BUG */ + +#define BUG() do { \ + __BUG_FLAGS(0); \ + unreachable(); \ +} while (0) + +#define __WARN_FLAGS(flags) __BUG_FLAGS(BUGFLAG_WARNING|(flags)) + +#define HAVE_ARCH_BUG + +#include <asm-generic/bug.h> + +struct pt_regs; +struct task_struct; + +void die(struct pt_regs *regs, const char *str); +void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr); + +#endif /* _ASM_RISCV_BUG_H */ diff --git a/arch/riscv/include/asm/cache.h b/arch/riscv/include/asm/cache.h new file mode 100644 index 000000000..9b58b1045 --- /dev/null +++ b/arch/riscv/include/asm/cache.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 Chen Liqin <liqin.chen@sunplusct.com> + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_CACHE_H +#define _ASM_RISCV_CACHE_H + +#define L1_CACHE_SHIFT 6 + +#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) + +/* + * RISC-V requires the stack pointer to be 16-byte aligned, so ensure that + * the flat loader aligns it accordingly. + */ +#ifndef CONFIG_MMU +#define ARCH_SLAB_MINALIGN 16 +#endif + +#endif /* _ASM_RISCV_CACHE_H */ diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h new file mode 100644 index 000000000..23ff70350 --- /dev/null +++ b/arch/riscv/include/asm/cacheflush.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2015 Regents of the University of California + */ + +#ifndef _ASM_RISCV_CACHEFLUSH_H +#define _ASM_RISCV_CACHEFLUSH_H + +#include <linux/mm.h> + +static inline void local_flush_icache_all(void) +{ + asm volatile ("fence.i" ::: "memory"); +} + +#define PG_dcache_clean PG_arch_1 + +static inline void flush_dcache_page(struct page *page) +{ + if (test_bit(PG_dcache_clean, &page->flags)) + clear_bit(PG_dcache_clean, &page->flags); +} +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 + +/* + * RISC-V doesn't have an instruction to flush parts of the instruction cache, + * so instead we just flush the whole thing. + */ +#define flush_icache_range(start, end) flush_icache_all() +#define flush_icache_user_page(vma, pg, addr, len) \ + flush_icache_mm(vma->vm_mm, 0) + +#ifndef CONFIG_SMP + +#define flush_icache_all() local_flush_icache_all() +#define flush_icache_mm(mm, local) flush_icache_all() + +#else /* CONFIG_SMP */ + +void flush_icache_all(void); +void flush_icache_mm(struct mm_struct *mm, bool local); + +#endif /* CONFIG_SMP */ + +/* + * Bits in sys_riscv_flush_icache()'s flags argument. + */ +#define SYS_RISCV_FLUSH_ICACHE_LOCAL 1UL +#define SYS_RISCV_FLUSH_ICACHE_ALL (SYS_RISCV_FLUSH_ICACHE_LOCAL) + +#include <asm-generic/cacheflush.h> + +#endif /* _ASM_RISCV_CACHEFLUSH_H */ diff --git a/arch/riscv/include/asm/cacheinfo.h b/arch/riscv/include/asm/cacheinfo.h new file mode 100644 index 000000000..d1a365215 --- /dev/null +++ b/arch/riscv/include/asm/cacheinfo.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 SiFive + */ + +#ifndef _ASM_RISCV_CACHEINFO_H +#define _ASM_RISCV_CACHEINFO_H + +#include <linux/cacheinfo.h> + +struct riscv_cacheinfo_ops { + const struct attribute_group * (*get_priv_group)(struct cacheinfo + *this_leaf); +}; + +void riscv_set_cacheinfo_ops(struct riscv_cacheinfo_ops *ops); +uintptr_t get_cache_size(u32 level, enum cache_type type); +uintptr_t get_cache_geometry(u32 level, enum cache_type type); + +#endif /* _ASM_RISCV_CACHEINFO_H */ diff --git a/arch/riscv/include/asm/clint.h b/arch/riscv/include/asm/clint.h new file mode 100644 index 000000000..0789fd37b --- /dev/null +++ b/arch/riscv/include/asm/clint.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Google, Inc + */ + +#ifndef _ASM_RISCV_CLINT_H +#define _ASM_RISCV_CLINT_H + +#include <linux/types.h> +#include <asm/mmio.h> + +#ifdef CONFIG_RISCV_M_MODE +/* + * This lives in the CLINT driver, but is accessed directly by timex.h to avoid + * any overhead when accessing the MMIO timer. + * + * The ISA defines mtime as a 64-bit memory-mapped register that increments at + * a constant frequency, but it doesn't define some other constraints we depend + * on (most notably ordering constraints, but also some simpler stuff like the + * memory layout). Thus, this is called "clint_time_val" instead of something + * like "riscv_mtime", to signify that these non-ISA assumptions must hold. + */ +extern u64 __iomem *clint_time_val; +#endif + +#endif diff --git a/arch/riscv/include/asm/clocksource.h b/arch/riscv/include/asm/clocksource.h new file mode 100644 index 000000000..482185566 --- /dev/null +++ b/arch/riscv/include/asm/clocksource.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_CLOCKSOURCE_H +#define _ASM_CLOCKSOURCE_H + +#include <asm/vdso/clocksource.h> + +#endif diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h new file mode 100644 index 000000000..262e5bbb2 --- /dev/null +++ b/arch/riscv/include/asm/cmpxchg.h @@ -0,0 +1,375 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2014 Regents of the University of California + */ + +#ifndef _ASM_RISCV_CMPXCHG_H +#define _ASM_RISCV_CMPXCHG_H + +#include <linux/bug.h> + +#include <asm/barrier.h> +#include <asm/fence.h> + +#define __xchg_relaxed(ptr, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(new) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + " amoswap.w %0, %2, %1\n" \ + : "=r" (__ret), "+A" (*__ptr) \ + : "r" (__new) \ + : "memory"); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + " amoswap.d %0, %2, %1\n" \ + : "=r" (__ret), "+A" (*__ptr) \ + : "r" (__new) \ + : "memory"); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define xchg_relaxed(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ + _x_, sizeof(*(ptr))); \ +}) + +#define __xchg_acquire(ptr, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(new) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + " amoswap.w %0, %2, %1\n" \ + RISCV_ACQUIRE_BARRIER \ + : "=r" (__ret), "+A" (*__ptr) \ + : "r" (__new) \ + : "memory"); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + " amoswap.d %0, %2, %1\n" \ + RISCV_ACQUIRE_BARRIER \ + : "=r" (__ret), "+A" (*__ptr) \ + : "r" (__new) \ + : "memory"); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define xchg_acquire(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __xchg_acquire((ptr), \ + _x_, sizeof(*(ptr))); \ +}) + +#define __xchg_release(ptr, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(new) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + RISCV_RELEASE_BARRIER \ + " amoswap.w %0, %2, %1\n" \ + : "=r" (__ret), "+A" (*__ptr) \ + : "r" (__new) \ + : "memory"); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + RISCV_RELEASE_BARRIER \ + " amoswap.d %0, %2, %1\n" \ + : "=r" (__ret), "+A" (*__ptr) \ + : "r" (__new) \ + : "memory"); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define xchg_release(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __xchg_release((ptr), \ + _x_, sizeof(*(ptr))); \ +}) + +#define __xchg(ptr, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(new) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + " amoswap.w.aqrl %0, %2, %1\n" \ + : "=r" (__ret), "+A" (*__ptr) \ + : "r" (__new) \ + : "memory"); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + " amoswap.d.aqrl %0, %2, %1\n" \ + : "=r" (__ret), "+A" (*__ptr) \ + : "r" (__new) \ + : "memory"); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define xchg(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \ +}) + +#define xchg32(ptr, x) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ + xchg((ptr), (x)); \ +}) + +#define xchg64(ptr, x) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + xchg((ptr), (x)); \ +}) + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + */ +#define __cmpxchg_relaxed(ptr, old, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(*(ptr)) __old = (old); \ + __typeof__(*(ptr)) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + register unsigned int __rc; \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + "0: lr.w %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.w %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + "1:\n" \ + : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ + : "rJ" ((long)__old), "rJ" (__new) \ + : "memory"); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + "0: lr.d %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.d %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + "1:\n" \ + : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ + : "rJ" (__old), "rJ" (__new) \ + : "memory"); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define cmpxchg_relaxed(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \ + _o_, _n_, sizeof(*(ptr))); \ +}) + +#define __cmpxchg_acquire(ptr, old, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(*(ptr)) __old = (old); \ + __typeof__(*(ptr)) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + register unsigned int __rc; \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + "0: lr.w %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.w %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + RISCV_ACQUIRE_BARRIER \ + "1:\n" \ + : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ + : "rJ" ((long)__old), "rJ" (__new) \ + : "memory"); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + "0: lr.d %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.d %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + RISCV_ACQUIRE_BARRIER \ + "1:\n" \ + : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ + : "rJ" (__old), "rJ" (__new) \ + : "memory"); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define cmpxchg_acquire(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ + _o_, _n_, sizeof(*(ptr))); \ +}) + +#define __cmpxchg_release(ptr, old, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(*(ptr)) __old = (old); \ + __typeof__(*(ptr)) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + register unsigned int __rc; \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + RISCV_RELEASE_BARRIER \ + "0: lr.w %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.w %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + "1:\n" \ + : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ + : "rJ" ((long)__old), "rJ" (__new) \ + : "memory"); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + RISCV_RELEASE_BARRIER \ + "0: lr.d %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.d %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + "1:\n" \ + : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ + : "rJ" (__old), "rJ" (__new) \ + : "memory"); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define cmpxchg_release(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg_release((ptr), \ + _o_, _n_, sizeof(*(ptr))); \ +}) + +#define __cmpxchg(ptr, old, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(*(ptr)) __old = (old); \ + __typeof__(*(ptr)) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + register unsigned int __rc; \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + "0: lr.w %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.w.rl %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + " fence rw, rw\n" \ + "1:\n" \ + : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ + : "rJ" ((long)__old), "rJ" (__new) \ + : "memory"); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + "0: lr.d %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.d.rl %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + " fence rw, rw\n" \ + "1:\n" \ + : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ + : "rJ" (__old), "rJ" (__new) \ + : "memory"); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define cmpxchg(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg((ptr), \ + _o_, _n_, sizeof(*(ptr))); \ +}) + +#define cmpxchg_local(ptr, o, n) \ + (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) + +#define cmpxchg32(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ + cmpxchg((ptr), (o), (n)); \ +}) + +#define cmpxchg32_local(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ + cmpxchg_relaxed((ptr), (o), (n)) \ +}) + +#define cmpxchg64(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + cmpxchg((ptr), (o), (n)); \ +}) + +#define cmpxchg64_local(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + cmpxchg_relaxed((ptr), (o), (n)); \ +}) + +#endif /* _ASM_RISCV_CMPXCHG_H */ diff --git a/arch/riscv/include/asm/cpu_ops.h b/arch/riscv/include/asm/cpu_ops.h new file mode 100644 index 000000000..a8ec3c5c1 --- /dev/null +++ b/arch/riscv/include/asm/cpu_ops.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020 Western Digital Corporation or its affiliates. + * Based on arch/arm64/include/asm/cpu_ops.h + */ +#ifndef __ASM_CPU_OPS_H +#define __ASM_CPU_OPS_H + +#include <linux/init.h> +#include <linux/sched.h> +#include <linux/threads.h> + +/** + * struct cpu_operations - Callback operations for hotplugging CPUs. + * + * @name: Name of the boot protocol. + * @cpu_prepare: Early one-time preparation step for a cpu. If there + * is a mechanism for doing so, tests whether it is + * possible to boot the given HART. + * @cpu_start: Boots a cpu into the kernel. + * @cpu_disable: Prepares a cpu to die. May fail for some + * mechanism-specific reason, which will cause the hot + * unplug to be aborted. Called from the cpu to be killed. + * @cpu_stop: Makes a cpu leave the kernel. Must not fail. Called from + * the cpu being stopped. + * @cpu_is_stopped: Ensures a cpu has left the kernel. Called from another + * cpu. + */ +struct cpu_operations { + const char *name; + int (*cpu_prepare)(unsigned int cpu); + int (*cpu_start)(unsigned int cpu, + struct task_struct *tidle); +#ifdef CONFIG_HOTPLUG_CPU + int (*cpu_disable)(unsigned int cpu); + void (*cpu_stop)(void); + int (*cpu_is_stopped)(unsigned int cpu); +#endif +}; + +extern const struct cpu_operations *cpu_ops[NR_CPUS]; +void __init cpu_set_ops(int cpu); +void cpu_update_secondary_bootdata(unsigned int cpuid, + struct task_struct *tidle); + +#endif /* ifndef __ASM_CPU_OPS_H */ diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h new file mode 100644 index 000000000..cec462e19 --- /dev/null +++ b/arch/riscv/include/asm/csr.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2015 Regents of the University of California + */ + +#ifndef _ASM_RISCV_CSR_H +#define _ASM_RISCV_CSR_H + +#include <asm/asm.h> +#include <linux/const.h> + +/* Status register flags */ +#define SR_SIE _AC(0x00000002, UL) /* Supervisor Interrupt Enable */ +#define SR_MIE _AC(0x00000008, UL) /* Machine Interrupt Enable */ +#define SR_SPIE _AC(0x00000020, UL) /* Previous Supervisor IE */ +#define SR_MPIE _AC(0x00000080, UL) /* Previous Machine IE */ +#define SR_SPP _AC(0x00000100, UL) /* Previously Supervisor */ +#define SR_MPP _AC(0x00001800, UL) /* Previously Machine */ +#define SR_SUM _AC(0x00040000, UL) /* Supervisor User Memory Access */ + +#define SR_FS _AC(0x00006000, UL) /* Floating-point Status */ +#define SR_FS_OFF _AC(0x00000000, UL) +#define SR_FS_INITIAL _AC(0x00002000, UL) +#define SR_FS_CLEAN _AC(0x00004000, UL) +#define SR_FS_DIRTY _AC(0x00006000, UL) + +#define SR_XS _AC(0x00018000, UL) /* Extension Status */ +#define SR_XS_OFF _AC(0x00000000, UL) +#define SR_XS_INITIAL _AC(0x00008000, UL) +#define SR_XS_CLEAN _AC(0x00010000, UL) +#define SR_XS_DIRTY _AC(0x00018000, UL) + +#ifndef CONFIG_64BIT +#define SR_SD _AC(0x80000000, UL) /* FS/XS dirty */ +#else +#define SR_SD _AC(0x8000000000000000, UL) /* FS/XS dirty */ +#endif + +/* SATP flags */ +#ifndef CONFIG_64BIT +#define SATP_PPN _AC(0x003FFFFF, UL) +#define SATP_MODE_32 _AC(0x80000000, UL) +#define SATP_MODE SATP_MODE_32 +#else +#define SATP_PPN _AC(0x00000FFFFFFFFFFF, UL) +#define SATP_MODE_39 _AC(0x8000000000000000, UL) +#define SATP_MODE SATP_MODE_39 +#endif + +/* Exception cause high bit - is an interrupt if set */ +#define CAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1)) + +/* Interrupt causes (minus the high bit) */ +#define IRQ_S_SOFT 1 +#define IRQ_M_SOFT 3 +#define IRQ_S_TIMER 5 +#define IRQ_M_TIMER 7 +#define IRQ_S_EXT 9 +#define IRQ_M_EXT 11 + +/* Exception causes */ +#define EXC_INST_MISALIGNED 0 +#define EXC_INST_ACCESS 1 +#define EXC_BREAKPOINT 3 +#define EXC_LOAD_ACCESS 5 +#define EXC_STORE_ACCESS 7 +#define EXC_SYSCALL 8 +#define EXC_INST_PAGE_FAULT 12 +#define EXC_LOAD_PAGE_FAULT 13 +#define EXC_STORE_PAGE_FAULT 15 + +/* PMP configuration */ +#define PMP_R 0x01 +#define PMP_W 0x02 +#define PMP_X 0x04 +#define PMP_A 0x18 +#define PMP_A_TOR 0x08 +#define PMP_A_NA4 0x10 +#define PMP_A_NAPOT 0x18 +#define PMP_L 0x80 + +/* symbolic CSR names: */ +#define CSR_CYCLE 0xc00 +#define CSR_TIME 0xc01 +#define CSR_INSTRET 0xc02 +#define CSR_CYCLEH 0xc80 +#define CSR_TIMEH 0xc81 +#define CSR_INSTRETH 0xc82 + +#define CSR_SSTATUS 0x100 +#define CSR_SIE 0x104 +#define CSR_STVEC 0x105 +#define CSR_SCOUNTEREN 0x106 +#define CSR_SSCRATCH 0x140 +#define CSR_SEPC 0x141 +#define CSR_SCAUSE 0x142 +#define CSR_STVAL 0x143 +#define CSR_SIP 0x144 +#define CSR_SATP 0x180 + +#define CSR_MSTATUS 0x300 +#define CSR_MISA 0x301 +#define CSR_MIE 0x304 +#define CSR_MTVEC 0x305 +#define CSR_MSCRATCH 0x340 +#define CSR_MEPC 0x341 +#define CSR_MCAUSE 0x342 +#define CSR_MTVAL 0x343 +#define CSR_MIP 0x344 +#define CSR_PMPCFG0 0x3a0 +#define CSR_PMPADDR0 0x3b0 +#define CSR_MHARTID 0xf14 + +#ifdef CONFIG_RISCV_M_MODE +# define CSR_STATUS CSR_MSTATUS +# define CSR_IE CSR_MIE +# define CSR_TVEC CSR_MTVEC +# define CSR_SCRATCH CSR_MSCRATCH +# define CSR_EPC CSR_MEPC +# define CSR_CAUSE CSR_MCAUSE +# define CSR_TVAL CSR_MTVAL +# define CSR_IP CSR_MIP + +# define SR_IE SR_MIE +# define SR_PIE SR_MPIE +# define SR_PP SR_MPP + +# define RV_IRQ_SOFT IRQ_M_SOFT +# define RV_IRQ_TIMER IRQ_M_TIMER +# define RV_IRQ_EXT IRQ_M_EXT +#else /* CONFIG_RISCV_M_MODE */ +# define CSR_STATUS CSR_SSTATUS +# define CSR_IE CSR_SIE +# define CSR_TVEC CSR_STVEC +# define CSR_SCRATCH CSR_SSCRATCH +# define CSR_EPC CSR_SEPC +# define CSR_CAUSE CSR_SCAUSE +# define CSR_TVAL CSR_STVAL +# define CSR_IP CSR_SIP + +# define SR_IE SR_SIE +# define SR_PIE SR_SPIE +# define SR_PP SR_SPP + +# define RV_IRQ_SOFT IRQ_S_SOFT +# define RV_IRQ_TIMER IRQ_S_TIMER +# define RV_IRQ_EXT IRQ_S_EXT +#endif /* CONFIG_RISCV_M_MODE */ + +/* IE/IP (Supervisor/Machine Interrupt Enable/Pending) flags */ +#define IE_SIE (_AC(0x1, UL) << RV_IRQ_SOFT) +#define IE_TIE (_AC(0x1, UL) << RV_IRQ_TIMER) +#define IE_EIE (_AC(0x1, UL) << RV_IRQ_EXT) + +#ifndef __ASSEMBLY__ + +#define csr_swap(csr, val) \ +({ \ + unsigned long __v = (unsigned long)(val); \ + __asm__ __volatile__ ("csrrw %0, " __ASM_STR(csr) ", %1"\ + : "=r" (__v) : "rK" (__v) \ + : "memory"); \ + __v; \ +}) + +#define csr_read(csr) \ +({ \ + register unsigned long __v; \ + __asm__ __volatile__ ("csrr %0, " __ASM_STR(csr) \ + : "=r" (__v) : \ + : "memory"); \ + __v; \ +}) + +#define csr_write(csr, val) \ +({ \ + unsigned long __v = (unsigned long)(val); \ + __asm__ __volatile__ ("csrw " __ASM_STR(csr) ", %0" \ + : : "rK" (__v) \ + : "memory"); \ +}) + +#define csr_read_set(csr, val) \ +({ \ + unsigned long __v = (unsigned long)(val); \ + __asm__ __volatile__ ("csrrs %0, " __ASM_STR(csr) ", %1"\ + : "=r" (__v) : "rK" (__v) \ + : "memory"); \ + __v; \ +}) + +#define csr_set(csr, val) \ +({ \ + unsigned long __v = (unsigned long)(val); \ + __asm__ __volatile__ ("csrs " __ASM_STR(csr) ", %0" \ + : : "rK" (__v) \ + : "memory"); \ +}) + +#define csr_read_clear(csr, val) \ +({ \ + unsigned long __v = (unsigned long)(val); \ + __asm__ __volatile__ ("csrrc %0, " __ASM_STR(csr) ", %1"\ + : "=r" (__v) : "rK" (__v) \ + : "memory"); \ + __v; \ +}) + +#define csr_clear(csr, val) \ +({ \ + unsigned long __v = (unsigned long)(val); \ + __asm__ __volatile__ ("csrc " __ASM_STR(csr) ", %0" \ + : : "rK" (__v) \ + : "memory"); \ +}) + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_RISCV_CSR_H */ diff --git a/arch/riscv/include/asm/current.h b/arch/riscv/include/asm/current.h new file mode 100644 index 000000000..1de233d8e --- /dev/null +++ b/arch/riscv/include/asm/current.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Based on arm/arm64/include/asm/current.h + * + * Copyright (C) 2016 ARM + * Copyright (C) 2017 SiFive + */ + + +#ifndef _ASM_RISCV_CURRENT_H +#define _ASM_RISCV_CURRENT_H + +#include <linux/bug.h> +#include <linux/compiler.h> + +#ifndef __ASSEMBLY__ + +struct task_struct; + +register struct task_struct *riscv_current_is_tp __asm__("tp"); + +/* + * This only works because "struct thread_info" is at offset 0 from "struct + * task_struct". This constraint seems to be necessary on other architectures + * as well, but __switch_to enforces it. We can't check TASK_TI here because + * <asm/asm-offsets.h> includes this, and I can't get the definition of "struct + * task_struct" here due to some header ordering problems. + */ +static __always_inline struct task_struct *get_current(void) +{ + return riscv_current_is_tp; +} + +#define current get_current() + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_RISCV_CURRENT_H */ diff --git a/arch/riscv/include/asm/delay.h b/arch/riscv/include/asm/delay.h new file mode 100644 index 000000000..524f8ef7c --- /dev/null +++ b/arch/riscv/include/asm/delay.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com> + * Copyright (C) 2016 Regents of the University of California + */ + +#ifndef _ASM_RISCV_DELAY_H +#define _ASM_RISCV_DELAY_H + +extern unsigned long riscv_timebase; + +#define udelay udelay +extern void udelay(unsigned long usecs); + +#define ndelay ndelay +extern void ndelay(unsigned long nsecs); + +extern void __delay(unsigned long cycles); + +#endif /* _ASM_RISCV_DELAY_H */ diff --git a/arch/riscv/include/asm/efi.h b/arch/riscv/include/asm/efi.h new file mode 100644 index 000000000..7542282f1 --- /dev/null +++ b/arch/riscv/include/asm/efi.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ +#ifndef _ASM_EFI_H +#define _ASM_EFI_H + +#include <asm/csr.h> +#include <asm/io.h> +#include <asm/mmu_context.h> +#include <asm/ptrace.h> +#include <asm/tlbflush.h> + +#ifdef CONFIG_EFI +extern void efi_init(void); +#else +#define efi_init() +#endif + +int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); +int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); + +#define arch_efi_call_virt_setup() efi_virtmap_load() +#define arch_efi_call_virt_teardown() efi_virtmap_unload() + +#define arch_efi_call_virt(p, f, args...) p->f(args) + +#define ARCH_EFI_IRQ_FLAGS_MASK (SR_IE | SR_SPIE) + +/* on RISC-V, the FDT may be located anywhere in system RAM */ +static inline unsigned long efi_get_max_fdt_addr(unsigned long image_addr) +{ + return ULONG_MAX; +} + +/* Load initrd at enough distance from DRAM start */ +static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr) +{ + return image_addr + SZ_256M; +} + +#define alloc_screen_info(x...) (&screen_info) + +static inline void free_screen_info(struct screen_info *si) +{ +} + +static inline void efifb_setup_from_dmi(struct screen_info *si, const char *opt) +{ +} + +void efi_virtmap_load(void); +void efi_virtmap_unload(void); + +#endif /* _ASM_EFI_H */ diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h new file mode 100644 index 000000000..5c725e1df --- /dev/null +++ b/arch/riscv/include/asm/elf.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2003 Matjaz Breskvar <phoenix@bsemi.com> + * Copyright (C) 2010-2011 Jonas Bonn <jonas@southpole.se> + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_ELF_H +#define _ASM_RISCV_ELF_H + +#include <uapi/asm/elf.h> +#include <asm/auxvec.h> +#include <asm/byteorder.h> +#include <asm/cacheinfo.h> + +/* + * These are used to set parameters in the core dumps. + */ +#define ELF_ARCH EM_RISCV + +#ifdef CONFIG_64BIT +#define ELF_CLASS ELFCLASS64 +#else +#define ELF_CLASS ELFCLASS32 +#endif + +#define ELF_DATA ELFDATA2LSB + +/* + * This is used to ensure we don't load something for the wrong architecture. + */ +#define elf_check_arch(x) ((x)->e_machine == EM_RISCV) + +#define CORE_DUMP_USE_REGSET +#define ELF_EXEC_PAGESIZE (PAGE_SIZE) + +/* + * This is the location that an ET_DYN program is loaded if exec'ed. Typical + * use of this is to invoke "./ld.so someprog" to test out a new version of + * the loader. We need to make sure that it is out of the way of the program + * that it will "exec", and that there is sufficient room for the brk. + */ +#define ELF_ET_DYN_BASE ((TASK_SIZE / 3) * 2) + +/* + * This yields a mask that user programs can use to figure out what + * instruction set this CPU supports. This could be done in user space, + * but it's not easy, and we've already done it here. + */ +#define ELF_HWCAP (elf_hwcap) +extern unsigned long elf_hwcap; + +/* + * This yields a string that ld.so will use to load implementation + * specific libraries for optimization. This is more specific in + * intent than poking at uname or /proc/cpuinfo. + */ +#define ELF_PLATFORM (NULL) + +#ifdef CONFIG_MMU +#define ARCH_DLINFO \ +do { \ + NEW_AUX_ENT(AT_SYSINFO_EHDR, \ + (elf_addr_t)current->mm->context.vdso); \ + NEW_AUX_ENT(AT_L1I_CACHESIZE, \ + get_cache_size(1, CACHE_TYPE_INST)); \ + NEW_AUX_ENT(AT_L1I_CACHEGEOMETRY, \ + get_cache_geometry(1, CACHE_TYPE_INST)); \ + NEW_AUX_ENT(AT_L1D_CACHESIZE, \ + get_cache_size(1, CACHE_TYPE_DATA)); \ + NEW_AUX_ENT(AT_L1D_CACHEGEOMETRY, \ + get_cache_geometry(1, CACHE_TYPE_DATA)); \ + NEW_AUX_ENT(AT_L2_CACHESIZE, \ + get_cache_size(2, CACHE_TYPE_UNIFIED)); \ + NEW_AUX_ENT(AT_L2_CACHEGEOMETRY, \ + get_cache_geometry(2, CACHE_TYPE_UNIFIED)); \ +} while (0) +#define ARCH_HAS_SETUP_ADDITIONAL_PAGES +struct linux_binprm; +extern int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp); +#endif /* CONFIG_MMU */ + +#endif /* _ASM_RISCV_ELF_H */ diff --git a/arch/riscv/include/asm/fence.h b/arch/riscv/include/asm/fence.h new file mode 100644 index 000000000..2b443a3a4 --- /dev/null +++ b/arch/riscv/include/asm/fence.h @@ -0,0 +1,12 @@ +#ifndef _ASM_RISCV_FENCE_H +#define _ASM_RISCV_FENCE_H + +#ifdef CONFIG_SMP +#define RISCV_ACQUIRE_BARRIER "\tfence r , rw\n" +#define RISCV_RELEASE_BARRIER "\tfence rw, w\n" +#else +#define RISCV_ACQUIRE_BARRIER +#define RISCV_RELEASE_BARRIER +#endif + +#endif /* _ASM_RISCV_FENCE_H */ diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h new file mode 100644 index 000000000..54cbf07fb --- /dev/null +++ b/arch/riscv/include/asm/fixmap.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 Western Digital Corporation or its affiliates. + */ + +#ifndef _ASM_RISCV_FIXMAP_H +#define _ASM_RISCV_FIXMAP_H + +#include <linux/kernel.h> +#include <linux/sizes.h> +#include <linux/pgtable.h> +#include <asm/page.h> + +#ifdef CONFIG_MMU +/* + * Here we define all the compile-time 'special' virtual addresses. + * The point is to have a constant address at compile time, but to + * set the physical address only in the boot process. + * + * These 'compile-time allocated' memory buffers are page-sized. Use + * set_fixmap(idx,phys) to associate physical memory with fixmap indices. + */ +enum fixed_addresses { + FIX_HOLE, + FIX_PTE, + FIX_PMD, + FIX_TEXT_POKE1, + FIX_TEXT_POKE0, + FIX_EARLYCON_MEM_BASE, + + __end_of_permanent_fixed_addresses, + /* + * Temporary boot-time mappings, used by early_ioremap(), + * before ioremap() is functional. + */ +#define NR_FIX_BTMAPS (SZ_256K / PAGE_SIZE) +#define FIX_BTMAPS_SLOTS 7 +#define TOTAL_FIX_BTMAPS (NR_FIX_BTMAPS * FIX_BTMAPS_SLOTS) + + FIX_BTMAP_END = __end_of_permanent_fixed_addresses, + FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1, + + __end_of_fixed_addresses +}; + +#define FIXMAP_PAGE_IO PAGE_KERNEL + +#define __early_set_fixmap __set_fixmap + +#define __late_set_fixmap __set_fixmap +#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) + +extern void __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t prot); + +#include <asm-generic/fixmap.h> + +#endif /* CONFIG_MMU */ +#endif /* _ASM_RISCV_FIXMAP_H */ diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h new file mode 100644 index 000000000..bc745900c --- /dev/null +++ b/arch/riscv/include/asm/ftrace.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2017 Andes Technology Corporation */ + +#ifndef _ASM_RISCV_FTRACE_H +#define _ASM_RISCV_FTRACE_H + +/* + * The graph frame test is not possible if CONFIG_FRAME_POINTER is not enabled. + * Check arch/riscv/kernel/mcount.S for detail. + */ +#if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_FRAME_POINTER) +#define HAVE_FUNCTION_GRAPH_FP_TEST +#endif +#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR + +/* + * Clang prior to 13 had "mcount" instead of "_mcount": + * https://reviews.llvm.org/D98881 + */ +#if defined(CONFIG_CC_IS_GCC) || CONFIG_CLANG_VERSION >= 130000 +#define MCOUNT_NAME _mcount +#else +#define MCOUNT_NAME mcount +#endif + +#define ARCH_SUPPORTS_FTRACE_OPS 1 +#ifndef __ASSEMBLY__ +void MCOUNT_NAME(void); +static inline unsigned long ftrace_call_adjust(unsigned long addr) +{ + return addr; +} + +struct dyn_arch_ftrace { +}; +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE +/* + * A general call in RISC-V is a pair of insts: + * 1) auipc: setting high-20 pc-related bits to ra register + * 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to + * return address (original pc + 4) + * + * Dynamic ftrace generates probes to call sites, so we must deal with + * both auipc and jalr at the same time. + */ + +#define MCOUNT_ADDR ((unsigned long)MCOUNT_NAME) +#define JALR_SIGN_MASK (0x00000800) +#define JALR_OFFSET_MASK (0x00000fff) +#define AUIPC_OFFSET_MASK (0xfffff000) +#define AUIPC_PAD (0x00001000) +#define JALR_SHIFT 20 +#define JALR_BASIC (0x000080e7) +#define AUIPC_BASIC (0x00000097) +#define NOP4 (0x00000013) + +#define make_call(caller, callee, call) \ +do { \ + call[0] = to_auipc_insn((unsigned int)((unsigned long)callee - \ + (unsigned long)caller)); \ + call[1] = to_jalr_insn((unsigned int)((unsigned long)callee - \ + (unsigned long)caller)); \ +} while (0) + +#define to_jalr_insn(offset) \ + (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_BASIC) + +#define to_auipc_insn(offset) \ + ((offset & JALR_SIGN_MASK) ? \ + (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_BASIC) : \ + ((offset & AUIPC_OFFSET_MASK) | AUIPC_BASIC)) + +/* + * Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here. + */ +#define MCOUNT_INSN_SIZE 8 + +#ifndef __ASSEMBLY__ +struct dyn_ftrace; +int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec); +#define ftrace_init_nop ftrace_init_nop +#endif + +#endif /* CONFIG_DYNAMIC_FTRACE */ + +#endif /* _ASM_RISCV_FTRACE_H */ diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h new file mode 100644 index 000000000..1b00badb9 --- /dev/null +++ b/arch/riscv/include/asm/futex.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2006 Ralf Baechle (ralf@linux-mips.org) + * Copyright (c) 2018 Jim Wilson (jimw@sifive.com) + */ + +#ifndef _ASM_RISCV_FUTEX_H +#define _ASM_RISCV_FUTEX_H + +#include <linux/futex.h> +#include <linux/uaccess.h> +#include <linux/errno.h> +#include <asm/asm.h> + +/* We don't even really need the extable code, but for now keep it simple */ +#ifndef CONFIG_MMU +#define __enable_user_access() do { } while (0) +#define __disable_user_access() do { } while (0) +#endif + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ +{ \ + uintptr_t tmp; \ + __enable_user_access(); \ + __asm__ __volatile__ ( \ + "1: " insn " \n" \ + "2: \n" \ + " .section .fixup,\"ax\" \n" \ + " .balign 4 \n" \ + "3: li %[r],%[e] \n" \ + " jump 2b,%[t] \n" \ + " .previous \n" \ + " .section __ex_table,\"a\" \n" \ + " .balign " RISCV_SZPTR " \n" \ + " " RISCV_PTR " 1b, 3b \n" \ + " .previous \n" \ + : [r] "+r" (ret), [ov] "=&r" (oldval), \ + [u] "+m" (*uaddr), [t] "=&r" (tmp) \ + : [op] "Jr" (oparg), [e] "i" (-EFAULT) \ + : "memory"); \ + __disable_user_access(); \ +} + +static inline int +arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) +{ + int oldval = 0, ret = 0; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + switch (op) { + case FUTEX_OP_SET: + __futex_atomic_op("amoswap.w.aqrl %[ov],%z[op],%[u]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ADD: + __futex_atomic_op("amoadd.w.aqrl %[ov],%z[op],%[u]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_OR: + __futex_atomic_op("amoor.w.aqrl %[ov],%z[op],%[u]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ANDN: + __futex_atomic_op("amoand.w.aqrl %[ov],%z[op],%[u]", + ret, oldval, uaddr, ~oparg); + break; + case FUTEX_OP_XOR: + __futex_atomic_op("amoxor.w.aqrl %[ov],%z[op],%[u]", + ret, oldval, uaddr, oparg); + break; + default: + ret = -ENOSYS; + } + + if (!ret) + *oval = oldval; + + return ret; +} + +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) +{ + int ret = 0; + u32 val; + uintptr_t tmp; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + __enable_user_access(); + __asm__ __volatile__ ( + "1: lr.w.aqrl %[v],%[u] \n" + " bne %[v],%z[ov],3f \n" + "2: sc.w.aqrl %[t],%z[nv],%[u] \n" + " bnez %[t],1b \n" + "3: \n" + " .section .fixup,\"ax\" \n" + " .balign 4 \n" + "4: li %[r],%[e] \n" + " jump 3b,%[t] \n" + " .previous \n" + " .section __ex_table,\"a\" \n" + " .balign " RISCV_SZPTR " \n" + " " RISCV_PTR " 1b, 4b \n" + " " RISCV_PTR " 2b, 4b \n" + " .previous \n" + : [r] "+r" (ret), [v] "=&r" (val), [u] "+m" (*uaddr), [t] "=&r" (tmp) + : [ov] "Jr" (oldval), [nv] "Jr" (newval), [e] "i" (-EFAULT) + : "memory"); + __disable_user_access(); + + *uval = val; + return ret; +} + +#endif /* _ASM_RISCV_FUTEX_H */ diff --git a/arch/riscv/include/asm/gdb_xml.h b/arch/riscv/include/asm/gdb_xml.h new file mode 100644 index 000000000..09342111f --- /dev/null +++ b/arch/riscv/include/asm/gdb_xml.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_GDB_XML_H_ +#define __ASM_GDB_XML_H_ + +const char riscv_gdb_stub_feature[64] = + "PacketSize=800;qXfer:features:read+;"; + +static const char gdb_xfer_read_target[31] = "qXfer:features:read:target.xml:"; + +#ifdef CONFIG_64BIT +static const char gdb_xfer_read_cpuxml[39] = + "qXfer:features:read:riscv-64bit-cpu.xml"; + +static const char riscv_gdb_stub_target_desc[256] = +"l<?xml version=\"1.0\"?>" +"<!DOCTYPE target SYSTEM \"gdb-target.dtd\">" +"<target>" +"<xi:include href=\"riscv-64bit-cpu.xml\"/>" +"</target>"; + +static const char riscv_gdb_stub_cpuxml[2048] = +"l<?xml version=\"1.0\"?>" +"<!DOCTYPE feature SYSTEM \"gdb-target.dtd\">" +"<feature name=\"org.gnu.gdb.riscv.cpu\">" +"<reg name=\""DBG_REG_ZERO"\" bitsize=\"64\" type=\"int\" regnum=\"0\"/>" +"<reg name=\""DBG_REG_RA"\" bitsize=\"64\" type=\"code_ptr\"/>" +"<reg name=\""DBG_REG_SP"\" bitsize=\"64\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_GP"\" bitsize=\"64\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_TP"\" bitsize=\"64\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_T0"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T1"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T2"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_FP"\" bitsize=\"64\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_S1"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A0"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A1"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A2"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A3"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A4"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A5"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A6"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A7"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S2"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S3"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S4"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S5"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S6"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S7"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S8"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S9"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S10"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S11"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T3"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T4"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T5"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T6"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_EPC"\" bitsize=\"64\" type=\"code_ptr\"/>" +"<reg name=\""DBG_REG_STATUS"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_BADADDR"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_CAUSE"\" bitsize=\"64\" type=\"int\"/>" +"</feature>"; +#else +static const char gdb_xfer_read_cpuxml[39] = + "qXfer:features:read:riscv-32bit-cpu.xml"; + +static const char riscv_gdb_stub_target_desc[256] = +"l<?xml version=\"1.0\"?>" +"<!DOCTYPE target SYSTEM \"gdb-target.dtd\">" +"<target>" +"<xi:include href=\"riscv-32bit-cpu.xml\"/>" +"</target>"; + +static const char riscv_gdb_stub_cpuxml[2048] = +"l<?xml version=\"1.0\"?>" +"<!DOCTYPE feature SYSTEM \"gdb-target.dtd\">" +"<feature name=\"org.gnu.gdb.riscv.cpu\">" +"<reg name=\""DBG_REG_ZERO"\" bitsize=\"32\" type=\"int\" regnum=\"0\"/>" +"<reg name=\""DBG_REG_RA"\" bitsize=\"32\" type=\"code_ptr\"/>" +"<reg name=\""DBG_REG_SP"\" bitsize=\"32\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_GP"\" bitsize=\"32\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_TP"\" bitsize=\"32\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_T0"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T1"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T2"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_FP"\" bitsize=\"32\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_S1"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A0"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A1"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A2"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A3"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A4"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A5"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A6"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A7"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S2"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S3"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S4"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S5"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S6"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S7"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S8"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S9"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S10"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S11"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T3"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T4"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T5"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T6"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_EPC"\" bitsize=\"32\" type=\"code_ptr\"/>" +"<reg name=\""DBG_REG_STATUS"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_BADADDR"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_CAUSE"\" bitsize=\"32\" type=\"int\"/>" +"</feature>"; +#endif +#endif diff --git a/arch/riscv/include/asm/hugetlb.h b/arch/riscv/include/asm/hugetlb.h new file mode 100644 index 000000000..ec19d6afc --- /dev/null +++ b/arch/riscv/include/asm/hugetlb.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_HUGETLB_H +#define _ASM_RISCV_HUGETLB_H + +#include <asm-generic/hugetlb.h> +#include <asm/page.h> + +static inline void arch_clear_hugepage_flags(struct page *page) +{ + clear_bit(PG_dcache_clean, &page->flags); +} +#define arch_clear_hugepage_flags arch_clear_hugepage_flags + +#endif /* _ASM_RISCV_HUGETLB_H */ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h new file mode 100644 index 000000000..5ce50468a --- /dev/null +++ b/arch/riscv/include/asm/hwcap.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copied from arch/arm64/include/asm/hwcap.h + * + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2017 SiFive + */ +#ifndef _ASM_RISCV_HWCAP_H +#define _ASM_RISCV_HWCAP_H + +#include <linux/bits.h> +#include <uapi/asm/hwcap.h> + +#ifndef __ASSEMBLY__ +/* + * This yields a mask that user programs can use to figure out what + * instruction set this cpu supports. + */ +#define ELF_HWCAP (elf_hwcap) + +enum { + CAP_HWCAP = 1, +}; + +extern unsigned long elf_hwcap; + +#define RISCV_ISA_EXT_a ('a' - 'a') +#define RISCV_ISA_EXT_c ('c' - 'a') +#define RISCV_ISA_EXT_d ('d' - 'a') +#define RISCV_ISA_EXT_f ('f' - 'a') +#define RISCV_ISA_EXT_h ('h' - 'a') +#define RISCV_ISA_EXT_i ('i' - 'a') +#define RISCV_ISA_EXT_m ('m' - 'a') +#define RISCV_ISA_EXT_s ('s' - 'a') +#define RISCV_ISA_EXT_u ('u' - 'a') + +#define RISCV_ISA_EXT_MAX 64 + +unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap); + +#define riscv_isa_extension_mask(ext) BIT_MASK(RISCV_ISA_EXT_##ext) + +bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit); +#define riscv_isa_extension_available(isa_bitmap, ext) \ + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext) + +#endif + +#endif /* _ASM_RISCV_HWCAP_H */ diff --git a/arch/riscv/include/asm/image.h b/arch/riscv/include/asm/image.h new file mode 100644 index 000000000..e0b319af3 --- /dev/null +++ b/arch/riscv/include/asm/image.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_IMAGE_H +#define _ASM_RISCV_IMAGE_H + +#define RISCV_IMAGE_MAGIC "RISCV\0\0\0" +#define RISCV_IMAGE_MAGIC2 "RSC\x05" + +#define RISCV_IMAGE_FLAG_BE_SHIFT 0 +#define RISCV_IMAGE_FLAG_BE_MASK 0x1 + +#define RISCV_IMAGE_FLAG_LE 0 +#define RISCV_IMAGE_FLAG_BE 1 + +#ifdef CONFIG_CPU_BIG_ENDIAN +#error conversion of header fields to LE not yet implemented +#else +#define __HEAD_FLAG_BE RISCV_IMAGE_FLAG_LE +#endif + +#define __HEAD_FLAG(field) (__HEAD_FLAG_##field << \ + RISCV_IMAGE_FLAG_##field##_SHIFT) + +#define __HEAD_FLAGS (__HEAD_FLAG(BE)) + +#define RISCV_HEADER_VERSION_MAJOR 0 +#define RISCV_HEADER_VERSION_MINOR 2 + +#define RISCV_HEADER_VERSION (RISCV_HEADER_VERSION_MAJOR << 16 | \ + RISCV_HEADER_VERSION_MINOR) + +#ifndef __ASSEMBLY__ +/** + * struct riscv_image_header - riscv kernel image header + * @code0: Executable code + * @code1: Executable code + * @text_offset: Image load offset (little endian) + * @image_size: Effective Image size (little endian) + * @flags: kernel flags (little endian) + * @version: version + * @res1: reserved + * @res2: reserved + * @magic: Magic number (RISC-V specific; deprecated) + * @magic2: Magic number 2 (to match the ARM64 'magic' field pos) + * @res3: reserved (will be used for PE COFF offset) + * + * The intention is for this header format to be shared between multiple + * architectures to avoid a proliferation of image header formats. + */ + +struct riscv_image_header { + u32 code0; + u32 code1; + u64 text_offset; + u64 image_size; + u64 flags; + u32 version; + u32 res1; + u64 res2; + u64 magic; + u32 magic2; + u32 res3; +}; +#endif /* __ASSEMBLY__ */ +#endif /* _ASM_RISCV_IMAGE_H */ diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h new file mode 100644 index 000000000..391dd869d --- /dev/null +++ b/arch/riscv/include/asm/io.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * {read,write}{b,w,l,q} based on arch/arm64/include/asm/io.h + * which was based on arch/arm/include/io.h + * + * Copyright (C) 1996-2000 Russell King + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2014 Regents of the University of California + */ + +#ifndef _ASM_RISCV_IO_H +#define _ASM_RISCV_IO_H + +#include <linux/types.h> +#include <linux/pgtable.h> +#include <asm/mmiowb.h> +#include <asm/early_ioremap.h> + +/* + * MMIO access functions are separated out to break dependency cycles + * when using {read,write}* fns in low-level headers + */ +#include <asm/mmio.h> + +/* + * I/O port access constants. + */ +#ifdef CONFIG_MMU +#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1) +#define PCI_IOBASE ((void __iomem *)PCI_IO_START) +#endif /* CONFIG_MMU */ + +/* + * Emulation routines for the port-mapped IO space used by some PCI drivers. + * These are defined as being "fully synchronous", but also "not guaranteed to + * be fully ordered with respect to other memory and I/O operations". We're + * going to be on the safe side here and just make them: + * - Fully ordered WRT each other, by bracketing them with two fences. The + * outer set contains both I/O so inX is ordered with outX, while the inner just + * needs the type of the access (I for inX and O for outX). + * - Ordered in the same manner as readX/writeX WRT memory by subsuming their + * fences. + * - Ordered WRT timer reads, so udelay and friends don't get elided by the + * implementation. + * Note that there is no way to actually enforce that outX is a non-posted + * operation on RISC-V, but hopefully the timer ordering constraint is + * sufficient to ensure this works sanely on controllers that support I/O + * writes. + */ +#define __io_pbr() __asm__ __volatile__ ("fence io,i" : : : "memory"); +#define __io_par(v) __asm__ __volatile__ ("fence i,ior" : : : "memory"); +#define __io_pbw() __asm__ __volatile__ ("fence iow,o" : : : "memory"); +#define __io_paw() __asm__ __volatile__ ("fence o,io" : : : "memory"); + +#define inb(c) ({ u8 __v; __io_pbr(); __v = readb_cpu((void*)(PCI_IOBASE + (c))); __io_par(__v); __v; }) +#define inw(c) ({ u16 __v; __io_pbr(); __v = readw_cpu((void*)(PCI_IOBASE + (c))); __io_par(__v); __v; }) +#define inl(c) ({ u32 __v; __io_pbr(); __v = readl_cpu((void*)(PCI_IOBASE + (c))); __io_par(__v); __v; }) + +#define outb(v,c) ({ __io_pbw(); writeb_cpu((v),(void*)(PCI_IOBASE + (c))); __io_paw(); }) +#define outw(v,c) ({ __io_pbw(); writew_cpu((v),(void*)(PCI_IOBASE + (c))); __io_paw(); }) +#define outl(v,c) ({ __io_pbw(); writel_cpu((v),(void*)(PCI_IOBASE + (c))); __io_paw(); }) + +#ifdef CONFIG_64BIT +#define inq(c) ({ u64 __v; __io_pbr(); __v = readq_cpu((void*)(c)); __io_par(__v); __v; }) +#define outq(v,c) ({ __io_pbw(); writeq_cpu((v),(void*)(c)); __io_paw(); }) +#endif + +/* + * Accesses from a single hart to a single I/O address must be ordered. This + * allows us to use the raw read macros, but we still need to fence before and + * after the block to ensure ordering WRT other macros. These are defined to + * perform host-endian accesses so we use __raw instead of __cpu. + */ +#define __io_reads_ins(port, ctype, len, bfence, afence) \ + static inline void __ ## port ## len(const volatile void __iomem *addr, \ + void *buffer, \ + unsigned int count) \ + { \ + bfence; \ + if (count) { \ + ctype *buf = buffer; \ + \ + do { \ + ctype x = __raw_read ## len(addr); \ + *buf++ = x; \ + } while (--count); \ + } \ + afence; \ + } + +#define __io_writes_outs(port, ctype, len, bfence, afence) \ + static inline void __ ## port ## len(volatile void __iomem *addr, \ + const void *buffer, \ + unsigned int count) \ + { \ + bfence; \ + if (count) { \ + const ctype *buf = buffer; \ + \ + do { \ + __raw_write ## len(*buf++, addr); \ + } while (--count); \ + } \ + afence; \ + } + +__io_reads_ins(reads, u8, b, __io_br(), __io_ar(addr)) +__io_reads_ins(reads, u16, w, __io_br(), __io_ar(addr)) +__io_reads_ins(reads, u32, l, __io_br(), __io_ar(addr)) +#define readsb(addr, buffer, count) __readsb(addr, buffer, count) +#define readsw(addr, buffer, count) __readsw(addr, buffer, count) +#define readsl(addr, buffer, count) __readsl(addr, buffer, count) + +__io_reads_ins(ins, u8, b, __io_pbr(), __io_par(addr)) +__io_reads_ins(ins, u16, w, __io_pbr(), __io_par(addr)) +__io_reads_ins(ins, u32, l, __io_pbr(), __io_par(addr)) +#define insb(addr, buffer, count) __insb(PCI_IOBASE + (addr), buffer, count) +#define insw(addr, buffer, count) __insw(PCI_IOBASE + (addr), buffer, count) +#define insl(addr, buffer, count) __insl(PCI_IOBASE + (addr), buffer, count) + +__io_writes_outs(writes, u8, b, __io_bw(), __io_aw()) +__io_writes_outs(writes, u16, w, __io_bw(), __io_aw()) +__io_writes_outs(writes, u32, l, __io_bw(), __io_aw()) +#define writesb(addr, buffer, count) __writesb(addr, buffer, count) +#define writesw(addr, buffer, count) __writesw(addr, buffer, count) +#define writesl(addr, buffer, count) __writesl(addr, buffer, count) + +__io_writes_outs(outs, u8, b, __io_pbw(), __io_paw()) +__io_writes_outs(outs, u16, w, __io_pbw(), __io_paw()) +__io_writes_outs(outs, u32, l, __io_pbw(), __io_paw()) +#define outsb(addr, buffer, count) __outsb(PCI_IOBASE + (addr), buffer, count) +#define outsw(addr, buffer, count) __outsw(PCI_IOBASE + (addr), buffer, count) +#define outsl(addr, buffer, count) __outsl(PCI_IOBASE + (addr), buffer, count) + +#ifdef CONFIG_64BIT +__io_reads_ins(reads, u64, q, __io_br(), __io_ar(addr)) +#define readsq(addr, buffer, count) __readsq(addr, buffer, count) + +__io_reads_ins(ins, u64, q, __io_pbr(), __io_par(addr)) +#define insq(addr, buffer, count) __insq(PCI_IOBASE + (addr), buffer, count) + +__io_writes_outs(writes, u64, q, __io_bw(), __io_aw()) +#define writesq(addr, buffer, count) __writesq(addr, buffer, count) + +__io_writes_outs(outs, u64, q, __io_pbr(), __io_paw()) +#define outsq(addr, buffer, count) __outsq(PCI_IOBASE + (addr), buffer, count) +#endif + +#include <asm-generic/io.h> + +#endif /* _ASM_RISCV_IO_H */ diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h new file mode 100644 index 000000000..9807ad164 --- /dev/null +++ b/arch/riscv/include/asm/irq.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#ifndef _ASM_RISCV_IRQ_H +#define _ASM_RISCV_IRQ_H + +#include <linux/interrupt.h> +#include <linux/linkage.h> + +#include <asm-generic/irq.h> + +#endif /* _ASM_RISCV_IRQ_H */ diff --git a/arch/riscv/include/asm/irq_work.h b/arch/riscv/include/asm/irq_work.h new file mode 100644 index 000000000..b53891964 --- /dev/null +++ b/arch/riscv/include/asm/irq_work.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_IRQ_WORK_H +#define _ASM_RISCV_IRQ_WORK_H + +static inline bool arch_irq_work_has_interrupt(void) +{ + return IS_ENABLED(CONFIG_SMP); +} +extern void arch_irq_work_raise(void); +#endif /* _ASM_RISCV_IRQ_WORK_H */ diff --git a/arch/riscv/include/asm/irqflags.h b/arch/riscv/include/asm/irqflags.h new file mode 100644 index 000000000..08d4d6a5b --- /dev/null +++ b/arch/riscv/include/asm/irqflags.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + + +#ifndef _ASM_RISCV_IRQFLAGS_H +#define _ASM_RISCV_IRQFLAGS_H + +#include <asm/processor.h> +#include <asm/csr.h> + +/* read interrupt enabled status */ +static inline unsigned long arch_local_save_flags(void) +{ + return csr_read(CSR_STATUS); +} + +/* unconditionally enable interrupts */ +static inline void arch_local_irq_enable(void) +{ + csr_set(CSR_STATUS, SR_IE); +} + +/* unconditionally disable interrupts */ +static inline void arch_local_irq_disable(void) +{ + csr_clear(CSR_STATUS, SR_IE); +} + +/* get status and disable interrupts */ +static inline unsigned long arch_local_irq_save(void) +{ + return csr_read_clear(CSR_STATUS, SR_IE); +} + +/* test flags */ +static inline int arch_irqs_disabled_flags(unsigned long flags) +{ + return !(flags & SR_IE); +} + +/* test hardware interrupt enable bit */ +static inline int arch_irqs_disabled(void) +{ + return arch_irqs_disabled_flags(arch_local_save_flags()); +} + +/* set interrupt enabled status */ +static inline void arch_local_irq_restore(unsigned long flags) +{ + csr_set(CSR_STATUS, flags & SR_IE); +} + +#endif /* _ASM_RISCV_IRQFLAGS_H */ diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h new file mode 100644 index 000000000..729991e8f --- /dev/null +++ b/arch/riscv/include/asm/jump_label.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Emil Renner Berthing + * + * Based on arch/arm64/include/asm/jump_label.h + */ +#ifndef __ASM_JUMP_LABEL_H +#define __ASM_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + +#include <linux/types.h> +#include <asm/asm.h> + +#define JUMP_LABEL_NOP_SIZE 4 + +static __always_inline bool arch_static_branch(struct static_key *key, + bool branch) +{ + asm_volatile_goto( + " .align 2 \n\t" + " .option push \n\t" + " .option norelax \n\t" + " .option norvc \n\t" + "1: nop \n\t" + " .option pop \n\t" + " .pushsection __jump_table, \"aw\" \n\t" + " .align " RISCV_LGPTR " \n\t" + " .long 1b - ., %l[label] - . \n\t" + " " RISCV_PTR " %0 - . \n\t" + " .popsection \n\t" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, + bool branch) +{ + asm_volatile_goto( + " .align 2 \n\t" + " .option push \n\t" + " .option norelax \n\t" + " .option norvc \n\t" + "1: jal zero, %l[label] \n\t" + " .option pop \n\t" + " .pushsection __jump_table, \"aw\" \n\t" + " .align " RISCV_LGPTR " \n\t" + " .long 1b - ., %l[label] - . \n\t" + " " RISCV_PTR " %0 - . \n\t" + " .popsection \n\t" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h new file mode 100644 index 000000000..db10a64e3 --- /dev/null +++ b/arch/riscv/include/asm/kasan.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2019 Andes Technology Corporation */ + +#ifndef __ASM_KASAN_H +#define __ASM_KASAN_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_KASAN + +#define KASAN_SHADOW_SCALE_SHIFT 3 + +#define KASAN_SHADOW_SIZE (UL(1) << (38 - KASAN_SHADOW_SCALE_SHIFT)) +#define KASAN_SHADOW_START KERN_VIRT_START /* 2^64 - 2^38 */ +#define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) + +#define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL) + +void kasan_init(void); +asmlinkage void kasan_early_init(void); + +#endif +#endif +#endif /* __ASM_KASAN_H */ diff --git a/arch/riscv/include/asm/kdebug.h b/arch/riscv/include/asm/kdebug.h new file mode 100644 index 000000000..85ac00411 --- /dev/null +++ b/arch/riscv/include/asm/kdebug.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_ARC_KDEBUG_H +#define _ASM_ARC_KDEBUG_H + +enum die_val { + DIE_UNUSED, + DIE_TRAP, + DIE_OOPS +}; + +#endif diff --git a/arch/riscv/include/asm/kgdb.h b/arch/riscv/include/asm/kgdb.h new file mode 100644 index 000000000..46677daf7 --- /dev/null +++ b/arch/riscv/include/asm/kgdb.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_KGDB_H_ +#define __ASM_KGDB_H_ + +#ifdef __KERNEL__ + +#define GDB_SIZEOF_REG sizeof(unsigned long) + +#define DBG_MAX_REG_NUM (36) +#define NUMREGBYTES ((DBG_MAX_REG_NUM) * GDB_SIZEOF_REG) +#define CACHE_FLUSH_IS_SAFE 1 +#define BUFMAX 2048 +#ifdef CONFIG_RISCV_ISA_C +#define BREAK_INSTR_SIZE 2 +#else +#define BREAK_INSTR_SIZE 4 +#endif + +#ifndef __ASSEMBLY__ + +extern unsigned long kgdb_compiled_break; + +static inline void arch_kgdb_breakpoint(void) +{ + asm(".global kgdb_compiled_break\n" + ".option norvc\n" + "kgdb_compiled_break: ebreak\n" + ".option rvc\n"); +} + +#endif /* !__ASSEMBLY__ */ + +#define DBG_REG_ZERO "zero" +#define DBG_REG_RA "ra" +#define DBG_REG_SP "sp" +#define DBG_REG_GP "gp" +#define DBG_REG_TP "tp" +#define DBG_REG_T0 "t0" +#define DBG_REG_T1 "t1" +#define DBG_REG_T2 "t2" +#define DBG_REG_FP "fp" +#define DBG_REG_S1 "s1" +#define DBG_REG_A0 "a0" +#define DBG_REG_A1 "a1" +#define DBG_REG_A2 "a2" +#define DBG_REG_A3 "a3" +#define DBG_REG_A4 "a4" +#define DBG_REG_A5 "a5" +#define DBG_REG_A6 "a6" +#define DBG_REG_A7 "a7" +#define DBG_REG_S2 "s2" +#define DBG_REG_S3 "s3" +#define DBG_REG_S4 "s4" +#define DBG_REG_S5 "s5" +#define DBG_REG_S6 "s6" +#define DBG_REG_S7 "s7" +#define DBG_REG_S8 "s8" +#define DBG_REG_S9 "s9" +#define DBG_REG_S10 "s10" +#define DBG_REG_S11 "s11" +#define DBG_REG_T3 "t3" +#define DBG_REG_T4 "t4" +#define DBG_REG_T5 "t5" +#define DBG_REG_T6 "t6" +#define DBG_REG_EPC "pc" +#define DBG_REG_STATUS "sstatus" +#define DBG_REG_BADADDR "stval" +#define DBG_REG_CAUSE "scause" + +#define DBG_REG_ZERO_OFF 0 +#define DBG_REG_RA_OFF 1 +#define DBG_REG_SP_OFF 2 +#define DBG_REG_GP_OFF 3 +#define DBG_REG_TP_OFF 4 +#define DBG_REG_T0_OFF 5 +#define DBG_REG_T1_OFF 6 +#define DBG_REG_T2_OFF 7 +#define DBG_REG_FP_OFF 8 +#define DBG_REG_S1_OFF 9 +#define DBG_REG_A0_OFF 10 +#define DBG_REG_A1_OFF 11 +#define DBG_REG_A2_OFF 12 +#define DBG_REG_A3_OFF 13 +#define DBG_REG_A4_OFF 14 +#define DBG_REG_A5_OFF 15 +#define DBG_REG_A6_OFF 16 +#define DBG_REG_A7_OFF 17 +#define DBG_REG_S2_OFF 18 +#define DBG_REG_S3_OFF 19 +#define DBG_REG_S4_OFF 20 +#define DBG_REG_S5_OFF 21 +#define DBG_REG_S6_OFF 22 +#define DBG_REG_S7_OFF 23 +#define DBG_REG_S8_OFF 24 +#define DBG_REG_S9_OFF 25 +#define DBG_REG_S10_OFF 26 +#define DBG_REG_S11_OFF 27 +#define DBG_REG_T3_OFF 28 +#define DBG_REG_T4_OFF 29 +#define DBG_REG_T5_OFF 30 +#define DBG_REG_T6_OFF 31 +#define DBG_REG_EPC_OFF 32 +#define DBG_REG_STATUS_OFF 33 +#define DBG_REG_BADADDR_OFF 34 +#define DBG_REG_CAUSE_OFF 35 + +extern const char riscv_gdb_stub_feature[64]; + +#define kgdb_arch_gdb_stub_feature riscv_gdb_stub_feature + +#endif +#endif diff --git a/arch/riscv/include/asm/kprobes.h b/arch/riscv/include/asm/kprobes.h new file mode 100644 index 000000000..56a98ea30 --- /dev/null +++ b/arch/riscv/include/asm/kprobes.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copied from arch/arm64/include/asm/kprobes.h + * + * Copyright (C) 2013 Linaro Limited + * Copyright (C) 2017 SiFive + */ + +#ifndef _ASM_RISCV_KPROBES_H +#define _ASM_RISCV_KPROBES_H + +#include <asm-generic/kprobes.h> + +#endif /* _ASM_RISCV_KPROBES_H */ diff --git a/arch/riscv/include/asm/linkage.h b/arch/riscv/include/asm/linkage.h new file mode 100644 index 000000000..9e88ba23c --- /dev/null +++ b/arch/riscv/include/asm/linkage.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2015 Regents of the University of California + */ + +#ifndef _ASM_RISCV_LINKAGE_H +#define _ASM_RISCV_LINKAGE_H + +#define __ALIGN .balign 4 +#define __ALIGN_STR ".balign 4" + +#endif /* _ASM_RISCV_LINKAGE_H */ diff --git a/arch/riscv/include/asm/mmio.h b/arch/riscv/include/asm/mmio.h new file mode 100644 index 000000000..4c58ee7f9 --- /dev/null +++ b/arch/riscv/include/asm/mmio.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * {read,write}{b,w,l,q} based on arch/arm64/include/asm/io.h + * which was based on arch/arm/include/io.h + * + * Copyright (C) 1996-2000 Russell King + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2014 Regents of the University of California + */ + +#ifndef _ASM_RISCV_MMIO_H +#define _ASM_RISCV_MMIO_H + +#include <linux/types.h> +#include <asm/mmiowb.h> + +/* Generic IO read/write. These perform native-endian accesses. */ +#define __raw_writeb __raw_writeb +static inline void __raw_writeb(u8 val, volatile void __iomem *addr) +{ + asm volatile("sb %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#define __raw_writew __raw_writew +static inline void __raw_writew(u16 val, volatile void __iomem *addr) +{ + asm volatile("sh %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#define __raw_writel __raw_writel +static inline void __raw_writel(u32 val, volatile void __iomem *addr) +{ + asm volatile("sw %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#ifdef CONFIG_64BIT +#define __raw_writeq __raw_writeq +static inline void __raw_writeq(u64 val, volatile void __iomem *addr) +{ + asm volatile("sd %0, 0(%1)" : : "r" (val), "r" (addr)); +} +#endif + +#define __raw_readb __raw_readb +static inline u8 __raw_readb(const volatile void __iomem *addr) +{ + u8 val; + + asm volatile("lb %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +#define __raw_readw __raw_readw +static inline u16 __raw_readw(const volatile void __iomem *addr) +{ + u16 val; + + asm volatile("lh %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +#define __raw_readl __raw_readl +static inline u32 __raw_readl(const volatile void __iomem *addr) +{ + u32 val; + + asm volatile("lw %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +#ifdef CONFIG_64BIT +#define __raw_readq __raw_readq +static inline u64 __raw_readq(const volatile void __iomem *addr) +{ + u64 val; + + asm volatile("ld %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} +#endif + +/* + * Unordered I/O memory access primitives. These are even more relaxed than + * the relaxed versions, as they don't even order accesses between successive + * operations to the I/O regions. + */ +#define readb_cpu(c) ({ u8 __r = __raw_readb(c); __r; }) +#define readw_cpu(c) ({ u16 __r = le16_to_cpu((__force __le16)__raw_readw(c)); __r; }) +#define readl_cpu(c) ({ u32 __r = le32_to_cpu((__force __le32)__raw_readl(c)); __r; }) + +#define writeb_cpu(v, c) ((void)__raw_writeb((v), (c))) +#define writew_cpu(v, c) ((void)__raw_writew((__force u16)cpu_to_le16(v), (c))) +#define writel_cpu(v, c) ((void)__raw_writel((__force u32)cpu_to_le32(v), (c))) + +#ifdef CONFIG_64BIT +#define readq_cpu(c) ({ u64 __r = le64_to_cpu((__force __le64)__raw_readq(c)); __r; }) +#define writeq_cpu(v, c) ((void)__raw_writeq((__force u64)cpu_to_le64(v), (c))) +#endif + +/* + * Relaxed I/O memory access primitives. These follow the Device memory + * ordering rules but do not guarantee any ordering relative to Normal memory + * accesses. These are defined to order the indicated access (either a read or + * write) with all other I/O memory accesses to the same peripheral. Since the + * platform specification defines that all I/O regions are strongly ordered on + * channel 0, no explicit fences are required to enforce this ordering. + */ +/* FIXME: These are now the same as asm-generic */ +#define __io_rbr() do {} while (0) +#define __io_rar() do {} while (0) +#define __io_rbw() do {} while (0) +#define __io_raw() do {} while (0) + +#define readb_relaxed(c) ({ u8 __v; __io_rbr(); __v = readb_cpu(c); __io_rar(); __v; }) +#define readw_relaxed(c) ({ u16 __v; __io_rbr(); __v = readw_cpu(c); __io_rar(); __v; }) +#define readl_relaxed(c) ({ u32 __v; __io_rbr(); __v = readl_cpu(c); __io_rar(); __v; }) + +#define writeb_relaxed(v, c) ({ __io_rbw(); writeb_cpu((v), (c)); __io_raw(); }) +#define writew_relaxed(v, c) ({ __io_rbw(); writew_cpu((v), (c)); __io_raw(); }) +#define writel_relaxed(v, c) ({ __io_rbw(); writel_cpu((v), (c)); __io_raw(); }) + +#ifdef CONFIG_64BIT +#define readq_relaxed(c) ({ u64 __v; __io_rbr(); __v = readq_cpu(c); __io_rar(); __v; }) +#define writeq_relaxed(v, c) ({ __io_rbw(); writeq_cpu((v), (c)); __io_raw(); }) +#endif + +/* + * I/O memory access primitives. Reads are ordered relative to any following + * Normal memory read and delay() loop. Writes are ordered relative to any + * prior Normal memory write. The memory barriers here are necessary as RISC-V + * doesn't define any ordering between the memory space and the I/O space. + */ +#define __io_br() do {} while (0) +#define __io_ar(v) ({ __asm__ __volatile__ ("fence i,ir" : : : "memory"); }) +#define __io_bw() ({ __asm__ __volatile__ ("fence w,o" : : : "memory"); }) +#define __io_aw() mmiowb_set_pending() + +#define readb(c) ({ u8 __v; __io_br(); __v = readb_cpu(c); __io_ar(__v); __v; }) +#define readw(c) ({ u16 __v; __io_br(); __v = readw_cpu(c); __io_ar(__v); __v; }) +#define readl(c) ({ u32 __v; __io_br(); __v = readl_cpu(c); __io_ar(__v); __v; }) + +#define writeb(v, c) ({ __io_bw(); writeb_cpu((v), (c)); __io_aw(); }) +#define writew(v, c) ({ __io_bw(); writew_cpu((v), (c)); __io_aw(); }) +#define writel(v, c) ({ __io_bw(); writel_cpu((v), (c)); __io_aw(); }) + +#ifdef CONFIG_64BIT +#define readq(c) ({ u64 __v; __io_br(); __v = readq_cpu(c); __io_ar(__v); __v; }) +#define writeq(v, c) ({ __io_bw(); writeq_cpu((v), (c)); __io_aw(); }) +#endif + +#endif /* _ASM_RISCV_MMIO_H */ diff --git a/arch/riscv/include/asm/mmiowb.h b/arch/riscv/include/asm/mmiowb.h new file mode 100644 index 000000000..0b2333e71 --- /dev/null +++ b/arch/riscv/include/asm/mmiowb.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_MMIOWB_H +#define _ASM_RISCV_MMIOWB_H + +/* + * "o,w" is sufficient to ensure that all writes to the device have completed + * before the write to the spinlock is allowed to commit. + */ +#define mmiowb() __asm__ __volatile__ ("fence o,w" : : : "memory"); + +#include <linux/smp.h> +#include <asm-generic/mmiowb.h> + +#endif /* _ASM_RISCV_MMIOWB_H */ diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h new file mode 100644 index 000000000..dabcf2cfb --- /dev/null +++ b/arch/riscv/include/asm/mmu.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + + +#ifndef _ASM_RISCV_MMU_H +#define _ASM_RISCV_MMU_H + +#ifndef __ASSEMBLY__ + +typedef struct { +#ifndef CONFIG_MMU + unsigned long end_brk; +#endif + void *vdso; +#ifdef CONFIG_SMP + /* A local icache flush is needed before user execution can resume. */ + cpumask_t icache_stale_mask; +#endif +} mm_context_t; + +void __init create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, + phys_addr_t sz, pgprot_t prot); +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_RISCV_MMU_H */ diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h new file mode 100644 index 000000000..67c463812 --- /dev/null +++ b/arch/riscv/include/asm/mmu_context.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#ifndef _ASM_RISCV_MMU_CONTEXT_H +#define _ASM_RISCV_MMU_CONTEXT_H + +#include <linux/mm_types.h> +#include <asm-generic/mm_hooks.h> + +#include <linux/mm.h> +#include <linux/sched.h> + +static inline void enter_lazy_tlb(struct mm_struct *mm, + struct task_struct *task) +{ +} + +/* Initialize context-related info for a new mm_struct */ +static inline int init_new_context(struct task_struct *task, + struct mm_struct *mm) +{ + return 0; +} + +static inline void destroy_context(struct mm_struct *mm) +{ +} + +void switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *task); + +static inline void activate_mm(struct mm_struct *prev, + struct mm_struct *next) +{ + switch_mm(prev, next, NULL); +} + +static inline void deactivate_mm(struct task_struct *task, + struct mm_struct *mm) +{ +} + +#endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/module.h b/arch/riscv/include/asm/module.h new file mode 100644 index 000000000..76aa96a9f --- /dev/null +++ b/arch/riscv/include/asm/module.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2017 Andes Technology Corporation */ + +#ifndef _ASM_RISCV_MODULE_H +#define _ASM_RISCV_MODULE_H + +#include <asm-generic/module.h> + +struct module; +unsigned long module_emit_got_entry(struct module *mod, unsigned long val); +unsigned long module_emit_plt_entry(struct module *mod, unsigned long val); + +#ifdef CONFIG_MODULE_SECTIONS +struct mod_section { + Elf_Shdr *shdr; + int num_entries; + int max_entries; +}; + +struct mod_arch_specific { + struct mod_section got; + struct mod_section plt; + struct mod_section got_plt; +}; + +struct got_entry { + unsigned long symbol_addr; /* the real variable address */ +}; + +static inline struct got_entry emit_got_entry(unsigned long val) +{ + return (struct got_entry) {val}; +} + +static inline struct got_entry *get_got_entry(unsigned long val, + const struct mod_section *sec) +{ + struct got_entry *got = (struct got_entry *)(sec->shdr->sh_addr); + int i; + for (i = 0; i < sec->num_entries; i++) { + if (got[i].symbol_addr == val) + return &got[i]; + } + return NULL; +} + +struct plt_entry { + /* + * Trampoline code to real target address. The return address + * should be the original (pc+4) before entring plt entry. + */ + u32 insn_auipc; /* auipc t0, 0x0 */ + u32 insn_ld; /* ld t1, 0x10(t0) */ + u32 insn_jr; /* jr t1 */ +}; + +#define OPC_AUIPC 0x0017 +#define OPC_LD 0x3003 +#define OPC_JALR 0x0067 +#define REG_T0 0x5 +#define REG_T1 0x6 + +static inline struct plt_entry emit_plt_entry(unsigned long val, + unsigned long plt, + unsigned long got_plt) +{ + /* + * U-Type encoding: + * +------------+----------+----------+ + * | imm[31:12] | rd[11:7] | opc[6:0] | + * +------------+----------+----------+ + * + * I-Type encoding: + * +------------+------------+--------+----------+----------+ + * | imm[31:20] | rs1[19:15] | funct3 | rd[11:7] | opc[6:0] | + * +------------+------------+--------+----------+----------+ + * + */ + unsigned long offset = got_plt - plt; + u32 hi20 = (offset + 0x800) & 0xfffff000; + u32 lo12 = (offset - hi20); + return (struct plt_entry) { + OPC_AUIPC | (REG_T0 << 7) | hi20, + OPC_LD | (lo12 << 20) | (REG_T0 << 15) | (REG_T1 << 7), + OPC_JALR | (REG_T1 << 15) + }; +} + +static inline int get_got_plt_idx(unsigned long val, const struct mod_section *sec) +{ + struct got_entry *got_plt = (struct got_entry *)sec->shdr->sh_addr; + int i; + for (i = 0; i < sec->num_entries; i++) { + if (got_plt[i].symbol_addr == val) + return i; + } + return -1; +} + +static inline struct plt_entry *get_plt_entry(unsigned long val, + const struct mod_section *sec_plt, + const struct mod_section *sec_got_plt) +{ + struct plt_entry *plt = (struct plt_entry *)sec_plt->shdr->sh_addr; + int got_plt_idx = get_got_plt_idx(val, sec_got_plt); + if (got_plt_idx >= 0) + return plt + got_plt_idx; + else + return NULL; +} + +#endif /* CONFIG_MODULE_SECTIONS */ + +#endif /* _ASM_RISCV_MODULE_H */ diff --git a/arch/riscv/include/asm/module.lds.h b/arch/riscv/include/asm/module.lds.h new file mode 100644 index 000000000..1075beae1 --- /dev/null +++ b/arch/riscv/include/asm/module.lds.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2017 Andes Technology Corporation */ +#ifdef CONFIG_MODULE_SECTIONS +SECTIONS { + .plt : { BYTE(0) } + .got : { BYTE(0) } + .got.plt : { BYTE(0) } +} +#endif diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h new file mode 100644 index 000000000..64a675c5c --- /dev/null +++ b/arch/riscv/include/asm/page.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com> + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + * Copyright (C) 2017 XiaojingZhu <zhuxiaoj@ict.ac.cn> + */ + +#ifndef _ASM_RISCV_PAGE_H +#define _ASM_RISCV_PAGE_H + +#include <linux/pfn.h> +#include <linux/const.h> + +#define PAGE_SHIFT (12) +#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) +#define PAGE_MASK (~(PAGE_SIZE - 1)) + +#ifdef CONFIG_64BIT +#define HUGE_MAX_HSTATE 2 +#else +#define HUGE_MAX_HSTATE 1 +#endif +#define HPAGE_SHIFT PMD_SHIFT +#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + +/* + * PAGE_OFFSET -- the first address of the first page of memory. + * When not using MMU this corresponds to the first free page in + * physical memory (aligned on a page boundary). + */ +#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) + +#define KERN_VIRT_SIZE (-PAGE_OFFSET) + +#ifndef __ASSEMBLY__ + +#define PAGE_UP(addr) (((addr)+((PAGE_SIZE)-1))&(~((PAGE_SIZE)-1))) +#define PAGE_DOWN(addr) ((addr)&(~((PAGE_SIZE)-1))) + +/* align addr on a size boundary - adjust address up/down if needed */ +#define _ALIGN_UP(addr, size) (((addr)+((size)-1))&(~((size)-1))) +#define _ALIGN_DOWN(addr, size) ((addr)&(~((size)-1))) + +/* align addr on a size boundary - adjust address up if needed */ +#define _ALIGN(addr, size) _ALIGN_UP(addr, size) + +#define clear_page(pgaddr) memset((pgaddr), 0, PAGE_SIZE) +#define copy_page(to, from) memcpy((to), (from), PAGE_SIZE) + +#define clear_user_page(pgaddr, vaddr, page) memset((pgaddr), 0, PAGE_SIZE) +#define copy_user_page(vto, vfrom, vaddr, topg) \ + memcpy((vto), (vfrom), PAGE_SIZE) + +/* + * Use struct definitions to apply C type checking + */ + +/* Page Global Directory entry */ +typedef struct { + unsigned long pgd; +} pgd_t; + +/* Page Table entry */ +typedef struct { + unsigned long pte; +} pte_t; + +typedef struct { + unsigned long pgprot; +} pgprot_t; + +typedef struct page *pgtable_t; + +#define pte_val(x) ((x).pte) +#define pgd_val(x) ((x).pgd) +#define pgprot_val(x) ((x).pgprot) + +#define __pte(x) ((pte_t) { (x) }) +#define __pgd(x) ((pgd_t) { (x) }) +#define __pgprot(x) ((pgprot_t) { (x) }) + +#ifdef CONFIG_64BIT +#define PTE_FMT "%016lx" +#else +#define PTE_FMT "%08lx" +#endif + +#ifdef CONFIG_MMU +extern unsigned long va_pa_offset; +extern unsigned long pfn_base; +#define ARCH_PFN_OFFSET (pfn_base) +#else +#define va_pa_offset 0 +#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) +#endif /* CONFIG_MMU */ + +extern unsigned long max_low_pfn; +extern unsigned long min_low_pfn; + +#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset)) +#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset) + +#ifdef CONFIG_DEBUG_VIRTUAL +extern phys_addr_t __virt_to_phys(unsigned long x); +extern phys_addr_t __phys_addr_symbol(unsigned long x); +#else +#define __virt_to_phys(x) __va_to_pa_nodebug(x) +#define __phys_addr_symbol(x) __va_to_pa_nodebug(x) +#endif /* CONFIG_DEBUG_VIRTUAL */ + +#define __pa_symbol(x) __phys_addr_symbol(RELOC_HIDE((unsigned long)(x), 0)) +#define __pa(x) __virt_to_phys((unsigned long)(x)) +#define __va(x) ((void *)__pa_to_va_nodebug((phys_addr_t)(x))) + +#define phys_to_pfn(phys) (PFN_DOWN(phys)) +#define pfn_to_phys(pfn) (PFN_PHYS(pfn)) + +#define virt_to_pfn(vaddr) (phys_to_pfn(__pa(vaddr))) +#define pfn_to_virt(pfn) (__va(pfn_to_phys(pfn))) + +#define virt_to_page(vaddr) (pfn_to_page(virt_to_pfn(vaddr))) +#define page_to_virt(page) (pfn_to_virt(page_to_pfn(page))) + +#define page_to_phys(page) (pfn_to_phys(page_to_pfn(page))) +#define page_to_bus(page) (page_to_phys(page)) +#define phys_to_page(paddr) (pfn_to_page(phys_to_pfn(paddr))) + +#ifdef CONFIG_FLATMEM +#define pfn_valid(pfn) \ + (((pfn) >= ARCH_PFN_OFFSET) && (((pfn) - ARCH_PFN_OFFSET) < max_mapnr)) +#endif + +#endif /* __ASSEMBLY__ */ + +#define virt_addr_valid(vaddr) ({ \ + unsigned long _addr = (unsigned long)vaddr; \ + (unsigned long)(_addr) >= PAGE_OFFSET && pfn_valid(virt_to_pfn(_addr)); \ +}) + +#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_NON_EXEC + +#include <asm-generic/memory_model.h> +#include <asm-generic/getorder.h> + +#endif /* _ASM_RISCV_PAGE_H */ diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/parse_asm.h new file mode 100644 index 000000000..ad254da85 --- /dev/null +++ b/arch/riscv/include/asm/parse_asm.h @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 SiFive + */ + +#ifndef _ASM_RISCV_INSN_H +#define _ASM_RISCV_INSN_H + +#include <linux/bits.h> + +/* The bit field of immediate value in I-type instruction */ +#define I_IMM_SIGN_OPOFF 31 +#define I_IMM_11_0_OPOFF 20 +#define I_IMM_SIGN_OFF 12 +#define I_IMM_11_0_OFF 0 +#define I_IMM_11_0_MASK GENMASK(11, 0) + +/* The bit field of immediate value in J-type instruction */ +#define J_IMM_SIGN_OPOFF 31 +#define J_IMM_10_1_OPOFF 21 +#define J_IMM_11_OPOFF 20 +#define J_IMM_19_12_OPOFF 12 +#define J_IMM_SIGN_OFF 20 +#define J_IMM_10_1_OFF 1 +#define J_IMM_11_OFF 11 +#define J_IMM_19_12_OFF 12 +#define J_IMM_10_1_MASK GENMASK(9, 0) +#define J_IMM_11_MASK GENMASK(0, 0) +#define J_IMM_19_12_MASK GENMASK(7, 0) + +/* The bit field of immediate value in B-type instruction */ +#define B_IMM_SIGN_OPOFF 31 +#define B_IMM_10_5_OPOFF 25 +#define B_IMM_4_1_OPOFF 8 +#define B_IMM_11_OPOFF 7 +#define B_IMM_SIGN_OFF 12 +#define B_IMM_10_5_OFF 5 +#define B_IMM_4_1_OFF 1 +#define B_IMM_11_OFF 11 +#define B_IMM_10_5_MASK GENMASK(5, 0) +#define B_IMM_4_1_MASK GENMASK(3, 0) +#define B_IMM_11_MASK GENMASK(0, 0) + +/* The register offset in RVG instruction */ +#define RVG_RS1_OPOFF 15 +#define RVG_RS2_OPOFF 20 +#define RVG_RD_OPOFF 7 + +/* The bit field of immediate value in RVC J instruction */ +#define RVC_J_IMM_SIGN_OPOFF 12 +#define RVC_J_IMM_4_OPOFF 11 +#define RVC_J_IMM_9_8_OPOFF 9 +#define RVC_J_IMM_10_OPOFF 8 +#define RVC_J_IMM_6_OPOFF 7 +#define RVC_J_IMM_7_OPOFF 6 +#define RVC_J_IMM_3_1_OPOFF 3 +#define RVC_J_IMM_5_OPOFF 2 +#define RVC_J_IMM_SIGN_OFF 11 +#define RVC_J_IMM_4_OFF 4 +#define RVC_J_IMM_9_8_OFF 8 +#define RVC_J_IMM_10_OFF 10 +#define RVC_J_IMM_6_OFF 6 +#define RVC_J_IMM_7_OFF 7 +#define RVC_J_IMM_3_1_OFF 1 +#define RVC_J_IMM_5_OFF 5 +#define RVC_J_IMM_4_MASK GENMASK(0, 0) +#define RVC_J_IMM_9_8_MASK GENMASK(1, 0) +#define RVC_J_IMM_10_MASK GENMASK(0, 0) +#define RVC_J_IMM_6_MASK GENMASK(0, 0) +#define RVC_J_IMM_7_MASK GENMASK(0, 0) +#define RVC_J_IMM_3_1_MASK GENMASK(2, 0) +#define RVC_J_IMM_5_MASK GENMASK(0, 0) + +/* The bit field of immediate value in RVC B instruction */ +#define RVC_B_IMM_SIGN_OPOFF 12 +#define RVC_B_IMM_4_3_OPOFF 10 +#define RVC_B_IMM_7_6_OPOFF 5 +#define RVC_B_IMM_2_1_OPOFF 3 +#define RVC_B_IMM_5_OPOFF 2 +#define RVC_B_IMM_SIGN_OFF 8 +#define RVC_B_IMM_4_3_OFF 3 +#define RVC_B_IMM_7_6_OFF 6 +#define RVC_B_IMM_2_1_OFF 1 +#define RVC_B_IMM_5_OFF 5 +#define RVC_B_IMM_4_3_MASK GENMASK(1, 0) +#define RVC_B_IMM_7_6_MASK GENMASK(1, 0) +#define RVC_B_IMM_2_1_MASK GENMASK(1, 0) +#define RVC_B_IMM_5_MASK GENMASK(0, 0) + +/* The register offset in RVC op=C0 instruction */ +#define RVC_C0_RS1_OPOFF 7 +#define RVC_C0_RS2_OPOFF 2 +#define RVC_C0_RD_OPOFF 2 + +/* The register offset in RVC op=C1 instruction */ +#define RVC_C1_RS1_OPOFF 7 +#define RVC_C1_RS2_OPOFF 2 +#define RVC_C1_RD_OPOFF 7 + +/* The register offset in RVC op=C2 instruction */ +#define RVC_C2_RS1_OPOFF 7 +#define RVC_C2_RS2_OPOFF 2 +#define RVC_C2_RD_OPOFF 7 + +/* parts of opcode for RVG*/ +#define OPCODE_BRANCH 0x63 +#define OPCODE_JALR 0x67 +#define OPCODE_JAL 0x6f +#define OPCODE_SYSTEM 0x73 + +/* parts of opcode for RVC*/ +#define OPCODE_C_0 0x0 +#define OPCODE_C_1 0x1 +#define OPCODE_C_2 0x2 + +/* parts of funct3 code for I, M, A extension*/ +#define FUNCT3_JALR 0x0 +#define FUNCT3_BEQ 0x0 +#define FUNCT3_BNE 0x1000 +#define FUNCT3_BLT 0x4000 +#define FUNCT3_BGE 0x5000 +#define FUNCT3_BLTU 0x6000 +#define FUNCT3_BGEU 0x7000 + +/* parts of funct3 code for C extension*/ +#define FUNCT3_C_BEQZ 0xc000 +#define FUNCT3_C_BNEZ 0xe000 +#define FUNCT3_C_J 0xa000 +#define FUNCT3_C_JAL 0x2000 +#define FUNCT4_C_JR 0x8000 +#define FUNCT4_C_JALR 0x9000 + +#define FUNCT12_SRET 0x10200000 + +#define MATCH_JALR (FUNCT3_JALR | OPCODE_JALR) +#define MATCH_JAL (OPCODE_JAL) +#define MATCH_BEQ (FUNCT3_BEQ | OPCODE_BRANCH) +#define MATCH_BNE (FUNCT3_BNE | OPCODE_BRANCH) +#define MATCH_BLT (FUNCT3_BLT | OPCODE_BRANCH) +#define MATCH_BGE (FUNCT3_BGE | OPCODE_BRANCH) +#define MATCH_BLTU (FUNCT3_BLTU | OPCODE_BRANCH) +#define MATCH_BGEU (FUNCT3_BGEU | OPCODE_BRANCH) +#define MATCH_SRET (FUNCT12_SRET | OPCODE_SYSTEM) +#define MATCH_C_BEQZ (FUNCT3_C_BEQZ | OPCODE_C_1) +#define MATCH_C_BNEZ (FUNCT3_C_BNEZ | OPCODE_C_1) +#define MATCH_C_J (FUNCT3_C_J | OPCODE_C_1) +#define MATCH_C_JAL (FUNCT3_C_JAL | OPCODE_C_1) +#define MATCH_C_JR (FUNCT4_C_JR | OPCODE_C_2) +#define MATCH_C_JALR (FUNCT4_C_JALR | OPCODE_C_2) + +#define MASK_JALR 0x707f +#define MASK_JAL 0x7f +#define MASK_C_JALR 0xf07f +#define MASK_C_JR 0xf07f +#define MASK_C_JAL 0xe003 +#define MASK_C_J 0xe003 +#define MASK_BEQ 0x707f +#define MASK_BNE 0x707f +#define MASK_BLT 0x707f +#define MASK_BGE 0x707f +#define MASK_BLTU 0x707f +#define MASK_BGEU 0x707f +#define MASK_C_BEQZ 0xe003 +#define MASK_C_BNEZ 0xe003 +#define MASK_SRET 0xffffffff + +#define __INSN_LENGTH_MASK _UL(0x3) +#define __INSN_LENGTH_GE_32 _UL(0x3) +#define __INSN_OPCODE_MASK _UL(0x7F) +#define __INSN_BRANCH_OPCODE _UL(OPCODE_BRANCH) + +/* Define a series of is_XXX_insn functions to check if the value INSN + * is an instance of instruction XXX. + */ +#define DECLARE_INSN(INSN_NAME, INSN_MATCH, INSN_MASK) \ +static inline bool is_ ## INSN_NAME ## _insn(long insn) \ +{ \ + return (insn & (INSN_MASK)) == (INSN_MATCH); \ +} + +#define RV_IMM_SIGN(x) (-(((x) >> 31) & 1)) +#define RVC_IMM_SIGN(x) (-(((x) >> 12) & 1)) +#define RV_X(X, s, mask) (((X) >> (s)) & (mask)) +#define RVC_X(X, s, mask) RV_X(X, s, mask) + +#define EXTRACT_JTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X(x_, J_IMM_10_1_OPOFF, J_IMM_10_1_MASK) << J_IMM_10_1_OFF) | \ + (RV_X(x_, J_IMM_11_OPOFF, J_IMM_11_MASK) << J_IMM_11_OFF) | \ + (RV_X(x_, J_IMM_19_12_OPOFF, J_IMM_19_12_MASK) << J_IMM_19_12_OFF) | \ + (RV_IMM_SIGN(x_) << J_IMM_SIGN_OFF); }) + +#define EXTRACT_ITYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X(x_, I_IMM_11_0_OPOFF, I_IMM_11_0_MASK)) | \ + (RV_IMM_SIGN(x_) << I_IMM_SIGN_OFF); }) + +#define EXTRACT_BTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X(x_, B_IMM_4_1_OPOFF, B_IMM_4_1_MASK) << B_IMM_4_1_OFF) | \ + (RV_X(x_, B_IMM_10_5_OPOFF, B_IMM_10_5_MASK) << B_IMM_10_5_OFF) | \ + (RV_X(x_, B_IMM_11_OPOFF, B_IMM_11_MASK) << B_IMM_11_OFF) | \ + (RV_IMM_SIGN(x_) << B_IMM_SIGN_OFF); }) + +#define EXTRACT_RVC_J_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RVC_X(x_, RVC_J_IMM_3_1_OPOFF, RVC_J_IMM_3_1_MASK) << RVC_J_IMM_3_1_OFF) | \ + (RVC_X(x_, RVC_J_IMM_4_OPOFF, RVC_J_IMM_4_MASK) << RVC_J_IMM_4_OFF) | \ + (RVC_X(x_, RVC_J_IMM_5_OPOFF, RVC_J_IMM_5_MASK) << RVC_J_IMM_5_OFF) | \ + (RVC_X(x_, RVC_J_IMM_6_OPOFF, RVC_J_IMM_6_MASK) << RVC_J_IMM_6_OFF) | \ + (RVC_X(x_, RVC_J_IMM_7_OPOFF, RVC_J_IMM_7_MASK) << RVC_J_IMM_7_OFF) | \ + (RVC_X(x_, RVC_J_IMM_9_8_OPOFF, RVC_J_IMM_9_8_MASK) << RVC_J_IMM_9_8_OFF) | \ + (RVC_X(x_, RVC_J_IMM_10_OPOFF, RVC_J_IMM_10_MASK) << RVC_J_IMM_10_OFF) | \ + (RVC_IMM_SIGN(x_) << RVC_J_IMM_SIGN_OFF); }) + +#define EXTRACT_RVC_B_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RVC_X(x_, RVC_B_IMM_2_1_OPOFF, RVC_B_IMM_2_1_MASK) << RVC_B_IMM_2_1_OFF) | \ + (RVC_X(x_, RVC_B_IMM_4_3_OPOFF, RVC_B_IMM_4_3_MASK) << RVC_B_IMM_4_3_OFF) | \ + (RVC_X(x_, RVC_B_IMM_5_OPOFF, RVC_B_IMM_5_MASK) << RVC_B_IMM_5_OFF) | \ + (RVC_X(x_, RVC_B_IMM_7_6_OPOFF, RVC_B_IMM_7_6_MASK) << RVC_B_IMM_7_6_OFF) | \ + (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) + +#endif /* _ASM_RISCV_INSN_H */ diff --git a/arch/riscv/include/asm/patch.h b/arch/riscv/include/asm/patch.h new file mode 100644 index 000000000..98d9de07c --- /dev/null +++ b/arch/riscv/include/asm/patch.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 SiFive + */ + +#ifndef _ASM_RISCV_PATCH_H +#define _ASM_RISCV_PATCH_H + +int patch_text_nosync(void *addr, const void *insns, size_t len); +int patch_text(void *addr, u32 insn); + +extern int riscv_patch_in_stop_machine; + +#endif /* _ASM_RISCV_PATCH_H */ diff --git a/arch/riscv/include/asm/pci.h b/arch/riscv/include/asm/pci.h new file mode 100644 index 000000000..1c473a1bd --- /dev/null +++ b/arch/riscv/include/asm/pci.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 SiFive + */ + +#ifndef _ASM_RISCV_PCI_H +#define _ASM_RISCV_PCI_H + +#include <linux/types.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> + +#include <asm/io.h> + +#define PCIBIOS_MIN_IO 0 +#define PCIBIOS_MIN_MEM 0 + +/* RISC-V shim does not initialize PCI bus */ +#define pcibios_assign_all_busses() 1 + +extern int isa_dma_bridge_buggy; + +#ifdef CONFIG_PCI +static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) +{ + /* no legacy IRQ on risc-v */ + return -ENODEV; +} + +static inline int pci_proc_domain(struct pci_bus *bus) +{ + /* always show the domain in /proc */ + return 1; +} +#endif /* CONFIG_PCI */ + +#endif /* _ASM_RISCV_PCI_H */ diff --git a/arch/riscv/include/asm/perf_event.h b/arch/riscv/include/asm/perf_event.h new file mode 100644 index 000000000..062efd3a1 --- /dev/null +++ b/arch/riscv/include/asm/perf_event.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018 SiFive + * Copyright (C) 2018 Andes Technology Corporation + * + */ + +#ifndef _ASM_RISCV_PERF_EVENT_H +#define _ASM_RISCV_PERF_EVENT_H + +#include <linux/perf_event.h> +#include <linux/ptrace.h> +#include <linux/interrupt.h> + +#ifdef CONFIG_RISCV_BASE_PMU +#define RISCV_BASE_COUNTERS 2 + +/* + * The RISCV_MAX_COUNTERS parameter should be specified. + */ + +#define RISCV_MAX_COUNTERS 2 + +/* + * These are the indexes of bits in counteren register *minus* 1, + * except for cycle. It would be coherent if it can directly mapped + * to counteren bit definition, but there is a *time* register at + * counteren[1]. Per-cpu structure is scarce resource here. + * + * According to the spec, an implementation can support counter up to + * mhpmcounter31, but many high-end processors has at most 6 general + * PMCs, we give the definition to MHPMCOUNTER8 here. + */ +#define RISCV_PMU_CYCLE 0 +#define RISCV_PMU_INSTRET 1 +#define RISCV_PMU_MHPMCOUNTER3 2 +#define RISCV_PMU_MHPMCOUNTER4 3 +#define RISCV_PMU_MHPMCOUNTER5 4 +#define RISCV_PMU_MHPMCOUNTER6 5 +#define RISCV_PMU_MHPMCOUNTER7 6 +#define RISCV_PMU_MHPMCOUNTER8 7 + +#define RISCV_OP_UNSUPP (-EOPNOTSUPP) + +struct cpu_hw_events { + /* # currently enabled events*/ + int n_events; + /* currently enabled events */ + struct perf_event *events[RISCV_MAX_COUNTERS]; + /* vendor-defined PMU data */ + void *platform; +}; + +struct riscv_pmu { + struct pmu *pmu; + + /* generic hw/cache events table */ + const int *hw_events; + const int (*cache_events)[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; + /* method used to map hw/cache events */ + int (*map_hw_event)(u64 config); + int (*map_cache_event)(u64 config); + + /* max generic hw events in map */ + int max_events; + /* number total counters, 2(base) + x(general) */ + int num_counters; + /* the width of the counter */ + int counter_width; + + /* vendor-defined PMU features */ + void *platform; + + irqreturn_t (*handle_irq)(int irq_num, void *dev); + int irq; +}; + +#endif +#ifdef CONFIG_PERF_EVENTS +#define perf_arch_bpf_user_pt_regs(regs) (struct user_regs_struct *)regs +#endif + +#endif /* _ASM_RISCV_PERF_EVENT_H */ diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h new file mode 100644 index 000000000..23b1544e0 --- /dev/null +++ b/arch/riscv/include/asm/pgalloc.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com> + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_PGALLOC_H +#define _ASM_RISCV_PGALLOC_H + +#include <linux/mm.h> +#include <asm/tlb.h> + +#ifdef CONFIG_MMU +#include <asm-generic/pgalloc.h> + +static inline void pmd_populate_kernel(struct mm_struct *mm, + pmd_t *pmd, pte_t *pte) +{ + unsigned long pfn = virt_to_pfn(pte); + + set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); +} + +static inline void pmd_populate(struct mm_struct *mm, + pmd_t *pmd, pgtable_t pte) +{ + unsigned long pfn = virt_to_pfn(page_address(pte)); + + set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); +} + +#ifndef __PAGETABLE_PMD_FOLDED +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + unsigned long pfn = virt_to_pfn(pmd); + + set_pud(pud, __pud((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); +} +#endif /* __PAGETABLE_PMD_FOLDED */ + +#define pmd_pgtable(pmd) pmd_page(pmd) + +static inline pgd_t *pgd_alloc(struct mm_struct *mm) +{ + pgd_t *pgd; + + pgd = (pgd_t *)__get_free_page(GFP_KERNEL); + if (likely(pgd != NULL)) { + memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); + /* Copy kernel mappings */ + memcpy(pgd + USER_PTRS_PER_PGD, + init_mm.pgd + USER_PTRS_PER_PGD, + (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); + } + return pgd; +} + +#ifndef __PAGETABLE_PMD_FOLDED + +#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) + +#endif /* __PAGETABLE_PMD_FOLDED */ + +#define __pte_free_tlb(tlb, pte, buf) \ +do { \ + pgtable_pte_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0) +#endif /* CONFIG_MMU */ + +#endif /* _ASM_RISCV_PGALLOC_H */ diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h new file mode 100644 index 000000000..5b2e79e5b --- /dev/null +++ b/arch/riscv/include/asm/pgtable-32.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_PGTABLE_32_H +#define _ASM_RISCV_PGTABLE_32_H + +#include <asm-generic/pgtable-nopmd.h> +#include <linux/const.h> + +/* Size of region mapped by a page global directory */ +#define PGDIR_SHIFT 22 +#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) + +#define MAX_POSSIBLE_PHYSMEM_BITS 34 + +#endif /* _ASM_RISCV_PGTABLE_32_H */ diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h new file mode 100644 index 000000000..0e863f3f7 --- /dev/null +++ b/arch/riscv/include/asm/pgtable-64.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_PGTABLE_64_H +#define _ASM_RISCV_PGTABLE_64_H + +#include <linux/const.h> + +#define PGDIR_SHIFT 30 +/* Size of region mapped by a page global directory */ +#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) + +#define PMD_SHIFT 21 +/* Size of region mapped by a page middle directory */ +#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE - 1)) + +/* Page Middle Directory entry */ +typedef struct { + unsigned long pmd; +} pmd_t; + +#define pmd_val(x) ((x).pmd) +#define __pmd(x) ((pmd_t) { (x) }) + +#define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) + +static inline int pud_present(pud_t pud) +{ + return (pud_val(pud) & _PAGE_PRESENT); +} + +static inline int pud_none(pud_t pud) +{ + return (pud_val(pud) == 0); +} + +static inline int pud_bad(pud_t pud) +{ + return !pud_present(pud); +} + +#define pud_leaf pud_leaf +static inline int pud_leaf(pud_t pud) +{ + return pud_present(pud) && + (pud_val(pud) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); +} + +static inline void set_pud(pud_t *pudp, pud_t pud) +{ + *pudp = pud; +} + +static inline void pud_clear(pud_t *pudp) +{ + set_pud(pudp, __pud(0)); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + return (pmd_t *)pfn_to_virt(pud_val(pud) >> _PAGE_PFN_SHIFT); +} + +static inline struct page *pud_page(pud_t pud) +{ + return pfn_to_page(pud_val(pud) >> _PAGE_PFN_SHIFT); +} + +static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) +{ + return __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); +} + +static inline unsigned long _pmd_pfn(pmd_t pmd) +{ + return pmd_val(pmd) >> _PAGE_PFN_SHIFT; +} + +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) + +#endif /* _ASM_RISCV_PGTABLE_64_H */ diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h new file mode 100644 index 000000000..bbaeb5d35 --- /dev/null +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_PGTABLE_BITS_H +#define _ASM_RISCV_PGTABLE_BITS_H + +/* + * PTE format: + * | XLEN-1 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 + * PFN reserved for SW D A G U X W R V + */ + +#define _PAGE_ACCESSED_OFFSET 6 + +#define _PAGE_PRESENT (1 << 0) +#define _PAGE_READ (1 << 1) /* Readable */ +#define _PAGE_WRITE (1 << 2) /* Writable */ +#define _PAGE_EXEC (1 << 3) /* Executable */ +#define _PAGE_USER (1 << 4) /* User */ +#define _PAGE_GLOBAL (1 << 5) /* Global */ +#define _PAGE_ACCESSED (1 << 6) /* Set by hardware on any access */ +#define _PAGE_DIRTY (1 << 7) /* Set by hardware on any write */ +#define _PAGE_SOFT (1 << 8) /* Reserved for software */ + +#define _PAGE_SPECIAL _PAGE_SOFT +#define _PAGE_TABLE _PAGE_PRESENT + +/* + * _PAGE_PROT_NONE is set on not-present pages (and ignored by the hardware) to + * distinguish them from swapped out pages + */ +#define _PAGE_PROT_NONE _PAGE_READ + +#define _PAGE_PFN_SHIFT 10 + +/* Set of bits to preserve across pte_modify() */ +#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ + _PAGE_WRITE | _PAGE_EXEC | \ + _PAGE_USER | _PAGE_GLOBAL)) + +#endif /* _ASM_RISCV_PGTABLE_BITS_H */ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h new file mode 100644 index 000000000..b16304fdf --- /dev/null +++ b/arch/riscv/include/asm/pgtable.h @@ -0,0 +1,486 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_PGTABLE_H +#define _ASM_RISCV_PGTABLE_H + +#include <linux/mmzone.h> +#include <linux/sizes.h> + +#include <asm/pgtable-bits.h> + +#ifndef __ASSEMBLY__ + +/* Page Upper Directory not used in RISC-V */ +#include <asm-generic/pgtable-nopud.h> +#include <asm/page.h> +#include <asm/tlbflush.h> +#include <linux/mm_types.h> + +#ifdef CONFIG_MMU + +#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) +#define VMALLOC_END (PAGE_OFFSET - 1) +#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) + +#define BPF_JIT_REGION_SIZE (SZ_128M) +#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE) +#define BPF_JIT_REGION_END (VMALLOC_END) + +/* + * Roughly size the vmemmap space to be large enough to fit enough + * struct pages to map half the virtual address space. Then + * position vmemmap directly below the VMALLOC region. + */ +#define VMEMMAP_SHIFT \ + (CONFIG_VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT) +#define VMEMMAP_SIZE BIT(VMEMMAP_SHIFT) +#define VMEMMAP_END (VMALLOC_START - 1) +#define VMEMMAP_START (VMALLOC_START - VMEMMAP_SIZE) + +/* + * Define vmemmap for pfn_to_page & page_to_pfn calls. Needed if kernel + * is configured with CONFIG_SPARSEMEM_VMEMMAP enabled. + */ +#define vmemmap ((struct page *)VMEMMAP_START) + +#define PCI_IO_SIZE SZ_16M +#define PCI_IO_END VMEMMAP_START +#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) + +#define FIXADDR_TOP PCI_IO_START +#ifdef CONFIG_64BIT +#define FIXADDR_SIZE PMD_SIZE +#else +#define FIXADDR_SIZE PGDIR_SIZE +#endif +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) + +#endif + +#ifdef CONFIG_64BIT +#include <asm/pgtable-64.h> +#else +#include <asm/pgtable-32.h> +#endif /* CONFIG_64BIT */ + +#ifdef CONFIG_MMU +/* Number of entries in the page global directory */ +#define PTRS_PER_PGD (PAGE_SIZE / sizeof(pgd_t)) +/* Number of entries in the page table */ +#define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t)) + +/* Number of PGD entries that a user-mode program can use */ +#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) + +/* Page protection bits */ +#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER) + +#define PAGE_NONE __pgprot(_PAGE_PROT_NONE) +#define PAGE_READ __pgprot(_PAGE_BASE | _PAGE_READ) +#define PAGE_WRITE __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_WRITE) +#define PAGE_EXEC __pgprot(_PAGE_BASE | _PAGE_EXEC) +#define PAGE_READ_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) +#define PAGE_WRITE_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | \ + _PAGE_EXEC | _PAGE_WRITE) + +#define PAGE_COPY PAGE_READ +#define PAGE_COPY_EXEC PAGE_EXEC +#define PAGE_COPY_READ_EXEC PAGE_READ_EXEC +#define PAGE_SHARED PAGE_WRITE +#define PAGE_SHARED_EXEC PAGE_WRITE_EXEC + +#define _PAGE_KERNEL (_PAGE_READ \ + | _PAGE_WRITE \ + | _PAGE_PRESENT \ + | _PAGE_ACCESSED \ + | _PAGE_DIRTY) + +#define PAGE_KERNEL __pgprot(_PAGE_KERNEL) +#define PAGE_KERNEL_READ __pgprot(_PAGE_KERNEL & ~_PAGE_WRITE) +#define PAGE_KERNEL_EXEC __pgprot(_PAGE_KERNEL | _PAGE_EXEC) +#define PAGE_KERNEL_READ_EXEC __pgprot((_PAGE_KERNEL & ~_PAGE_WRITE) \ + | _PAGE_EXEC) + +#define PAGE_TABLE __pgprot(_PAGE_TABLE) + +/* + * The RISC-V ISA doesn't yet specify how to query or modify PMAs, so we can't + * change the properties of memory regions. + */ +#define _PAGE_IOREMAP _PAGE_KERNEL + +extern pgd_t swapper_pg_dir[]; + +/* MAP_PRIVATE permissions: xwr (copy-on-write) */ +#define __P000 PAGE_NONE +#define __P001 PAGE_READ +#define __P010 PAGE_COPY +#define __P011 PAGE_COPY +#define __P100 PAGE_EXEC +#define __P101 PAGE_READ_EXEC +#define __P110 PAGE_COPY_EXEC +#define __P111 PAGE_COPY_READ_EXEC + +/* MAP_SHARED permissions: xwr */ +#define __S000 PAGE_NONE +#define __S001 PAGE_READ +#define __S010 PAGE_SHARED +#define __S011 PAGE_SHARED +#define __S100 PAGE_EXEC +#define __S101 PAGE_READ_EXEC +#define __S110 PAGE_SHARED_EXEC +#define __S111 PAGE_SHARED_EXEC + +static inline int pmd_present(pmd_t pmd) +{ + return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE)); +} + +static inline int pmd_none(pmd_t pmd) +{ + return (pmd_val(pmd) == 0); +} + +static inline int pmd_bad(pmd_t pmd) +{ + return !pmd_present(pmd); +} + +#define pmd_leaf pmd_leaf +static inline int pmd_leaf(pmd_t pmd) +{ + return pmd_present(pmd) && + (pmd_val(pmd) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); +} + +static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) +{ + *pmdp = pmd; +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pmd(pmdp, __pmd(0)); +} + +static inline pgd_t pfn_pgd(unsigned long pfn, pgprot_t prot) +{ + return __pgd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); +} + +static inline unsigned long _pgd_pfn(pgd_t pgd) +{ + return pgd_val(pgd) >> _PAGE_PFN_SHIFT; +} + +static inline struct page *pmd_page(pmd_t pmd) +{ + return pfn_to_page(pmd_val(pmd) >> _PAGE_PFN_SHIFT); +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)pfn_to_virt(pmd_val(pmd) >> _PAGE_PFN_SHIFT); +} + +/* Yields the page frame number (PFN) of a page table entry */ +static inline unsigned long pte_pfn(pte_t pte) +{ + return (pte_val(pte) >> _PAGE_PFN_SHIFT); +} + +#define pte_page(x) pfn_to_page(pte_pfn(x)) + +/* Constructs a page table entry */ +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) +{ + return __pte((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); +} + +#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) + +static inline int pte_present(pte_t pte) +{ + return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)); +} + +static inline int pte_none(pte_t pte) +{ + return (pte_val(pte) == 0); +} + +static inline int pte_write(pte_t pte) +{ + return pte_val(pte) & _PAGE_WRITE; +} + +static inline int pte_exec(pte_t pte) +{ + return pte_val(pte) & _PAGE_EXEC; +} + +static inline int pte_huge(pte_t pte) +{ + return pte_present(pte) + && (pte_val(pte) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); +} + +static inline int pte_dirty(pte_t pte) +{ + return pte_val(pte) & _PAGE_DIRTY; +} + +static inline int pte_young(pte_t pte) +{ + return pte_val(pte) & _PAGE_ACCESSED; +} + +static inline int pte_special(pte_t pte) +{ + return pte_val(pte) & _PAGE_SPECIAL; +} + +/* static inline pte_t pte_rdprotect(pte_t pte) */ + +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_WRITE)); +} + +/* static inline pte_t pte_mkread(pte_t pte) */ + +static inline pte_t pte_mkwrite(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_WRITE); +} + +/* static inline pte_t pte_mkexec(pte_t pte) */ + +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_DIRTY); +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_DIRTY)); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_ACCESSED); +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_ACCESSED)); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SPECIAL); +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return pte; +} + +/* Modify page protection bits */ +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); +} + +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e)) + + +/* Commit new configuration to MMU hardware */ +static inline void update_mmu_cache(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ + /* + * The kernel assumes that TLBs don't cache invalid entries, but + * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a + * cache flush; it is necessary even after writing invalid entries. + * Relying on flush_tlb_fix_spurious_fault would suffice, but + * the extra traps reduce performance. So, eagerly SFENCE.VMA. + */ + local_flush_tlb_page(address); +} + +#define __HAVE_ARCH_PTE_SAME +static inline int pte_same(pte_t pte_a, pte_t pte_b) +{ + return pte_val(pte_a) == pte_val(pte_b); +} + +/* + * Certain architectures need to do special things when PTEs within + * a page table are directly modified. Thus, the following hook is + * made available. + */ +static inline void set_pte(pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; +} + +void flush_icache_pte(pte_t pte); + +static inline void set_pte_at(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, pte_t pteval) +{ + if (pte_present(pteval) && pte_exec(pteval)) + flush_icache_pte(pteval); + + set_pte(ptep, pteval); +} + +static inline void pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + set_pte_at(mm, addr, ptep, __pte(0)); +} + +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +static inline int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty) +{ + if (!pte_same(*ptep, entry)) + set_pte_at(vma->vm_mm, address, ptep, entry); + /* + * update_mmu_cache will unconditionally execute, handling both + * the case that the PTE changed and the spurious fault case. + */ + return true; +} + +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + return __pte(atomic_long_xchg((atomic_long_t *)ptep, 0)); +} + +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pte_t *ptep) +{ + if (!pte_young(*ptep)) + return 0; + return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep)); +} + +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +static inline void ptep_set_wrprotect(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + atomic_long_and(~(unsigned long)_PAGE_WRITE, (atomic_long_t *)ptep); +} + +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +static inline int ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ + /* + * This comment is borrowed from x86, but applies equally to RISC-V: + * + * Clearing the accessed bit without a TLB flush + * doesn't cause data corruption. [ It could cause incorrect + * page aging and the (mistaken) reclaim of hot pages, but the + * chance of that should be relatively low. ] + * + * So as a performance optimization don't flush the TLB when + * clearing the accessed bit, it will eventually be flushed by + * a context switch or a VM operation anyway. [ In the rare + * event of it not getting flushed for a long time the delay + * shouldn't really matter because there's no real memory + * pressure for swapout to react to. ] + */ + return ptep_test_and_clear_young(vma, address, ptep); +} + +/* + * Encode and decode a swap entry + * + * Format of swap PTE: + * bit 0: _PAGE_PRESENT (zero) + * bit 1: _PAGE_PROT_NONE (zero) + * bits 2 to 6: swap type + * bits 7 to XLEN-1: swap offset + */ +#define __SWP_TYPE_SHIFT 2 +#define __SWP_TYPE_BITS 5 +#define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1) +#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) + +#define MAX_SWAPFILES_CHECK() \ + BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS) + +#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) +#define __swp_offset(x) ((x).val >> __SWP_OFFSET_SHIFT) +#define __swp_entry(type, offset) ((swp_entry_t) \ + { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) }) + +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) +#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) + +/* + * In the RV64 Linux scheme, we give the user half of the virtual-address space + * and give the kernel the other (upper) half. + */ +#ifdef CONFIG_64BIT +#define KERN_VIRT_START (-(BIT(CONFIG_VA_BITS)) + TASK_SIZE) +#else +#define KERN_VIRT_START FIXADDR_START +#endif + +/* + * Task size is 0x4000000000 for RV64 or 0x9fc00000 for RV32. + * Note that PGDIR_SIZE must evenly divide TASK_SIZE. + */ +#ifdef CONFIG_64BIT +#define TASK_SIZE (PGDIR_SIZE * PTRS_PER_PGD / 2) +#else +#define TASK_SIZE FIXADDR_START +#endif + +#else /* CONFIG_MMU */ + +#define PAGE_SHARED __pgprot(0) +#define PAGE_KERNEL __pgprot(0) +#define swapper_pg_dir NULL +#define TASK_SIZE 0xffffffffUL +#define VMALLOC_START 0 +#define VMALLOC_END TASK_SIZE + +static inline void __kernel_map_pages(struct page *page, int numpages, int enable) {} + +#endif /* !CONFIG_MMU */ + +#define kern_addr_valid(addr) (1) /* FIXME */ + +extern void *dtb_early_va; +extern uintptr_t dtb_early_pa; +void setup_bootmem(void); +void paging_init(void); +void misc_mem_init(void); + +#define FIRST_USER_ADDRESS 0 + +/* + * ZERO_PAGE is a global shared page that is always zero, + * used for zero-mapped memory areas, etc. + */ +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; +#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_RISCV_PGTABLE_H */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h new file mode 100644 index 000000000..bdddcd5c1 --- /dev/null +++ b/arch/riscv/include/asm/processor.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_PROCESSOR_H +#define _ASM_RISCV_PROCESSOR_H + +#include <linux/const.h> + +#include <vdso/processor.h> + +#include <asm/ptrace.h> + +/* + * This decides where the kernel will search for a free chunk of vm + * space during mmap's. + */ +#define TASK_UNMAPPED_BASE PAGE_ALIGN(TASK_SIZE / 3) + +#define STACK_TOP TASK_SIZE +#define STACK_TOP_MAX STACK_TOP +#define STACK_ALIGN 16 + +#ifndef __ASSEMBLY__ + +struct task_struct; +struct pt_regs; + +/* CPU-specific state of a task */ +struct thread_struct { + /* Callee-saved registers */ + unsigned long ra; + unsigned long sp; /* Kernel mode stack */ + unsigned long s[12]; /* s[0]: frame pointer */ + struct __riscv_d_ext_state fstate; +}; + +#define INIT_THREAD { \ + .sp = sizeof(init_stack) + (long)&init_stack, \ +} + +#define task_pt_regs(tsk) \ + ((struct pt_regs *)(task_stack_page(tsk) + THREAD_SIZE \ + - ALIGN(sizeof(struct pt_regs), STACK_ALIGN))) + +#define KSTK_EIP(tsk) (task_pt_regs(tsk)->epc) +#define KSTK_ESP(tsk) (task_pt_regs(tsk)->sp) + + +/* Do necessary setup to start up a newly executed thread. */ +extern void start_thread(struct pt_regs *regs, + unsigned long pc, unsigned long sp); + +/* Free all resources held by a thread. */ +static inline void release_thread(struct task_struct *dead_task) +{ +} + +extern unsigned long get_wchan(struct task_struct *p); + + +static inline void wait_for_interrupt(void) +{ + __asm__ __volatile__ ("wfi"); +} + +struct device_node; +int riscv_of_processor_hartid(struct device_node *node); +int riscv_of_parent_hartid(struct device_node *node); + +extern void riscv_fill_hwcap(void); + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_RISCV_PROCESSOR_H */ diff --git a/arch/riscv/include/asm/ptdump.h b/arch/riscv/include/asm/ptdump.h new file mode 100644 index 000000000..3c9ea6dd5 --- /dev/null +++ b/arch/riscv/include/asm/ptdump.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 SiFive + */ + +#ifndef _ASM_RISCV_PTDUMP_H +#define _ASM_RISCV_PTDUMP_H + +void ptdump_check_wx(void); + +#ifdef CONFIG_DEBUG_WX +static inline void debug_checkwx(void) +{ + ptdump_check_wx(); +} +#else +static inline void debug_checkwx(void) +{ +} +#endif + +#endif /* _ASM_RISCV_PTDUMP_H */ diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h new file mode 100644 index 000000000..ee49f80c9 --- /dev/null +++ b/arch/riscv/include/asm/ptrace.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_PTRACE_H +#define _ASM_RISCV_PTRACE_H + +#include <uapi/asm/ptrace.h> +#include <asm/csr.h> + +#ifndef __ASSEMBLY__ + +struct pt_regs { + unsigned long epc; + unsigned long ra; + unsigned long sp; + unsigned long gp; + unsigned long tp; + unsigned long t0; + unsigned long t1; + unsigned long t2; + unsigned long s0; + unsigned long s1; + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long a5; + unsigned long a6; + unsigned long a7; + unsigned long s2; + unsigned long s3; + unsigned long s4; + unsigned long s5; + unsigned long s6; + unsigned long s7; + unsigned long s8; + unsigned long s9; + unsigned long s10; + unsigned long s11; + unsigned long t3; + unsigned long t4; + unsigned long t5; + unsigned long t6; + /* Supervisor/Machine CSRs */ + unsigned long status; + unsigned long badaddr; + unsigned long cause; + /* a0 value before the syscall */ + unsigned long orig_a0; +}; + +#ifdef CONFIG_64BIT +#define REG_FMT "%016lx" +#else +#define REG_FMT "%08lx" +#endif + +#define user_mode(regs) (((regs)->status & SR_PP) == 0) + + +/* Helpers for working with the instruction pointer */ +static inline unsigned long instruction_pointer(struct pt_regs *regs) +{ + return regs->epc; +} +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->epc = val; +} + +#define profile_pc(regs) instruction_pointer(regs) + +/* Helpers for working with the user stack pointer */ +static inline unsigned long user_stack_pointer(struct pt_regs *regs) +{ + return regs->sp; +} +static inline void user_stack_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->sp = val; +} + +/* Helpers for working with the frame pointer */ +static inline unsigned long frame_pointer(struct pt_regs *regs) +{ + return regs->s0; +} +static inline void frame_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->s0 = val; +} + +static inline unsigned long regs_return_value(struct pt_regs *regs) +{ + return regs->a0; +} + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_RISCV_PTRACE_H */ diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h new file mode 100644 index 000000000..c0fdb05ff --- /dev/null +++ b/arch/riscv/include/asm/sbi.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2015 Regents of the University of California + * Copyright (c) 2020 Western Digital Corporation or its affiliates. + */ + +#ifndef _ASM_RISCV_SBI_H +#define _ASM_RISCV_SBI_H + +#include <linux/types.h> + +#ifdef CONFIG_RISCV_SBI +enum sbi_ext_id { +#ifdef CONFIG_RISCV_SBI_V01 + SBI_EXT_0_1_SET_TIMER = 0x0, + SBI_EXT_0_1_CONSOLE_PUTCHAR = 0x1, + SBI_EXT_0_1_CONSOLE_GETCHAR = 0x2, + SBI_EXT_0_1_CLEAR_IPI = 0x3, + SBI_EXT_0_1_SEND_IPI = 0x4, + SBI_EXT_0_1_REMOTE_FENCE_I = 0x5, + SBI_EXT_0_1_REMOTE_SFENCE_VMA = 0x6, + SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID = 0x7, + SBI_EXT_0_1_SHUTDOWN = 0x8, +#endif + SBI_EXT_BASE = 0x10, + SBI_EXT_TIME = 0x54494D45, + SBI_EXT_IPI = 0x735049, + SBI_EXT_RFENCE = 0x52464E43, + SBI_EXT_HSM = 0x48534D, +}; + +enum sbi_ext_base_fid { + SBI_EXT_BASE_GET_SPEC_VERSION = 0, + SBI_EXT_BASE_GET_IMP_ID, + SBI_EXT_BASE_GET_IMP_VERSION, + SBI_EXT_BASE_PROBE_EXT, + SBI_EXT_BASE_GET_MVENDORID, + SBI_EXT_BASE_GET_MARCHID, + SBI_EXT_BASE_GET_MIMPID, +}; + +enum sbi_ext_time_fid { + SBI_EXT_TIME_SET_TIMER = 0, +}; + +enum sbi_ext_ipi_fid { + SBI_EXT_IPI_SEND_IPI = 0, +}; + +enum sbi_ext_rfence_fid { + SBI_EXT_RFENCE_REMOTE_FENCE_I = 0, + SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, + SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, + SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, + SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, + SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, + SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, +}; + +enum sbi_ext_hsm_fid { + SBI_EXT_HSM_HART_START = 0, + SBI_EXT_HSM_HART_STOP, + SBI_EXT_HSM_HART_STATUS, +}; + +enum sbi_hsm_hart_status { + SBI_HSM_HART_STATUS_STARTED = 0, + SBI_HSM_HART_STATUS_STOPPED, + SBI_HSM_HART_STATUS_START_PENDING, + SBI_HSM_HART_STATUS_STOP_PENDING, +}; + +#define SBI_SPEC_VERSION_DEFAULT 0x1 +#define SBI_SPEC_VERSION_MAJOR_SHIFT 24 +#define SBI_SPEC_VERSION_MAJOR_MASK 0x7f +#define SBI_SPEC_VERSION_MINOR_MASK 0xffffff + +/* SBI return error codes */ +#define SBI_SUCCESS 0 +#define SBI_ERR_FAILURE -1 +#define SBI_ERR_NOT_SUPPORTED -2 +#define SBI_ERR_INVALID_PARAM -3 +#define SBI_ERR_DENIED -4 +#define SBI_ERR_INVALID_ADDRESS -5 + +extern unsigned long sbi_spec_version; +struct sbiret { + long error; + long value; +}; + +int sbi_init(void); +struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, + unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4, + unsigned long arg5); + +void sbi_console_putchar(int ch); +int sbi_console_getchar(void); +void sbi_set_timer(uint64_t stime_value); +void sbi_shutdown(void); +void sbi_clear_ipi(void); +void sbi_send_ipi(const unsigned long *hart_mask); +void sbi_remote_fence_i(const unsigned long *hart_mask); +void sbi_remote_sfence_vma(const unsigned long *hart_mask, + unsigned long start, + unsigned long size); + +void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, + unsigned long start, + unsigned long size, + unsigned long asid); +int sbi_remote_hfence_gvma(const unsigned long *hart_mask, + unsigned long start, + unsigned long size); +int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask, + unsigned long start, + unsigned long size, + unsigned long vmid); +int sbi_remote_hfence_vvma(const unsigned long *hart_mask, + unsigned long start, + unsigned long size); +int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask, + unsigned long start, + unsigned long size, + unsigned long asid); +int sbi_probe_extension(int ext); + +/* Check if current SBI specification version is 0.1 or not */ +static inline int sbi_spec_is_0_1(void) +{ + return (sbi_spec_version == SBI_SPEC_VERSION_DEFAULT) ? 1 : 0; +} + +/* Get the major version of SBI */ +static inline unsigned long sbi_major_version(void) +{ + return (sbi_spec_version >> SBI_SPEC_VERSION_MAJOR_SHIFT) & + SBI_SPEC_VERSION_MAJOR_MASK; +} + +/* Get the minor version of SBI */ +static inline unsigned long sbi_minor_version(void) +{ + return sbi_spec_version & SBI_SPEC_VERSION_MINOR_MASK; +} + +int sbi_err_map_linux_errno(int err); +#else /* CONFIG_RISCV_SBI */ +/* stubs for code that is only reachable under IS_ENABLED(CONFIG_RISCV_SBI): */ +void sbi_set_timer(uint64_t stime_value); +void sbi_clear_ipi(void); +void sbi_send_ipi(const unsigned long *hart_mask); +void sbi_remote_fence_i(const unsigned long *hart_mask); +void sbi_init(void); +#endif /* CONFIG_RISCV_SBI */ +#endif /* _ASM_RISCV_SBI_H */ diff --git a/arch/riscv/include/asm/seccomp.h b/arch/riscv/include/asm/seccomp.h new file mode 100644 index 000000000..bf7744ee3 --- /dev/null +++ b/arch/riscv/include/asm/seccomp.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SECCOMP_H +#define _ASM_SECCOMP_H + +#include <asm/unistd.h> + +#include <asm-generic/seccomp.h> + +#endif /* _ASM_SECCOMP_H */ diff --git a/arch/riscv/include/asm/sections.h b/arch/riscv/include/asm/sections.h new file mode 100644 index 000000000..3a9971b12 --- /dev/null +++ b/arch/riscv/include/asm/sections.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ +#ifndef __ASM_SECTIONS_H +#define __ASM_SECTIONS_H + +#include <asm-generic/sections.h> + +extern char _start[]; +extern char _start_kernel[]; + +#endif /* __ASM_SECTIONS_H */ diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h new file mode 100644 index 000000000..4c5bae7ca --- /dev/null +++ b/arch/riscv/include/asm/set_memory.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2019 SiFive + */ + +#ifndef _ASM_RISCV_SET_MEMORY_H +#define _ASM_RISCV_SET_MEMORY_H + +#ifndef __ASSEMBLY__ +/* + * Functions to change memory attributes. + */ +#ifdef CONFIG_MMU +int set_memory_ro(unsigned long addr, int numpages); +int set_memory_rw(unsigned long addr, int numpages); +int set_memory_x(unsigned long addr, int numpages); +int set_memory_nx(unsigned long addr, int numpages); +#else +static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; } +static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; } +static inline int set_memory_x(unsigned long addr, int numpages) { return 0; } +static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; } +#endif + +int set_direct_map_invalid_noflush(struct page *page); +int set_direct_map_default_noflush(struct page *page); + +#endif /* __ASSEMBLY__ */ + +#ifdef CONFIG_ARCH_HAS_STRICT_KERNEL_RWX +#ifdef CONFIG_64BIT +#define SECTION_ALIGN (1 << 21) +#else +#define SECTION_ALIGN (1 << 22) +#endif +#else /* !CONFIG_ARCH_HAS_STRICT_KERNEL_RWX */ +#define SECTION_ALIGN L1_CACHE_BYTES +#endif /* CONFIG_ARCH_HAS_STRICT_KERNEL_RWX */ + +#endif /* _ASM_RISCV_SET_MEMORY_H */ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h new file mode 100644 index 000000000..df1f7c4cd --- /dev/null +++ b/arch/riscv/include/asm/smp.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_SMP_H +#define _ASM_RISCV_SMP_H + +#include <linux/cpumask.h> +#include <linux/irqreturn.h> +#include <linux/thread_info.h> + +#define INVALID_HARTID ULONG_MAX + +struct seq_file; +extern unsigned long boot_cpu_hartid; + +struct riscv_ipi_ops { + void (*ipi_inject)(const struct cpumask *target); + void (*ipi_clear)(void); +}; + +#ifdef CONFIG_SMP +/* + * Mapping between linux logical cpu index and hartid. + */ +extern unsigned long __cpuid_to_hartid_map[NR_CPUS]; +#define cpuid_to_hartid_map(cpu) __cpuid_to_hartid_map[cpu] + +/* print IPI stats */ +void show_ipi_stats(struct seq_file *p, int prec); + +/* SMP initialization hook for setup_arch */ +void __init setup_smp(void); + +/* Called from C code, this handles an IPI. */ +void handle_IPI(struct pt_regs *regs); + +/* Hook for the generic smp_call_function_many() routine. */ +void arch_send_call_function_ipi_mask(struct cpumask *mask); + +/* Hook for the generic smp_call_function_single() routine. */ +void arch_send_call_function_single_ipi(int cpu); + +int riscv_hartid_to_cpuid(int hartid); +void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out); + +/* Set custom IPI operations */ +void riscv_set_ipi_ops(struct riscv_ipi_ops *ops); + +/* Clear IPI for current CPU */ +void riscv_clear_ipi(void); + +/* Secondary hart entry */ +asmlinkage void smp_callin(void); + +/* + * Obtains the hart ID of the currently executing task. This relies on + * THREAD_INFO_IN_TASK, but we define that unconditionally. + */ +#define raw_smp_processor_id() (current_thread_info()->cpu) + +#if defined CONFIG_HOTPLUG_CPU +int __cpu_disable(void); +void __cpu_die(unsigned int cpu); +void cpu_stop(void); +#else +#endif /* CONFIG_HOTPLUG_CPU */ + +#else + +static inline void show_ipi_stats(struct seq_file *p, int prec) +{ +} + +static inline int riscv_hartid_to_cpuid(int hartid) +{ + if (hartid == boot_cpu_hartid) + return 0; + + return -1; +} +static inline unsigned long cpuid_to_hartid_map(int cpu) +{ + return boot_cpu_hartid; +} + +static inline void riscv_cpuid_to_hartid_mask(const struct cpumask *in, + struct cpumask *out) +{ + cpumask_clear(out); + cpumask_set_cpu(boot_cpu_hartid, out); +} + +static inline void riscv_set_ipi_ops(struct riscv_ipi_ops *ops) +{ +} + +static inline void riscv_clear_ipi(void) +{ +} + +#endif /* CONFIG_SMP */ + +#if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP) +bool cpu_has_hotplug(unsigned int cpu); +#else +static inline bool cpu_has_hotplug(unsigned int cpu) +{ + return false; +} +#endif + +#endif /* _ASM_RISCV_SMP_H */ diff --git a/arch/riscv/include/asm/soc.h b/arch/riscv/include/asm/soc.h new file mode 100644 index 000000000..6c8363b1f --- /dev/null +++ b/arch/riscv/include/asm/soc.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + * Copyright (C) 2020 Google, Inc + */ + +#ifndef _ASM_RISCV_SOC_H +#define _ASM_RISCV_SOC_H + +#include <linux/of.h> +#include <linux/linkage.h> +#include <linux/types.h> + +#define SOC_EARLY_INIT_DECLARE(name, compat, fn) \ + static const struct of_device_id __soc_early_init__##name \ + __used __section("__soc_early_init_table") \ + = { .compatible = compat, .data = fn } + +void soc_early_init(void); + +extern unsigned long __soc_early_init_table_start; +extern unsigned long __soc_early_init_table_end; + +/* + * Allows Linux to provide a device tree, which is necessary for SOCs that + * don't provide a useful one on their own. + */ +struct soc_builtin_dtb { + unsigned long vendor_id; + unsigned long arch_id; + unsigned long imp_id; + void *(*dtb_func)(void); +}; + +/* + * The argument name must specify a valid DTS file name without the dts + * extension. + */ +#define SOC_BUILTIN_DTB_DECLARE(name, vendor, arch, impl) \ + extern void *__dtb_##name##_begin; \ + \ + static __init __used \ + void *__soc_builtin_dtb_f__##name(void) \ + { \ + return (void *)&__dtb_##name##_begin; \ + } \ + \ + static const struct soc_builtin_dtb __soc_builtin_dtb__##name \ + __used __section("__soc_builtin_dtb_table") = \ + { \ + .vendor_id = vendor, \ + .arch_id = arch, \ + .imp_id = impl, \ + .dtb_func = __soc_builtin_dtb_f__##name, \ + } + +extern unsigned long __soc_builtin_dtb_table_start; +extern unsigned long __soc_builtin_dtb_table_end; + +void *soc_lookup_builtin_dtb(void); + +#endif diff --git a/arch/riscv/include/asm/sparsemem.h b/arch/riscv/include/asm/sparsemem.h new file mode 100644 index 000000000..45a7018a8 --- /dev/null +++ b/arch/riscv/include/asm/sparsemem.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_SPARSEMEM_H +#define _ASM_RISCV_SPARSEMEM_H + +#ifdef CONFIG_SPARSEMEM +#define MAX_PHYSMEM_BITS CONFIG_PA_BITS +#define SECTION_SIZE_BITS 27 +#endif /* CONFIG_SPARSEMEM */ + +#endif /* _ASM_RISCV_SPARSEMEM_H */ diff --git a/arch/riscv/include/asm/spinlock.h b/arch/riscv/include/asm/spinlock.h new file mode 100644 index 000000000..f4f7fa1b7 --- /dev/null +++ b/arch/riscv/include/asm/spinlock.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2015 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#ifndef _ASM_RISCV_SPINLOCK_H +#define _ASM_RISCV_SPINLOCK_H + +#include <linux/kernel.h> +#include <asm/current.h> +#include <asm/fence.h> + +/* + * Simple spin lock operations. These provide no fairness guarantees. + */ + +/* FIXME: Replace this with a ticket lock, like MIPS. */ + +#define arch_spin_is_locked(x) (READ_ONCE((x)->lock) != 0) + +static inline void arch_spin_unlock(arch_spinlock_t *lock) +{ + smp_store_release(&lock->lock, 0); +} + +static inline int arch_spin_trylock(arch_spinlock_t *lock) +{ + int tmp = 1, busy; + + __asm__ __volatile__ ( + " amoswap.w %0, %2, %1\n" + RISCV_ACQUIRE_BARRIER + : "=r" (busy), "+A" (lock->lock) + : "r" (tmp) + : "memory"); + + return !busy; +} + +static inline void arch_spin_lock(arch_spinlock_t *lock) +{ + while (1) { + if (arch_spin_is_locked(lock)) + continue; + + if (arch_spin_trylock(lock)) + break; + } +} + +/***********************************************************/ + +static inline void arch_read_lock(arch_rwlock_t *lock) +{ + int tmp; + + __asm__ __volatile__( + "1: lr.w %1, %0\n" + " bltz %1, 1b\n" + " addi %1, %1, 1\n" + " sc.w %1, %1, %0\n" + " bnez %1, 1b\n" + RISCV_ACQUIRE_BARRIER + : "+A" (lock->lock), "=&r" (tmp) + :: "memory"); +} + +static inline void arch_write_lock(arch_rwlock_t *lock) +{ + int tmp; + + __asm__ __volatile__( + "1: lr.w %1, %0\n" + " bnez %1, 1b\n" + " li %1, -1\n" + " sc.w %1, %1, %0\n" + " bnez %1, 1b\n" + RISCV_ACQUIRE_BARRIER + : "+A" (lock->lock), "=&r" (tmp) + :: "memory"); +} + +static inline int arch_read_trylock(arch_rwlock_t *lock) +{ + int busy; + + __asm__ __volatile__( + "1: lr.w %1, %0\n" + " bltz %1, 1f\n" + " addi %1, %1, 1\n" + " sc.w %1, %1, %0\n" + " bnez %1, 1b\n" + RISCV_ACQUIRE_BARRIER + "1:\n" + : "+A" (lock->lock), "=&r" (busy) + :: "memory"); + + return !busy; +} + +static inline int arch_write_trylock(arch_rwlock_t *lock) +{ + int busy; + + __asm__ __volatile__( + "1: lr.w %1, %0\n" + " bnez %1, 1f\n" + " li %1, -1\n" + " sc.w %1, %1, %0\n" + " bnez %1, 1b\n" + RISCV_ACQUIRE_BARRIER + "1:\n" + : "+A" (lock->lock), "=&r" (busy) + :: "memory"); + + return !busy; +} + +static inline void arch_read_unlock(arch_rwlock_t *lock) +{ + __asm__ __volatile__( + RISCV_RELEASE_BARRIER + " amoadd.w x0, %1, %0\n" + : "+A" (lock->lock) + : "r" (-1) + : "memory"); +} + +static inline void arch_write_unlock(arch_rwlock_t *lock) +{ + smp_store_release(&lock->lock, 0); +} + +#endif /* _ASM_RISCV_SPINLOCK_H */ diff --git a/arch/riscv/include/asm/spinlock_types.h b/arch/riscv/include/asm/spinlock_types.h new file mode 100644 index 000000000..f398e7638 --- /dev/null +++ b/arch/riscv/include/asm/spinlock_types.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2015 Regents of the University of California + */ + +#ifndef _ASM_RISCV_SPINLOCK_TYPES_H +#define _ASM_RISCV_SPINLOCK_TYPES_H + +#ifndef __LINUX_SPINLOCK_TYPES_H +# error "please don't include this file directly" +#endif + +typedef struct { + volatile unsigned int lock; +} arch_spinlock_t; + +#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } + +typedef struct { + volatile unsigned int lock; +} arch_rwlock_t; + +#define __ARCH_RW_LOCK_UNLOCKED { 0 } + +#endif /* _ASM_RISCV_SPINLOCK_TYPES_H */ diff --git a/arch/riscv/include/asm/stackprotector.h b/arch/riscv/include/asm/stackprotector.h new file mode 100644 index 000000000..5962f8891 --- /dev/null +++ b/arch/riscv/include/asm/stackprotector.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_STACKPROTECTOR_H +#define _ASM_RISCV_STACKPROTECTOR_H + +#include <linux/random.h> +#include <linux/version.h> + +extern unsigned long __stack_chk_guard; + +/* + * Initialize the stackprotector canary value. + * + * NOTE: this must only be called from functions that never return, + * and it must always be inlined. + */ +static __always_inline void boot_init_stack_canary(void) +{ + unsigned long canary; + + /* Try to get a semi random initial value. */ + get_random_bytes(&canary, sizeof(canary)); + canary ^= LINUX_VERSION_CODE; + canary &= CANARY_MASK; + + current->stack_canary = canary; + __stack_chk_guard = current->stack_canary; +} +#endif /* _ASM_RISCV_STACKPROTECTOR_H */ diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h new file mode 100644 index 000000000..924af13f8 --- /dev/null +++ b/arch/riscv/include/asm/string.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2013 Regents of the University of California + */ + +#ifndef _ASM_RISCV_STRING_H +#define _ASM_RISCV_STRING_H + +#include <linux/types.h> +#include <linux/linkage.h> + +#define __HAVE_ARCH_MEMSET +extern asmlinkage void *memset(void *, int, size_t); +extern asmlinkage void *__memset(void *, int, size_t); + +#define __HAVE_ARCH_MEMCPY +extern asmlinkage void *memcpy(void *, const void *, size_t); +extern asmlinkage void *__memcpy(void *, const void *, size_t); + +/* For those files which don't want to check by kasan. */ +#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) + +#define memcpy(dst, src, len) __memcpy(dst, src, len) +#define memset(s, c, n) __memset(s, c, n) + +#endif +#endif /* _ASM_RISCV_STRING_H */ diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h new file mode 100644 index 000000000..407bcc96a --- /dev/null +++ b/arch/riscv/include/asm/switch_to.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_SWITCH_TO_H +#define _ASM_RISCV_SWITCH_TO_H + +#include <linux/sched/task_stack.h> +#include <asm/processor.h> +#include <asm/ptrace.h> +#include <asm/csr.h> + +#ifdef CONFIG_FPU +extern void __fstate_save(struct task_struct *save_to); +extern void __fstate_restore(struct task_struct *restore_from); + +static inline void __fstate_clean(struct pt_regs *regs) +{ + regs->status = (regs->status & ~SR_FS) | SR_FS_CLEAN; +} + +static inline void fstate_off(struct task_struct *task, + struct pt_regs *regs) +{ + regs->status = (regs->status & ~SR_FS) | SR_FS_OFF; +} + +static inline void fstate_save(struct task_struct *task, + struct pt_regs *regs) +{ + if ((regs->status & SR_FS) == SR_FS_DIRTY) { + __fstate_save(task); + __fstate_clean(regs); + } +} + +static inline void fstate_restore(struct task_struct *task, + struct pt_regs *regs) +{ + if ((regs->status & SR_FS) != SR_FS_OFF) { + __fstate_restore(task); + __fstate_clean(regs); + } +} + +static inline void __switch_to_aux(struct task_struct *prev, + struct task_struct *next) +{ + struct pt_regs *regs; + + regs = task_pt_regs(prev); + if (unlikely(regs->status & SR_SD)) + fstate_save(prev, regs); + fstate_restore(next, task_pt_regs(next)); +} + +extern bool has_fpu; +#else +#define has_fpu false +#define fstate_save(task, regs) do { } while (0) +#define fstate_restore(task, regs) do { } while (0) +#define __switch_to_aux(__prev, __next) do { } while (0) +#endif + +extern struct task_struct *__switch_to(struct task_struct *, + struct task_struct *); + +#define switch_to(prev, next, last) \ +do { \ + struct task_struct *__prev = (prev); \ + struct task_struct *__next = (next); \ + if (has_fpu) \ + __switch_to_aux(__prev, __next); \ + ((last) = __switch_to(__prev, __next)); \ +} while (0) + +#endif /* _ASM_RISCV_SWITCH_TO_H */ diff --git a/arch/riscv/include/asm/syscall.h b/arch/riscv/include/asm/syscall.h new file mode 100644 index 000000000..49350c8bd --- /dev/null +++ b/arch/riscv/include/asm/syscall.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2008-2009 Red Hat, Inc. All rights reserved. + * Copyright 2010 Tilera Corporation. All Rights Reserved. + * Copyright 2015 Regents of the University of California, Berkeley + * + * See asm-generic/syscall.h for descriptions of what we must do here. + */ + +#ifndef _ASM_RISCV_SYSCALL_H +#define _ASM_RISCV_SYSCALL_H + +#include <uapi/linux/audit.h> +#include <linux/sched.h> +#include <linux/err.h> + +/* The array of function pointers for syscalls. */ +extern void *sys_call_table[]; + +/* + * Only the low 32 bits of orig_r0 are meaningful, so we return int. + * This importantly ignores the high bits on 64-bit, so comparisons + * sign-extend the low 32 bits. + */ +static inline int syscall_get_nr(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->a7; +} + +static inline void syscall_rollback(struct task_struct *task, + struct pt_regs *regs) +{ + regs->a0 = regs->orig_a0; +} + +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + unsigned long error = regs->a0; + + return IS_ERR_VALUE(error) ? error : 0; +} + +static inline long syscall_get_return_value(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->a0; +} + +static inline void syscall_set_return_value(struct task_struct *task, + struct pt_regs *regs, + int error, long val) +{ + regs->a0 = (long) error ?: val; +} + +static inline void syscall_get_arguments(struct task_struct *task, + struct pt_regs *regs, + unsigned long *args) +{ + args[0] = regs->orig_a0; + args++; + memcpy(args, ®s->a1, 5 * sizeof(args[0])); +} + +static inline void syscall_set_arguments(struct task_struct *task, + struct pt_regs *regs, + const unsigned long *args) +{ + regs->orig_a0 = args[0]; + args++; + memcpy(®s->a1, args, 5 * sizeof(regs->a1)); +} + +static inline int syscall_get_arch(struct task_struct *task) +{ +#ifdef CONFIG_64BIT + return AUDIT_ARCH_RISCV64; +#else + return AUDIT_ARCH_RISCV32; +#endif +} + +#endif /* _ASM_RISCV_SYSCALL_H */ diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h new file mode 100644 index 000000000..c55e0a1f0 --- /dev/null +++ b/arch/riscv/include/asm/thread_info.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com> + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#ifndef _ASM_RISCV_THREAD_INFO_H +#define _ASM_RISCV_THREAD_INFO_H + +#include <asm/page.h> +#include <linux/const.h> + +#ifdef CONFIG_KASAN +#define KASAN_STACK_ORDER 1 +#else +#define KASAN_STACK_ORDER 0 +#endif + +/* thread information allocation */ +#ifdef CONFIG_64BIT +#define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER) +#else +#define THREAD_SIZE_ORDER (1 + KASAN_STACK_ORDER) +#endif +#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) + +#ifndef __ASSEMBLY__ + +#include <asm/processor.h> +#include <asm/csr.h> + +/* + * low level task data that entry.S needs immediate access to + * - this struct should fit entirely inside of one cache line + * - if the members of this struct changes, the assembly constants + * in asm-offsets.c must be updated accordingly + * - thread_info is included in task_struct at an offset of 0. This means that + * tp points to both thread_info and task_struct. + */ +struct thread_info { + unsigned long flags; /* low level flags */ + int preempt_count; /* 0=>preemptible, <0=>BUG */ + /* + * These stack pointers are overwritten on every system call or + * exception. SP is also saved to the stack it can be recovered when + * overwritten. + */ + long kernel_sp; /* Kernel stack pointer */ + long user_sp; /* User stack pointer */ + int cpu; +}; + +/* + * macros/functions for gaining access to the thread information structure + * + * preempt_count needs to be 1 initially, until the scheduler is functional. + */ +#define INIT_THREAD_INFO(tsk) \ +{ \ + .flags = 0, \ + .preempt_count = INIT_PREEMPT_COUNT, \ +} + +#endif /* !__ASSEMBLY__ */ + +/* + * thread information flags + * - these are process state flags that various assembly files may need to + * access + * - pending work-to-be-done flags are in lowest half-word + * - other flags in upper half-word(s) + */ +#define TIF_SYSCALL_TRACE 0 /* syscall trace active */ +#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ +#define TIF_SIGPENDING 2 /* signal pending */ +#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ +#define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ +#define TIF_MEMDIE 5 /* is terminating due to OOM killer */ +#define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ +#define TIF_SYSCALL_AUDIT 7 /* syscall auditing */ +#define TIF_SECCOMP 8 /* syscall secure computing */ +#define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */ + +#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) +#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) +#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) +#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) +#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) +#define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + +#define _TIF_WORK_MASK \ + (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \ + _TIF_NOTIFY_SIGNAL) + +#define _TIF_SYSCALL_WORK \ + (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \ + _TIF_SECCOMP) + +#endif /* _ASM_RISCV_THREAD_INFO_H */ diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h new file mode 100644 index 000000000..a06697846 --- /dev/null +++ b/arch/riscv/include/asm/timex.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_TIMEX_H +#define _ASM_RISCV_TIMEX_H + +#include <asm/csr.h> + +typedef unsigned long cycles_t; + +#ifdef CONFIG_RISCV_M_MODE + +#include <asm/clint.h> + +#ifdef CONFIG_64BIT +static inline cycles_t get_cycles(void) +{ + return readq_relaxed(clint_time_val); +} +#else /* !CONFIG_64BIT */ +static inline u32 get_cycles(void) +{ + return readl_relaxed(((u32 *)clint_time_val)); +} +#define get_cycles get_cycles + +static inline u32 get_cycles_hi(void) +{ + return readl_relaxed(((u32 *)clint_time_val) + 1); +} +#define get_cycles_hi get_cycles_hi +#endif /* CONFIG_64BIT */ + +/* + * Much like MIPS, we may not have a viable counter to use at an early point + * in the boot process. Unfortunately we don't have a fallback, so instead + * we just return 0. + */ +static inline unsigned long random_get_entropy(void) +{ + if (unlikely(clint_time_val == NULL)) + return random_get_entropy_fallback(); + return get_cycles(); +} +#define random_get_entropy() random_get_entropy() + +#else /* CONFIG_RISCV_M_MODE */ + +static inline cycles_t get_cycles(void) +{ + return csr_read(CSR_TIME); +} +#define get_cycles get_cycles + +static inline u32 get_cycles_hi(void) +{ + return csr_read(CSR_TIMEH); +} +#define get_cycles_hi get_cycles_hi + +#endif /* !CONFIG_RISCV_M_MODE */ + +#ifdef CONFIG_64BIT +static inline u64 get_cycles64(void) +{ + return get_cycles(); +} +#else /* CONFIG_64BIT */ +static inline u64 get_cycles64(void) +{ + u32 hi, lo; + + do { + hi = get_cycles_hi(); + lo = get_cycles(); + } while (hi != get_cycles_hi()); + + return ((u64)hi << 32) | lo; +} +#endif /* CONFIG_64BIT */ + +#define ARCH_HAS_READ_CURRENT_TIMER +static inline int read_current_timer(unsigned long *timer_val) +{ + *timer_val = get_cycles(); + return 0; +} + +#endif /* _ASM_RISCV_TIMEX_H */ diff --git a/arch/riscv/include/asm/tlb.h b/arch/riscv/include/asm/tlb.h new file mode 100644 index 000000000..120bcf2ed --- /dev/null +++ b/arch/riscv/include/asm/tlb.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_TLB_H +#define _ASM_RISCV_TLB_H + +struct mmu_gather; + +static void tlb_flush(struct mmu_gather *tlb); + +#define tlb_flush tlb_flush +#include <asm-generic/tlb.h> + +static inline void tlb_flush(struct mmu_gather *tlb) +{ + flush_tlb_mm(tlb->mm); +} + +#endif /* _ASM_RISCV_TLB_H */ diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h new file mode 100644 index 000000000..394cfbccd --- /dev/null +++ b/arch/riscv/include/asm/tlbflush.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com> + * Copyright (C) 2012 Regents of the University of California + */ + +#ifndef _ASM_RISCV_TLBFLUSH_H +#define _ASM_RISCV_TLBFLUSH_H + +#include <linux/mm_types.h> +#include <asm/smp.h> + +#ifdef CONFIG_MMU +static inline void local_flush_tlb_all(void) +{ + __asm__ __volatile__ ("sfence.vma" : : : "memory"); +} + +/* Flush one page from local TLB */ +static inline void local_flush_tlb_page(unsigned long addr) +{ + __asm__ __volatile__ ("sfence.vma %0" : : "r" (addr) : "memory"); +} +#else /* CONFIG_MMU */ +#define local_flush_tlb_all() do { } while (0) +#define local_flush_tlb_page(addr) do { } while (0) +#endif /* CONFIG_MMU */ + +#if defined(CONFIG_SMP) && defined(CONFIG_MMU) +void flush_tlb_all(void); +void flush_tlb_mm(struct mm_struct *mm); +void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); +void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +#else /* CONFIG_SMP && CONFIG_MMU */ + +#define flush_tlb_all() local_flush_tlb_all() +#define flush_tlb_page(vma, addr) local_flush_tlb_page(addr) + +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + local_flush_tlb_all(); +} + +#define flush_tlb_mm(mm) flush_tlb_all() +#endif /* !CONFIG_SMP || !CONFIG_MMU */ + +/* Flush a range of kernel pages */ +static inline void flush_tlb_kernel_range(unsigned long start, + unsigned long end) +{ + flush_tlb_all(); +} + +#endif /* _ASM_RISCV_TLBFLUSH_H */ diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h new file mode 100644 index 000000000..66af6abfe --- /dev/null +++ b/arch/riscv/include/asm/uaccess.h @@ -0,0 +1,490 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + * + * This file was copied from include/asm-generic/uaccess.h + */ + +#ifndef _ASM_RISCV_UACCESS_H +#define _ASM_RISCV_UACCESS_H + +#include <asm/pgtable.h> /* for TASK_SIZE */ + +/* + * User space memory access functions + */ +#ifdef CONFIG_MMU +#include <linux/errno.h> +#include <linux/compiler.h> +#include <linux/thread_info.h> +#include <asm/byteorder.h> +#include <asm/extable.h> +#include <asm/asm.h> + +#define __enable_user_access() \ + __asm__ __volatile__ ("csrs sstatus, %0" : : "r" (SR_SUM) : "memory") +#define __disable_user_access() \ + __asm__ __volatile__ ("csrc sstatus, %0" : : "r" (SR_SUM) : "memory") + +/** + * access_ok: - Checks if a user space pointer is valid + * @addr: User space pointer to start of block to check + * @size: Size of block to check + * + * Context: User context only. This function may sleep. + * + * Checks if a pointer to a block of memory in user space is valid. + * + * Returns true (nonzero) if the memory block may be valid, false (zero) + * if it is definitely invalid. + * + * Note that, depending on architecture, this function probably just + * checks that the pointer is in the user space range - after calling + * this function, memory access functions may still return -EFAULT. + */ +#define access_ok(addr, size) ({ \ + __chk_user_ptr(addr); \ + likely(__access_ok((unsigned long __force)(addr), (size))); \ +}) + +/* + * Ensure that the range [addr, addr+size) is within the process's + * address space + */ +static inline int __access_ok(unsigned long addr, unsigned long size) +{ + return size <= TASK_SIZE && addr <= TASK_SIZE - size; +} + +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +#define __LSW 0 +#define __MSW 1 + +/* + * The "__xxx" versions of the user access functions do not verify the address + * space - it must have been done previously with a separate "access_ok()" + * call. + */ + +#define __get_user_asm(insn, x, ptr, err) \ +do { \ + uintptr_t __tmp; \ + __typeof__(x) __x; \ + __asm__ __volatile__ ( \ + "1:\n" \ + " " insn " %1, %3\n" \ + "2:\n" \ + " .section .fixup,\"ax\"\n" \ + " .balign 4\n" \ + "3:\n" \ + " li %0, %4\n" \ + " li %1, 0\n" \ + " jump 2b, %2\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .balign " RISCV_SZPTR "\n" \ + " " RISCV_PTR " 1b, 3b\n" \ + " .previous" \ + : "+r" (err), "=&r" (__x), "=r" (__tmp) \ + : "m" (*(ptr)), "i" (-EFAULT)); \ + (x) = __x; \ +} while (0) + +#ifdef CONFIG_64BIT +#define __get_user_8(x, ptr, err) \ + __get_user_asm("ld", x, ptr, err) +#else /* !CONFIG_64BIT */ +#define __get_user_8(x, ptr, err) \ +do { \ + u32 __user *__ptr = (u32 __user *)(ptr); \ + u32 __lo, __hi; \ + uintptr_t __tmp; \ + __asm__ __volatile__ ( \ + "1:\n" \ + " lw %1, %4\n" \ + "2:\n" \ + " lw %2, %5\n" \ + "3:\n" \ + " .section .fixup,\"ax\"\n" \ + " .balign 4\n" \ + "4:\n" \ + " li %0, %6\n" \ + " li %1, 0\n" \ + " li %2, 0\n" \ + " jump 3b, %3\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .balign " RISCV_SZPTR "\n" \ + " " RISCV_PTR " 1b, 4b\n" \ + " " RISCV_PTR " 2b, 4b\n" \ + " .previous" \ + : "+r" (err), "=&r" (__lo), "=r" (__hi), \ + "=r" (__tmp) \ + : "m" (__ptr[__LSW]), "m" (__ptr[__MSW]), \ + "i" (-EFAULT)); \ + (x) = (__typeof__(x))((__typeof__((x)-(x)))( \ + (((u64)__hi << 32) | __lo))); \ +} while (0) +#endif /* CONFIG_64BIT */ + +#define __get_user_nocheck(x, __gu_ptr, __gu_err) \ +do { \ + switch (sizeof(*__gu_ptr)) { \ + case 1: \ + __get_user_asm("lb", (x), __gu_ptr, __gu_err); \ + break; \ + case 2: \ + __get_user_asm("lh", (x), __gu_ptr, __gu_err); \ + break; \ + case 4: \ + __get_user_asm("lw", (x), __gu_ptr, __gu_err); \ + break; \ + case 8: \ + __get_user_8((x), __gu_ptr, __gu_err); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ +} while (0) + +/** + * __get_user: - Get a simple variable from user space, with less checking. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ +#define __get_user(x, ptr) \ +({ \ + const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ + long __gu_err = 0; \ + \ + __chk_user_ptr(__gu_ptr); \ + \ + __enable_user_access(); \ + __get_user_nocheck(x, __gu_ptr, __gu_err); \ + __disable_user_access(); \ + \ + __gu_err; \ +}) + +/** + * get_user: - Get a simple variable from user space. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ +#define get_user(x, ptr) \ +({ \ + const __typeof__(*(ptr)) __user *__p = (ptr); \ + might_fault(); \ + access_ok(__p, sizeof(*__p)) ? \ + __get_user((x), __p) : \ + ((x) = (__force __typeof__(x))0, -EFAULT); \ +}) + +#define __put_user_asm(insn, x, ptr, err) \ +do { \ + uintptr_t __tmp; \ + __typeof__(*(ptr)) __x = x; \ + __asm__ __volatile__ ( \ + "1:\n" \ + " " insn " %z3, %2\n" \ + "2:\n" \ + " .section .fixup,\"ax\"\n" \ + " .balign 4\n" \ + "3:\n" \ + " li %0, %4\n" \ + " jump 2b, %1\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .balign " RISCV_SZPTR "\n" \ + " " RISCV_PTR " 1b, 3b\n" \ + " .previous" \ + : "+r" (err), "=r" (__tmp), "=m" (*(ptr)) \ + : "rJ" (__x), "i" (-EFAULT)); \ +} while (0) + +#ifdef CONFIG_64BIT +#define __put_user_8(x, ptr, err) \ + __put_user_asm("sd", x, ptr, err) +#else /* !CONFIG_64BIT */ +#define __put_user_8(x, ptr, err) \ +do { \ + u32 __user *__ptr = (u32 __user *)(ptr); \ + u64 __x = (__typeof__((x)-(x)))(x); \ + uintptr_t __tmp; \ + __asm__ __volatile__ ( \ + "1:\n" \ + " sw %z4, %2\n" \ + "2:\n" \ + " sw %z5, %3\n" \ + "3:\n" \ + " .section .fixup,\"ax\"\n" \ + " .balign 4\n" \ + "4:\n" \ + " li %0, %6\n" \ + " jump 3b, %1\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .balign " RISCV_SZPTR "\n" \ + " " RISCV_PTR " 1b, 4b\n" \ + " " RISCV_PTR " 2b, 4b\n" \ + " .previous" \ + : "+r" (err), "=r" (__tmp), \ + "=m" (__ptr[__LSW]), \ + "=m" (__ptr[__MSW]) \ + : "rJ" (__x), "rJ" (__x >> 32), "i" (-EFAULT)); \ +} while (0) +#endif /* CONFIG_64BIT */ + +#define __put_user_nocheck(x, __gu_ptr, __pu_err) \ +do { \ + switch (sizeof(*__gu_ptr)) { \ + case 1: \ + __put_user_asm("sb", (x), __gu_ptr, __pu_err); \ + break; \ + case 2: \ + __put_user_asm("sh", (x), __gu_ptr, __pu_err); \ + break; \ + case 4: \ + __put_user_asm("sw", (x), __gu_ptr, __pu_err); \ + break; \ + case 8: \ + __put_user_8((x), __gu_ptr, __pu_err); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ +} while (0) + +/** + * __put_user: - Write a simple value into user space, with less checking. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. The value of @x is copied to avoid + * re-ordering where @x is evaluated inside the block that enables user-space + * access (thus bypassing user space protection if @x is a function). + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + */ +#define __put_user(x, ptr) \ +({ \ + __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ + __typeof__(*__gu_ptr) __val = (x); \ + long __pu_err = 0; \ + \ + __chk_user_ptr(__gu_ptr); \ + \ + __enable_user_access(); \ + __put_user_nocheck(__val, __gu_ptr, __pu_err); \ + __disable_user_access(); \ + \ + __pu_err; \ +}) + +/** + * put_user: - Write a simple value into user space. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Returns zero on success, or -EFAULT on error. + */ +#define put_user(x, ptr) \ +({ \ + __typeof__(*(ptr)) __user *__p = (ptr); \ + might_fault(); \ + access_ok(__p, sizeof(*__p)) ? \ + __put_user((x), __p) : \ + -EFAULT; \ +}) + + +unsigned long __must_check __asm_copy_to_user(void __user *to, + const void *from, unsigned long n); +unsigned long __must_check __asm_copy_from_user(void *to, + const void __user *from, unsigned long n); + +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return __asm_copy_from_user(to, from, n); +} + +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return __asm_copy_to_user(to, from, n); +} + +extern long strncpy_from_user(char *dest, const char __user *src, long count); + +extern long __must_check strlen_user(const char __user *str); +extern long __must_check strnlen_user(const char __user *str, long n); + +extern +unsigned long __must_check __clear_user(void __user *addr, unsigned long n); + +static inline +unsigned long __must_check clear_user(void __user *to, unsigned long n) +{ + might_fault(); + return access_ok(to, n) ? + __clear_user(to, n) : n; +} + +/* + * Atomic compare-and-exchange, but with a fixup for userspace faults. Faults + * will set "err" to -EFAULT, while successful accesses return the previous + * value. + */ +#define __cmpxchg_user(ptr, old, new, err, size, lrb, scb) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(*(ptr)) __old = (old); \ + __typeof__(*(ptr)) __new = (new); \ + __typeof__(*(ptr)) __ret; \ + __typeof__(err) __err = 0; \ + register unsigned int __rc; \ + __enable_user_access(); \ + switch (size) { \ + case 4: \ + __asm__ __volatile__ ( \ + "0:\n" \ + " lr.w" #scb " %[ret], %[ptr]\n" \ + " bne %[ret], %z[old], 1f\n" \ + " sc.w" #lrb " %[rc], %z[new], %[ptr]\n" \ + " bnez %[rc], 0b\n" \ + "1:\n" \ + ".section .fixup,\"ax\"\n" \ + ".balign 4\n" \ + "2:\n" \ + " li %[err], %[efault]\n" \ + " jump 1b, %[rc]\n" \ + ".previous\n" \ + ".section __ex_table,\"a\"\n" \ + ".balign " RISCV_SZPTR "\n" \ + " " RISCV_PTR " 1b, 2b\n" \ + ".previous\n" \ + : [ret] "=&r" (__ret), \ + [rc] "=&r" (__rc), \ + [ptr] "+A" (*__ptr), \ + [err] "=&r" (__err) \ + : [old] "rJ" (__old), \ + [new] "rJ" (__new), \ + [efault] "i" (-EFAULT)); \ + break; \ + case 8: \ + __asm__ __volatile__ ( \ + "0:\n" \ + " lr.d" #scb " %[ret], %[ptr]\n" \ + " bne %[ret], %z[old], 1f\n" \ + " sc.d" #lrb " %[rc], %z[new], %[ptr]\n" \ + " bnez %[rc], 0b\n" \ + "1:\n" \ + ".section .fixup,\"ax\"\n" \ + ".balign 4\n" \ + "2:\n" \ + " li %[err], %[efault]\n" \ + " jump 1b, %[rc]\n" \ + ".previous\n" \ + ".section __ex_table,\"a\"\n" \ + ".balign " RISCV_SZPTR "\n" \ + " " RISCV_PTR " 1b, 2b\n" \ + ".previous\n" \ + : [ret] "=&r" (__ret), \ + [rc] "=&r" (__rc), \ + [ptr] "+A" (*__ptr), \ + [err] "=&r" (__err) \ + : [old] "rJ" (__old), \ + [new] "rJ" (__new), \ + [efault] "i" (-EFAULT)); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __disable_user_access(); \ + (err) = __err; \ + __ret; \ +}) + +#define HAVE_GET_KERNEL_NOFAULT + +#define __get_kernel_nofault(dst, src, type, err_label) \ +do { \ + long __kr_err; \ + \ + __get_user_nocheck(*((type *)(dst)), (type *)(src), __kr_err); \ + if (unlikely(__kr_err)) \ + goto err_label; \ +} while (0) + +#define __put_kernel_nofault(dst, src, type, err_label) \ +do { \ + long __kr_err; \ + \ + __put_user_nocheck(*((type *)(src)), (type *)(dst), __kr_err); \ + if (unlikely(__kr_err)) \ + goto err_label; \ +} while (0) + +#else /* CONFIG_MMU */ +#include <asm-generic/uaccess.h> +#endif /* CONFIG_MMU */ +#endif /* _ASM_RISCV_UACCESS_H */ diff --git a/arch/riscv/include/asm/unistd.h b/arch/riscv/include/asm/unistd.h new file mode 100644 index 000000000..977ee6181 --- /dev/null +++ b/arch/riscv/include/asm/unistd.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +/* + * There is explicitly no include guard here because this file is expected to + * be included multiple times. + */ + +#define __ARCH_WANT_SYS_CLONE + +#include <uapi/asm/unistd.h> + +#define NR_syscalls (__NR_syscalls) diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h new file mode 100644 index 000000000..1453a2f56 --- /dev/null +++ b/arch/riscv/include/asm/vdso.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 ARM Limited + * Copyright (C) 2014 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#ifndef _ASM_RISCV_VDSO_H +#define _ASM_RISCV_VDSO_H + +#include <linux/types.h> + +#ifndef CONFIG_GENERIC_TIME_VSYSCALL +struct vdso_data { +}; +#endif + +/* + * The VDSO symbols are mapped into Linux so we can just use regular symbol + * addressing to get their offsets in userspace. The symbols are mapped at an + * offset of 0, but since the linker must support setting weak undefined + * symbols to the absolute address 0 it also happens to support other low + * addresses even when the code model suggests those low addresses would not + * otherwise be availiable. + */ +#define VDSO_SYMBOL(base, name) \ +({ \ + extern const char __vdso_##name[]; \ + (void __user *)((unsigned long)(base) + __vdso_##name); \ +}) + +asmlinkage long sys_riscv_flush_icache(uintptr_t, uintptr_t, uintptr_t); + +#endif /* _ASM_RISCV_VDSO_H */ diff --git a/arch/riscv/include/asm/vdso/clocksource.h b/arch/riscv/include/asm/vdso/clocksource.h new file mode 100644 index 000000000..df6ea65c1 --- /dev/null +++ b/arch/riscv/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000..f839f16e0 --- /dev/null +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_GETTIMEOFDAY_H +#define __ASM_VDSO_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include <asm/barrier.h> +#include <asm/unistd.h> +#include <asm/csr.h> +#include <uapi/linux/time.h> + +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + register struct __kernel_old_timeval *tv asm("a0") = _tv; + register struct timezone *tz asm("a1") = _tz; + register long ret asm("a0"); + register long nr asm("a7") = __NR_gettimeofday; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(tv), "r"(tz), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm("a7") = __NR_clock_gettime; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm("a7") = __NR_clock_getres; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_data *vd) +{ + /* + * The purpose of csr_read(CSR_TIME) is to trap the system into + * M-mode to obtain the value of CSR_TIME. Hence, unlike other + * architecture, no fence instructions surround the csr_read() + */ + return csr_read(CSR_TIME); +} + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return _vdso_data; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h new file mode 100644 index 000000000..134388cba --- /dev/null +++ b/arch/riscv/include/asm/vdso/processor.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#include <asm/barrier.h> + +static inline void cpu_relax(void) +{ +#ifdef __riscv_muldiv + int dummy; + /* In lieu of a halt instruction, induce a long-latency stall. */ + __asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy)); +#endif + barrier(); +} + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/riscv/include/asm/vdso/vsyscall.h b/arch/riscv/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000..82fd5d83b --- /dev/null +++ b/arch/riscv/include/asm/vdso/vsyscall.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_VSYSCALL_H +#define __ASM_VDSO_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include <linux/timekeeper_internal.h> +#include <vdso/datapage.h> + +extern struct vdso_data *vdso_data; + +/* + * Update the vDSO data page to keep in sync with kernel timekeeping. + */ +static __always_inline struct vdso_data *__riscv_get_k_vdso_data(void) +{ + return vdso_data; +} + +#define __arch_get_k_vdso_data __riscv_get_k_vdso_data + +/* The asm-generic header needs to be included after the definitions above */ +#include <asm-generic/vdso/vsyscall.h> + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_VSYSCALL_H */ diff --git a/arch/riscv/include/asm/vermagic.h b/arch/riscv/include/asm/vermagic.h new file mode 100644 index 000000000..7b9441a57 --- /dev/null +++ b/arch/riscv/include/asm/vermagic.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2017 Andes Technology Corporation */ + +#ifndef _ASM_VERMAGIC_H +#define _ASM_VERMAGIC_H + +#define MODULE_ARCH_VERMAGIC "riscv" + +#endif /* _ASM_VERMAGIC_H */ diff --git a/arch/riscv/include/asm/vmalloc.h b/arch/riscv/include/asm/vmalloc.h new file mode 100644 index 000000000..ff9abc00d --- /dev/null +++ b/arch/riscv/include/asm/vmalloc.h @@ -0,0 +1,4 @@ +#ifndef _ASM_RISCV_VMALLOC_H +#define _ASM_RISCV_VMALLOC_H + +#endif /* _ASM_RISCV_VMALLOC_H */ diff --git a/arch/riscv/include/asm/word-at-a-time.h b/arch/riscv/include/asm/word-at-a-time.h new file mode 100644 index 000000000..7c086ac6e --- /dev/null +++ b/arch/riscv/include/asm/word-at-a-time.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + * + * Derived from arch/x86/include/asm/word-at-a-time.h + */ + +#ifndef _ASM_RISCV_WORD_AT_A_TIME_H +#define _ASM_RISCV_WORD_AT_A_TIME_H + + +#include <linux/kernel.h> + +struct word_at_a_time { + const unsigned long one_bits, high_bits; +}; + +#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) } + +static inline unsigned long has_zero(unsigned long val, + unsigned long *bits, const struct word_at_a_time *c) +{ + unsigned long mask = ((val - c->one_bits) & ~val) & c->high_bits; + *bits = mask; + return mask; +} + +static inline unsigned long prep_zero_mask(unsigned long val, + unsigned long bits, const struct word_at_a_time *c) +{ + return bits; +} + +static inline unsigned long create_zero_mask(unsigned long bits) +{ + bits = (bits - 1) & ~bits; + return bits >> 7; +} + +static inline unsigned long find_zero(unsigned long mask) +{ + return fls64(mask) >> 3; +} + +/* The mask we created is directly usable as a bytemask */ +#define zero_bytemask(mask) (mask) + +#endif /* _ASM_RISCV_WORD_AT_A_TIME_H */ |