diff options
Diffstat (limited to 'arch/riscv/kernel')
59 files changed, 8173 insertions, 0 deletions
diff --git a/arch/riscv/kernel/.gitignore b/arch/riscv/kernel/.gitignore new file mode 100644 index 000000000..e052ed331 --- /dev/null +++ b/arch/riscv/kernel/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +/vmlinux.lds diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile new file mode 100644 index 000000000..bc49d5f23 --- /dev/null +++ b/arch/riscv/kernel/Makefile @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the RISC-V Linux kernel +# + +ifdef CONFIG_FTRACE +CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_sbi.o = $(CC_FLAGS_FTRACE) +endif + +extra-y += head.o +extra-y += vmlinux.lds + +obj-y += soc.o +obj-y += cpu.o +obj-y += cpufeature.o +obj-y += entry.o +obj-y += irq.o +obj-y += process.o +obj-y += ptrace.o +obj-y += reset.o +obj-y += setup.o +obj-y += signal.o +obj-y += syscall_table.o +obj-y += sys_riscv.o +obj-y += time.o +obj-y += traps.o +obj-y += riscv_ksyms.o +obj-y += stacktrace.o +obj-y += cacheinfo.o +obj-y += patch.o +obj-$(CONFIG_MMU) += vdso.o vdso/ + +obj-$(CONFIG_RISCV_M_MODE) += traps_misaligned.o +obj-$(CONFIG_FPU) += fpu.o +obj-$(CONFIG_SMP) += smpboot.o +obj-$(CONFIG_SMP) += smp.o +obj-$(CONFIG_SMP) += cpu_ops.o +obj-$(CONFIG_SMP) += cpu_ops_spinwait.o +obj-$(CONFIG_MODULES) += module.o +obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o + +obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o +obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o + +obj-$(CONFIG_TRACE_IRQFLAGS) += trace_irq.o + +obj-$(CONFIG_RISCV_BASE_PMU) += perf_event.o +obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o +obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o +obj-$(CONFIG_RISCV_SBI) += sbi.o +ifeq ($(CONFIG_RISCV_SBI), y) +obj-$(CONFIG_SMP) += cpu_ops_sbi.o +endif +obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o +obj-$(CONFIG_KGDB) += kgdb.o + +obj-$(CONFIG_JUMP_LABEL) += jump_label.o + +obj-$(CONFIG_EFI) += efi.o + +clean: diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c new file mode 100644 index 000000000..db203442c --- /dev/null +++ b/arch/riscv/kernel/asm-offsets.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#define GENERATING_ASM_OFFSETS + +#include <linux/kbuild.h> +#include <linux/sched.h> +#include <asm/thread_info.h> +#include <asm/ptrace.h> + +void asm_offsets(void) +{ + OFFSET(TASK_THREAD_RA, task_struct, thread.ra); + OFFSET(TASK_THREAD_SP, task_struct, thread.sp); + OFFSET(TASK_THREAD_S0, task_struct, thread.s[0]); + OFFSET(TASK_THREAD_S1, task_struct, thread.s[1]); + OFFSET(TASK_THREAD_S2, task_struct, thread.s[2]); + OFFSET(TASK_THREAD_S3, task_struct, thread.s[3]); + OFFSET(TASK_THREAD_S4, task_struct, thread.s[4]); + OFFSET(TASK_THREAD_S5, task_struct, thread.s[5]); + OFFSET(TASK_THREAD_S6, task_struct, thread.s[6]); + OFFSET(TASK_THREAD_S7, task_struct, thread.s[7]); + OFFSET(TASK_THREAD_S8, task_struct, thread.s[8]); + OFFSET(TASK_THREAD_S9, task_struct, thread.s[9]); + OFFSET(TASK_THREAD_S10, task_struct, thread.s[10]); + OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]); + OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags); + OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); + OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); + OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp); + OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu); + + OFFSET(TASK_THREAD_F0, task_struct, thread.fstate.f[0]); + OFFSET(TASK_THREAD_F1, task_struct, thread.fstate.f[1]); + OFFSET(TASK_THREAD_F2, task_struct, thread.fstate.f[2]); + OFFSET(TASK_THREAD_F3, task_struct, thread.fstate.f[3]); + OFFSET(TASK_THREAD_F4, task_struct, thread.fstate.f[4]); + OFFSET(TASK_THREAD_F5, task_struct, thread.fstate.f[5]); + OFFSET(TASK_THREAD_F6, task_struct, thread.fstate.f[6]); + OFFSET(TASK_THREAD_F7, task_struct, thread.fstate.f[7]); + OFFSET(TASK_THREAD_F8, task_struct, thread.fstate.f[8]); + OFFSET(TASK_THREAD_F9, task_struct, thread.fstate.f[9]); + OFFSET(TASK_THREAD_F10, task_struct, thread.fstate.f[10]); + OFFSET(TASK_THREAD_F11, task_struct, thread.fstate.f[11]); + OFFSET(TASK_THREAD_F12, task_struct, thread.fstate.f[12]); + OFFSET(TASK_THREAD_F13, task_struct, thread.fstate.f[13]); + OFFSET(TASK_THREAD_F14, task_struct, thread.fstate.f[14]); + OFFSET(TASK_THREAD_F15, task_struct, thread.fstate.f[15]); + OFFSET(TASK_THREAD_F16, task_struct, thread.fstate.f[16]); + OFFSET(TASK_THREAD_F17, task_struct, thread.fstate.f[17]); + OFFSET(TASK_THREAD_F18, task_struct, thread.fstate.f[18]); + OFFSET(TASK_THREAD_F19, task_struct, thread.fstate.f[19]); + OFFSET(TASK_THREAD_F20, task_struct, thread.fstate.f[20]); + OFFSET(TASK_THREAD_F21, task_struct, thread.fstate.f[21]); + OFFSET(TASK_THREAD_F22, task_struct, thread.fstate.f[22]); + OFFSET(TASK_THREAD_F23, task_struct, thread.fstate.f[23]); + OFFSET(TASK_THREAD_F24, task_struct, thread.fstate.f[24]); + OFFSET(TASK_THREAD_F25, task_struct, thread.fstate.f[25]); + OFFSET(TASK_THREAD_F26, task_struct, thread.fstate.f[26]); + OFFSET(TASK_THREAD_F27, task_struct, thread.fstate.f[27]); + OFFSET(TASK_THREAD_F28, task_struct, thread.fstate.f[28]); + OFFSET(TASK_THREAD_F29, task_struct, thread.fstate.f[29]); + OFFSET(TASK_THREAD_F30, task_struct, thread.fstate.f[30]); + OFFSET(TASK_THREAD_F31, task_struct, thread.fstate.f[31]); + OFFSET(TASK_THREAD_FCSR, task_struct, thread.fstate.fcsr); + + DEFINE(PT_SIZE, sizeof(struct pt_regs)); + OFFSET(PT_EPC, pt_regs, epc); + OFFSET(PT_RA, pt_regs, ra); + OFFSET(PT_FP, pt_regs, s0); + OFFSET(PT_S0, pt_regs, s0); + OFFSET(PT_S1, pt_regs, s1); + OFFSET(PT_S2, pt_regs, s2); + OFFSET(PT_S3, pt_regs, s3); + OFFSET(PT_S4, pt_regs, s4); + OFFSET(PT_S5, pt_regs, s5); + OFFSET(PT_S6, pt_regs, s6); + OFFSET(PT_S7, pt_regs, s7); + OFFSET(PT_S8, pt_regs, s8); + OFFSET(PT_S9, pt_regs, s9); + OFFSET(PT_S10, pt_regs, s10); + OFFSET(PT_S11, pt_regs, s11); + OFFSET(PT_SP, pt_regs, sp); + OFFSET(PT_TP, pt_regs, tp); + OFFSET(PT_A0, pt_regs, a0); + OFFSET(PT_A1, pt_regs, a1); + OFFSET(PT_A2, pt_regs, a2); + OFFSET(PT_A3, pt_regs, a3); + OFFSET(PT_A4, pt_regs, a4); + OFFSET(PT_A5, pt_regs, a5); + OFFSET(PT_A6, pt_regs, a6); + OFFSET(PT_A7, pt_regs, a7); + OFFSET(PT_T0, pt_regs, t0); + OFFSET(PT_T1, pt_regs, t1); + OFFSET(PT_T2, pt_regs, t2); + OFFSET(PT_T3, pt_regs, t3); + OFFSET(PT_T4, pt_regs, t4); + OFFSET(PT_T5, pt_regs, t5); + OFFSET(PT_T6, pt_regs, t6); + OFFSET(PT_GP, pt_regs, gp); + OFFSET(PT_ORIG_A0, pt_regs, orig_a0); + OFFSET(PT_STATUS, pt_regs, status); + OFFSET(PT_BADADDR, pt_regs, badaddr); + OFFSET(PT_CAUSE, pt_regs, cause); + + /* + * THREAD_{F,X}* might be larger than a S-type offset can handle, but + * these are used in performance-sensitive assembly so we can't resort + * to loading the long immediate every time. + */ + DEFINE(TASK_THREAD_RA_RA, + offsetof(struct task_struct, thread.ra) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_SP_RA, + offsetof(struct task_struct, thread.sp) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S0_RA, + offsetof(struct task_struct, thread.s[0]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S1_RA, + offsetof(struct task_struct, thread.s[1]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S2_RA, + offsetof(struct task_struct, thread.s[2]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S3_RA, + offsetof(struct task_struct, thread.s[3]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S4_RA, + offsetof(struct task_struct, thread.s[4]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S5_RA, + offsetof(struct task_struct, thread.s[5]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S6_RA, + offsetof(struct task_struct, thread.s[6]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S7_RA, + offsetof(struct task_struct, thread.s[7]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S8_RA, + offsetof(struct task_struct, thread.s[8]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S9_RA, + offsetof(struct task_struct, thread.s[9]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S10_RA, + offsetof(struct task_struct, thread.s[10]) + - offsetof(struct task_struct, thread.ra) + ); + DEFINE(TASK_THREAD_S11_RA, + offsetof(struct task_struct, thread.s[11]) + - offsetof(struct task_struct, thread.ra) + ); + + DEFINE(TASK_THREAD_F0_F0, + offsetof(struct task_struct, thread.fstate.f[0]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F1_F0, + offsetof(struct task_struct, thread.fstate.f[1]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F2_F0, + offsetof(struct task_struct, thread.fstate.f[2]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F3_F0, + offsetof(struct task_struct, thread.fstate.f[3]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F4_F0, + offsetof(struct task_struct, thread.fstate.f[4]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F5_F0, + offsetof(struct task_struct, thread.fstate.f[5]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F6_F0, + offsetof(struct task_struct, thread.fstate.f[6]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F7_F0, + offsetof(struct task_struct, thread.fstate.f[7]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F8_F0, + offsetof(struct task_struct, thread.fstate.f[8]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F9_F0, + offsetof(struct task_struct, thread.fstate.f[9]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F10_F0, + offsetof(struct task_struct, thread.fstate.f[10]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F11_F0, + offsetof(struct task_struct, thread.fstate.f[11]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F12_F0, + offsetof(struct task_struct, thread.fstate.f[12]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F13_F0, + offsetof(struct task_struct, thread.fstate.f[13]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F14_F0, + offsetof(struct task_struct, thread.fstate.f[14]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F15_F0, + offsetof(struct task_struct, thread.fstate.f[15]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F16_F0, + offsetof(struct task_struct, thread.fstate.f[16]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F17_F0, + offsetof(struct task_struct, thread.fstate.f[17]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F18_F0, + offsetof(struct task_struct, thread.fstate.f[18]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F19_F0, + offsetof(struct task_struct, thread.fstate.f[19]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F20_F0, + offsetof(struct task_struct, thread.fstate.f[20]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F21_F0, + offsetof(struct task_struct, thread.fstate.f[21]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F22_F0, + offsetof(struct task_struct, thread.fstate.f[22]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F23_F0, + offsetof(struct task_struct, thread.fstate.f[23]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F24_F0, + offsetof(struct task_struct, thread.fstate.f[24]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F25_F0, + offsetof(struct task_struct, thread.fstate.f[25]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F26_F0, + offsetof(struct task_struct, thread.fstate.f[26]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F27_F0, + offsetof(struct task_struct, thread.fstate.f[27]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F28_F0, + offsetof(struct task_struct, thread.fstate.f[28]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F29_F0, + offsetof(struct task_struct, thread.fstate.f[29]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F30_F0, + offsetof(struct task_struct, thread.fstate.f[30]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_F31_F0, + offsetof(struct task_struct, thread.fstate.f[31]) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + DEFINE(TASK_THREAD_FCSR_F0, + offsetof(struct task_struct, thread.fstate.fcsr) + - offsetof(struct task_struct, thread.fstate.f[0]) + ); + + /* + * We allocate a pt_regs on the stack when entering the kernel. This + * ensures the alignment is sane. + */ + DEFINE(PT_SIZE_ON_STACK, ALIGN(sizeof(struct pt_regs), STACK_ALIGN)); +} diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c new file mode 100644 index 000000000..90deabfe6 --- /dev/null +++ b/arch/riscv/kernel/cacheinfo.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017 SiFive + */ + +#include <linux/cpu.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <asm/cacheinfo.h> + +static struct riscv_cacheinfo_ops *rv_cache_ops; + +void riscv_set_cacheinfo_ops(struct riscv_cacheinfo_ops *ops) +{ + rv_cache_ops = ops; +} +EXPORT_SYMBOL_GPL(riscv_set_cacheinfo_ops); + +const struct attribute_group * +cache_get_priv_group(struct cacheinfo *this_leaf) +{ + if (rv_cache_ops && rv_cache_ops->get_priv_group) + return rv_cache_ops->get_priv_group(this_leaf); + return NULL; +} + +static struct cacheinfo *get_cacheinfo(u32 level, enum cache_type type) +{ + /* + * Using raw_smp_processor_id() elides a preemptability check, but this + * is really indicative of a larger problem: the cacheinfo UABI assumes + * that cores have a homonogenous view of the cache hierarchy. That + * happens to be the case for the current set of RISC-V systems, but + * likely won't be true in general. Since there's no way to provide + * correct information for these systems via the current UABI we're + * just eliding the check for now. + */ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(raw_smp_processor_id()); + struct cacheinfo *this_leaf; + int index; + + for (index = 0; index < this_cpu_ci->num_leaves; index++) { + this_leaf = this_cpu_ci->info_list + index; + if (this_leaf->level == level && this_leaf->type == type) + return this_leaf; + } + + return NULL; +} + +uintptr_t get_cache_size(u32 level, enum cache_type type) +{ + struct cacheinfo *this_leaf = get_cacheinfo(level, type); + + return this_leaf ? this_leaf->size : 0; +} + +uintptr_t get_cache_geometry(u32 level, enum cache_type type) +{ + struct cacheinfo *this_leaf = get_cacheinfo(level, type); + + return this_leaf ? (this_leaf->ways_of_associativity << 16 | + this_leaf->coherency_line_size) : + 0; +} + +static void ci_leaf_init(struct cacheinfo *this_leaf, enum cache_type type, + unsigned int level, unsigned int size, + unsigned int sets, unsigned int line_size) +{ + this_leaf->level = level; + this_leaf->type = type; + this_leaf->size = size; + this_leaf->number_of_sets = sets; + this_leaf->coherency_line_size = line_size; + + /* + * If the cache is fully associative, there is no need to + * check the other properties. + */ + if (sets == 1) + return; + + /* + * Set the ways number for n-ways associative, make sure + * all properties are big than zero. + */ + if (sets > 0 && size > 0 && line_size > 0) + this_leaf->ways_of_associativity = (size / sets) / line_size; +} + +static void fill_cacheinfo(struct cacheinfo **this_leaf, + struct device_node *node, unsigned int level) +{ + unsigned int size, sets, line_size; + + if (!of_property_read_u32(node, "cache-size", &size) && + !of_property_read_u32(node, "cache-block-size", &line_size) && + !of_property_read_u32(node, "cache-sets", &sets)) { + ci_leaf_init((*this_leaf)++, CACHE_TYPE_UNIFIED, level, size, sets, line_size); + } + + if (!of_property_read_u32(node, "i-cache-size", &size) && + !of_property_read_u32(node, "i-cache-sets", &sets) && + !of_property_read_u32(node, "i-cache-block-size", &line_size)) { + ci_leaf_init((*this_leaf)++, CACHE_TYPE_INST, level, size, sets, line_size); + } + + if (!of_property_read_u32(node, "d-cache-size", &size) && + !of_property_read_u32(node, "d-cache-sets", &sets) && + !of_property_read_u32(node, "d-cache-block-size", &line_size)) { + ci_leaf_init((*this_leaf)++, CACHE_TYPE_DATA, level, size, sets, line_size); + } +} + +int init_cache_level(unsigned int cpu) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct device_node *np = of_cpu_device_node_get(cpu); + struct device_node *prev = NULL; + int levels = 0, leaves = 0, level; + + if (of_property_read_bool(np, "cache-size")) + ++leaves; + if (of_property_read_bool(np, "i-cache-size")) + ++leaves; + if (of_property_read_bool(np, "d-cache-size")) + ++leaves; + if (leaves > 0) + levels = 1; + + prev = np; + while ((np = of_find_next_cache_node(np))) { + of_node_put(prev); + prev = np; + if (!of_device_is_compatible(np, "cache")) + break; + if (of_property_read_u32(np, "cache-level", &level)) + break; + if (level <= levels) + break; + if (of_property_read_bool(np, "cache-size")) + ++leaves; + if (of_property_read_bool(np, "i-cache-size")) + ++leaves; + if (of_property_read_bool(np, "d-cache-size")) + ++leaves; + levels = level; + } + + of_node_put(np); + this_cpu_ci->num_levels = levels; + this_cpu_ci->num_leaves = leaves; + + return 0; +} + +int populate_cache_leaves(unsigned int cpu) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf = this_cpu_ci->info_list; + struct device_node *np = of_cpu_device_node_get(cpu); + struct device_node *prev = NULL; + int levels = 1, level = 1; + + /* Level 1 caches in cpu node */ + fill_cacheinfo(&this_leaf, np, level); + + /* Next level caches in cache nodes */ + prev = np; + while ((np = of_find_next_cache_node(np))) { + of_node_put(prev); + prev = np; + + if (!of_device_is_compatible(np, "cache")) + break; + if (of_property_read_u32(np, "cache-level", &level)) + break; + if (level <= levels) + break; + + fill_cacheinfo(&this_leaf, np, level); + + levels = level; + } + of_node_put(np); + + return 0; +} diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c new file mode 100644 index 000000000..df84e0c13 --- /dev/null +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/err.h> +#include <linux/irq.h> +#include <linux/cpu.h> +#include <linux/sched/hotplug.h> +#include <asm/irq.h> +#include <asm/cpu_ops.h> +#include <asm/sbi.h> + +void cpu_stop(void); +void arch_cpu_idle_dead(void) +{ + cpu_stop(); +} + +bool cpu_has_hotplug(unsigned int cpu) +{ + if (cpu_ops[cpu]->cpu_stop) + return true; + + return false; +} + +/* + * __cpu_disable runs on the processor to be shutdown. + */ +int __cpu_disable(void) +{ + int ret = 0; + unsigned int cpu = smp_processor_id(); + + if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_stop) + return -EOPNOTSUPP; + + if (cpu_ops[cpu]->cpu_disable) + ret = cpu_ops[cpu]->cpu_disable(cpu); + + if (ret) + return ret; + + remove_cpu_topology(cpu); + set_cpu_online(cpu, false); + irq_migrate_all_off_this_cpu(); + + return ret; +} + +/* + * Called on the thread which is asking for a CPU to be shutdown. + */ +void __cpu_die(unsigned int cpu) +{ + int ret = 0; + + if (!cpu_wait_death(cpu, 5)) { + pr_err("CPU %u: didn't die\n", cpu); + return; + } + pr_notice("CPU%u: off\n", cpu); + + /* Verify from the firmware if the cpu is really stopped*/ + if (cpu_ops[cpu]->cpu_is_stopped) + ret = cpu_ops[cpu]->cpu_is_stopped(cpu); + if (ret) + pr_warn("CPU%d may not have stopped: %d\n", cpu, ret); +} + +/* + * Called from the idle thread for the CPU which has been shutdown. + */ +void cpu_stop(void) +{ + idle_task_exit(); + + (void)cpu_report_death(); + + cpu_ops[smp_processor_id()]->cpu_stop(); + /* It should never reach here */ + BUG(); +} diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c new file mode 100644 index 000000000..6d59e6906 --- /dev/null +++ b/arch/riscv/kernel/cpu.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#include <linux/init.h> +#include <linux/seq_file.h> +#include <linux/of.h> +#include <asm/smp.h> + +/* + * Returns the hart ID of the given device tree node, or -ENODEV if the node + * isn't an enabled and valid RISC-V hart node. + */ +int riscv_of_processor_hartid(struct device_node *node) +{ + const char *isa; + u32 hart; + + if (!of_device_is_compatible(node, "riscv")) { + pr_warn("Found incompatible CPU\n"); + return -ENODEV; + } + + if (of_property_read_u32(node, "reg", &hart)) { + pr_warn("Found CPU without hart ID\n"); + return -ENODEV; + } + + if (!of_device_is_available(node)) { + pr_info("CPU with hartid=%d is not available\n", hart); + return -ENODEV; + } + + if (of_property_read_string(node, "riscv,isa", &isa)) { + pr_warn("CPU with hartid=%d has no \"riscv,isa\" property\n", hart); + return -ENODEV; + } + if (isa[0] != 'r' || isa[1] != 'v') { + pr_warn("CPU with hartid=%d has an invalid ISA of \"%s\"\n", hart, isa); + return -ENODEV; + } + + return hart; +} + +/* + * Find hart ID of the CPU DT node under which given DT node falls. + * + * To achieve this, we walk up the DT tree until we find an active + * RISC-V core (HART) node and extract the cpuid from it. + */ +int riscv_of_parent_hartid(struct device_node *node) +{ + for (; node; node = node->parent) { + if (of_device_is_compatible(node, "riscv")) + return riscv_of_processor_hartid(node); + } + + return -1; +} + +#ifdef CONFIG_PROC_FS + +static void print_isa(struct seq_file *f, const char *isa) +{ + /* Print the entire ISA as it is */ + seq_puts(f, "isa\t\t: "); + seq_write(f, isa, strlen(isa)); + seq_puts(f, "\n"); +} + +static void print_mmu(struct seq_file *f, const char *mmu_type) +{ +#if defined(CONFIG_32BIT) + if (strcmp(mmu_type, "riscv,sv32") != 0) + return; +#elif defined(CONFIG_64BIT) + if (strcmp(mmu_type, "riscv,sv39") != 0 && + strcmp(mmu_type, "riscv,sv48") != 0) + return; +#endif + + seq_printf(f, "mmu\t\t: %s\n", mmu_type+6); +} + +static void *c_start(struct seq_file *m, loff_t *pos) +{ + *pos = cpumask_next(*pos - 1, cpu_online_mask); + if ((*pos) < nr_cpu_ids) + return (void *)(uintptr_t)(1 + *pos); + return NULL; +} + +static void *c_next(struct seq_file *m, void *v, loff_t *pos) +{ + (*pos)++; + return c_start(m, pos); +} + +static void c_stop(struct seq_file *m, void *v) +{ +} + +static int c_show(struct seq_file *m, void *v) +{ + unsigned long cpu_id = (unsigned long)v - 1; + struct device_node *node = of_get_cpu_node(cpu_id, NULL); + const char *compat, *isa, *mmu; + + seq_printf(m, "processor\t: %lu\n", cpu_id); + seq_printf(m, "hart\t\t: %lu\n", cpuid_to_hartid_map(cpu_id)); + if (!of_property_read_string(node, "riscv,isa", &isa)) + print_isa(m, isa); + if (!of_property_read_string(node, "mmu-type", &mmu)) + print_mmu(m, mmu); + if (!of_property_read_string(node, "compatible", &compat) + && strcmp(compat, "riscv")) + seq_printf(m, "uarch\t\t: %s\n", compat); + seq_puts(m, "\n"); + of_node_put(node); + + return 0; +} + +const struct seq_operations cpuinfo_op = { + .start = c_start, + .next = c_next, + .stop = c_stop, + .show = c_show +}; + +#endif /* CONFIG_PROC_FS */ diff --git a/arch/riscv/kernel/cpu_ops.c b/arch/riscv/kernel/cpu_ops.c new file mode 100644 index 000000000..1985884fe --- /dev/null +++ b/arch/riscv/kernel/cpu_ops.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020 Western Digital Corporation or its affiliates. + */ + +#include <linux/errno.h> +#include <linux/mm.h> +#include <linux/of.h> +#include <linux/string.h> +#include <linux/sched.h> +#include <linux/sched/task_stack.h> +#include <asm/cpu_ops.h> +#include <asm/sbi.h> +#include <asm/smp.h> + +const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; + +void *__cpu_up_stack_pointer[NR_CPUS] __section(".data"); +void *__cpu_up_task_pointer[NR_CPUS] __section(".data"); + +extern const struct cpu_operations cpu_ops_sbi; +extern const struct cpu_operations cpu_ops_spinwait; + +void cpu_update_secondary_bootdata(unsigned int cpuid, + struct task_struct *tidle) +{ + int hartid = cpuid_to_hartid_map(cpuid); + + /* Make sure tidle is updated */ + smp_mb(); + WRITE_ONCE(__cpu_up_stack_pointer[hartid], + task_stack_page(tidle) + THREAD_SIZE); + WRITE_ONCE(__cpu_up_task_pointer[hartid], tidle); +} + +void __init cpu_set_ops(int cpuid) +{ +#if IS_ENABLED(CONFIG_RISCV_SBI) + if (sbi_probe_extension(SBI_EXT_HSM) > 0) { + if (!cpuid) + pr_info("SBI v0.2 HSM extension detected\n"); + cpu_ops[cpuid] = &cpu_ops_sbi; + } else +#endif + cpu_ops[cpuid] = &cpu_ops_spinwait; +} diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c new file mode 100644 index 000000000..685fae72b --- /dev/null +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * HSM extension and cpu_ops implementation. + * + * Copyright (c) 2020 Western Digital Corporation or its affiliates. + */ + +#include <linux/init.h> +#include <linux/mm.h> +#include <asm/cpu_ops.h> +#include <asm/sbi.h> +#include <asm/smp.h> + +extern char secondary_start_sbi[]; +const struct cpu_operations cpu_ops_sbi; + +static int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr, + unsigned long priv) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_START, + hartid, saddr, priv, 0, 0, 0); + if (ret.error) + return sbi_err_map_linux_errno(ret.error); + else + return 0; +} + +#ifdef CONFIG_HOTPLUG_CPU +static int sbi_hsm_hart_stop(void) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STOP, 0, 0, 0, 0, 0, 0); + + if (ret.error) + return sbi_err_map_linux_errno(ret.error); + else + return 0; +} + +static int sbi_hsm_hart_get_status(unsigned long hartid) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STATUS, + hartid, 0, 0, 0, 0, 0); + if (ret.error) + return sbi_err_map_linux_errno(ret.error); + else + return ret.value; +} +#endif + +static int sbi_cpu_start(unsigned int cpuid, struct task_struct *tidle) +{ + int rc; + unsigned long boot_addr = __pa_symbol(secondary_start_sbi); + int hartid = cpuid_to_hartid_map(cpuid); + + cpu_update_secondary_bootdata(cpuid, tidle); + rc = sbi_hsm_hart_start(hartid, boot_addr, 0); + + return rc; +} + +static int sbi_cpu_prepare(unsigned int cpuid) +{ + if (!cpu_ops_sbi.cpu_start) { + pr_err("cpu start method not defined for CPU [%d]\n", cpuid); + return -ENODEV; + } + return 0; +} + +#ifdef CONFIG_HOTPLUG_CPU +static int sbi_cpu_disable(unsigned int cpuid) +{ + if (!cpu_ops_sbi.cpu_stop) + return -EOPNOTSUPP; + return 0; +} + +static void sbi_cpu_stop(void) +{ + int ret; + + ret = sbi_hsm_hart_stop(); + pr_crit("Unable to stop the cpu %u (%d)\n", smp_processor_id(), ret); +} + +static int sbi_cpu_is_stopped(unsigned int cpuid) +{ + int rc; + int hartid = cpuid_to_hartid_map(cpuid); + + rc = sbi_hsm_hart_get_status(hartid); + + if (rc == SBI_HSM_HART_STATUS_STOPPED) + return 0; + return rc; +} +#endif + +const struct cpu_operations cpu_ops_sbi = { + .name = "sbi", + .cpu_prepare = sbi_cpu_prepare, + .cpu_start = sbi_cpu_start, +#ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = sbi_cpu_disable, + .cpu_stop = sbi_cpu_stop, + .cpu_is_stopped = sbi_cpu_is_stopped, +#endif +}; diff --git a/arch/riscv/kernel/cpu_ops_spinwait.c b/arch/riscv/kernel/cpu_ops_spinwait.c new file mode 100644 index 000000000..b2c957bb6 --- /dev/null +++ b/arch/riscv/kernel/cpu_ops_spinwait.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020 Western Digital Corporation or its affiliates. + */ + +#include <linux/errno.h> +#include <linux/of.h> +#include <linux/string.h> +#include <asm/cpu_ops.h> +#include <asm/sbi.h> +#include <asm/smp.h> + +const struct cpu_operations cpu_ops_spinwait; + +static int spinwait_cpu_prepare(unsigned int cpuid) +{ + if (!cpu_ops_spinwait.cpu_start) { + pr_err("cpu start method not defined for CPU [%d]\n", cpuid); + return -ENODEV; + } + return 0; +} + +static int spinwait_cpu_start(unsigned int cpuid, struct task_struct *tidle) +{ + /* + * In this protocol, all cpus boot on their own accord. _start + * selects the first cpu to boot the kernel and causes the remainder + * of the cpus to spin in a loop waiting for their stack pointer to be + * setup by that main cpu. Writing to bootdata + * (i.e __cpu_up_stack_pointer) signals to the spinning cpus that they + * can continue the boot process. + */ + cpu_update_secondary_bootdata(cpuid, tidle); + + return 0; +} + +const struct cpu_operations cpu_ops_spinwait = { + .name = "spinwait", + .cpu_prepare = spinwait_cpu_prepare, + .cpu_start = spinwait_cpu_start, +}; diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c new file mode 100644 index 000000000..ac202f44a --- /dev/null +++ b/arch/riscv/kernel/cpufeature.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copied from arch/arm64/kernel/cpufeature.c + * + * Copyright (C) 2015 ARM Ltd. + * Copyright (C) 2017 SiFive + */ + +#include <linux/bitmap.h> +#include <linux/of.h> +#include <asm/processor.h> +#include <asm/hwcap.h> +#include <asm/smp.h> +#include <asm/switch_to.h> + +unsigned long elf_hwcap __read_mostly; + +/* Host ISA bitmap */ +static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; + +#ifdef CONFIG_FPU +bool has_fpu __read_mostly; +#endif + +/** + * riscv_isa_extension_base() - Get base extension word + * + * @isa_bitmap: ISA bitmap to use + * Return: base extension word as unsigned long value + * + * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. + */ +unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) +{ + if (!isa_bitmap) + return riscv_isa[0]; + return isa_bitmap[0]; +} +EXPORT_SYMBOL_GPL(riscv_isa_extension_base); + +/** + * __riscv_isa_extension_available() - Check whether given extension + * is available or not + * + * @isa_bitmap: ISA bitmap to use + * @bit: bit position of the desired extension + * Return: true or false + * + * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. + */ +bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) +{ + const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; + + if (bit >= RISCV_ISA_EXT_MAX) + return false; + + return test_bit(bit, bmap) ? true : false; +} +EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); + +void riscv_fill_hwcap(void) +{ + struct device_node *node; + const char *isa; + char print_str[BITS_PER_LONG + 1]; + size_t i, j, isa_len; + static unsigned long isa2hwcap[256] = {0}; + + isa2hwcap['i'] = isa2hwcap['I'] = COMPAT_HWCAP_ISA_I; + isa2hwcap['m'] = isa2hwcap['M'] = COMPAT_HWCAP_ISA_M; + isa2hwcap['a'] = isa2hwcap['A'] = COMPAT_HWCAP_ISA_A; + isa2hwcap['f'] = isa2hwcap['F'] = COMPAT_HWCAP_ISA_F; + isa2hwcap['d'] = isa2hwcap['D'] = COMPAT_HWCAP_ISA_D; + isa2hwcap['c'] = isa2hwcap['C'] = COMPAT_HWCAP_ISA_C; + + elf_hwcap = 0; + + bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); + + for_each_of_cpu_node(node) { + unsigned long this_hwcap = 0; + unsigned long this_isa = 0; + + if (riscv_of_processor_hartid(node) < 0) + continue; + + if (of_property_read_string(node, "riscv,isa", &isa)) { + pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); + continue; + } + + i = 0; + isa_len = strlen(isa); +#if IS_ENABLED(CONFIG_32BIT) + if (!strncmp(isa, "rv32", 4)) + i += 4; +#elif IS_ENABLED(CONFIG_64BIT) + if (!strncmp(isa, "rv64", 4)) + i += 4; +#endif + for (; i < isa_len; ++i) { + this_hwcap |= isa2hwcap[(unsigned char)(isa[i])]; + /* + * TODO: X, Y and Z extension parsing for Host ISA + * bitmap will be added in-future. + */ + if ('a' <= isa[i] && isa[i] < 'x') + this_isa |= (1UL << (isa[i] - 'a')); + } + + /* + * All "okay" hart should have same isa. Set HWCAP based on + * common capabilities of every "okay" hart, in case they don't + * have. + */ + if (elf_hwcap) + elf_hwcap &= this_hwcap; + else + elf_hwcap = this_hwcap; + + if (riscv_isa[0]) + riscv_isa[0] &= this_isa; + else + riscv_isa[0] = this_isa; + } + + /* We don't support systems with F but without D, so mask those out + * here. */ + if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { + pr_info("This kernel does not support systems with F but not D\n"); + elf_hwcap &= ~COMPAT_HWCAP_ISA_F; + } + + memset(print_str, 0, sizeof(print_str)); + for (i = 0, j = 0; i < BITS_PER_LONG; i++) + if (riscv_isa[0] & BIT_MASK(i)) + print_str[j++] = (char)('a' + i); + pr_info("riscv: ISA extensions %s\n", print_str); + + memset(print_str, 0, sizeof(print_str)); + for (i = 0, j = 0; i < BITS_PER_LONG; i++) + if (elf_hwcap & BIT_MASK(i)) + print_str[j++] = (char)('a' + i); + pr_info("riscv: ELF capabilities %s\n", print_str); + +#ifdef CONFIG_FPU + if (elf_hwcap & (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)) + has_fpu = true; +#endif +} diff --git a/arch/riscv/kernel/efi-header.S b/arch/riscv/kernel/efi-header.S new file mode 100644 index 000000000..8e733aa48 --- /dev/null +++ b/arch/riscv/kernel/efi-header.S @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + * Adapted from arch/arm64/kernel/efi-header.S + */ + +#include <linux/pe.h> +#include <linux/sizes.h> + + .macro __EFI_PE_HEADER + .long PE_MAGIC +coff_header: +#ifdef CONFIG_64BIT + .short IMAGE_FILE_MACHINE_RISCV64 // Machine +#else + .short IMAGE_FILE_MACHINE_RISCV32 // Machine +#endif + .short section_count // NumberOfSections + .long 0 // TimeDateStamp + .long 0 // PointerToSymbolTable + .long 0 // NumberOfSymbols + .short section_table - optional_header // SizeOfOptionalHeader + .short IMAGE_FILE_DEBUG_STRIPPED | \ + IMAGE_FILE_EXECUTABLE_IMAGE | \ + IMAGE_FILE_LINE_NUMS_STRIPPED // Characteristics + +optional_header: +#ifdef CONFIG_64BIT + .short PE_OPT_MAGIC_PE32PLUS // PE32+ format +#else + .short PE_OPT_MAGIC_PE32 // PE32 format +#endif + .byte 0x02 // MajorLinkerVersion + .byte 0x14 // MinorLinkerVersion + .long __pecoff_text_end - efi_header_end // SizeOfCode + .long __pecoff_data_virt_size // SizeOfInitializedData + .long 0 // SizeOfUninitializedData + .long __efistub_efi_pe_entry - _start // AddressOfEntryPoint + .long efi_header_end - _start // BaseOfCode +#ifdef CONFIG_32BIT + .long __pecoff_text_end - _start // BaseOfData +#endif + +extra_header_fields: + .quad 0 // ImageBase + .long PECOFF_SECTION_ALIGNMENT // SectionAlignment + .long PECOFF_FILE_ALIGNMENT // FileAlignment + .short 0 // MajorOperatingSystemVersion + .short 0 // MinorOperatingSystemVersion + .short LINUX_EFISTUB_MAJOR_VERSION // MajorImageVersion + .short LINUX_EFISTUB_MINOR_VERSION // MinorImageVersion + .short 0 // MajorSubsystemVersion + .short 0 // MinorSubsystemVersion + .long 0 // Win32VersionValue + + .long _end - _start // SizeOfImage + + // Everything before the kernel image is considered part of the header + .long efi_header_end - _start // SizeOfHeaders + .long 0 // CheckSum + .short IMAGE_SUBSYSTEM_EFI_APPLICATION // Subsystem + .short 0 // DllCharacteristics + .quad 0 // SizeOfStackReserve + .quad 0 // SizeOfStackCommit + .quad 0 // SizeOfHeapReserve + .quad 0 // SizeOfHeapCommit + .long 0 // LoaderFlags + .long (section_table - .) / 8 // NumberOfRvaAndSizes + + .quad 0 // ExportTable + .quad 0 // ImportTable + .quad 0 // ResourceTable + .quad 0 // ExceptionTable + .quad 0 // CertificationTable + .quad 0 // BaseRelocationTable + + // Section table +section_table: + .ascii ".text\0\0\0" + .long __pecoff_text_end - efi_header_end // VirtualSize + .long efi_header_end - _start // VirtualAddress + .long __pecoff_text_end - efi_header_end // SizeOfRawData + .long efi_header_end - _start // PointerToRawData + + .long 0 // PointerToRelocations + .long 0 // PointerToLineNumbers + .short 0 // NumberOfRelocations + .short 0 // NumberOfLineNumbers + .long IMAGE_SCN_CNT_CODE | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_EXECUTE // Characteristics + + .ascii ".data\0\0\0" + .long __pecoff_data_virt_size // VirtualSize + .long __pecoff_text_end - _start // VirtualAddress + .long __pecoff_data_raw_size // SizeOfRawData + .long __pecoff_text_end - _start // PointerToRawData + + .long 0 // PointerToRelocations + .long 0 // PointerToLineNumbers + .short 0 // NumberOfRelocations + .short 0 // NumberOfLineNumbers + .long IMAGE_SCN_CNT_INITIALIZED_DATA | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_WRITE // Characteristics + + .set section_count, (. - section_table) / 40 + + .balign 0x1000 +efi_header_end: + .endm diff --git a/arch/riscv/kernel/efi.c b/arch/riscv/kernel/efi.c new file mode 100644 index 000000000..1aa540350 --- /dev/null +++ b/arch/riscv/kernel/efi.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + * Adapted from arch/arm64/kernel/efi.c + */ + +#include <linux/efi.h> +#include <linux/init.h> + +#include <asm/efi.h> +#include <asm/pgtable.h> +#include <asm/pgtable-bits.h> + +/* + * Only regions of type EFI_RUNTIME_SERVICES_CODE need to be + * executable, everything else can be mapped with the XN bits + * set. Also take the new (optional) RO/XP bits into account. + */ +static __init pgprot_t efimem_to_pgprot_map(efi_memory_desc_t *md) +{ + u64 attr = md->attribute; + u32 type = md->type; + + if (type == EFI_MEMORY_MAPPED_IO) + return PAGE_KERNEL; + + /* R-- */ + if ((attr & (EFI_MEMORY_XP | EFI_MEMORY_RO)) == + (EFI_MEMORY_XP | EFI_MEMORY_RO)) + return PAGE_KERNEL_READ; + + /* R-X */ + if (attr & EFI_MEMORY_RO) + return PAGE_KERNEL_READ_EXEC; + + /* RW- */ + if (((attr & (EFI_MEMORY_RP | EFI_MEMORY_WP | EFI_MEMORY_XP)) == + EFI_MEMORY_XP) || + type != EFI_RUNTIME_SERVICES_CODE) + return PAGE_KERNEL; + + /* RWX */ + return PAGE_KERNEL_EXEC; +} + +int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) +{ + pgprot_t prot = __pgprot(pgprot_val(efimem_to_pgprot_map(md)) & + ~(_PAGE_GLOBAL)); + int i; + + /* RISC-V maps one page at a time */ + for (i = 0; i < md->num_pages; i++) + create_pgd_mapping(mm->pgd, md->virt_addr + i * PAGE_SIZE, + md->phys_addr + i * PAGE_SIZE, + PAGE_SIZE, prot); + return 0; +} + +static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data) +{ + efi_memory_desc_t *md = data; + pte_t pte = READ_ONCE(*ptep); + unsigned long val; + + if (md->attribute & EFI_MEMORY_RO) { + val = pte_val(pte) & ~_PAGE_WRITE; + val |= _PAGE_READ; + pte = __pte(val); + } + if (md->attribute & EFI_MEMORY_XP) { + val = pte_val(pte) & ~_PAGE_EXEC; + pte = __pte(val); + } + set_pte(ptep, pte); + + return 0; +} + +int __init efi_set_mapping_permissions(struct mm_struct *mm, + efi_memory_desc_t *md) +{ + BUG_ON(md->type != EFI_RUNTIME_SERVICES_CODE && + md->type != EFI_RUNTIME_SERVICES_DATA); + + /* + * Calling apply_to_page_range() is only safe on regions that are + * guaranteed to be mapped down to pages. Since we are only called + * for regions that have been mapped using efi_create_mapping() above + * (and this is checked by the generic Memory Attributes table parsing + * routines), there is no need to check that again here. + */ + return apply_to_page_range(mm, md->virt_addr, + md->num_pages << EFI_PAGE_SHIFT, + set_permissions, md); +} diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S new file mode 100644 index 000000000..5214c578a --- /dev/null +++ b/arch/riscv/kernel/entry.S @@ -0,0 +1,477 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#include <linux/init.h> +#include <linux/linkage.h> + +#include <asm/asm.h> +#include <asm/csr.h> +#include <asm/unistd.h> +#include <asm/thread_info.h> +#include <asm/asm-offsets.h> + +#if !IS_ENABLED(CONFIG_PREEMPTION) +.set resume_kernel, restore_all +#endif + +ENTRY(handle_exception) + /* + * If coming from userspace, preserve the user thread pointer and load + * the kernel thread pointer. If we came from the kernel, the scratch + * register will contain 0, and we should continue on the current TP. + */ + csrrw tp, CSR_SCRATCH, tp + bnez tp, _save_context + +_restore_kernel_tpsp: + csrr tp, CSR_SCRATCH + REG_S sp, TASK_TI_KERNEL_SP(tp) +_save_context: + REG_S sp, TASK_TI_USER_SP(tp) + REG_L sp, TASK_TI_KERNEL_SP(tp) + addi sp, sp, -(PT_SIZE_ON_STACK) + REG_S x1, PT_RA(sp) + REG_S x3, PT_GP(sp) + REG_S x5, PT_T0(sp) + REG_S x6, PT_T1(sp) + REG_S x7, PT_T2(sp) + REG_S x8, PT_S0(sp) + REG_S x9, PT_S1(sp) + REG_S x10, PT_A0(sp) + REG_S x11, PT_A1(sp) + REG_S x12, PT_A2(sp) + REG_S x13, PT_A3(sp) + REG_S x14, PT_A4(sp) + REG_S x15, PT_A5(sp) + REG_S x16, PT_A6(sp) + REG_S x17, PT_A7(sp) + REG_S x18, PT_S2(sp) + REG_S x19, PT_S3(sp) + REG_S x20, PT_S4(sp) + REG_S x21, PT_S5(sp) + REG_S x22, PT_S6(sp) + REG_S x23, PT_S7(sp) + REG_S x24, PT_S8(sp) + REG_S x25, PT_S9(sp) + REG_S x26, PT_S10(sp) + REG_S x27, PT_S11(sp) + REG_S x28, PT_T3(sp) + REG_S x29, PT_T4(sp) + REG_S x30, PT_T5(sp) + REG_S x31, PT_T6(sp) + + /* + * Disable user-mode memory access as it should only be set in the + * actual user copy routines. + * + * Disable the FPU to detect illegal usage of floating point in kernel + * space. + */ + li t0, SR_SUM | SR_FS + + REG_L s0, TASK_TI_USER_SP(tp) + csrrc s1, CSR_STATUS, t0 + csrr s2, CSR_EPC + csrr s3, CSR_TVAL + csrr s4, CSR_CAUSE + csrr s5, CSR_SCRATCH + REG_S s0, PT_SP(sp) + REG_S s1, PT_STATUS(sp) + REG_S s2, PT_EPC(sp) + REG_S s3, PT_BADADDR(sp) + REG_S s4, PT_CAUSE(sp) + REG_S s5, PT_TP(sp) + + /* + * Set the scratch register to 0, so that if a recursive exception + * occurs, the exception vector knows it came from the kernel + */ + csrw CSR_SCRATCH, x0 + + /* Load the global pointer */ +.option push +.option norelax + la gp, __global_pointer$ +.option pop + +#ifdef CONFIG_TRACE_IRQFLAGS + call __trace_hardirqs_off +#endif + +#ifdef CONFIG_CONTEXT_TRACKING + /* If previous state is in user mode, call context_tracking_user_exit. */ + li a0, SR_PP + and a0, s1, a0 + bnez a0, skip_context_tracking + call context_tracking_user_exit +skip_context_tracking: +#endif + + /* + * MSB of cause differentiates between + * interrupts and exceptions + */ + bge s4, zero, 1f + + la ra, ret_from_exception + + /* Handle interrupts */ + move a0, sp /* pt_regs */ + la a1, handle_arch_irq + REG_L a1, (a1) + jr a1 +1: + /* + * Exceptions run with interrupts enabled or disabled depending on the + * state of SR_PIE in m/sstatus. + */ + andi t0, s1, SR_PIE + beqz t0, 1f +#ifdef CONFIG_TRACE_IRQFLAGS + call __trace_hardirqs_on +#endif + csrs CSR_STATUS, SR_IE + +1: + la ra, ret_from_exception + /* Handle syscalls */ + li t0, EXC_SYSCALL + beq s4, t0, handle_syscall + + /* Handle other exceptions */ + slli t0, s4, RISCV_LGPTR + la t1, excp_vect_table + la t2, excp_vect_table_end + move a0, sp /* pt_regs */ + add t0, t1, t0 + /* Check if exception code lies within bounds */ + bgeu t0, t2, 1f + REG_L t0, 0(t0) + jr t0 +1: + tail do_trap_unknown + +handle_syscall: +#ifdef CONFIG_RISCV_M_MODE + /* + * When running is M-Mode (no MMU config), MPIE does not get set. + * As a result, we need to force enable interrupts here because + * handle_exception did not do set SR_IE as it always sees SR_PIE + * being cleared. + */ + csrs CSR_STATUS, SR_IE +#endif +#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) + /* Recover a0 - a7 for system calls */ + REG_L a0, PT_A0(sp) + REG_L a1, PT_A1(sp) + REG_L a2, PT_A2(sp) + REG_L a3, PT_A3(sp) + REG_L a4, PT_A4(sp) + REG_L a5, PT_A5(sp) + REG_L a6, PT_A6(sp) + REG_L a7, PT_A7(sp) +#endif + /* save the initial A0 value (needed in signal handlers) */ + REG_S a0, PT_ORIG_A0(sp) + /* + * Advance SEPC to avoid executing the original + * scall instruction on sret + */ + addi s2, s2, 0x4 + REG_S s2, PT_EPC(sp) + /* Trace syscalls, but only if requested by the user. */ + REG_L t0, TASK_TI_FLAGS(tp) + andi t0, t0, _TIF_SYSCALL_WORK + bnez t0, handle_syscall_trace_enter +check_syscall_nr: + /* Check to make sure we don't jump to a bogus syscall number. */ + li t0, __NR_syscalls + la s0, sys_ni_syscall + /* + * Syscall number held in a7. + * If syscall number is above allowed value, redirect to ni_syscall. + */ + bgeu a7, t0, 1f + /* Call syscall */ + la s0, sys_call_table + slli t0, a7, RISCV_LGPTR + add s0, s0, t0 + REG_L s0, 0(s0) +1: + jalr s0 + +ret_from_syscall: + /* Set user a0 to kernel a0 */ + REG_S a0, PT_A0(sp) + /* + * We didn't execute the actual syscall. + * Seccomp already set return value for the current task pt_regs. + * (If it was configured with SECCOMP_RET_ERRNO/TRACE) + */ +ret_from_syscall_rejected: + /* Trace syscalls, but only if requested by the user. */ + REG_L t0, TASK_TI_FLAGS(tp) + andi t0, t0, _TIF_SYSCALL_WORK + bnez t0, handle_syscall_trace_exit + +ret_from_exception: + REG_L s0, PT_STATUS(sp) + csrc CSR_STATUS, SR_IE +#ifdef CONFIG_TRACE_IRQFLAGS + call __trace_hardirqs_off +#endif +#ifdef CONFIG_RISCV_M_MODE + /* the MPP value is too large to be used as an immediate arg for addi */ + li t0, SR_MPP + and s0, s0, t0 +#else + andi s0, s0, SR_SPP +#endif + bnez s0, resume_kernel + +resume_userspace: + /* Interrupts must be disabled here so flags are checked atomically */ + REG_L s0, TASK_TI_FLAGS(tp) /* current_thread_info->flags */ + andi s1, s0, _TIF_WORK_MASK + bnez s1, work_pending + +#ifdef CONFIG_CONTEXT_TRACKING + call context_tracking_user_enter +#endif + + /* Save unwound kernel stack pointer in thread_info */ + addi s0, sp, PT_SIZE_ON_STACK + REG_S s0, TASK_TI_KERNEL_SP(tp) + + /* + * Save TP into the scratch register , so we can find the kernel data + * structures again. + */ + csrw CSR_SCRATCH, tp + +restore_all: +#ifdef CONFIG_TRACE_IRQFLAGS + REG_L s1, PT_STATUS(sp) + andi t0, s1, SR_PIE + beqz t0, 1f + call __trace_hardirqs_on + j 2f +1: + call __trace_hardirqs_off +2: +#endif + REG_L a0, PT_STATUS(sp) + /* + * The current load reservation is effectively part of the processor's + * state, in the sense that load reservations cannot be shared between + * different hart contexts. We can't actually save and restore a load + * reservation, so instead here we clear any existing reservation -- + * it's always legal for implementations to clear load reservations at + * any point (as long as the forward progress guarantee is kept, but + * we'll ignore that here). + * + * Dangling load reservations can be the result of taking a trap in the + * middle of an LR/SC sequence, but can also be the result of a taken + * forward branch around an SC -- which is how we implement CAS. As a + * result we need to clear reservations between the last CAS and the + * jump back to the new context. While it is unlikely the store + * completes, implementations are allowed to expand reservations to be + * arbitrarily large. + */ + REG_L a2, PT_EPC(sp) + REG_SC x0, a2, PT_EPC(sp) + + csrw CSR_STATUS, a0 + csrw CSR_EPC, a2 + + REG_L x1, PT_RA(sp) + REG_L x3, PT_GP(sp) + REG_L x4, PT_TP(sp) + REG_L x5, PT_T0(sp) + REG_L x6, PT_T1(sp) + REG_L x7, PT_T2(sp) + REG_L x8, PT_S0(sp) + REG_L x9, PT_S1(sp) + REG_L x10, PT_A0(sp) + REG_L x11, PT_A1(sp) + REG_L x12, PT_A2(sp) + REG_L x13, PT_A3(sp) + REG_L x14, PT_A4(sp) + REG_L x15, PT_A5(sp) + REG_L x16, PT_A6(sp) + REG_L x17, PT_A7(sp) + REG_L x18, PT_S2(sp) + REG_L x19, PT_S3(sp) + REG_L x20, PT_S4(sp) + REG_L x21, PT_S5(sp) + REG_L x22, PT_S6(sp) + REG_L x23, PT_S7(sp) + REG_L x24, PT_S8(sp) + REG_L x25, PT_S9(sp) + REG_L x26, PT_S10(sp) + REG_L x27, PT_S11(sp) + REG_L x28, PT_T3(sp) + REG_L x29, PT_T4(sp) + REG_L x30, PT_T5(sp) + REG_L x31, PT_T6(sp) + + REG_L x2, PT_SP(sp) + +#ifdef CONFIG_RISCV_M_MODE + mret +#else + sret +#endif + +#if IS_ENABLED(CONFIG_PREEMPTION) +resume_kernel: + REG_L s0, TASK_TI_PREEMPT_COUNT(tp) + bnez s0, restore_all + REG_L s0, TASK_TI_FLAGS(tp) + andi s0, s0, _TIF_NEED_RESCHED + beqz s0, restore_all + call preempt_schedule_irq + j restore_all +#endif + +work_pending: + /* Enter slow path for supplementary processing */ + la ra, ret_from_exception + andi s1, s0, _TIF_NEED_RESCHED + bnez s1, work_resched +work_notifysig: + /* Handle pending signals and notify-resume requests */ + csrs CSR_STATUS, SR_IE /* Enable interrupts for do_notify_resume() */ + move a0, sp /* pt_regs */ + move a1, s0 /* current_thread_info->flags */ + tail do_notify_resume +work_resched: + tail schedule + +/* Slow paths for ptrace. */ +handle_syscall_trace_enter: + move a0, sp + call do_syscall_trace_enter + move t0, a0 + REG_L a0, PT_A0(sp) + REG_L a1, PT_A1(sp) + REG_L a2, PT_A2(sp) + REG_L a3, PT_A3(sp) + REG_L a4, PT_A4(sp) + REG_L a5, PT_A5(sp) + REG_L a6, PT_A6(sp) + REG_L a7, PT_A7(sp) + bnez t0, ret_from_syscall_rejected + j check_syscall_nr +handle_syscall_trace_exit: + move a0, sp + call do_syscall_trace_exit + j ret_from_exception + +END(handle_exception) + +ENTRY(ret_from_fork) + la ra, ret_from_exception + tail schedule_tail +ENDPROC(ret_from_fork) + +ENTRY(ret_from_kernel_thread) + call schedule_tail + /* Call fn(arg) */ + la ra, ret_from_exception + move a0, s1 + jr s0 +ENDPROC(ret_from_kernel_thread) + + +/* + * Integer register context switch + * The callee-saved registers must be saved and restored. + * + * a0: previous task_struct (must be preserved across the switch) + * a1: next task_struct + * + * The value of a0 and a1 must be preserved by this function, as that's how + * arguments are passed to schedule_tail. + */ +ENTRY(__switch_to) + /* Save context into prev->thread */ + li a4, TASK_THREAD_RA + add a3, a0, a4 + add a4, a1, a4 + REG_S ra, TASK_THREAD_RA_RA(a3) + REG_S sp, TASK_THREAD_SP_RA(a3) + REG_S s0, TASK_THREAD_S0_RA(a3) + REG_S s1, TASK_THREAD_S1_RA(a3) + REG_S s2, TASK_THREAD_S2_RA(a3) + REG_S s3, TASK_THREAD_S3_RA(a3) + REG_S s4, TASK_THREAD_S4_RA(a3) + REG_S s5, TASK_THREAD_S5_RA(a3) + REG_S s6, TASK_THREAD_S6_RA(a3) + REG_S s7, TASK_THREAD_S7_RA(a3) + REG_S s8, TASK_THREAD_S8_RA(a3) + REG_S s9, TASK_THREAD_S9_RA(a3) + REG_S s10, TASK_THREAD_S10_RA(a3) + REG_S s11, TASK_THREAD_S11_RA(a3) + /* Restore context from next->thread */ + REG_L ra, TASK_THREAD_RA_RA(a4) + REG_L sp, TASK_THREAD_SP_RA(a4) + REG_L s0, TASK_THREAD_S0_RA(a4) + REG_L s1, TASK_THREAD_S1_RA(a4) + REG_L s2, TASK_THREAD_S2_RA(a4) + REG_L s3, TASK_THREAD_S3_RA(a4) + REG_L s4, TASK_THREAD_S4_RA(a4) + REG_L s5, TASK_THREAD_S5_RA(a4) + REG_L s6, TASK_THREAD_S6_RA(a4) + REG_L s7, TASK_THREAD_S7_RA(a4) + REG_L s8, TASK_THREAD_S8_RA(a4) + REG_L s9, TASK_THREAD_S9_RA(a4) + REG_L s10, TASK_THREAD_S10_RA(a4) + REG_L s11, TASK_THREAD_S11_RA(a4) + /* Swap the CPU entry around. */ + lw a3, TASK_TI_CPU(a0) + lw a4, TASK_TI_CPU(a1) + sw a3, TASK_TI_CPU(a1) + sw a4, TASK_TI_CPU(a0) + /* The offset of thread_info in task_struct is zero. */ + move tp, a1 + ret +ENDPROC(__switch_to) + +#ifndef CONFIG_MMU +#define do_page_fault do_trap_unknown +#endif + + .section ".rodata" + .align LGREG + /* Exception vector table */ +ENTRY(excp_vect_table) + RISCV_PTR do_trap_insn_misaligned + RISCV_PTR do_trap_insn_fault + RISCV_PTR do_trap_insn_illegal + RISCV_PTR do_trap_break + RISCV_PTR do_trap_load_misaligned + RISCV_PTR do_trap_load_fault + RISCV_PTR do_trap_store_misaligned + RISCV_PTR do_trap_store_fault + RISCV_PTR do_trap_ecall_u /* system call, gets intercepted */ + RISCV_PTR do_trap_ecall_s + RISCV_PTR do_trap_unknown + RISCV_PTR do_trap_ecall_m + RISCV_PTR do_page_fault /* instruction page fault */ + RISCV_PTR do_page_fault /* load page fault */ + RISCV_PTR do_trap_unknown + RISCV_PTR do_page_fault /* store page fault */ +excp_vect_table_end: +END(excp_vect_table) + +#ifndef CONFIG_MMU +ENTRY(__user_rt_sigreturn) + li a7, __NR_rt_sigreturn + scall +END(__user_rt_sigreturn) +#endif diff --git a/arch/riscv/kernel/fpu.S b/arch/riscv/kernel/fpu.S new file mode 100644 index 000000000..dd2205473 --- /dev/null +++ b/arch/riscv/kernel/fpu.S @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/linkage.h> + +#include <asm/asm.h> +#include <asm/csr.h> +#include <asm/asm-offsets.h> + +ENTRY(__fstate_save) + li a2, TASK_THREAD_F0 + add a0, a0, a2 + li t1, SR_FS + csrs CSR_STATUS, t1 + frcsr t0 + fsd f0, TASK_THREAD_F0_F0(a0) + fsd f1, TASK_THREAD_F1_F0(a0) + fsd f2, TASK_THREAD_F2_F0(a0) + fsd f3, TASK_THREAD_F3_F0(a0) + fsd f4, TASK_THREAD_F4_F0(a0) + fsd f5, TASK_THREAD_F5_F0(a0) + fsd f6, TASK_THREAD_F6_F0(a0) + fsd f7, TASK_THREAD_F7_F0(a0) + fsd f8, TASK_THREAD_F8_F0(a0) + fsd f9, TASK_THREAD_F9_F0(a0) + fsd f10, TASK_THREAD_F10_F0(a0) + fsd f11, TASK_THREAD_F11_F0(a0) + fsd f12, TASK_THREAD_F12_F0(a0) + fsd f13, TASK_THREAD_F13_F0(a0) + fsd f14, TASK_THREAD_F14_F0(a0) + fsd f15, TASK_THREAD_F15_F0(a0) + fsd f16, TASK_THREAD_F16_F0(a0) + fsd f17, TASK_THREAD_F17_F0(a0) + fsd f18, TASK_THREAD_F18_F0(a0) + fsd f19, TASK_THREAD_F19_F0(a0) + fsd f20, TASK_THREAD_F20_F0(a0) + fsd f21, TASK_THREAD_F21_F0(a0) + fsd f22, TASK_THREAD_F22_F0(a0) + fsd f23, TASK_THREAD_F23_F0(a0) + fsd f24, TASK_THREAD_F24_F0(a0) + fsd f25, TASK_THREAD_F25_F0(a0) + fsd f26, TASK_THREAD_F26_F0(a0) + fsd f27, TASK_THREAD_F27_F0(a0) + fsd f28, TASK_THREAD_F28_F0(a0) + fsd f29, TASK_THREAD_F29_F0(a0) + fsd f30, TASK_THREAD_F30_F0(a0) + fsd f31, TASK_THREAD_F31_F0(a0) + sw t0, TASK_THREAD_FCSR_F0(a0) + csrc CSR_STATUS, t1 + ret +ENDPROC(__fstate_save) + +ENTRY(__fstate_restore) + li a2, TASK_THREAD_F0 + add a0, a0, a2 + li t1, SR_FS + lw t0, TASK_THREAD_FCSR_F0(a0) + csrs CSR_STATUS, t1 + fld f0, TASK_THREAD_F0_F0(a0) + fld f1, TASK_THREAD_F1_F0(a0) + fld f2, TASK_THREAD_F2_F0(a0) + fld f3, TASK_THREAD_F3_F0(a0) + fld f4, TASK_THREAD_F4_F0(a0) + fld f5, TASK_THREAD_F5_F0(a0) + fld f6, TASK_THREAD_F6_F0(a0) + fld f7, TASK_THREAD_F7_F0(a0) + fld f8, TASK_THREAD_F8_F0(a0) + fld f9, TASK_THREAD_F9_F0(a0) + fld f10, TASK_THREAD_F10_F0(a0) + fld f11, TASK_THREAD_F11_F0(a0) + fld f12, TASK_THREAD_F12_F0(a0) + fld f13, TASK_THREAD_F13_F0(a0) + fld f14, TASK_THREAD_F14_F0(a0) + fld f15, TASK_THREAD_F15_F0(a0) + fld f16, TASK_THREAD_F16_F0(a0) + fld f17, TASK_THREAD_F17_F0(a0) + fld f18, TASK_THREAD_F18_F0(a0) + fld f19, TASK_THREAD_F19_F0(a0) + fld f20, TASK_THREAD_F20_F0(a0) + fld f21, TASK_THREAD_F21_F0(a0) + fld f22, TASK_THREAD_F22_F0(a0) + fld f23, TASK_THREAD_F23_F0(a0) + fld f24, TASK_THREAD_F24_F0(a0) + fld f25, TASK_THREAD_F25_F0(a0) + fld f26, TASK_THREAD_F26_F0(a0) + fld f27, TASK_THREAD_F27_F0(a0) + fld f28, TASK_THREAD_F28_F0(a0) + fld f29, TASK_THREAD_F29_F0(a0) + fld f30, TASK_THREAD_F30_F0(a0) + fld f31, TASK_THREAD_F31_F0(a0) + fscsr t0 + csrc CSR_STATUS, t1 + ret +ENDPROC(__fstate_restore) diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c new file mode 100644 index 000000000..8693dfcff --- /dev/null +++ b/arch/riscv/kernel/ftrace.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> + * Copyright (C) 2017 Andes Technology Corporation + */ + +#include <linux/ftrace.h> +#include <linux/uaccess.h> +#include <linux/memory.h> +#include <asm/cacheflush.h> +#include <asm/patch.h> + +#ifdef CONFIG_DYNAMIC_FTRACE +int ftrace_arch_code_modify_prepare(void) __acquires(&text_mutex) +{ + mutex_lock(&text_mutex); + + /* + * The code sequences we use for ftrace can't be patched while the + * kernel is running, so we need to use stop_machine() to modify them + * for now. This doesn't play nice with text_mutex, we use this flag + * to elide the check. + */ + riscv_patch_in_stop_machine = true; + + return 0; +} + +int ftrace_arch_code_modify_post_process(void) __releases(&text_mutex) +{ + riscv_patch_in_stop_machine = false; + mutex_unlock(&text_mutex); + return 0; +} + +static int ftrace_check_current_call(unsigned long hook_pos, + unsigned int *expected) +{ + unsigned int replaced[2]; + unsigned int nops[2] = {NOP4, NOP4}; + + /* we expect nops at the hook position */ + if (!expected) + expected = nops; + + /* + * Read the text we want to modify; + * return must be -EFAULT on read error + */ + if (copy_from_kernel_nofault(replaced, (void *)hook_pos, + MCOUNT_INSN_SIZE)) + return -EFAULT; + + /* + * Make sure it is what we expect it to be; + * return must be -EINVAL on failed comparison + */ + if (memcmp(expected, replaced, sizeof(replaced))) { + pr_err("%p: expected (%08x %08x) but got (%08x %08x)\n", + (void *)hook_pos, expected[0], expected[1], replaced[0], + replaced[1]); + return -EINVAL; + } + + return 0; +} + +static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, + bool enable) +{ + unsigned int call[2]; + unsigned int nops[2] = {NOP4, NOP4}; + + make_call(hook_pos, target, call); + + /* Replace the auipc-jalr pair at once. Return -EPERM on write error. */ + if (patch_text_nosync + ((void *)hook_pos, enable ? call : nops, MCOUNT_INSN_SIZE)) + return -EPERM; + + return 0; +} + +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + int ret = ftrace_check_current_call(rec->ip, NULL); + + if (ret) + return ret; + + return __ftrace_modify_call(rec->ip, addr, true); +} + +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, + unsigned long addr) +{ + unsigned int call[2]; + int ret; + + make_call(rec->ip, addr, call); + ret = ftrace_check_current_call(rec->ip, call); + + if (ret) + return ret; + + return __ftrace_modify_call(rec->ip, addr, false); +} + + +/* + * This is called early on, and isn't wrapped by + * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold + * text_mutex, which triggers a lockdep failure. SMP isn't running so we could + * just directly poke the text, but it's simpler to just take the lock + * ourselves. + */ +int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) +{ + int out; + + mutex_lock(&text_mutex); + out = ftrace_make_nop(mod, rec, MCOUNT_ADDR); + mutex_unlock(&text_mutex); + + return out; +} + +int ftrace_update_ftrace_func(ftrace_func_t func) +{ + int ret = __ftrace_modify_call((unsigned long)&ftrace_call, + (unsigned long)func, true); + if (!ret) { + ret = __ftrace_modify_call((unsigned long)&ftrace_regs_call, + (unsigned long)func, true); + } + + return ret; +} + +int __init ftrace_dyn_arch_init(void) +{ + return 0; +} +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, + unsigned long addr) +{ + unsigned int call[2]; + int ret; + + make_call(rec->ip, old_addr, call); + ret = ftrace_check_current_call(rec->ip, call); + + if (ret) + return ret; + + return __ftrace_modify_call(rec->ip, addr, true); +} +#endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * Most of this function is copied from arm64. + */ +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, + unsigned long frame_pointer) +{ + unsigned long return_hooker = (unsigned long)&return_to_handler; + unsigned long old; + + if (unlikely(atomic_read(¤t->tracing_graph_pause))) + return; + + /* + * We don't suffer access faults, so no extra fault-recovery assembly + * is needed here. + */ + old = *parent; + + if (!function_graph_enter(old, self_addr, frame_pointer, parent)) + *parent = return_hooker; +} + +#ifdef CONFIG_DYNAMIC_FTRACE +extern void ftrace_graph_call(void); +int ftrace_enable_ftrace_graph_caller(void) +{ + unsigned int call[2]; + static int init_graph = 1; + int ret; + + make_call(&ftrace_graph_call, &ftrace_stub, call); + + /* + * When enabling graph tracer for the first time, ftrace_graph_call + * should contains a call to ftrace_stub. Once it has been disabled, + * the 8-bytes at the position becomes NOPs. + */ + if (init_graph) { + ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, + call); + init_graph = 0; + } else { + ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, + NULL); + } + + if (ret) + return ret; + + return __ftrace_modify_call((unsigned long)&ftrace_graph_call, + (unsigned long)&prepare_ftrace_return, true); +} + +int ftrace_disable_ftrace_graph_caller(void) +{ + unsigned int call[2]; + int ret; + + make_call(&ftrace_graph_call, &prepare_ftrace_return, call); + + /* + * This is to make sure that ftrace_enable_ftrace_graph_caller + * did the right thing. + */ + ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, + call); + + if (ret) + return ret; + + return __ftrace_modify_call((unsigned long)&ftrace_graph_call, + (unsigned long)&prepare_ftrace_return, false); +} +#endif /* CONFIG_DYNAMIC_FTRACE */ +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S new file mode 100644 index 000000000..47d1411db --- /dev/null +++ b/arch/riscv/kernel/head.S @@ -0,0 +1,396 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#include <asm/asm-offsets.h> +#include <asm/asm.h> +#include <linux/init.h> +#include <linux/linkage.h> +#include <asm/thread_info.h> +#include <asm/page.h> +#include <asm/csr.h> +#include <asm/hwcap.h> +#include <asm/image.h> +#include "efi-header.S" + +__HEAD +ENTRY(_start) + /* + * Image header expected by Linux boot-loaders. The image header data + * structure is described in asm/image.h. + * Do not modify it without modifying the structure and all bootloaders + * that expects this header format!! + */ +#ifdef CONFIG_EFI + /* + * This instruction decodes to "MZ" ASCII required by UEFI. + */ + c.li s4,-13 + j _start_kernel +#else + /* jump to start kernel */ + j _start_kernel + /* reserved */ + .word 0 +#endif + .balign 8 +#ifdef CONFIG_RISCV_M_MODE + /* Image load offset (0MB) from start of RAM for M-mode */ + .dword 0 +#else +#if __riscv_xlen == 64 + /* Image load offset(2MB) from start of RAM */ + .dword 0x200000 +#else + /* Image load offset(4MB) from start of RAM */ + .dword 0x400000 +#endif +#endif + /* Effective size of kernel image */ + .dword _end - _start + .dword __HEAD_FLAGS + .word RISCV_HEADER_VERSION + .word 0 + .dword 0 + .ascii RISCV_IMAGE_MAGIC + .balign 4 + .ascii RISCV_IMAGE_MAGIC2 +#ifdef CONFIG_EFI + .word pe_head_start - _start +pe_head_start: + + __EFI_PE_HEADER +#else + .word 0 +#endif + +.align 2 +#ifdef CONFIG_MMU +relocate: + /* Relocate return address */ + li a1, PAGE_OFFSET + la a2, _start + sub a1, a1, a2 + add ra, ra, a1 + + /* Point stvec to virtual address of intruction after satp write */ + la a2, 1f + add a2, a2, a1 + csrw CSR_TVEC, a2 + + /* Compute satp for kernel page tables, but don't load it yet */ + srl a2, a0, PAGE_SHIFT + li a1, SATP_MODE + or a2, a2, a1 + + /* + * Load trampoline page directory, which will cause us to trap to + * stvec if VA != PA, or simply fall through if VA == PA. We need a + * full fence here because setup_vm() just wrote these PTEs and we need + * to ensure the new translations are in use. + */ + la a0, trampoline_pg_dir + srl a0, a0, PAGE_SHIFT + or a0, a0, a1 + sfence.vma + csrw CSR_SATP, a0 +.align 2 +1: + /* Set trap vector to spin forever to help debug */ + la a0, .Lsecondary_park + csrw CSR_TVEC, a0 + + /* Reload the global pointer */ +.option push +.option norelax + la gp, __global_pointer$ +.option pop + + /* + * Switch to kernel page tables. A full fence is necessary in order to + * avoid using the trampoline translations, which are only correct for + * the first superpage. Fetching the fence is guarnteed to work + * because that first superpage is translated the same way. + */ + csrw CSR_SATP, a2 + sfence.vma + + ret +#endif /* CONFIG_MMU */ +#ifdef CONFIG_SMP + .global secondary_start_sbi +secondary_start_sbi: + /* Mask all interrupts */ + csrw CSR_IE, zero + csrw CSR_IP, zero + + /* Load the global pointer */ + .option push + .option norelax + la gp, __global_pointer$ + .option pop + + /* + * Disable FPU to detect illegal usage of + * floating point in kernel space + */ + li t0, SR_FS + csrc CSR_STATUS, t0 + + /* Set trap vector to spin forever to help debug */ + la a3, .Lsecondary_park + csrw CSR_TVEC, a3 + + slli a3, a0, LGREG + la a4, __cpu_up_stack_pointer + la a5, __cpu_up_task_pointer + add a4, a3, a4 + add a5, a3, a5 + REG_L sp, (a4) + REG_L tp, (a5) + + .global secondary_start_common +secondary_start_common: + +#ifdef CONFIG_MMU + /* Enable virtual memory and relocate to virtual address */ + la a0, swapper_pg_dir + call relocate +#endif + call setup_trap_vector + tail smp_callin +#endif /* CONFIG_SMP */ + +.align 2 +setup_trap_vector: + /* Set trap vector to exception handler */ + la a0, handle_exception + csrw CSR_TVEC, a0 + + /* + * Set sup0 scratch register to 0, indicating to exception vector that + * we are presently executing in kernel. + */ + csrw CSR_SCRATCH, zero + ret + +.align 2 +.Lsecondary_park: + /* We lack SMP support or have too many harts, so park this hart */ + wfi + j .Lsecondary_park + +END(_start) + + __INIT +ENTRY(_start_kernel) + /* Mask all interrupts */ + csrw CSR_IE, zero + csrw CSR_IP, zero + +#ifdef CONFIG_RISCV_M_MODE + /* flush the instruction cache */ + fence.i + + /* Reset all registers except ra, a0, a1 */ + call reset_regs + + /* + * Setup a PMP to permit access to all of memory. Some machines may + * not implement PMPs, so we set up a quick trap handler to just skip + * touching the PMPs on any trap. + */ + la a0, pmp_done + csrw CSR_TVEC, a0 + + li a0, -1 + csrw CSR_PMPADDR0, a0 + li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X) + csrw CSR_PMPCFG0, a0 +.align 2 +pmp_done: + + /* + * The hartid in a0 is expected later on, and we have no firmware + * to hand it to us. + */ + csrr a0, CSR_MHARTID +#endif /* CONFIG_RISCV_M_MODE */ + + /* Load the global pointer */ +.option push +.option norelax + la gp, __global_pointer$ +.option pop + + /* + * Disable FPU to detect illegal usage of + * floating point in kernel space + */ + li t0, SR_FS + csrc CSR_STATUS, t0 + +#ifdef CONFIG_SMP + li t0, CONFIG_NR_CPUS + blt a0, t0, .Lgood_cores + tail .Lsecondary_park +.Lgood_cores: +#endif + + /* Pick one hart to run the main boot sequence */ + la a3, hart_lottery + li a2, 1 + amoadd.w a3, a2, (a3) + bnez a3, .Lsecondary_start + + /* Clear BSS for flat non-ELF images */ + la a3, __bss_start + la a4, __bss_stop + ble a4, a3, clear_bss_done +clear_bss: + REG_S zero, (a3) + add a3, a3, RISCV_SZPTR + blt a3, a4, clear_bss +clear_bss_done: + + /* Save hart ID and DTB physical address */ + mv s0, a0 + mv s1, a1 + la a2, boot_cpu_hartid + REG_S a0, (a2) + + /* Initialize page tables and relocate to virtual addresses */ + la tp, init_task + la sp, init_thread_union + THREAD_SIZE + mv a0, s1 + call setup_vm +#ifdef CONFIG_MMU + la a0, early_pg_dir + call relocate +#endif /* CONFIG_MMU */ + + call setup_trap_vector + /* Restore C environment */ + la tp, init_task + sw zero, TASK_TI_CPU(tp) + la sp, init_thread_union + THREAD_SIZE + +#ifdef CONFIG_KASAN + call kasan_early_init +#endif + /* Start the kernel */ + call soc_early_init + tail start_kernel + +.Lsecondary_start: +#ifdef CONFIG_SMP + /* Set trap vector to spin forever to help debug */ + la a3, .Lsecondary_park + csrw CSR_TVEC, a3 + + slli a3, a0, LGREG + la a1, __cpu_up_stack_pointer + la a2, __cpu_up_task_pointer + add a1, a3, a1 + add a2, a3, a2 + + /* + * This hart didn't win the lottery, so we wait for the winning hart to + * get far enough along the boot process that it should continue. + */ +.Lwait_for_cpu_up: + /* FIXME: We should WFI to save some energy here. */ + REG_L sp, (a1) + REG_L tp, (a2) + beqz sp, .Lwait_for_cpu_up + beqz tp, .Lwait_for_cpu_up + fence + + tail secondary_start_common +#endif + +END(_start_kernel) + +#ifdef CONFIG_RISCV_M_MODE +ENTRY(reset_regs) + li sp, 0 + li gp, 0 + li tp, 0 + li t0, 0 + li t1, 0 + li t2, 0 + li s0, 0 + li s1, 0 + li a2, 0 + li a3, 0 + li a4, 0 + li a5, 0 + li a6, 0 + li a7, 0 + li s2, 0 + li s3, 0 + li s4, 0 + li s5, 0 + li s6, 0 + li s7, 0 + li s8, 0 + li s9, 0 + li s10, 0 + li s11, 0 + li t3, 0 + li t4, 0 + li t5, 0 + li t6, 0 + csrw CSR_SCRATCH, 0 + +#ifdef CONFIG_FPU + csrr t0, CSR_MISA + andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) + beqz t0, .Lreset_regs_done + + li t1, SR_FS + csrs CSR_STATUS, t1 + fmv.s.x f0, zero + fmv.s.x f1, zero + fmv.s.x f2, zero + fmv.s.x f3, zero + fmv.s.x f4, zero + fmv.s.x f5, zero + fmv.s.x f6, zero + fmv.s.x f7, zero + fmv.s.x f8, zero + fmv.s.x f9, zero + fmv.s.x f10, zero + fmv.s.x f11, zero + fmv.s.x f12, zero + fmv.s.x f13, zero + fmv.s.x f14, zero + fmv.s.x f15, zero + fmv.s.x f16, zero + fmv.s.x f17, zero + fmv.s.x f18, zero + fmv.s.x f19, zero + fmv.s.x f20, zero + fmv.s.x f21, zero + fmv.s.x f22, zero + fmv.s.x f23, zero + fmv.s.x f24, zero + fmv.s.x f25, zero + fmv.s.x f26, zero + fmv.s.x f27, zero + fmv.s.x f28, zero + fmv.s.x f29, zero + fmv.s.x f30, zero + fmv.s.x f31, zero + csrw fcsr, 0 + /* note that the caller must clear SR_FS */ +#endif /* CONFIG_FPU */ +.Lreset_regs_done: + ret +END(reset_regs) +#endif /* CONFIG_RISCV_M_MODE */ + +__PAGE_ALIGNED_BSS + /* Empty zero page */ + .balign PAGE_SIZE diff --git a/arch/riscv/kernel/head.h b/arch/riscv/kernel/head.h new file mode 100644 index 000000000..b48dda3d0 --- /dev/null +++ b/arch/riscv/kernel/head.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 SiFive, Inc. + */ +#ifndef __ASM_HEAD_H +#define __ASM_HEAD_H + +#include <linux/linkage.h> +#include <linux/init.h> + +extern atomic_t hart_lottery; + +asmlinkage void do_page_fault(struct pt_regs *regs); +asmlinkage void __init setup_vm(uintptr_t dtb_pa); + +extern void *__cpu_up_stack_pointer[]; +extern void *__cpu_up_task_pointer[]; + +#endif /* __ASM_HEAD_H */ diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h new file mode 100644 index 000000000..8c212efb3 --- /dev/null +++ b/arch/riscv/kernel/image-vars.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + * Linker script variables to be set after section resolution, as + * ld.lld does not like variables assigned before SECTIONS is processed. + * Based on arch/arm64/kerne/image-vars.h + */ +#ifndef __RISCV_KERNEL_IMAGE_VARS_H +#define __RISCV_KERNEL_IMAGE_VARS_H + +#ifndef LINKER_SCRIPT +#error This file should only be included in vmlinux.lds.S +#endif + +#ifdef CONFIG_EFI + +/* + * The EFI stub has its own symbol namespace prefixed by __efistub_, to + * isolate it from the kernel proper. The following symbols are legally + * accessed by the stub, so provide some aliases to make them accessible. + * Only include data symbols here, or text symbols of functions that are + * guaranteed to be safe when executed at another offset than they were + * linked at. The routines below are all implemented in assembler in a + * position independent manner + */ +__efistub_memcmp = memcmp; +__efistub_memchr = memchr; +__efistub_memcpy = memcpy; +__efistub_memmove = memmove; +__efistub_memset = memset; +__efistub_strlen = strlen; +__efistub_strnlen = strnlen; +__efistub_strcmp = strcmp; +__efistub_strncmp = strncmp; +__efistub_strrchr = strrchr; + +#ifdef CONFIG_KASAN +__efistub___memcpy = memcpy; +__efistub___memmove = memmove; +__efistub___memset = memset; +#endif + +__efistub__start = _start; +__efistub__start_kernel = _start_kernel; +__efistub__end = _end; +__efistub__edata = _edata; +__efistub_screen_info = screen_info; + +#endif + +#endif /* __RISCV_KERNEL_IMAGE_VARS_H */ diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c new file mode 100644 index 000000000..7207fa08d --- /dev/null +++ b/arch/riscv/kernel/irq.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + * Copyright (C) 2018 Christoph Hellwig + */ + +#include <linux/interrupt.h> +#include <linux/irqchip.h> +#include <linux/seq_file.h> +#include <asm/smp.h> + +int arch_show_interrupts(struct seq_file *p, int prec) +{ + show_ipi_stats(p, prec); + return 0; +} + +void __init init_IRQ(void) +{ + irqchip_init(); + if (!handle_arch_irq) + panic("No interrupt controller found."); +} diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c new file mode 100644 index 000000000..20e09056d --- /dev/null +++ b/arch/riscv/kernel/jump_label.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Emil Renner Berthing + * + * Based on arch/arm64/kernel/jump_label.c + */ +#include <linux/jump_label.h> +#include <linux/kernel.h> +#include <linux/memory.h> +#include <linux/mutex.h> +#include <asm/bug.h> +#include <asm/patch.h> + +#define RISCV_INSN_NOP 0x00000013U +#define RISCV_INSN_JAL 0x0000006fU + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + void *addr = (void *)jump_entry_code(entry); + u32 insn; + + if (type == JUMP_LABEL_JMP) { + long offset = jump_entry_target(entry) - jump_entry_code(entry); + + if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288)) + return; + + insn = RISCV_INSN_JAL | + (((u32)offset & GENMASK(19, 12)) << (12 - 12)) | + (((u32)offset & GENMASK(11, 11)) << (20 - 11)) | + (((u32)offset & GENMASK(10, 1)) << (21 - 1)) | + (((u32)offset & GENMASK(20, 20)) << (31 - 20)); + } else { + insn = RISCV_INSN_NOP; + } + + mutex_lock(&text_mutex); + patch_text_nosync(addr, &insn, sizeof(insn)); + mutex_unlock(&text_mutex); +} + +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + /* + * We use the same instructions in the arch_static_branch and + * arch_static_branch_jump inline functions, so there's no + * need to patch them up here. + * The core will call arch_jump_label_transform when those + * instructions need to be replaced. + */ +} diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c new file mode 100644 index 000000000..963ed7edc --- /dev/null +++ b/arch/riscv/kernel/kgdb.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 SiFive + */ + +#include <linux/ptrace.h> +#include <linux/kdebug.h> +#include <linux/bug.h> +#include <linux/kgdb.h> +#include <linux/irqflags.h> +#include <linux/string.h> +#include <asm/cacheflush.h> +#include <asm/gdb_xml.h> +#include <asm/parse_asm.h> + +enum { + NOT_KGDB_BREAK = 0, + KGDB_SW_BREAK, + KGDB_COMPILED_BREAK, + KGDB_SW_SINGLE_STEP +}; + +static unsigned long stepped_address; +static unsigned int stepped_opcode; + +#if __riscv_xlen == 32 +/* C.JAL is an RV32C-only instruction */ +DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) +#else +#define is_c_jal_insn(opcode) 0 +#endif +DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR) +DECLARE_INSN(jal, MATCH_JAL, MASK_JAL) +DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) +DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR) +DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J) +DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) +DECLARE_INSN(bne, MATCH_BNE, MASK_BNE) +DECLARE_INSN(blt, MATCH_BLT, MASK_BLT) +DECLARE_INSN(bge, MATCH_BGE, MASK_BGE) +DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU) +DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU) +DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ) +DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ) +DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) + +static int decode_register_index(unsigned long opcode, int offset) +{ + return (opcode >> offset) & 0x1F; +} + +static int decode_register_index_short(unsigned long opcode, int offset) +{ + return ((opcode >> offset) & 0x7) + 8; +} + +/* Calculate the new address for after a step */ +static int get_step_address(struct pt_regs *regs, unsigned long *next_addr) +{ + unsigned long pc = regs->epc; + unsigned long *regs_ptr = (unsigned long *)regs; + unsigned int rs1_num, rs2_num; + int op_code; + + if (get_kernel_nofault(op_code, (void *)pc)) + return -EINVAL; + if ((op_code & __INSN_LENGTH_MASK) != __INSN_LENGTH_GE_32) { + if (is_c_jalr_insn(op_code) || is_c_jr_insn(op_code)) { + rs1_num = decode_register_index(op_code, RVC_C2_RS1_OPOFF); + *next_addr = regs_ptr[rs1_num]; + } else if (is_c_j_insn(op_code) || is_c_jal_insn(op_code)) { + *next_addr = EXTRACT_RVC_J_IMM(op_code) + pc; + } else if (is_c_beqz_insn(op_code)) { + rs1_num = decode_register_index_short(op_code, + RVC_C1_RS1_OPOFF); + if (!rs1_num || regs_ptr[rs1_num] == 0) + *next_addr = EXTRACT_RVC_B_IMM(op_code) + pc; + else + *next_addr = pc + 2; + } else if (is_c_bnez_insn(op_code)) { + rs1_num = + decode_register_index_short(op_code, RVC_C1_RS1_OPOFF); + if (rs1_num && regs_ptr[rs1_num] != 0) + *next_addr = EXTRACT_RVC_B_IMM(op_code) + pc; + else + *next_addr = pc + 2; + } else { + *next_addr = pc + 2; + } + } else { + if ((op_code & __INSN_OPCODE_MASK) == __INSN_BRANCH_OPCODE) { + bool result = false; + long imm = EXTRACT_BTYPE_IMM(op_code); + unsigned long rs1_val = 0, rs2_val = 0; + + rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); + rs2_num = decode_register_index(op_code, RVG_RS2_OPOFF); + if (rs1_num) + rs1_val = regs_ptr[rs1_num]; + if (rs2_num) + rs2_val = regs_ptr[rs2_num]; + + if (is_beq_insn(op_code)) + result = (rs1_val == rs2_val) ? true : false; + else if (is_bne_insn(op_code)) + result = (rs1_val != rs2_val) ? true : false; + else if (is_blt_insn(op_code)) + result = + ((long)rs1_val < + (long)rs2_val) ? true : false; + else if (is_bge_insn(op_code)) + result = + ((long)rs1_val >= + (long)rs2_val) ? true : false; + else if (is_bltu_insn(op_code)) + result = (rs1_val < rs2_val) ? true : false; + else if (is_bgeu_insn(op_code)) + result = (rs1_val >= rs2_val) ? true : false; + if (result) + *next_addr = imm + pc; + else + *next_addr = pc + 4; + } else if (is_jal_insn(op_code)) { + *next_addr = EXTRACT_JTYPE_IMM(op_code) + pc; + } else if (is_jalr_insn(op_code)) { + rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); + if (rs1_num) + *next_addr = ((unsigned long *)regs)[rs1_num]; + *next_addr += EXTRACT_ITYPE_IMM(op_code); + } else if (is_sret_insn(op_code)) { + *next_addr = pc; + } else { + *next_addr = pc + 4; + } + } + return 0; +} + +static int do_single_step(struct pt_regs *regs) +{ + /* Determine where the target instruction will send us to */ + unsigned long addr = 0; + int error = get_step_address(regs, &addr); + + if (error) + return error; + + /* Store the op code in the stepped address */ + error = get_kernel_nofault(stepped_opcode, (void *)addr); + if (error) + return error; + + stepped_address = addr; + + /* Replace the op code with the break instruction */ + error = copy_to_kernel_nofault((void *)stepped_address, + arch_kgdb_ops.gdb_bpt_instr, + BREAK_INSTR_SIZE); + /* Flush and return */ + if (!error) { + flush_icache_range(addr, addr + BREAK_INSTR_SIZE); + kgdb_single_step = 1; + atomic_set(&kgdb_cpu_doing_single_step, + raw_smp_processor_id()); + } else { + stepped_address = 0; + stepped_opcode = 0; + } + return error; +} + +/* Undo a single step */ +static void undo_single_step(struct pt_regs *regs) +{ + if (stepped_opcode != 0) { + copy_to_kernel_nofault((void *)stepped_address, + (void *)&stepped_opcode, BREAK_INSTR_SIZE); + flush_icache_range(stepped_address, + stepped_address + BREAK_INSTR_SIZE); + } + stepped_address = 0; + stepped_opcode = 0; + kgdb_single_step = 0; + atomic_set(&kgdb_cpu_doing_single_step, -1); +} + +struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { + {DBG_REG_ZERO, GDB_SIZEOF_REG, -1}, + {DBG_REG_RA, GDB_SIZEOF_REG, offsetof(struct pt_regs, ra)}, + {DBG_REG_SP, GDB_SIZEOF_REG, offsetof(struct pt_regs, sp)}, + {DBG_REG_GP, GDB_SIZEOF_REG, offsetof(struct pt_regs, gp)}, + {DBG_REG_TP, GDB_SIZEOF_REG, offsetof(struct pt_regs, tp)}, + {DBG_REG_T0, GDB_SIZEOF_REG, offsetof(struct pt_regs, t0)}, + {DBG_REG_T1, GDB_SIZEOF_REG, offsetof(struct pt_regs, t1)}, + {DBG_REG_T2, GDB_SIZEOF_REG, offsetof(struct pt_regs, t2)}, + {DBG_REG_FP, GDB_SIZEOF_REG, offsetof(struct pt_regs, s0)}, + {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, + {DBG_REG_A0, GDB_SIZEOF_REG, offsetof(struct pt_regs, a0)}, + {DBG_REG_A1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, + {DBG_REG_A2, GDB_SIZEOF_REG, offsetof(struct pt_regs, a2)}, + {DBG_REG_A3, GDB_SIZEOF_REG, offsetof(struct pt_regs, a3)}, + {DBG_REG_A4, GDB_SIZEOF_REG, offsetof(struct pt_regs, a4)}, + {DBG_REG_A5, GDB_SIZEOF_REG, offsetof(struct pt_regs, a5)}, + {DBG_REG_A6, GDB_SIZEOF_REG, offsetof(struct pt_regs, a6)}, + {DBG_REG_A7, GDB_SIZEOF_REG, offsetof(struct pt_regs, a7)}, + {DBG_REG_S2, GDB_SIZEOF_REG, offsetof(struct pt_regs, s2)}, + {DBG_REG_S3, GDB_SIZEOF_REG, offsetof(struct pt_regs, s3)}, + {DBG_REG_S4, GDB_SIZEOF_REG, offsetof(struct pt_regs, s4)}, + {DBG_REG_S5, GDB_SIZEOF_REG, offsetof(struct pt_regs, s5)}, + {DBG_REG_S6, GDB_SIZEOF_REG, offsetof(struct pt_regs, s6)}, + {DBG_REG_S7, GDB_SIZEOF_REG, offsetof(struct pt_regs, s7)}, + {DBG_REG_S8, GDB_SIZEOF_REG, offsetof(struct pt_regs, s8)}, + {DBG_REG_S9, GDB_SIZEOF_REG, offsetof(struct pt_regs, s9)}, + {DBG_REG_S10, GDB_SIZEOF_REG, offsetof(struct pt_regs, s10)}, + {DBG_REG_S11, GDB_SIZEOF_REG, offsetof(struct pt_regs, s11)}, + {DBG_REG_T3, GDB_SIZEOF_REG, offsetof(struct pt_regs, t3)}, + {DBG_REG_T4, GDB_SIZEOF_REG, offsetof(struct pt_regs, t4)}, + {DBG_REG_T5, GDB_SIZEOF_REG, offsetof(struct pt_regs, t5)}, + {DBG_REG_T6, GDB_SIZEOF_REG, offsetof(struct pt_regs, t6)}, + {DBG_REG_EPC, GDB_SIZEOF_REG, offsetof(struct pt_regs, epc)}, + {DBG_REG_STATUS, GDB_SIZEOF_REG, offsetof(struct pt_regs, status)}, + {DBG_REG_BADADDR, GDB_SIZEOF_REG, offsetof(struct pt_regs, badaddr)}, + {DBG_REG_CAUSE, GDB_SIZEOF_REG, offsetof(struct pt_regs, cause)}, +}; + +char *dbg_get_reg(int regno, void *mem, struct pt_regs *regs) +{ + if (regno >= DBG_MAX_REG_NUM || regno < 0) + return NULL; + + if (dbg_reg_def[regno].offset != -1) + memcpy(mem, (void *)regs + dbg_reg_def[regno].offset, + dbg_reg_def[regno].size); + else + memset(mem, 0, dbg_reg_def[regno].size); + return dbg_reg_def[regno].name; +} + +int dbg_set_reg(int regno, void *mem, struct pt_regs *regs) +{ + if (regno >= DBG_MAX_REG_NUM || regno < 0) + return -EINVAL; + + if (dbg_reg_def[regno].offset != -1) + memcpy((void *)regs + dbg_reg_def[regno].offset, mem, + dbg_reg_def[regno].size); + return 0; +} + +void +sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task) +{ + /* Initialize to zero */ + memset((char *)gdb_regs, 0, NUMREGBYTES); + + gdb_regs[DBG_REG_SP_OFF] = task->thread.sp; + gdb_regs[DBG_REG_FP_OFF] = task->thread.s[0]; + gdb_regs[DBG_REG_S1_OFF] = task->thread.s[1]; + gdb_regs[DBG_REG_S2_OFF] = task->thread.s[2]; + gdb_regs[DBG_REG_S3_OFF] = task->thread.s[3]; + gdb_regs[DBG_REG_S4_OFF] = task->thread.s[4]; + gdb_regs[DBG_REG_S5_OFF] = task->thread.s[5]; + gdb_regs[DBG_REG_S6_OFF] = task->thread.s[6]; + gdb_regs[DBG_REG_S7_OFF] = task->thread.s[7]; + gdb_regs[DBG_REG_S8_OFF] = task->thread.s[8]; + gdb_regs[DBG_REG_S9_OFF] = task->thread.s[10]; + gdb_regs[DBG_REG_S10_OFF] = task->thread.s[11]; + gdb_regs[DBG_REG_EPC_OFF] = task->thread.ra; +} + +void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) +{ + regs->epc = pc; +} + +void kgdb_arch_handle_qxfer_pkt(char *remcom_in_buffer, + char *remcom_out_buffer) +{ + if (!strncmp(remcom_in_buffer, gdb_xfer_read_target, + sizeof(gdb_xfer_read_target))) + strcpy(remcom_out_buffer, riscv_gdb_stub_target_desc); + else if (!strncmp(remcom_in_buffer, gdb_xfer_read_cpuxml, + sizeof(gdb_xfer_read_cpuxml))) + strcpy(remcom_out_buffer, riscv_gdb_stub_cpuxml); +} + +static inline void kgdb_arch_update_addr(struct pt_regs *regs, + char *remcom_in_buffer) +{ + unsigned long addr; + char *ptr; + + ptr = &remcom_in_buffer[1]; + if (kgdb_hex2long(&ptr, &addr)) + regs->epc = addr; +} + +int kgdb_arch_handle_exception(int vector, int signo, int err_code, + char *remcom_in_buffer, char *remcom_out_buffer, + struct pt_regs *regs) +{ + int err = 0; + + undo_single_step(regs); + + switch (remcom_in_buffer[0]) { + case 'c': + case 'D': + case 'k': + if (remcom_in_buffer[0] == 'c') + kgdb_arch_update_addr(regs, remcom_in_buffer); + break; + case 's': + kgdb_arch_update_addr(regs, remcom_in_buffer); + err = do_single_step(regs); + break; + default: + err = -1; + } + return err; +} + +static int kgdb_riscv_kgdbbreak(unsigned long addr) +{ + if (stepped_address == addr) + return KGDB_SW_SINGLE_STEP; + if (atomic_read(&kgdb_setting_breakpoint)) + if (addr == (unsigned long)&kgdb_compiled_break) + return KGDB_COMPILED_BREAK; + + return kgdb_has_hit_break(addr); +} + +static int kgdb_riscv_notify(struct notifier_block *self, unsigned long cmd, + void *ptr) +{ + struct die_args *args = (struct die_args *)ptr; + struct pt_regs *regs = args->regs; + unsigned long flags; + int type; + + if (user_mode(regs)) + return NOTIFY_DONE; + + type = kgdb_riscv_kgdbbreak(regs->epc); + if (type == NOT_KGDB_BREAK && cmd == DIE_TRAP) + return NOTIFY_DONE; + + local_irq_save(flags); + + if (kgdb_handle_exception(type == KGDB_SW_SINGLE_STEP ? 0 : 1, + args->signr, cmd, regs)) + return NOTIFY_DONE; + + if (type == KGDB_COMPILED_BREAK) + regs->epc += 4; + + local_irq_restore(flags); + + return NOTIFY_STOP; +} + +static struct notifier_block kgdb_notifier = { + .notifier_call = kgdb_riscv_notify, +}; + +int kgdb_arch_init(void) +{ + register_die_notifier(&kgdb_notifier); + + return 0; +} + +void kgdb_arch_exit(void) +{ + unregister_die_notifier(&kgdb_notifier); +} + +/* + * Global data + */ +#ifdef CONFIG_RISCV_ISA_C +const struct kgdb_arch arch_kgdb_ops = { + .gdb_bpt_instr = {0x02, 0x90}, /* c.ebreak */ +}; +#else +const struct kgdb_arch arch_kgdb_ops = { + .gdb_bpt_instr = {0x73, 0x00, 0x10, 0x00}, /* ebreak */ +}; +#endif diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S new file mode 100644 index 000000000..35a6ed76c --- /dev/null +++ b/arch/riscv/kernel/mcount-dyn.S @@ -0,0 +1,239 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2017 Andes Technology Corporation */ + +#include <linux/init.h> +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm/csr.h> +#include <asm/unistd.h> +#include <asm/thread_info.h> +#include <asm/asm-offsets.h> +#include <asm-generic/export.h> +#include <asm/ftrace.h> + + .text + + .macro SAVE_ABI_STATE +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + addi sp, sp, -48 + sd s0, 32(sp) + sd ra, 40(sp) + addi s0, sp, 48 + sd t0, 24(sp) + sd t1, 16(sp) +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + sd t2, 8(sp) +#endif +#else + addi sp, sp, -16 + sd s0, 0(sp) + sd ra, 8(sp) + addi s0, sp, 16 +#endif + .endm + + .macro RESTORE_ABI_STATE +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + ld s0, 32(sp) + ld ra, 40(sp) + addi sp, sp, 48 +#else + ld ra, 8(sp) + ld s0, 0(sp) + addi sp, sp, 16 +#endif + .endm + + .macro RESTORE_GRAPH_ARGS + ld a0, 24(sp) + ld a1, 16(sp) +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld a2, 8(sp) +#endif + .endm + +ENTRY(ftrace_graph_caller) + addi sp, sp, -16 + sd s0, 0(sp) + sd ra, 8(sp) + addi s0, sp, 16 +ftrace_graph_call: + .global ftrace_graph_call + /* + * Calling ftrace_enable/disable_ftrace_graph_caller would overwrite the + * call below. Check ftrace_modify_all_code for details. + */ + call ftrace_stub + ld ra, 8(sp) + ld s0, 0(sp) + addi sp, sp, 16 + ret +ENDPROC(ftrace_graph_caller) + +ENTRY(ftrace_caller) + /* + * a0: the address in the caller when calling ftrace_caller + * a1: the caller's return address + * a2: the address of global variable function_trace_op + */ + ld a1, -8(s0) + addi a0, ra, -MCOUNT_INSN_SIZE + la t5, function_trace_op + ld a2, 0(t5) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + /* + * the graph tracer (specifically, prepare_ftrace_return) needs these + * arguments but for now the function tracer occupies the regs, so we + * save them in temporary regs to recover later. + */ + addi t0, s0, -8 + mv t1, a0 +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld t2, -16(s0) +#endif +#endif + + SAVE_ABI_STATE +ftrace_call: + .global ftrace_call + /* + * For the dynamic ftrace to work, here we should reserve at least + * 8 bytes for a functional auipc-jalr pair. The following call + * serves this purpose. + * + * Calling ftrace_update_ftrace_func would overwrite the nops below. + * Check ftrace_modify_all_code for details. + */ + call ftrace_stub + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + RESTORE_GRAPH_ARGS + call ftrace_graph_caller +#endif + + RESTORE_ABI_STATE + ret +ENDPROC(ftrace_caller) + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + .macro SAVE_ALL + addi sp, sp, -(PT_SIZE_ON_STACK+16) + sd s0, (PT_SIZE_ON_STACK)(sp) + sd ra, (PT_SIZE_ON_STACK+8)(sp) + addi s0, sp, (PT_SIZE_ON_STACK+16) + + sd x1, PT_RA(sp) + sd x2, PT_SP(sp) + sd x3, PT_GP(sp) + sd x4, PT_TP(sp) + sd x5, PT_T0(sp) + sd x6, PT_T1(sp) + sd x7, PT_T2(sp) + sd x8, PT_S0(sp) + sd x9, PT_S1(sp) + sd x10, PT_A0(sp) + sd x11, PT_A1(sp) + sd x12, PT_A2(sp) + sd x13, PT_A3(sp) + sd x14, PT_A4(sp) + sd x15, PT_A5(sp) + sd x16, PT_A6(sp) + sd x17, PT_A7(sp) + sd x18, PT_S2(sp) + sd x19, PT_S3(sp) + sd x20, PT_S4(sp) + sd x21, PT_S5(sp) + sd x22, PT_S6(sp) + sd x23, PT_S7(sp) + sd x24, PT_S8(sp) + sd x25, PT_S9(sp) + sd x26, PT_S10(sp) + sd x27, PT_S11(sp) + sd x28, PT_T3(sp) + sd x29, PT_T4(sp) + sd x30, PT_T5(sp) + sd x31, PT_T6(sp) + .endm + + .macro RESTORE_ALL + ld x1, PT_RA(sp) + ld x2, PT_SP(sp) + ld x3, PT_GP(sp) + ld x4, PT_TP(sp) + ld x5, PT_T0(sp) + ld x6, PT_T1(sp) + ld x7, PT_T2(sp) + ld x8, PT_S0(sp) + ld x9, PT_S1(sp) + ld x10, PT_A0(sp) + ld x11, PT_A1(sp) + ld x12, PT_A2(sp) + ld x13, PT_A3(sp) + ld x14, PT_A4(sp) + ld x15, PT_A5(sp) + ld x16, PT_A6(sp) + ld x17, PT_A7(sp) + ld x18, PT_S2(sp) + ld x19, PT_S3(sp) + ld x20, PT_S4(sp) + ld x21, PT_S5(sp) + ld x22, PT_S6(sp) + ld x23, PT_S7(sp) + ld x24, PT_S8(sp) + ld x25, PT_S9(sp) + ld x26, PT_S10(sp) + ld x27, PT_S11(sp) + ld x28, PT_T3(sp) + ld x29, PT_T4(sp) + ld x30, PT_T5(sp) + ld x31, PT_T6(sp) + + ld s0, (PT_SIZE_ON_STACK)(sp) + ld ra, (PT_SIZE_ON_STACK+8)(sp) + addi sp, sp, (PT_SIZE_ON_STACK+16) + .endm + + .macro RESTORE_GRAPH_REG_ARGS + ld a0, PT_T0(sp) + ld a1, PT_T1(sp) +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld a2, PT_T2(sp) +#endif + .endm + +/* + * Most of the contents are the same as ftrace_caller. + */ +ENTRY(ftrace_regs_caller) + /* + * a3: the address of all registers in the stack + */ + ld a1, -8(s0) + addi a0, ra, -MCOUNT_INSN_SIZE + la t5, function_trace_op + ld a2, 0(t5) + addi a3, sp, -(PT_SIZE_ON_STACK+16) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + addi t0, s0, -8 + mv t1, a0 +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld t2, -16(s0) +#endif +#endif + SAVE_ALL + +ftrace_regs_call: + .global ftrace_regs_call + call ftrace_stub + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + RESTORE_GRAPH_REG_ARGS + call ftrace_graph_caller +#endif + + RESTORE_ALL + ret +ENDPROC(ftrace_regs_caller) +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S new file mode 100644 index 000000000..6d462681c --- /dev/null +++ b/arch/riscv/kernel/mcount.S @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2017 Andes Technology Corporation */ + +#include <linux/init.h> +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm/csr.h> +#include <asm/unistd.h> +#include <asm/thread_info.h> +#include <asm/asm-offsets.h> +#include <asm-generic/export.h> +#include <asm/ftrace.h> + + .text + + .macro SAVE_ABI_STATE + addi sp, sp, -16 + sd s0, 0(sp) + sd ra, 8(sp) + addi s0, sp, 16 + .endm + + /* + * The call to ftrace_return_to_handler would overwrite the return + * register if a0 was not saved. + */ + .macro SAVE_RET_ABI_STATE + addi sp, sp, -32 + sd s0, 16(sp) + sd ra, 24(sp) + sd a0, 8(sp) + addi s0, sp, 32 + .endm + + .macro RESTORE_ABI_STATE + ld ra, 8(sp) + ld s0, 0(sp) + addi sp, sp, 16 + .endm + + .macro RESTORE_RET_ABI_STATE + ld ra, 24(sp) + ld s0, 16(sp) + ld a0, 8(sp) + addi sp, sp, 32 + .endm + +ENTRY(ftrace_stub) +#ifdef CONFIG_DYNAMIC_FTRACE + .global MCOUNT_NAME + .set MCOUNT_NAME, ftrace_stub +#endif + ret +ENDPROC(ftrace_stub) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +ENTRY(return_to_handler) +/* + * On implementing the frame point test, the ideal way is to compare the + * s0 (frame pointer, if enabled) on entry and the sp (stack pointer) on return. + * However, the psABI of variable-length-argument functions does not allow this. + * + * So alternatively we check the *old* frame pointer position, that is, the + * value stored in -16(s0) on entry, and the s0 on return. + */ +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + mv t6, s0 +#endif + SAVE_RET_ABI_STATE +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + mv a0, t6 +#endif + call ftrace_return_to_handler + mv a1, a0 + RESTORE_RET_ABI_STATE + jalr a1 +ENDPROC(return_to_handler) +#endif + +#ifndef CONFIG_DYNAMIC_FTRACE +ENTRY(MCOUNT_NAME) + la t4, ftrace_stub +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + la t0, ftrace_graph_return + ld t1, 0(t0) + bne t1, t4, do_ftrace_graph_caller + + la t3, ftrace_graph_entry + ld t2, 0(t3) + la t6, ftrace_graph_entry_stub + bne t2, t6, do_ftrace_graph_caller +#endif + la t3, ftrace_trace_function + ld t5, 0(t3) + bne t5, t4, do_trace + ret + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * A pseudo representation for the function graph tracer: + * prepare_to_return(&ra_to_caller_of_caller, ra_to_caller) + */ +do_ftrace_graph_caller: + addi a0, s0, -8 + mv a1, ra +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld a2, -16(s0) +#endif + SAVE_ABI_STATE + call prepare_ftrace_return + RESTORE_ABI_STATE + ret +#endif + +/* + * A pseudo representation for the function tracer: + * (*ftrace_trace_function)(ra_to_caller, ra_to_caller_of_caller) + */ +do_trace: + ld a1, -8(s0) + mv a0, ra + + SAVE_ABI_STATE + jalr t5 + RESTORE_ABI_STATE + ret +ENDPROC(MCOUNT_NAME) +#endif +EXPORT_SYMBOL(MCOUNT_NAME) diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c new file mode 100644 index 000000000..e264e59e5 --- /dev/null +++ b/arch/riscv/kernel/module-sections.c @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org> + * + * Copyright (C) 2018 Andes Technology Corporation <zong@andestech.com> + */ + +#include <linux/elf.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/moduleloader.h> + +unsigned long module_emit_got_entry(struct module *mod, unsigned long val) +{ + struct mod_section *got_sec = &mod->arch.got; + int i = got_sec->num_entries; + struct got_entry *got = get_got_entry(val, got_sec); + + if (got) + return (unsigned long)got; + + /* There is no duplicate entry, create a new one */ + got = (struct got_entry *)got_sec->shdr->sh_addr; + got[i] = emit_got_entry(val); + + got_sec->num_entries++; + BUG_ON(got_sec->num_entries > got_sec->max_entries); + + return (unsigned long)&got[i]; +} + +unsigned long module_emit_plt_entry(struct module *mod, unsigned long val) +{ + struct mod_section *got_plt_sec = &mod->arch.got_plt; + struct got_entry *got_plt; + struct mod_section *plt_sec = &mod->arch.plt; + struct plt_entry *plt = get_plt_entry(val, plt_sec, got_plt_sec); + int i = plt_sec->num_entries; + + if (plt) + return (unsigned long)plt; + + /* There is no duplicate entry, create a new one */ + got_plt = (struct got_entry *)got_plt_sec->shdr->sh_addr; + got_plt[i] = emit_got_entry(val); + plt = (struct plt_entry *)plt_sec->shdr->sh_addr; + plt[i] = emit_plt_entry(val, + (unsigned long)&plt[i], + (unsigned long)&got_plt[i]); + + plt_sec->num_entries++; + got_plt_sec->num_entries++; + BUG_ON(plt_sec->num_entries > plt_sec->max_entries); + + return (unsigned long)&plt[i]; +} + +static int is_rela_equal(const Elf_Rela *x, const Elf_Rela *y) +{ + return x->r_info == y->r_info && x->r_addend == y->r_addend; +} + +static bool duplicate_rela(const Elf_Rela *rela, int idx) +{ + int i; + for (i = 0; i < idx; i++) { + if (is_rela_equal(&rela[i], &rela[idx])) + return true; + } + return false; +} + +static void count_max_entries(Elf_Rela *relas, int num, + unsigned int *plts, unsigned int *gots) +{ + unsigned int type, i; + + for (i = 0; i < num; i++) { + type = ELF_RISCV_R_TYPE(relas[i].r_info); + if (type == R_RISCV_CALL_PLT) { + if (!duplicate_rela(relas, i)) + (*plts)++; + } else if (type == R_RISCV_GOT_HI20) { + if (!duplicate_rela(relas, i)) + (*gots)++; + } + } +} + +int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, + char *secstrings, struct module *mod) +{ + unsigned int num_plts = 0; + unsigned int num_gots = 0; + int i; + + /* + * Find the empty .got and .plt sections. + */ + for (i = 0; i < ehdr->e_shnum; i++) { + if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt")) + mod->arch.plt.shdr = sechdrs + i; + else if (!strcmp(secstrings + sechdrs[i].sh_name, ".got")) + mod->arch.got.shdr = sechdrs + i; + else if (!strcmp(secstrings + sechdrs[i].sh_name, ".got.plt")) + mod->arch.got_plt.shdr = sechdrs + i; + } + + if (!mod->arch.plt.shdr) { + pr_err("%s: module PLT section(s) missing\n", mod->name); + return -ENOEXEC; + } + if (!mod->arch.got.shdr) { + pr_err("%s: module GOT section(s) missing\n", mod->name); + return -ENOEXEC; + } + if (!mod->arch.got_plt.shdr) { + pr_err("%s: module GOT.PLT section(s) missing\n", mod->name); + return -ENOEXEC; + } + + /* Calculate the maxinum number of entries */ + for (i = 0; i < ehdr->e_shnum; i++) { + Elf_Rela *relas = (void *)ehdr + sechdrs[i].sh_offset; + int num_rela = sechdrs[i].sh_size / sizeof(Elf_Rela); + Elf_Shdr *dst_sec = sechdrs + sechdrs[i].sh_info; + + if (sechdrs[i].sh_type != SHT_RELA) + continue; + + /* ignore relocations that operate on non-exec sections */ + if (!(dst_sec->sh_flags & SHF_EXECINSTR)) + continue; + + count_max_entries(relas, num_rela, &num_plts, &num_gots); + } + + mod->arch.plt.shdr->sh_type = SHT_NOBITS; + mod->arch.plt.shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC; + mod->arch.plt.shdr->sh_addralign = L1_CACHE_BYTES; + mod->arch.plt.shdr->sh_size = (num_plts + 1) * sizeof(struct plt_entry); + mod->arch.plt.num_entries = 0; + mod->arch.plt.max_entries = num_plts; + + mod->arch.got.shdr->sh_type = SHT_NOBITS; + mod->arch.got.shdr->sh_flags = SHF_ALLOC; + mod->arch.got.shdr->sh_addralign = L1_CACHE_BYTES; + mod->arch.got.shdr->sh_size = (num_gots + 1) * sizeof(struct got_entry); + mod->arch.got.num_entries = 0; + mod->arch.got.max_entries = num_gots; + + mod->arch.got_plt.shdr->sh_type = SHT_NOBITS; + mod->arch.got_plt.shdr->sh_flags = SHF_ALLOC; + mod->arch.got_plt.shdr->sh_addralign = L1_CACHE_BYTES; + mod->arch.got_plt.shdr->sh_size = (num_plts + 1) * sizeof(struct got_entry); + mod->arch.got_plt.num_entries = 0; + mod->arch.got_plt.max_entries = num_plts; + return 0; +} diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c new file mode 100644 index 000000000..c3310a68a --- /dev/null +++ b/arch/riscv/kernel/module.c @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (C) 2017 Zihao Yu + */ + +#include <linux/elf.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/moduleloader.h> +#include <linux/vmalloc.h> +#include <linux/sizes.h> +#include <linux/pgtable.h> +#include <asm/sections.h> + +/* + * The auipc+jalr instruction pair can reach any PC-relative offset + * in the range [-2^31 - 2^11, 2^31 - 2^11) + */ +static bool riscv_insn_valid_32bit_offset(ptrdiff_t val) +{ +#ifdef CONFIG_32BIT + return true; +#else + return (-(1L << 31) - (1L << 11)) <= val && val < ((1L << 31) - (1L << 11)); +#endif +} + +static int apply_r_riscv_32_rela(struct module *me, u32 *location, Elf_Addr v) +{ + if (v != (u32)v) { + pr_err("%s: value %016llx out of range for 32-bit field\n", + me->name, (long long)v); + return -EINVAL; + } + *location = v; + return 0; +} + +static int apply_r_riscv_64_rela(struct module *me, u32 *location, Elf_Addr v) +{ + *(u64 *)location = v; + return 0; +} + +static int apply_r_riscv_branch_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + ptrdiff_t offset = (void *)v - (void *)location; + u32 imm12 = (offset & 0x1000) << (31 - 12); + u32 imm11 = (offset & 0x800) >> (11 - 7); + u32 imm10_5 = (offset & 0x7e0) << (30 - 10); + u32 imm4_1 = (offset & 0x1e) << (11 - 4); + + *location = (*location & 0x1fff07f) | imm12 | imm11 | imm10_5 | imm4_1; + return 0; +} + +static int apply_r_riscv_jal_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + ptrdiff_t offset = (void *)v - (void *)location; + u32 imm20 = (offset & 0x100000) << (31 - 20); + u32 imm19_12 = (offset & 0xff000); + u32 imm11 = (offset & 0x800) << (20 - 11); + u32 imm10_1 = (offset & 0x7fe) << (30 - 10); + + *location = (*location & 0xfff) | imm20 | imm19_12 | imm11 | imm10_1; + return 0; +} + +static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + ptrdiff_t offset = (void *)v - (void *)location; + u16 imm8 = (offset & 0x100) << (12 - 8); + u16 imm7_6 = (offset & 0xc0) >> (6 - 5); + u16 imm5 = (offset & 0x20) >> (5 - 2); + u16 imm4_3 = (offset & 0x18) << (12 - 5); + u16 imm2_1 = (offset & 0x6) << (12 - 10); + + *(u16 *)location = (*(u16 *)location & 0xe383) | + imm8 | imm7_6 | imm5 | imm4_3 | imm2_1; + return 0; +} + +static int apply_r_riscv_rvc_jump_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + ptrdiff_t offset = (void *)v - (void *)location; + u16 imm11 = (offset & 0x800) << (12 - 11); + u16 imm10 = (offset & 0x400) >> (10 - 8); + u16 imm9_8 = (offset & 0x300) << (12 - 11); + u16 imm7 = (offset & 0x80) >> (7 - 6); + u16 imm6 = (offset & 0x40) << (12 - 11); + u16 imm5 = (offset & 0x20) >> (5 - 2); + u16 imm4 = (offset & 0x10) << (12 - 5); + u16 imm3_1 = (offset & 0xe) << (12 - 10); + + *(u16 *)location = (*(u16 *)location & 0xe003) | + imm11 | imm10 | imm9_8 | imm7 | imm6 | imm5 | imm4 | imm3_1; + return 0; +} + +static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + ptrdiff_t offset = (void *)v - (void *)location; + s32 hi20; + + if (!riscv_insn_valid_32bit_offset(offset)) { + pr_err( + "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", + me->name, (long long)v, location); + return -EINVAL; + } + + hi20 = (offset + 0x800) & 0xfffff000; + *location = (*location & 0xfff) | hi20; + return 0; +} + +static int apply_r_riscv_pcrel_lo12_i_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + /* + * v is the lo12 value to fill. It is calculated before calling this + * handler. + */ + *location = (*location & 0xfffff) | ((v & 0xfff) << 20); + return 0; +} + +static int apply_r_riscv_pcrel_lo12_s_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + /* + * v is the lo12 value to fill. It is calculated before calling this + * handler. + */ + u32 imm11_5 = (v & 0xfe0) << (31 - 11); + u32 imm4_0 = (v & 0x1f) << (11 - 4); + + *location = (*location & 0x1fff07f) | imm11_5 | imm4_0; + return 0; +} + +static int apply_r_riscv_hi20_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + s32 hi20; + + if (IS_ENABLED(CONFIG_CMODEL_MEDLOW)) { + pr_err( + "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", + me->name, (long long)v, location); + return -EINVAL; + } + + hi20 = ((s32)v + 0x800) & 0xfffff000; + *location = (*location & 0xfff) | hi20; + return 0; +} + +static int apply_r_riscv_lo12_i_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + /* Skip medlow checking because of filtering by HI20 already */ + s32 hi20 = ((s32)v + 0x800) & 0xfffff000; + s32 lo12 = ((s32)v - hi20); + *location = (*location & 0xfffff) | ((lo12 & 0xfff) << 20); + return 0; +} + +static int apply_r_riscv_lo12_s_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + /* Skip medlow checking because of filtering by HI20 already */ + s32 hi20 = ((s32)v + 0x800) & 0xfffff000; + s32 lo12 = ((s32)v - hi20); + u32 imm11_5 = (lo12 & 0xfe0) << (31 - 11); + u32 imm4_0 = (lo12 & 0x1f) << (11 - 4); + *location = (*location & 0x1fff07f) | imm11_5 | imm4_0; + return 0; +} + +static int apply_r_riscv_got_hi20_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + ptrdiff_t offset = (void *)v - (void *)location; + s32 hi20; + + /* Always emit the got entry */ + if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) { + offset = module_emit_got_entry(me, v); + offset = (void *)offset - (void *)location; + } else { + pr_err( + "%s: can not generate the GOT entry for symbol = %016llx from PC = %p\n", + me->name, (long long)v, location); + return -EINVAL; + } + + hi20 = (offset + 0x800) & 0xfffff000; + *location = (*location & 0xfff) | hi20; + return 0; +} + +static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + ptrdiff_t offset = (void *)v - (void *)location; + u32 hi20, lo12; + + if (!riscv_insn_valid_32bit_offset(offset)) { + /* Only emit the plt entry if offset over 32-bit range */ + if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) { + offset = module_emit_plt_entry(me, v); + offset = (void *)offset - (void *)location; + } else { + pr_err( + "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", + me->name, (long long)v, location); + return -EINVAL; + } + } + + hi20 = (offset + 0x800) & 0xfffff000; + lo12 = (offset - hi20) & 0xfff; + *location = (*location & 0xfff) | hi20; + *(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20); + return 0; +} + +static int apply_r_riscv_call_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + ptrdiff_t offset = (void *)v - (void *)location; + u32 hi20, lo12; + + if (!riscv_insn_valid_32bit_offset(offset)) { + pr_err( + "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", + me->name, (long long)v, location); + return -EINVAL; + } + + hi20 = (offset + 0x800) & 0xfffff000; + lo12 = (offset - hi20) & 0xfff; + *location = (*location & 0xfff) | hi20; + *(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20); + return 0; +} + +static int apply_r_riscv_relax_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + return 0; +} + +static int apply_r_riscv_align_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + pr_err( + "%s: The unexpected relocation type 'R_RISCV_ALIGN' from PC = %p\n", + me->name, location); + return -EINVAL; +} + +static int apply_r_riscv_add32_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u32 *)location += (u32)v; + return 0; +} + +static int apply_r_riscv_add64_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u64 *)location += (u64)v; + return 0; +} + +static int apply_r_riscv_sub32_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u32 *)location -= (u32)v; + return 0; +} + +static int apply_r_riscv_sub64_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u64 *)location -= (u64)v; + return 0; +} + +static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, + Elf_Addr v) = { + [R_RISCV_32] = apply_r_riscv_32_rela, + [R_RISCV_64] = apply_r_riscv_64_rela, + [R_RISCV_BRANCH] = apply_r_riscv_branch_rela, + [R_RISCV_JAL] = apply_r_riscv_jal_rela, + [R_RISCV_RVC_BRANCH] = apply_r_riscv_rcv_branch_rela, + [R_RISCV_RVC_JUMP] = apply_r_riscv_rvc_jump_rela, + [R_RISCV_PCREL_HI20] = apply_r_riscv_pcrel_hi20_rela, + [R_RISCV_PCREL_LO12_I] = apply_r_riscv_pcrel_lo12_i_rela, + [R_RISCV_PCREL_LO12_S] = apply_r_riscv_pcrel_lo12_s_rela, + [R_RISCV_HI20] = apply_r_riscv_hi20_rela, + [R_RISCV_LO12_I] = apply_r_riscv_lo12_i_rela, + [R_RISCV_LO12_S] = apply_r_riscv_lo12_s_rela, + [R_RISCV_GOT_HI20] = apply_r_riscv_got_hi20_rela, + [R_RISCV_CALL_PLT] = apply_r_riscv_call_plt_rela, + [R_RISCV_CALL] = apply_r_riscv_call_rela, + [R_RISCV_RELAX] = apply_r_riscv_relax_rela, + [R_RISCV_ALIGN] = apply_r_riscv_align_rela, + [R_RISCV_ADD32] = apply_r_riscv_add32_rela, + [R_RISCV_ADD64] = apply_r_riscv_add64_rela, + [R_RISCV_SUB32] = apply_r_riscv_sub32_rela, + [R_RISCV_SUB64] = apply_r_riscv_sub64_rela, +}; + +int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, + unsigned int symindex, unsigned int relsec, + struct module *me) +{ + Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; + int (*handler)(struct module *me, u32 *location, Elf_Addr v); + Elf_Sym *sym; + u32 *location; + unsigned int i, type; + Elf_Addr v; + int res; + + pr_debug("Applying relocate section %u to %u\n", relsec, + sechdrs[relsec].sh_info); + + for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { + /* This is where to make the change */ + location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + + rel[i].r_offset; + /* This is the symbol it is referring to */ + sym = (Elf_Sym *)sechdrs[symindex].sh_addr + + ELF_RISCV_R_SYM(rel[i].r_info); + if (IS_ERR_VALUE(sym->st_value)) { + /* Ignore unresolved weak symbol */ + if (ELF_ST_BIND(sym->st_info) == STB_WEAK) + continue; + pr_warn("%s: Unknown symbol %s\n", + me->name, strtab + sym->st_name); + return -ENOENT; + } + + type = ELF_RISCV_R_TYPE(rel[i].r_info); + + if (type < ARRAY_SIZE(reloc_handlers_rela)) + handler = reloc_handlers_rela[type]; + else + handler = NULL; + + if (!handler) { + pr_err("%s: Unknown relocation type %u\n", + me->name, type); + return -EINVAL; + } + + v = sym->st_value + rel[i].r_addend; + + if (type == R_RISCV_PCREL_LO12_I || type == R_RISCV_PCREL_LO12_S) { + unsigned int j; + + for (j = 0; j < sechdrs[relsec].sh_size / sizeof(*rel); j++) { + unsigned long hi20_loc = + sechdrs[sechdrs[relsec].sh_info].sh_addr + + rel[j].r_offset; + u32 hi20_type = ELF_RISCV_R_TYPE(rel[j].r_info); + + /* Find the corresponding HI20 relocation entry */ + if (hi20_loc == sym->st_value + && (hi20_type == R_RISCV_PCREL_HI20 + || hi20_type == R_RISCV_GOT_HI20)) { + s32 hi20, lo12; + Elf_Sym *hi20_sym = + (Elf_Sym *)sechdrs[symindex].sh_addr + + ELF_RISCV_R_SYM(rel[j].r_info); + unsigned long hi20_sym_val = + hi20_sym->st_value + + rel[j].r_addend; + + /* Calculate lo12 */ + size_t offset = hi20_sym_val - hi20_loc; + if (IS_ENABLED(CONFIG_MODULE_SECTIONS) + && hi20_type == R_RISCV_GOT_HI20) { + offset = module_emit_got_entry( + me, hi20_sym_val); + offset = offset - hi20_loc; + } + hi20 = (offset + 0x800) & 0xfffff000; + lo12 = offset - hi20; + v = lo12; + + break; + } + } + if (j == sechdrs[relsec].sh_size / sizeof(*rel)) { + pr_err( + "%s: Can not find HI20 relocation information\n", + me->name); + return -EINVAL; + } + } + + res = handler(me, location, v); + if (res) + return res; + } + + return 0; +} + +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) +#define VMALLOC_MODULE_START \ + max(PFN_ALIGN((unsigned long)&_end - SZ_2G), VMALLOC_START) +void *module_alloc(unsigned long size) +{ + return __vmalloc_node_range(size, 1, VMALLOC_MODULE_START, + VMALLOC_END, GFP_KERNEL, + PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, + __builtin_return_address(0)); +} +#endif diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c new file mode 100644 index 000000000..c3fced410 --- /dev/null +++ b/arch/riscv/kernel/patch.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 SiFive + */ + +#include <linux/spinlock.h> +#include <linux/mm.h> +#include <linux/memory.h> +#include <linux/uaccess.h> +#include <linux/stop_machine.h> +#include <asm/kprobes.h> +#include <asm/cacheflush.h> +#include <asm/fixmap.h> +#include <asm/ftrace.h> +#include <asm/patch.h> + +struct patch_insn { + void *addr; + u32 insn; + atomic_t cpu_count; +}; + +int riscv_patch_in_stop_machine = false; + +#ifdef CONFIG_MMU +static void *patch_map(void *addr, int fixmap) +{ + uintptr_t uintaddr = (uintptr_t) addr; + struct page *page; + + if (core_kernel_text(uintaddr)) + page = phys_to_page(__pa_symbol(addr)); + else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) + page = vmalloc_to_page(addr); + else + return addr; + + BUG_ON(!page); + + return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + + (uintaddr & ~PAGE_MASK)); +} +NOKPROBE_SYMBOL(patch_map); + +static void patch_unmap(int fixmap) +{ + clear_fixmap(fixmap); +} +NOKPROBE_SYMBOL(patch_unmap); + +static int patch_insn_write(void *addr, const void *insn, size_t len) +{ + void *waddr = addr; + bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; + int ret; + + /* + * Before reaching here, it was expected to lock the text_mutex + * already, so we don't need to give another lock here and could + * ensure that it was safe between each cores. + * + * We're currently using stop_machine() for ftrace & kprobes, and while + * that ensures text_mutex is held before installing the mappings it + * does not ensure text_mutex is held by the calling thread. That's + * safe but triggers a lockdep failure, so just elide it for that + * specific case. + */ + if (!riscv_patch_in_stop_machine) + lockdep_assert_held(&text_mutex); + + if (across_pages) + patch_map(addr + len, FIX_TEXT_POKE1); + + waddr = patch_map(addr, FIX_TEXT_POKE0); + + ret = copy_to_kernel_nofault(waddr, insn, len); + + patch_unmap(FIX_TEXT_POKE0); + + if (across_pages) + patch_unmap(FIX_TEXT_POKE1); + + return ret; +} +NOKPROBE_SYMBOL(patch_insn_write); +#else +static int patch_insn_write(void *addr, const void *insn, size_t len) +{ + return copy_to_kernel_nofault(addr, insn, len); +} +NOKPROBE_SYMBOL(patch_insn_write); +#endif /* CONFIG_MMU */ + +int patch_text_nosync(void *addr, const void *insns, size_t len) +{ + u32 *tp = addr; + int ret; + + ret = patch_insn_write(tp, insns, len); + + if (!ret) + flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len); + + return ret; +} +NOKPROBE_SYMBOL(patch_text_nosync); + +static int patch_text_cb(void *data) +{ + struct patch_insn *patch = data; + int ret = 0; + + if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) { + ret = + patch_text_nosync(patch->addr, &patch->insn, + GET_INSN_LENGTH(patch->insn)); + atomic_inc(&patch->cpu_count); + } else { + while (atomic_read(&patch->cpu_count) <= num_online_cpus()) + cpu_relax(); + smp_mb(); + } + + return ret; +} +NOKPROBE_SYMBOL(patch_text_cb); + +int patch_text(void *addr, u32 insn) +{ + int ret; + struct patch_insn patch = { + .addr = addr, + .insn = insn, + .cpu_count = ATOMIC_INIT(0), + }; + + /* + * kprobes takes text_mutex, before calling patch_text(), but as we call + * calls stop_machine(), the lockdep assertion in patch_insn_write() + * gets confused by the context in which the lock is taken. + * Instead, ensure the lock is held before calling stop_machine(), and + * set riscv_patch_in_stop_machine to skip the check in + * patch_insn_write(). + */ + lockdep_assert_held(&text_mutex); + riscv_patch_in_stop_machine = true; + ret = stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask); + riscv_patch_in_stop_machine = false; + return ret; +} +NOKPROBE_SYMBOL(patch_text); diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c new file mode 100644 index 000000000..fb02811df --- /dev/null +++ b/arch/riscv/kernel/perf_callchain.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */ + +#include <linux/perf_event.h> +#include <linux/uaccess.h> + +/* Kernel callchain */ +struct stackframe { + unsigned long fp; + unsigned long ra; +}; + +/* + * Get the return address for a single stackframe and return a pointer to the + * next frame tail. + */ +static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, + unsigned long fp, unsigned long reg_ra) +{ + struct stackframe buftail; + unsigned long ra = 0; + unsigned long __user *user_frame_tail = + (unsigned long __user *)(fp - sizeof(struct stackframe)); + + /* Check accessibility of one struct frame_tail beyond */ + if (!access_ok(user_frame_tail, sizeof(buftail))) + return 0; + if (__copy_from_user_inatomic(&buftail, user_frame_tail, + sizeof(buftail))) + return 0; + + if (reg_ra != 0) + ra = reg_ra; + else + ra = buftail.ra; + + fp = buftail.fp; + if (ra != 0) + perf_callchain_store(entry, ra); + else + return 0; + + return fp; +} + +/* + * This will be called when the target is in user mode + * This function will only be called when we use + * "PERF_SAMPLE_CALLCHAIN" in + * kernel/events/core.c:perf_prepare_sample() + * + * How to trigger perf_callchain_[user/kernel] : + * $ perf record -e cpu-clock --call-graph fp ./program + * $ perf report --call-graph + * + * On RISC-V platform, the program being sampled and the C library + * need to be compiled with -fno-omit-frame-pointer, otherwise + * the user stack will not contain function frame. + */ +void perf_callchain_user(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + unsigned long fp = 0; + + /* RISC-V does not support perf in guest mode. */ + if (guest_cbs && guest_cbs->is_in_guest()) + return; + + fp = regs->s0; + perf_callchain_store(entry, regs->epc); + + fp = user_backtrace(entry, fp, regs->ra); + while (fp && !(fp & 0x3) && entry->nr < entry->max_stack) + fp = user_backtrace(entry, fp, 0); +} + +bool fill_callchain(unsigned long pc, void *entry) +{ + return perf_callchain_store(entry, pc) == 0; +} + +void notrace walk_stackframe(struct task_struct *task, + struct pt_regs *regs, bool (*fn)(unsigned long, void *), void *arg); +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + + /* RISC-V does not support perf in guest mode. */ + if (guest_cbs && guest_cbs->is_in_guest()) { + pr_warn("RISC-V does not support perf in guest mode!"); + return; + } + + walk_stackframe(NULL, regs, fill_callchain, entry); +} diff --git a/arch/riscv/kernel/perf_event.c b/arch/riscv/kernel/perf_event.c new file mode 100644 index 000000000..c835f0362 --- /dev/null +++ b/arch/riscv/kernel/perf_event.c @@ -0,0 +1,485 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> + * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar + * Copyright (C) 2009 Jaswinder Singh Rajput + * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter + * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra + * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> + * Copyright (C) 2009 Google, Inc., Stephane Eranian + * Copyright 2014 Tilera Corporation. All Rights Reserved. + * Copyright (C) 2018 Andes Technology Corporation + * + * Perf_events support for RISC-V platforms. + * + * Since the spec. (as of now, Priv-Spec 1.10) does not provide enough + * functionality for perf event to fully work, this file provides + * the very basic framework only. + * + * For platform portings, please check Documentations/riscv/pmu.txt. + * + * The Copyright line includes x86 and tile ones. + */ + +#include <linux/kprobes.h> +#include <linux/kernel.h> +#include <linux/kdebug.h> +#include <linux/mutex.h> +#include <linux/bitmap.h> +#include <linux/irq.h> +#include <linux/perf_event.h> +#include <linux/atomic.h> +#include <linux/of.h> +#include <asm/perf_event.h> + +static const struct riscv_pmu *riscv_pmu __read_mostly; +static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); + +/* + * Hardware & cache maps and their methods + */ + +static const int riscv_hw_event_map[] = { + [PERF_COUNT_HW_CPU_CYCLES] = RISCV_PMU_CYCLE, + [PERF_COUNT_HW_INSTRUCTIONS] = RISCV_PMU_INSTRET, + [PERF_COUNT_HW_CACHE_REFERENCES] = RISCV_OP_UNSUPP, + [PERF_COUNT_HW_CACHE_MISSES] = RISCV_OP_UNSUPP, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = RISCV_OP_UNSUPP, + [PERF_COUNT_HW_BRANCH_MISSES] = RISCV_OP_UNSUPP, + [PERF_COUNT_HW_BUS_CYCLES] = RISCV_OP_UNSUPP, +}; + +#define C(x) PERF_COUNT_HW_CACHE_##x +static const int riscv_cache_event_map[PERF_COUNT_HW_CACHE_MAX] +[PERF_COUNT_HW_CACHE_OP_MAX] +[PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, + [C(RESULT_MISS)] = RISCV_OP_UNSUPP, + }, + }, +}; + +static int riscv_map_hw_event(u64 config) +{ + if (config >= riscv_pmu->max_events) + return -EINVAL; + + return riscv_pmu->hw_events[config]; +} + +static int riscv_map_cache_decode(u64 config, unsigned int *type, + unsigned int *op, unsigned int *result) +{ + return -ENOENT; +} + +static int riscv_map_cache_event(u64 config) +{ + unsigned int type, op, result; + int err = -ENOENT; + int code; + + err = riscv_map_cache_decode(config, &type, &op, &result); + if (!riscv_pmu->cache_events || err) + return err; + + if (type >= PERF_COUNT_HW_CACHE_MAX || + op >= PERF_COUNT_HW_CACHE_OP_MAX || + result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return -EINVAL; + + code = (*riscv_pmu->cache_events)[type][op][result]; + if (code == RISCV_OP_UNSUPP) + return -EINVAL; + + return code; +} + +/* + * Low-level functions: reading/writing counters + */ + +static inline u64 read_counter(int idx) +{ + u64 val = 0; + + switch (idx) { + case RISCV_PMU_CYCLE: + val = csr_read(CSR_CYCLE); + break; + case RISCV_PMU_INSTRET: + val = csr_read(CSR_INSTRET); + break; + default: + WARN_ON_ONCE(idx < 0 || idx > RISCV_MAX_COUNTERS); + return -EINVAL; + } + + return val; +} + +static inline void write_counter(int idx, u64 value) +{ + /* currently not supported */ + WARN_ON_ONCE(1); +} + +/* + * pmu->read: read and update the counter + * + * Other architectures' implementation often have a xxx_perf_event_update + * routine, which can return counter values when called in the IRQ, but + * return void when being called by the pmu->read method. + */ +static void riscv_pmu_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + u64 prev_raw_count, new_raw_count; + u64 oldval; + int idx = hwc->idx; + u64 delta; + + do { + prev_raw_count = local64_read(&hwc->prev_count); + new_raw_count = read_counter(idx); + + oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count); + } while (oldval != prev_raw_count); + + /* + * delta is the value to update the counter we maintain in the kernel. + */ + delta = (new_raw_count - prev_raw_count) & + ((1ULL << riscv_pmu->counter_width) - 1); + local64_add(delta, &event->count); + /* + * Something like local64_sub(delta, &hwc->period_left) here is + * needed if there is an interrupt for perf. + */ +} + +/* + * State transition functions: + * + * stop()/start() & add()/del() + */ + +/* + * pmu->stop: stop the counter + */ +static void riscv_pmu_stop(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); + hwc->state |= PERF_HES_STOPPED; + + if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { + riscv_pmu->pmu->read(event); + hwc->state |= PERF_HES_UPTODATE; + } +} + +/* + * pmu->start: start the event. + */ +static void riscv_pmu_start(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) + return; + + if (flags & PERF_EF_RELOAD) { + WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); + + /* + * Set the counter to the period to the next interrupt here, + * if you have any. + */ + } + + hwc->state = 0; + perf_event_update_userpage(event); + + /* + * Since we cannot write to counters, this serves as an initialization + * to the delta-mechanism in pmu->read(); otherwise, the delta would be + * wrong when pmu->read is called for the first time. + */ + local64_set(&hwc->prev_count, read_counter(hwc->idx)); +} + +/* + * pmu->add: add the event to PMU. + */ +static int riscv_pmu_add(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + + if (cpuc->n_events == riscv_pmu->num_counters) + return -ENOSPC; + + /* + * We don't have general conunters, so no binding-event-to-counter + * process here. + * + * Indexing using hwc->config generally not works, since config may + * contain extra information, but here the only info we have in + * hwc->config is the event index. + */ + hwc->idx = hwc->config; + cpuc->events[hwc->idx] = event; + cpuc->n_events++; + + hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; + + if (flags & PERF_EF_START) + riscv_pmu->pmu->start(event, PERF_EF_RELOAD); + + return 0; +} + +/* + * pmu->del: delete the event from PMU. + */ +static void riscv_pmu_del(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + + cpuc->events[hwc->idx] = NULL; + cpuc->n_events--; + riscv_pmu->pmu->stop(event, PERF_EF_UPDATE); + perf_event_update_userpage(event); +} + +/* + * Interrupt: a skeletion for reference. + */ + +static DEFINE_MUTEX(pmc_reserve_mutex); + +static irqreturn_t riscv_base_pmu_handle_irq(int irq_num, void *dev) +{ + return IRQ_NONE; +} + +static int reserve_pmc_hardware(void) +{ + int err = 0; + + mutex_lock(&pmc_reserve_mutex); + if (riscv_pmu->irq >= 0 && riscv_pmu->handle_irq) { + err = request_irq(riscv_pmu->irq, riscv_pmu->handle_irq, + IRQF_PERCPU, "riscv-base-perf", NULL); + } + mutex_unlock(&pmc_reserve_mutex); + + return err; +} + +static void release_pmc_hardware(void) +{ + mutex_lock(&pmc_reserve_mutex); + if (riscv_pmu->irq >= 0) + free_irq(riscv_pmu->irq, NULL); + mutex_unlock(&pmc_reserve_mutex); +} + +/* + * Event Initialization/Finalization + */ + +static atomic_t riscv_active_events = ATOMIC_INIT(0); + +static void riscv_event_destroy(struct perf_event *event) +{ + if (atomic_dec_return(&riscv_active_events) == 0) + release_pmc_hardware(); +} + +static int riscv_event_init(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + struct hw_perf_event *hwc = &event->hw; + int err; + int code; + + if (atomic_inc_return(&riscv_active_events) == 1) { + err = reserve_pmc_hardware(); + + if (err) { + pr_warn("PMC hardware not available\n"); + atomic_dec(&riscv_active_events); + return -EBUSY; + } + } + + switch (event->attr.type) { + case PERF_TYPE_HARDWARE: + code = riscv_pmu->map_hw_event(attr->config); + break; + case PERF_TYPE_HW_CACHE: + code = riscv_pmu->map_cache_event(attr->config); + break; + case PERF_TYPE_RAW: + return -EOPNOTSUPP; + default: + return -ENOENT; + } + + event->destroy = riscv_event_destroy; + if (code < 0) { + event->destroy(event); + return code; + } + + /* + * idx is set to -1 because the index of a general event should not be + * decided until binding to some counter in pmu->add(). + * + * But since we don't have such support, later in pmu->add(), we just + * use hwc->config as the index instead. + */ + hwc->config = code; + hwc->idx = -1; + + return 0; +} + +/* + * Initialization + */ + +static struct pmu min_pmu = { + .name = "riscv-base", + .event_init = riscv_event_init, + .add = riscv_pmu_add, + .del = riscv_pmu_del, + .start = riscv_pmu_start, + .stop = riscv_pmu_stop, + .read = riscv_pmu_read, +}; + +static const struct riscv_pmu riscv_base_pmu = { + .pmu = &min_pmu, + .max_events = ARRAY_SIZE(riscv_hw_event_map), + .map_hw_event = riscv_map_hw_event, + .hw_events = riscv_hw_event_map, + .map_cache_event = riscv_map_cache_event, + .cache_events = &riscv_cache_event_map, + .counter_width = 63, + .num_counters = RISCV_BASE_COUNTERS + 0, + .handle_irq = &riscv_base_pmu_handle_irq, + + /* This means this PMU has no IRQ. */ + .irq = -1, +}; + +static const struct of_device_id riscv_pmu_of_ids[] = { + {.compatible = "riscv,base-pmu", .data = &riscv_base_pmu}, + { /* sentinel value */ } +}; + +static int __init init_hw_perf_events(void) +{ + struct device_node *node = of_find_node_by_type(NULL, "pmu"); + const struct of_device_id *of_id; + + riscv_pmu = &riscv_base_pmu; + + if (node) { + of_id = of_match_node(riscv_pmu_of_ids, node); + + if (of_id) + riscv_pmu = of_id->data; + of_node_put(node); + } + + perf_pmu_register(riscv_pmu->pmu, "cpu", PERF_TYPE_RAW); + return 0; +} +arch_initcall(init_hw_perf_events); diff --git a/arch/riscv/kernel/perf_regs.c b/arch/riscv/kernel/perf_regs.c new file mode 100644 index 000000000..fd304a248 --- /dev/null +++ b/arch/riscv/kernel/perf_regs.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */ + +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/perf_event.h> +#include <linux/bug.h> +#include <asm/perf_regs.h> +#include <asm/ptrace.h> + +u64 perf_reg_value(struct pt_regs *regs, int idx) +{ + if (WARN_ON_ONCE((u32)idx >= PERF_REG_RISCV_MAX)) + return 0; + + return ((unsigned long *)regs)[idx]; +} + +#define REG_RESERVED (~((1ULL << PERF_REG_RISCV_MAX) - 1)) + +int perf_reg_validate(u64 mask) +{ + if (!mask || mask & REG_RESERVED) + return -EINVAL; + + return 0; +} + +u64 perf_reg_abi(struct task_struct *task) +{ +#if __riscv_xlen == 64 + return PERF_SAMPLE_REGS_ABI_64; +#else + return PERF_SAMPLE_REGS_ABI_32; +#endif +} + +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c new file mode 100644 index 000000000..7868050ff --- /dev/null +++ b/arch/riscv/kernel/process.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2009 Sunplus Core Technology Co., Ltd. + * Chen Liqin <liqin.chen@sunplusct.com> + * Lennox Wu <lennox.wu@sunplusct.com> + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#include <linux/cpu.h> +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/sched/task_stack.h> +#include <linux/tick.h> +#include <linux/ptrace.h> +#include <linux/uaccess.h> + +#include <asm/unistd.h> +#include <asm/processor.h> +#include <asm/csr.h> +#include <asm/string.h> +#include <asm/switch_to.h> +#include <asm/thread_info.h> + +register unsigned long gp_in_global __asm__("gp"); + +#ifdef CONFIG_STACKPROTECTOR +#include <linux/stackprotector.h> +unsigned long __stack_chk_guard __read_mostly; +EXPORT_SYMBOL(__stack_chk_guard); +#endif + +extern asmlinkage void ret_from_fork(void); +extern asmlinkage void ret_from_kernel_thread(void); + +void arch_cpu_idle(void) +{ + wait_for_interrupt(); + raw_local_irq_enable(); +} + +void show_regs(struct pt_regs *regs) +{ + show_regs_print_info(KERN_DEFAULT); + + pr_cont("epc: " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n", + regs->epc, regs->ra, regs->sp); + pr_cont(" gp : " REG_FMT " tp : " REG_FMT " t0 : " REG_FMT "\n", + regs->gp, regs->tp, regs->t0); + pr_cont(" t1 : " REG_FMT " t2 : " REG_FMT " s0 : " REG_FMT "\n", + regs->t1, regs->t2, regs->s0); + pr_cont(" s1 : " REG_FMT " a0 : " REG_FMT " a1 : " REG_FMT "\n", + regs->s1, regs->a0, regs->a1); + pr_cont(" a2 : " REG_FMT " a3 : " REG_FMT " a4 : " REG_FMT "\n", + regs->a2, regs->a3, regs->a4); + pr_cont(" a5 : " REG_FMT " a6 : " REG_FMT " a7 : " REG_FMT "\n", + regs->a5, regs->a6, regs->a7); + pr_cont(" s2 : " REG_FMT " s3 : " REG_FMT " s4 : " REG_FMT "\n", + regs->s2, regs->s3, regs->s4); + pr_cont(" s5 : " REG_FMT " s6 : " REG_FMT " s7 : " REG_FMT "\n", + regs->s5, regs->s6, regs->s7); + pr_cont(" s8 : " REG_FMT " s9 : " REG_FMT " s10: " REG_FMT "\n", + regs->s8, regs->s9, regs->s10); + pr_cont(" s11: " REG_FMT " t3 : " REG_FMT " t4 : " REG_FMT "\n", + regs->s11, regs->t3, regs->t4); + pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT "\n", + regs->t5, regs->t6); + + pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n", + regs->status, regs->badaddr, regs->cause); +} + +void start_thread(struct pt_regs *regs, unsigned long pc, + unsigned long sp) +{ + regs->status = SR_PIE; + if (has_fpu) { + regs->status |= SR_FS_INITIAL; + /* + * Restore the initial value to the FP register + * before starting the user program. + */ + fstate_restore(current, regs); + } + regs->epc = pc; + regs->sp = sp; +} + +void flush_thread(void) +{ +#ifdef CONFIG_FPU + /* + * Reset FPU state and context + * frm: round to nearest, ties to even (IEEE default) + * fflags: accrued exceptions cleared + */ + fstate_off(current, task_pt_regs(current)); + memset(¤t->thread.fstate, 0, sizeof(current->thread.fstate)); +#endif +} + +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) +{ + fstate_save(src, task_pt_regs(src)); + *dst = *src; + return 0; +} + +int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, + struct task_struct *p, unsigned long tls) +{ + struct pt_regs *childregs = task_pt_regs(p); + + memset(&p->thread.s, 0, sizeof(p->thread.s)); + + /* p->thread holds context to be restored by __switch_to() */ + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* Kernel thread */ + memset(childregs, 0, sizeof(struct pt_regs)); + childregs->gp = gp_in_global; + /* Supervisor/Machine, irqs on: */ + childregs->status = SR_PP | SR_PIE; + + p->thread.ra = (unsigned long)ret_from_kernel_thread; + p->thread.s[0] = usp; /* fn */ + p->thread.s[1] = arg; + } else { + *childregs = *(current_pt_regs()); + if (usp) /* User fork */ + childregs->sp = usp; + if (clone_flags & CLONE_SETTLS) + childregs->tp = tls; + childregs->a0 = 0; /* Return value of fork() */ + p->thread.ra = (unsigned long)ret_from_fork; + } + p->thread.sp = (unsigned long)childregs; /* kernel sp */ + return 0; +} diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c new file mode 100644 index 000000000..69678ab64 --- /dev/null +++ b/arch/riscv/kernel/ptrace.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2010 Tilera Corporation. All Rights Reserved. + * Copyright 2015 Regents of the University of California + * Copyright 2017 SiFive + * + * Copied from arch/tile/kernel/ptrace.c + */ + +#include <asm/ptrace.h> +#include <asm/syscall.h> +#include <asm/thread_info.h> +#include <asm/switch_to.h> +#include <linux/audit.h> +#include <linux/ptrace.h> +#include <linux/elf.h> +#include <linux/regset.h> +#include <linux/sched.h> +#include <linux/sched/task_stack.h> +#include <linux/tracehook.h> + +#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h> + +enum riscv_regset { + REGSET_X, +#ifdef CONFIG_FPU + REGSET_F, +#endif +}; + +static int riscv_gpr_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + return membuf_write(&to, task_pt_regs(target), + sizeof(struct user_regs_struct)); +} + +static int riscv_gpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret; + struct pt_regs *regs; + + regs = task_pt_regs(target); + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, regs, 0, -1); + return ret; +} + +#ifdef CONFIG_FPU +static int riscv_fpr_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct __riscv_d_ext_state *fstate = &target->thread.fstate; + + if (target == current) + fstate_save(current, task_pt_regs(current)); + + membuf_write(&to, fstate, offsetof(struct __riscv_d_ext_state, fcsr)); + membuf_store(&to, fstate->fcsr); + return membuf_zero(&to, 4); // explicitly pad +} + +static int riscv_fpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret; + struct __riscv_d_ext_state *fstate = &target->thread.fstate; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, fstate, 0, + offsetof(struct __riscv_d_ext_state, fcsr)); + if (!ret) { + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, fstate, 0, + offsetof(struct __riscv_d_ext_state, fcsr) + + sizeof(fstate->fcsr)); + } + + return ret; +} +#endif + +static const struct user_regset riscv_user_regset[] = { + [REGSET_X] = { + .core_note_type = NT_PRSTATUS, + .n = ELF_NGREG, + .size = sizeof(elf_greg_t), + .align = sizeof(elf_greg_t), + .regset_get = riscv_gpr_get, + .set = riscv_gpr_set, + }, +#ifdef CONFIG_FPU + [REGSET_F] = { + .core_note_type = NT_PRFPREG, + .n = ELF_NFPREG, + .size = sizeof(elf_fpreg_t), + .align = sizeof(elf_fpreg_t), + .regset_get = riscv_fpr_get, + .set = riscv_fpr_set, + }, +#endif +}; + +static const struct user_regset_view riscv_user_native_view = { + .name = "riscv", + .e_machine = EM_RISCV, + .regsets = riscv_user_regset, + .n = ARRAY_SIZE(riscv_user_regset), +}; + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ + return &riscv_user_native_view; +} + +void ptrace_disable(struct task_struct *child) +{ + clear_tsk_thread_flag(child, TIF_SYSCALL_TRACE); +} + +long arch_ptrace(struct task_struct *child, long request, + unsigned long addr, unsigned long data) +{ + long ret = -EIO; + + switch (request) { + default: + ret = ptrace_request(child, request, addr, data); + break; + } + + return ret; +} + +/* + * Allows PTRACE_SYSCALL to work. These are called from entry.S in + * {handle,ret_from}_syscall. + */ +__visible int do_syscall_trace_enter(struct pt_regs *regs) +{ + if (test_thread_flag(TIF_SYSCALL_TRACE)) + if (tracehook_report_syscall_entry(regs)) + return -1; + + /* + * Do the secure computing after ptrace; failures should be fast. + * If this fails we might have return value in a0 from seccomp + * (via SECCOMP_RET_ERRNO/TRACE). + */ + if (secure_computing() == -1) + return -1; + +#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS + if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) + trace_sys_enter(regs, syscall_get_nr(current, regs)); +#endif + + audit_syscall_entry(regs->a7, regs->a0, regs->a1, regs->a2, regs->a3); + return 0; +} + +__visible void do_syscall_trace_exit(struct pt_regs *regs) +{ + audit_syscall_exit(regs); + + if (test_thread_flag(TIF_SYSCALL_TRACE)) + tracehook_report_syscall_exit(regs, 0); + +#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS + if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) + trace_sys_exit(regs, regs_return_value(regs)); +#endif +} diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c new file mode 100644 index 000000000..9c842c416 --- /dev/null +++ b/arch/riscv/kernel/reset.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#include <linux/reboot.h> +#include <linux/pm.h> + +static void default_power_off(void) +{ + while (1) + wait_for_interrupt(); +} + +void (*pm_power_off)(void) = NULL; +EXPORT_SYMBOL(pm_power_off); + +void machine_restart(char *cmd) +{ + do_kernel_restart(cmd); + while (1); +} + +void machine_halt(void) +{ + if (pm_power_off != NULL) + pm_power_off(); + else + default_power_off(); +} + +void machine_power_off(void) +{ + if (pm_power_off != NULL) + pm_power_off(); + else + default_power_off(); +} diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c new file mode 100644 index 000000000..450492e1c --- /dev/null +++ b/arch/riscv/kernel/riscv_ksyms.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017 Zihao Yu + */ + +#include <linux/export.h> +#include <linux/uaccess.h> + +/* + * Assembly functions that may be used (directly or indirectly) by modules + */ +EXPORT_SYMBOL(memset); +EXPORT_SYMBOL(memcpy); +EXPORT_SYMBOL(__memset); +EXPORT_SYMBOL(__memcpy); diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c new file mode 100644 index 000000000..226ccce0f --- /dev/null +++ b/arch/riscv/kernel/sbi.c @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SBI initialilization and all extension implementation. + * + * Copyright (c) 2020 Western Digital Corporation or its affiliates. + */ + +#include <linux/init.h> +#include <linux/pm.h> +#include <asm/sbi.h> +#include <asm/smp.h> + +/* default SBI version is 0.1 */ +unsigned long sbi_spec_version = SBI_SPEC_VERSION_DEFAULT; +EXPORT_SYMBOL(sbi_spec_version); + +static void (*__sbi_set_timer)(uint64_t stime); +static int (*__sbi_send_ipi)(const unsigned long *hart_mask); +static int (*__sbi_rfence)(int fid, const unsigned long *hart_mask, + unsigned long start, unsigned long size, + unsigned long arg4, unsigned long arg5); + +struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, + unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4, + unsigned long arg5) +{ + struct sbiret ret; + + register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); + register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); + register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); + register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); + register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); + register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); + register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); + register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); + asm volatile ("ecall" + : "+r" (a0), "+r" (a1) + : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) + : "memory"); + ret.error = a0; + ret.value = a1; + + return ret; +} +EXPORT_SYMBOL(sbi_ecall); + +int sbi_err_map_linux_errno(int err) +{ + switch (err) { + case SBI_SUCCESS: + return 0; + case SBI_ERR_DENIED: + return -EPERM; + case SBI_ERR_INVALID_PARAM: + return -EINVAL; + case SBI_ERR_INVALID_ADDRESS: + return -EFAULT; + case SBI_ERR_NOT_SUPPORTED: + case SBI_ERR_FAILURE: + default: + return -ENOTSUPP; + }; +} +EXPORT_SYMBOL(sbi_err_map_linux_errno); + +#ifdef CONFIG_RISCV_SBI_V01 +/** + * sbi_console_putchar() - Writes given character to the console device. + * @ch: The data to be written to the console. + * + * Return: None + */ +void sbi_console_putchar(int ch) +{ + sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0); +} +EXPORT_SYMBOL(sbi_console_putchar); + +/** + * sbi_console_getchar() - Reads a byte from console device. + * + * Returns the value read from console. + */ +int sbi_console_getchar(void) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0); + + return ret.error; +} +EXPORT_SYMBOL(sbi_console_getchar); + +/** + * sbi_shutdown() - Remove all the harts from executing supervisor code. + * + * Return: None + */ +void sbi_shutdown(void) +{ + sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0); +} +EXPORT_SYMBOL(sbi_shutdown); + +/** + * sbi_clear_ipi() - Clear any pending IPIs for the calling hart. + * + * Return: None + */ +void sbi_clear_ipi(void) +{ + sbi_ecall(SBI_EXT_0_1_CLEAR_IPI, 0, 0, 0, 0, 0, 0, 0); +} +EXPORT_SYMBOL(sbi_clear_ipi); + +/** + * sbi_set_timer_v01() - Program the timer for next timer event. + * @stime_value: The value after which next timer event should fire. + * + * Return: None + */ +static void __sbi_set_timer_v01(uint64_t stime_value) +{ +#if __riscv_xlen == 32 + sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, + stime_value >> 32, 0, 0, 0, 0); +#else + sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0); +#endif +} + +static int __sbi_send_ipi_v01(const unsigned long *hart_mask) +{ + sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask, + 0, 0, 0, 0, 0); + return 0; +} + +static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, + unsigned long start, unsigned long size, + unsigned long arg4, unsigned long arg5) +{ + int result = 0; + + /* v0.2 function IDs are equivalent to v0.1 extension IDs */ + switch (fid) { + case SBI_EXT_RFENCE_REMOTE_FENCE_I: + sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0, + (unsigned long)hart_mask, 0, 0, 0, 0, 0); + break; + case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: + sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0, + (unsigned long)hart_mask, start, size, + 0, 0, 0); + break; + case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: + sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0, + (unsigned long)hart_mask, start, size, + arg4, 0, 0); + break; + default: + pr_err("SBI call [%d]not supported in SBI v0.1\n", fid); + result = -EINVAL; + } + + return result; +} + +static void sbi_set_power_off(void) +{ + pm_power_off = sbi_shutdown; +} +#else +static void __sbi_set_timer_v01(uint64_t stime_value) +{ + pr_warn("Timer extension is not available in SBI v%lu.%lu\n", + sbi_major_version(), sbi_minor_version()); +} + +static int __sbi_send_ipi_v01(const unsigned long *hart_mask) +{ + pr_warn("IPI extension is not available in SBI v%lu.%lu\n", + sbi_major_version(), sbi_minor_version()); + + return 0; +} + +static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, + unsigned long start, unsigned long size, + unsigned long arg4, unsigned long arg5) +{ + pr_warn("remote fence extension is not available in SBI v%lu.%lu\n", + sbi_major_version(), sbi_minor_version()); + + return 0; +} + +static void sbi_set_power_off(void) {} +#endif /* CONFIG_RISCV_SBI_V01 */ + +static void __sbi_set_timer_v02(uint64_t stime_value) +{ +#if __riscv_xlen == 32 + sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, + stime_value >> 32, 0, 0, 0, 0); +#else + sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, + 0, 0, 0, 0); +#endif +} + +static int __sbi_send_ipi_v02(const unsigned long *hart_mask) +{ + unsigned long hartid, hmask_val, hbase; + struct cpumask tmask; + struct sbiret ret = {0}; + int result; + + if (!hart_mask || !(*hart_mask)) { + riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); + hart_mask = cpumask_bits(&tmask); + } + + hmask_val = 0; + hbase = 0; + for_each_set_bit(hartid, hart_mask, NR_CPUS) { + if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { + ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, + hmask_val, hbase, 0, 0, 0, 0); + if (ret.error) + goto ecall_failed; + hmask_val = 0; + hbase = 0; + } + if (!hmask_val) + hbase = hartid; + hmask_val |= 1UL << (hartid - hbase); + } + + if (hmask_val) { + ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, + hmask_val, hbase, 0, 0, 0, 0); + if (ret.error) + goto ecall_failed; + } + + return 0; + +ecall_failed: + result = sbi_err_map_linux_errno(ret.error); + pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", + __func__, hbase, hmask_val, result); + return result; +} + +static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val, + unsigned long hbase, unsigned long start, + unsigned long size, unsigned long arg4, + unsigned long arg5) +{ + struct sbiret ret = {0}; + int ext = SBI_EXT_RFENCE; + int result = 0; + + switch (fid) { + case SBI_EXT_RFENCE_REMOTE_FENCE_I: + ret = sbi_ecall(ext, fid, hmask_val, hbase, 0, 0, 0, 0); + break; + case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + size, 0, 0); + break; + case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + size, arg4, 0); + break; + + case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + size, 0, 0); + break; + case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + size, arg4, 0); + break; + case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + size, 0, 0); + break; + case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + size, arg4, 0); + break; + default: + pr_err("unknown function ID [%lu] for SBI extension [%d]\n", + fid, ext); + result = -EINVAL; + } + + if (ret.error) { + result = sbi_err_map_linux_errno(ret.error); + pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", + __func__, hbase, hmask_val, result); + } + + return result; +} + +static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask, + unsigned long start, unsigned long size, + unsigned long arg4, unsigned long arg5) +{ + unsigned long hmask_val, hartid, hbase; + struct cpumask tmask; + int result; + + if (!hart_mask || !(*hart_mask)) { + riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); + hart_mask = cpumask_bits(&tmask); + } + + hmask_val = 0; + hbase = 0; + for_each_set_bit(hartid, hart_mask, NR_CPUS) { + if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { + result = __sbi_rfence_v02_call(fid, hmask_val, hbase, + start, size, arg4, arg5); + if (result) + return result; + hmask_val = 0; + hbase = 0; + } + if (!hmask_val) + hbase = hartid; + hmask_val |= 1UL << (hartid - hbase); + } + + if (hmask_val) { + result = __sbi_rfence_v02_call(fid, hmask_val, hbase, + start, size, arg4, arg5); + if (result) + return result; + } + + return 0; +} + +/** + * sbi_set_timer() - Program the timer for next timer event. + * @stime_value: The value after which next timer event should fire. + * + * Return: None + */ +void sbi_set_timer(uint64_t stime_value) +{ + __sbi_set_timer(stime_value); +} + +/** + * sbi_send_ipi() - Send an IPI to any hart. + * @hart_mask: A cpu mask containing all the target harts. + * + * Return: None + */ +void sbi_send_ipi(const unsigned long *hart_mask) +{ + __sbi_send_ipi(hart_mask); +} +EXPORT_SYMBOL(sbi_send_ipi); + +/** + * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. + * @hart_mask: A cpu mask containing all the target harts. + * + * Return: None + */ +void sbi_remote_fence_i(const unsigned long *hart_mask) +{ + __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, + hart_mask, 0, 0, 0, 0); +} +EXPORT_SYMBOL(sbi_remote_fence_i); + +/** + * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote + * harts for the specified virtual address range. + * @hart_mask: A cpu mask containing all the target harts. + * @start: Start of the virtual address + * @size: Total size of the virtual address range. + * + * Return: None + */ +void sbi_remote_sfence_vma(const unsigned long *hart_mask, + unsigned long start, + unsigned long size) +{ + __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, + hart_mask, start, size, 0, 0); +} +EXPORT_SYMBOL(sbi_remote_sfence_vma); + +/** + * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given + * remote harts for a virtual address range belonging to a specific ASID. + * + * @hart_mask: A cpu mask containing all the target harts. + * @start: Start of the virtual address + * @size: Total size of the virtual address range. + * @asid: The value of address space identifier (ASID). + * + * Return: None + */ +void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, + unsigned long start, + unsigned long size, + unsigned long asid) +{ + __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, + hart_mask, start, size, asid, 0); +} +EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); + +/** + * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote + * harts for the specified guest physical address range. + * @hart_mask: A cpu mask containing all the target harts. + * @start: Start of the guest physical address + * @size: Total size of the guest physical address range. + * + * Return: None + */ +int sbi_remote_hfence_gvma(const unsigned long *hart_mask, + unsigned long start, + unsigned long size) +{ + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, + hart_mask, start, size, 0, 0); +} +EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); + +/** + * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given + * remote harts for a guest physical address range belonging to a specific VMID. + * + * @hart_mask: A cpu mask containing all the target harts. + * @start: Start of the guest physical address + * @size: Total size of the guest physical address range. + * @vmid: The value of guest ID (VMID). + * + * Return: 0 if success, Error otherwise. + */ +int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask, + unsigned long start, + unsigned long size, + unsigned long vmid) +{ + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, + hart_mask, start, size, vmid, 0); +} +EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid); + +/** + * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote + * harts for the current guest virtual address range. + * @hart_mask: A cpu mask containing all the target harts. + * @start: Start of the current guest virtual address + * @size: Total size of the current guest virtual address range. + * + * Return: None + */ +int sbi_remote_hfence_vvma(const unsigned long *hart_mask, + unsigned long start, + unsigned long size) +{ + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, + hart_mask, start, size, 0, 0); +} +EXPORT_SYMBOL(sbi_remote_hfence_vvma); + +/** + * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given + * remote harts for current guest virtual address range belonging to a specific + * ASID. + * + * @hart_mask: A cpu mask containing all the target harts. + * @start: Start of the current guest virtual address + * @size: Total size of the current guest virtual address range. + * @asid: The value of address space identifier (ASID). + * + * Return: None + */ +int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask, + unsigned long start, + unsigned long size, + unsigned long asid) +{ + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, + hart_mask, start, size, asid, 0); +} +EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); + +/** + * sbi_probe_extension() - Check if an SBI extension ID is supported or not. + * @extid: The extension ID to be probed. + * + * Return: Extension specific nonzero value f yes, -ENOTSUPP otherwise. + */ +int sbi_probe_extension(int extid) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, + 0, 0, 0, 0, 0); + if (!ret.error) + if (ret.value) + return ret.value; + + return -ENOTSUPP; +} +EXPORT_SYMBOL(sbi_probe_extension); + +static long __sbi_base_ecall(int fid) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0); + if (!ret.error) + return ret.value; + else + return sbi_err_map_linux_errno(ret.error); +} + +static inline long sbi_get_spec_version(void) +{ + return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION); +} + +static inline long sbi_get_firmware_id(void) +{ + return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID); +} + +static inline long sbi_get_firmware_version(void) +{ + return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION); +} + +static void sbi_send_cpumask_ipi(const struct cpumask *target) +{ + struct cpumask hartid_mask; + + riscv_cpuid_to_hartid_mask(target, &hartid_mask); + + sbi_send_ipi(cpumask_bits(&hartid_mask)); +} + +static struct riscv_ipi_ops sbi_ipi_ops = { + .ipi_inject = sbi_send_cpumask_ipi +}; + +int __init sbi_init(void) +{ + int ret; + + sbi_set_power_off(); + ret = sbi_get_spec_version(); + if (ret > 0) + sbi_spec_version = ret; + + pr_info("SBI specification v%lu.%lu detected\n", + sbi_major_version(), sbi_minor_version()); + + if (!sbi_spec_is_0_1()) { + pr_info("SBI implementation ID=0x%lx Version=0x%lx\n", + sbi_get_firmware_id(), sbi_get_firmware_version()); + if (sbi_probe_extension(SBI_EXT_TIME) > 0) { + __sbi_set_timer = __sbi_set_timer_v02; + pr_info("SBI v0.2 TIME extension detected\n"); + } else { + __sbi_set_timer = __sbi_set_timer_v01; + } + if (sbi_probe_extension(SBI_EXT_IPI) > 0) { + __sbi_send_ipi = __sbi_send_ipi_v02; + pr_info("SBI v0.2 IPI extension detected\n"); + } else { + __sbi_send_ipi = __sbi_send_ipi_v01; + } + if (sbi_probe_extension(SBI_EXT_RFENCE) > 0) { + __sbi_rfence = __sbi_rfence_v02; + pr_info("SBI v0.2 RFENCE extension detected\n"); + } else { + __sbi_rfence = __sbi_rfence_v01; + } + } else { + __sbi_set_timer = __sbi_set_timer_v01; + __sbi_send_ipi = __sbi_send_ipi_v01; + __sbi_rfence = __sbi_rfence_v01; + } + + riscv_set_ipi_ops(&sbi_ipi_ops); + + return 0; +} diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c new file mode 100644 index 000000000..8e78a8ab6 --- /dev/null +++ b/arch/riscv/kernel/setup.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2009 Sunplus Core Technology Co., Ltd. + * Chen Liqin <liqin.chen@sunplusct.com> + * Lennox Wu <lennox.wu@sunplusct.com> + * Copyright (C) 2012 Regents of the University of California + */ + +#include <linux/init.h> +#include <linux/mm.h> +#include <linux/memblock.h> +#include <linux/sched.h> +#include <linux/console.h> +#include <linux/screen_info.h> +#include <linux/of_fdt.h> +#include <linux/of_platform.h> +#include <linux/sched/task.h> +#include <linux/swiotlb.h> +#include <linux/smp.h> +#include <linux/efi.h> + +#include <asm/cpu_ops.h> +#include <asm/early_ioremap.h> +#include <asm/setup.h> +#include <asm/sections.h> +#include <asm/sbi.h> +#include <asm/tlbflush.h> +#include <asm/thread_info.h> +#include <asm/kasan.h> +#include <asm/efi.h> + +#include "head.h" + +#if defined(CONFIG_DUMMY_CONSOLE) || defined(CONFIG_EFI) +struct screen_info screen_info __section(".data") = { + .orig_video_lines = 30, + .orig_video_cols = 80, + .orig_video_mode = 0, + .orig_video_ega_bx = 0, + .orig_video_isVGA = 1, + .orig_video_points = 8 +}; +#endif + +/* + * The lucky hart to first increment this variable will boot the other cores. + * This is used before the kernel initializes the BSS so it can't be in the + * BSS. + */ +atomic_t hart_lottery __section(".sdata"); +unsigned long boot_cpu_hartid; +static DEFINE_PER_CPU(struct cpu, cpu_devices); + +static void __init parse_dtb(void) +{ + /* Early scan of device tree from init memory */ + if (early_init_dt_scan(dtb_early_va)) { + const char *name = of_flat_dt_get_machine_name(); + + if (name) { + pr_info("Machine model: %s\n", name); + dump_stack_set_arch_desc("%s (DT)", name); + } + } else { + pr_err("No DTB passed to the kernel\n"); + } + +#ifdef CONFIG_CMDLINE_FORCE + strlcpy(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE); + pr_info("Forcing kernel command line to: %s\n", boot_command_line); +#endif +} + +void __init setup_arch(char **cmdline_p) +{ + parse_dtb(); + init_mm.start_code = (unsigned long) _stext; + init_mm.end_code = (unsigned long) _etext; + init_mm.end_data = (unsigned long) _edata; + init_mm.brk = (unsigned long) _end; + + *cmdline_p = boot_command_line; + + early_ioremap_setup(); + jump_label_init(); + parse_early_param(); + + efi_init(); + setup_bootmem(); + paging_init(); +#if IS_ENABLED(CONFIG_BUILTIN_DTB) + unflatten_and_copy_device_tree(); +#else + if (early_init_dt_verify(__va(dtb_early_pa))) + unflatten_device_tree(); + else + pr_err("No DTB found in kernel mappings\n"); +#endif + early_init_fdt_scan_reserved_mem(); + misc_mem_init(); + +#ifdef CONFIG_SWIOTLB + swiotlb_init(1); +#endif + +#ifdef CONFIG_KASAN + kasan_init(); +#endif + +#if IS_ENABLED(CONFIG_RISCV_SBI) + sbi_init(); +#endif + +#ifdef CONFIG_SMP + setup_smp(); +#endif + + riscv_fill_hwcap(); +} + +static int __init topology_init(void) +{ + int i; + + for_each_possible_cpu(i) { + struct cpu *cpu = &per_cpu(cpu_devices, i); + + cpu->hotpluggable = cpu_has_hotplug(i); + register_cpu(cpu, i); + } + + return 0; +} +subsys_initcall(topology_init); diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c new file mode 100644 index 000000000..dd63407e8 --- /dev/null +++ b/arch/riscv/kernel/signal.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2009 Sunplus Core Technology Co., Ltd. + * Chen Liqin <liqin.chen@sunplusct.com> + * Lennox Wu <lennox.wu@sunplusct.com> + * Copyright (C) 2012 Regents of the University of California + */ + +#include <linux/signal.h> +#include <linux/uaccess.h> +#include <linux/syscalls.h> +#include <linux/tracehook.h> +#include <linux/linkage.h> + +#include <asm/ucontext.h> +#include <asm/vdso.h> +#include <asm/switch_to.h> +#include <asm/csr.h> +#include <asm/cacheflush.h> + +extern u32 __user_rt_sigreturn[2]; + +#define DEBUG_SIG 0 + +struct rt_sigframe { + struct siginfo info; + struct ucontext uc; +#ifndef CONFIG_MMU + u32 sigreturn_code[2]; +#endif +}; + +#ifdef CONFIG_FPU +static long restore_fp_state(struct pt_regs *regs, + union __riscv_fp_state __user *sc_fpregs) +{ + long err; + struct __riscv_d_ext_state __user *state = &sc_fpregs->d; + size_t i; + + err = __copy_from_user(¤t->thread.fstate, state, sizeof(*state)); + if (unlikely(err)) + return err; + + fstate_restore(current, regs); + + /* We support no other extension state at this time. */ + for (i = 0; i < ARRAY_SIZE(sc_fpregs->q.reserved); i++) { + u32 value; + + err = __get_user(value, &sc_fpregs->q.reserved[i]); + if (unlikely(err)) + break; + if (value != 0) + return -EINVAL; + } + + return err; +} + +static long save_fp_state(struct pt_regs *regs, + union __riscv_fp_state __user *sc_fpregs) +{ + long err; + struct __riscv_d_ext_state __user *state = &sc_fpregs->d; + size_t i; + + fstate_save(current, regs); + err = __copy_to_user(state, ¤t->thread.fstate, sizeof(*state)); + if (unlikely(err)) + return err; + + /* We support no other extension state at this time. */ + for (i = 0; i < ARRAY_SIZE(sc_fpregs->q.reserved); i++) { + err = __put_user(0, &sc_fpregs->q.reserved[i]); + if (unlikely(err)) + break; + } + + return err; +} +#else +#define save_fp_state(task, regs) (0) +#define restore_fp_state(task, regs) (0) +#endif + +static long restore_sigcontext(struct pt_regs *regs, + struct sigcontext __user *sc) +{ + long err; + /* sc_regs is structured the same as the start of pt_regs */ + err = __copy_from_user(regs, &sc->sc_regs, sizeof(sc->sc_regs)); + /* Restore the floating-point state. */ + if (has_fpu) + err |= restore_fp_state(regs, &sc->sc_fpregs); + return err; +} + +SYSCALL_DEFINE0(rt_sigreturn) +{ + struct pt_regs *regs = current_pt_regs(); + struct rt_sigframe __user *frame; + struct task_struct *task; + sigset_t set; + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + frame = (struct rt_sigframe __user *)regs->sp; + + if (!access_ok(frame, sizeof(*frame))) + goto badframe; + + if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) + goto badframe; + + set_current_blocked(&set); + + if (restore_sigcontext(regs, &frame->uc.uc_mcontext)) + goto badframe; + + if (restore_altstack(&frame->uc.uc_stack)) + goto badframe; + + regs->cause = -1UL; + + return regs->a0; + +badframe: + task = current; + if (show_unhandled_signals) { + pr_info_ratelimited( + "%s[%d]: bad frame in %s: frame=%p pc=%p sp=%p\n", + task->comm, task_pid_nr(task), __func__, + frame, (void *)regs->epc, (void *)regs->sp); + } + force_sig(SIGSEGV); + return 0; +} + +static long setup_sigcontext(struct rt_sigframe __user *frame, + struct pt_regs *regs) +{ + struct sigcontext __user *sc = &frame->uc.uc_mcontext; + long err; + /* sc_regs is structured the same as the start of pt_regs */ + err = __copy_to_user(&sc->sc_regs, regs, sizeof(sc->sc_regs)); + /* Save the floating-point state. */ + if (has_fpu) + err |= save_fp_state(regs, &sc->sc_fpregs); + return err; +} + +static inline void __user *get_sigframe(struct ksignal *ksig, + struct pt_regs *regs, size_t framesize) +{ + unsigned long sp; + /* Default to using normal stack */ + sp = regs->sp; + + /* + * If we are on the alternate signal stack and would overflow it, don't. + * Return an always-bogus address instead so we will die with SIGSEGV. + */ + if (on_sig_stack(sp) && !likely(on_sig_stack(sp - framesize))) + return (void __user __force *)(-1UL); + + /* This is the X/Open sanctioned signal stack switching. */ + sp = sigsp(sp, ksig) - framesize; + + /* Align the stack frame. */ + sp &= ~0xfUL; + + return (void __user *)sp; +} + +static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, + struct pt_regs *regs) +{ + struct rt_sigframe __user *frame; + long err = 0; + unsigned long __maybe_unused addr; + + frame = get_sigframe(ksig, regs, sizeof(*frame)); + if (!access_ok(frame, sizeof(*frame))) + return -EFAULT; + + err |= copy_siginfo_to_user(&frame->info, &ksig->info); + + /* Create the ucontext. */ + err |= __put_user(0, &frame->uc.uc_flags); + err |= __put_user(NULL, &frame->uc.uc_link); + err |= __save_altstack(&frame->uc.uc_stack, regs->sp); + err |= setup_sigcontext(frame, regs); + err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); + if (err) + return -EFAULT; + + /* Set up to return from userspace. */ +#ifdef CONFIG_MMU + regs->ra = (unsigned long)VDSO_SYMBOL( + current->mm->context.vdso, rt_sigreturn); +#else + /* + * For the nommu case we don't have a VDSO. Instead we push two + * instructions to call the rt_sigreturn syscall onto the user stack. + */ + if (copy_to_user(&frame->sigreturn_code, __user_rt_sigreturn, + sizeof(frame->sigreturn_code))) + return -EFAULT; + + addr = (unsigned long)&frame->sigreturn_code; + /* Make sure the two instructions are pushed to icache. */ + flush_icache_range(addr, addr + sizeof(frame->sigreturn_code)); + + regs->ra = addr; +#endif /* CONFIG_MMU */ + + /* + * Set up registers for signal handler. + * Registers that we don't modify keep the value they had from + * user-space at the time we took the signal. + * We always pass siginfo and mcontext, regardless of SA_SIGINFO, + * since some things rely on this (e.g. glibc's debug/segfault.c). + */ + regs->epc = (unsigned long)ksig->ka.sa.sa_handler; + regs->sp = (unsigned long)frame; + regs->a0 = ksig->sig; /* a0: signal number */ + regs->a1 = (unsigned long)(&frame->info); /* a1: siginfo pointer */ + regs->a2 = (unsigned long)(&frame->uc); /* a2: ucontext pointer */ + +#if DEBUG_SIG + pr_info("SIG deliver (%s:%d): sig=%d pc=%p ra=%p sp=%p\n", + current->comm, task_pid_nr(current), ksig->sig, + (void *)regs->epc, (void *)regs->ra, frame); +#endif + + return 0; +} + +static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) +{ + sigset_t *oldset = sigmask_to_save(); + int ret; + + /* Are we from a system call? */ + if (regs->cause == EXC_SYSCALL) { + /* Avoid additional syscall restarting via ret_from_exception */ + regs->cause = -1UL; + /* If so, check system call restarting.. */ + switch (regs->a0) { + case -ERESTART_RESTARTBLOCK: + case -ERESTARTNOHAND: + regs->a0 = -EINTR; + break; + + case -ERESTARTSYS: + if (!(ksig->ka.sa.sa_flags & SA_RESTART)) { + regs->a0 = -EINTR; + break; + } + fallthrough; + case -ERESTARTNOINTR: + regs->a0 = regs->orig_a0; + regs->epc -= 0x4; + break; + } + } + + /* Set up the stack frame */ + ret = setup_rt_frame(ksig, oldset, regs); + + signal_setup_done(ret, ksig, 0); +} + +static void do_signal(struct pt_regs *regs) +{ + struct ksignal ksig; + + if (get_signal(&ksig)) { + /* Actually deliver the signal */ + handle_signal(&ksig, regs); + return; + } + + /* Did we come from a system call? */ + if (regs->cause == EXC_SYSCALL) { + /* Avoid additional syscall restarting via ret_from_exception */ + regs->cause = -1UL; + + /* Restart the system call - no handlers present */ + switch (regs->a0) { + case -ERESTARTNOHAND: + case -ERESTARTSYS: + case -ERESTARTNOINTR: + regs->a0 = regs->orig_a0; + regs->epc -= 0x4; + break; + case -ERESTART_RESTARTBLOCK: + regs->a0 = regs->orig_a0; + regs->a7 = __NR_restart_syscall; + regs->epc -= 0x4; + break; + } + } + + /* + * If there is no signal to deliver, we just put the saved + * sigmask back. + */ + restore_saved_sigmask(); +} + +/* + * notification of userspace execution resumption + * - triggered by the _TIF_WORK_MASK flags + */ +asmlinkage __visible void do_notify_resume(struct pt_regs *regs, + unsigned long thread_info_flags) +{ + /* Handle pending signal delivery */ + if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_info_flags & _TIF_NOTIFY_RESUME) + tracehook_notify_resume(regs); +} diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c new file mode 100644 index 000000000..d44567490 --- /dev/null +++ b/arch/riscv/kernel/smp.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SMP initialisation and IPI support + * Based on arch/arm64/kernel/smp.c + * + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2015 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/profile.h> +#include <linux/smp.h> +#include <linux/sched.h> +#include <linux/seq_file.h> +#include <linux/delay.h> +#include <linux/irq_work.h> + +#include <asm/sbi.h> +#include <asm/tlbflush.h> +#include <asm/cacheflush.h> + +enum ipi_message_type { + IPI_RESCHEDULE, + IPI_CALL_FUNC, + IPI_CPU_STOP, + IPI_IRQ_WORK, + IPI_MAX +}; + +unsigned long __cpuid_to_hartid_map[NR_CPUS] = { + [0 ... NR_CPUS-1] = INVALID_HARTID +}; + +void __init smp_setup_processor_id(void) +{ + cpuid_to_hartid_map(0) = boot_cpu_hartid; +} + +/* A collection of single bit ipi messages. */ +static struct { + unsigned long stats[IPI_MAX] ____cacheline_aligned; + unsigned long bits ____cacheline_aligned; +} ipi_data[NR_CPUS] __cacheline_aligned; + +int riscv_hartid_to_cpuid(int hartid) +{ + int i; + + for (i = 0; i < NR_CPUS; i++) + if (cpuid_to_hartid_map(i) == hartid) + return i; + + pr_err("Couldn't find cpu id for hartid [%d]\n", hartid); + return -ENOENT; +} + +void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out) +{ + int cpu; + + cpumask_clear(out); + for_each_cpu(cpu, in) + cpumask_set_cpu(cpuid_to_hartid_map(cpu), out); +} +EXPORT_SYMBOL_GPL(riscv_cpuid_to_hartid_mask); + +bool arch_match_cpu_phys_id(int cpu, u64 phys_id) +{ + return phys_id == cpuid_to_hartid_map(cpu); +} + +/* Unsupported */ +int setup_profiling_timer(unsigned int multiplier) +{ + return -EINVAL; +} + +static void ipi_stop(void) +{ + set_cpu_online(smp_processor_id(), false); + while (1) + wait_for_interrupt(); +} + +static struct riscv_ipi_ops *ipi_ops; + +void riscv_set_ipi_ops(struct riscv_ipi_ops *ops) +{ + ipi_ops = ops; +} +EXPORT_SYMBOL_GPL(riscv_set_ipi_ops); + +void riscv_clear_ipi(void) +{ + if (ipi_ops && ipi_ops->ipi_clear) + ipi_ops->ipi_clear(); + + csr_clear(CSR_IP, IE_SIE); +} +EXPORT_SYMBOL_GPL(riscv_clear_ipi); + +static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op) +{ + int cpu; + + smp_mb__before_atomic(); + for_each_cpu(cpu, mask) + set_bit(op, &ipi_data[cpu].bits); + smp_mb__after_atomic(); + + if (ipi_ops && ipi_ops->ipi_inject) + ipi_ops->ipi_inject(mask); + else + pr_warn("SMP: IPI inject method not available\n"); +} + +static void send_ipi_single(int cpu, enum ipi_message_type op) +{ + smp_mb__before_atomic(); + set_bit(op, &ipi_data[cpu].bits); + smp_mb__after_atomic(); + + if (ipi_ops && ipi_ops->ipi_inject) + ipi_ops->ipi_inject(cpumask_of(cpu)); + else + pr_warn("SMP: IPI inject method not available\n"); +} + +#ifdef CONFIG_IRQ_WORK +void arch_irq_work_raise(void) +{ + send_ipi_single(smp_processor_id(), IPI_IRQ_WORK); +} +#endif + +void handle_IPI(struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; + unsigned long *stats = ipi_data[smp_processor_id()].stats; + + irq_enter(); + + riscv_clear_ipi(); + + while (true) { + unsigned long ops; + + /* Order bit clearing and data access. */ + mb(); + + ops = xchg(pending_ipis, 0); + if (ops == 0) + goto done; + + if (ops & (1 << IPI_RESCHEDULE)) { + stats[IPI_RESCHEDULE]++; + scheduler_ipi(); + } + + if (ops & (1 << IPI_CALL_FUNC)) { + stats[IPI_CALL_FUNC]++; + generic_smp_call_function_interrupt(); + } + + if (ops & (1 << IPI_CPU_STOP)) { + stats[IPI_CPU_STOP]++; + ipi_stop(); + } + + if (ops & (1 << IPI_IRQ_WORK)) { + stats[IPI_IRQ_WORK]++; + irq_work_run(); + } + + BUG_ON((ops >> IPI_MAX) != 0); + + /* Order data access and bit testing. */ + mb(); + } + +done: + irq_exit(); + set_irq_regs(old_regs); +} + +static const char * const ipi_names[] = { + [IPI_RESCHEDULE] = "Rescheduling interrupts", + [IPI_CALL_FUNC] = "Function call interrupts", + [IPI_CPU_STOP] = "CPU stop interrupts", + [IPI_IRQ_WORK] = "IRQ work interrupts", +}; + +void show_ipi_stats(struct seq_file *p, int prec) +{ + unsigned int cpu, i; + + for (i = 0; i < IPI_MAX; i++) { + seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, + prec >= 4 ? " " : ""); + for_each_online_cpu(cpu) + seq_printf(p, "%10lu ", ipi_data[cpu].stats[i]); + seq_printf(p, " %s\n", ipi_names[i]); + } +} + +void arch_send_call_function_ipi_mask(struct cpumask *mask) +{ + send_ipi_mask(mask, IPI_CALL_FUNC); +} + +void arch_send_call_function_single_ipi(int cpu) +{ + send_ipi_single(cpu, IPI_CALL_FUNC); +} + +void smp_send_stop(void) +{ + unsigned long timeout; + + if (num_online_cpus() > 1) { + cpumask_t mask; + + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + + if (system_state <= SYSTEM_RUNNING) + pr_crit("SMP: stopping secondary CPUs\n"); + send_ipi_mask(&mask, IPI_CPU_STOP); + } + + /* Wait up to one second for other CPUs to stop */ + timeout = USEC_PER_SEC; + while (num_online_cpus() > 1 && timeout--) + udelay(1); + + if (num_online_cpus() > 1) + pr_warn("SMP: failed to stop secondary CPUs %*pbl\n", + cpumask_pr_args(cpu_online_mask)); +} + +void smp_send_reschedule(int cpu) +{ + send_ipi_single(cpu, IPI_RESCHEDULE); +} +EXPORT_SYMBOL_GPL(smp_send_reschedule); diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c new file mode 100644 index 000000000..0e0aed380 --- /dev/null +++ b/arch/riscv/kernel/smpboot.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SMP initialisation and IPI support + * Based on arch/arm64/kernel/smp.c + * + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2015 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#include <linux/arch_topology.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/kernel_stat.h> +#include <linux/notifier.h> +#include <linux/cpu.h> +#include <linux/percpu.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/irq.h> +#include <linux/of.h> +#include <linux/sched/task_stack.h> +#include <linux/sched/mm.h> +#include <asm/cpu_ops.h> +#include <asm/irq.h> +#include <asm/mmu_context.h> +#include <asm/tlbflush.h> +#include <asm/sections.h> +#include <asm/sbi.h> +#include <asm/smp.h> + +#include "head.h" + +static DECLARE_COMPLETION(cpu_running); + +void __init smp_prepare_boot_cpu(void) +{ + init_cpu_topology(); +} + +void __init smp_prepare_cpus(unsigned int max_cpus) +{ + int cpuid; + int ret; + + store_cpu_topology(smp_processor_id()); + + /* This covers non-smp usecase mandated by "nosmp" option */ + if (max_cpus == 0) + return; + + for_each_possible_cpu(cpuid) { + if (cpuid == smp_processor_id()) + continue; + if (cpu_ops[cpuid]->cpu_prepare) { + ret = cpu_ops[cpuid]->cpu_prepare(cpuid); + if (ret) + continue; + } + set_cpu_present(cpuid, true); + } +} + +void __init setup_smp(void) +{ + struct device_node *dn; + int hart; + bool found_boot_cpu = false; + int cpuid = 1; + + cpu_set_ops(0); + + for_each_of_cpu_node(dn) { + hart = riscv_of_processor_hartid(dn); + if (hart < 0) + continue; + + if (hart == cpuid_to_hartid_map(0)) { + BUG_ON(found_boot_cpu); + found_boot_cpu = 1; + continue; + } + if (cpuid >= NR_CPUS) { + pr_warn("Invalid cpuid [%d] for hartid [%d]\n", + cpuid, hart); + break; + } + + cpuid_to_hartid_map(cpuid) = hart; + cpuid++; + } + + BUG_ON(!found_boot_cpu); + + if (cpuid > nr_cpu_ids) + pr_warn("Total number of cpus [%d] is greater than nr_cpus option value [%d]\n", + cpuid, nr_cpu_ids); + + for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++) { + if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID) { + cpu_set_ops(cpuid); + set_cpu_possible(cpuid, true); + } + } +} + +static int start_secondary_cpu(int cpu, struct task_struct *tidle) +{ + if (cpu_ops[cpu]->cpu_start) + return cpu_ops[cpu]->cpu_start(cpu, tidle); + + return -EOPNOTSUPP; +} + +int __cpu_up(unsigned int cpu, struct task_struct *tidle) +{ + int ret = 0; + tidle->thread_info.cpu = cpu; + + ret = start_secondary_cpu(cpu, tidle); + if (!ret) { + wait_for_completion_timeout(&cpu_running, + msecs_to_jiffies(1000)); + + if (!cpu_online(cpu)) { + pr_crit("CPU%u: failed to come online\n", cpu); + ret = -EIO; + } + } else { + pr_crit("CPU%u: failed to start\n", cpu); + } + + return ret; +} + +void __init smp_cpus_done(unsigned int max_cpus) +{ +} + +/* + * C entry point for a secondary processor. + */ +asmlinkage __visible void smp_callin(void) +{ + struct mm_struct *mm = &init_mm; + unsigned int curr_cpuid = smp_processor_id(); + + riscv_clear_ipi(); + + /* All kernel threads share the same mm context. */ + mmgrab(mm); + current->active_mm = mm; + + store_cpu_topology(curr_cpuid); + notify_cpu_starting(curr_cpuid); + set_cpu_online(curr_cpuid, 1); + + /* + * Remote TLB flushes are ignored while the CPU is offline, so emit + * a local TLB flush right now just in case. + */ + local_flush_tlb_all(); + complete(&cpu_running); + /* + * Disable preemption before enabling interrupts, so we don't try to + * schedule a CPU that hasn't actually started yet. + */ + local_irq_enable(); + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); +} diff --git a/arch/riscv/kernel/soc.c b/arch/riscv/kernel/soc.c new file mode 100644 index 000000000..c7b0a73e3 --- /dev/null +++ b/arch/riscv/kernel/soc.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ +#include <linux/init.h> +#include <linux/libfdt.h> +#include <linux/pgtable.h> +#include <asm/soc.h> + +/* + * This is called extremly early, before parse_dtb(), to allow initializing + * SoC hardware before memory or any device driver initialization. + */ +void __init soc_early_init(void) +{ + void (*early_fn)(const void *fdt); + const struct of_device_id *s; + const void *fdt = dtb_early_va; + + for (s = (void *)&__soc_early_init_table_start; + (void *)s < (void *)&__soc_early_init_table_end; s++) { + if (!fdt_node_check_compatible(fdt, 0, s->compatible)) { + early_fn = s->data; + early_fn(fdt); + return; + } + } +} + +static bool soc_builtin_dtb_match(unsigned long vendor_id, + unsigned long arch_id, unsigned long imp_id, + const struct soc_builtin_dtb *entry) +{ + return entry->vendor_id == vendor_id && + entry->arch_id == arch_id && + entry->imp_id == imp_id; +} + +void * __init soc_lookup_builtin_dtb(void) +{ + unsigned long vendor_id, arch_id, imp_id; + const struct soc_builtin_dtb *s; + + __asm__ ("csrr %0, mvendorid" : "=r"(vendor_id)); + __asm__ ("csrr %0, marchid" : "=r"(arch_id)); + __asm__ ("csrr %0, mimpid" : "=r"(imp_id)); + + for (s = (void *)&__soc_builtin_dtb_table_start; + (void *)s < (void *)&__soc_builtin_dtb_table_end; s++) { + if (soc_builtin_dtb_match(vendor_id, arch_id, imp_id, s)) + return s->dtb_func(); + } + + return NULL; +} diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c new file mode 100644 index 000000000..9c34735c1 --- /dev/null +++ b/arch/riscv/kernel/stacktrace.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2008 ARM Limited + * Copyright (C) 2014 Regents of the University of California + */ + +#include <linux/export.h> +#include <linux/kallsyms.h> +#include <linux/sched.h> +#include <linux/sched/debug.h> +#include <linux/sched/task_stack.h> +#include <linux/stacktrace.h> +#include <linux/ftrace.h> + +register unsigned long sp_in_global __asm__("sp"); + +#ifdef CONFIG_FRAME_POINTER + +struct stackframe { + unsigned long fp; + unsigned long ra; +}; + +void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, + bool (*fn)(unsigned long, void *), void *arg) +{ + unsigned long fp, sp, pc; + + if (regs) { + fp = frame_pointer(regs); + sp = user_stack_pointer(regs); + pc = instruction_pointer(regs); + } else if (task == NULL || task == current) { + const register unsigned long current_sp = sp_in_global; + fp = (unsigned long)__builtin_frame_address(0); + sp = current_sp; + pc = (unsigned long)walk_stackframe; + } else { + /* task blocked in __switch_to */ + fp = task->thread.s[0]; + sp = task->thread.sp; + pc = task->thread.ra; + } + + for (;;) { + unsigned long low, high; + struct stackframe *frame; + + if (unlikely(!__kernel_text_address(pc) || fn(pc, arg))) + break; + + /* Validate frame pointer */ + low = sp + sizeof(struct stackframe); + high = ALIGN(sp, THREAD_SIZE); + if (unlikely(fp < low || fp > high || fp & 0x7)) + break; + /* Unwind stack frame */ + frame = (struct stackframe *)fp - 1; + sp = fp; + if (regs && (regs->epc == pc) && (frame->fp & 0x7)) { + fp = frame->ra; + pc = regs->ra; + } else { + fp = frame->fp; + pc = ftrace_graph_ret_addr(current, NULL, frame->ra, + &frame->ra); + } + + } +} + +#else /* !CONFIG_FRAME_POINTER */ + +void notrace walk_stackframe(struct task_struct *task, + struct pt_regs *regs, bool (*fn)(unsigned long, void *), void *arg) +{ + unsigned long sp, pc; + unsigned long *ksp; + + if (regs) { + sp = user_stack_pointer(regs); + pc = instruction_pointer(regs); + } else if (task == NULL || task == current) { + sp = sp_in_global; + pc = (unsigned long)walk_stackframe; + } else { + /* task blocked in __switch_to */ + sp = task->thread.sp; + pc = task->thread.ra; + } + + if (unlikely(sp & 0x7)) + return; + + ksp = (unsigned long *)sp; + while (!kstack_end(ksp)) { + if (__kernel_text_address(pc) && unlikely(fn(pc, arg))) + break; + pc = READ_ONCE_NOCHECK(*ksp++) - 0x4; + } +} + +#endif /* CONFIG_FRAME_POINTER */ + + +static bool print_trace_address(unsigned long pc, void *arg) +{ + const char *loglvl = arg; + + print_ip_sym(loglvl, pc); + return false; +} + +void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) +{ + pr_cont("Call Trace:\n"); + walk_stackframe(task, NULL, print_trace_address, (void *)loglvl); +} + +static bool save_wchan(unsigned long pc, void *arg) +{ + if (!in_sched_functions(pc)) { + unsigned long *p = arg; + *p = pc; + return true; + } + return false; +} + +unsigned long get_wchan(struct task_struct *task) +{ + unsigned long pc = 0; + + if (likely(task && task != current && task->state != TASK_RUNNING)) + walk_stackframe(task, NULL, save_wchan, &pc); + return pc; +} + + +#ifdef CONFIG_STACKTRACE + +static bool __save_trace(unsigned long pc, void *arg, bool nosched) +{ + struct stack_trace *trace = arg; + + if (unlikely(nosched && in_sched_functions(pc))) + return false; + if (unlikely(trace->skip > 0)) { + trace->skip--; + return false; + } + + trace->entries[trace->nr_entries++] = pc; + return (trace->nr_entries >= trace->max_entries); +} + +static bool save_trace(unsigned long pc, void *arg) +{ + return __save_trace(pc, arg, false); +} + +/* + * Save stack-backtrace addresses into a stack_trace buffer. + */ +void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +{ + walk_stackframe(tsk, NULL, save_trace, trace); +} +EXPORT_SYMBOL_GPL(save_stack_trace_tsk); + +void save_stack_trace(struct stack_trace *trace) +{ + save_stack_trace_tsk(NULL, trace); +} +EXPORT_SYMBOL_GPL(save_stack_trace); + +#endif /* CONFIG_STACKTRACE */ diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c new file mode 100644 index 000000000..bb4026850 --- /dev/null +++ b/arch/riscv/kernel/sys_riscv.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2014 Darius Rad <darius@bluespec.com> + * Copyright (C) 2017 SiFive + */ + +#include <linux/syscalls.h> +#include <asm/unistd.h> +#include <asm/cacheflush.h> +#include <asm-generic/mman-common.h> + +static long riscv_sys_mmap(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, + unsigned long fd, off_t offset, + unsigned long page_shift_offset) +{ + if (unlikely(offset & (~PAGE_MASK >> page_shift_offset))) + return -EINVAL; + + return ksys_mmap_pgoff(addr, len, prot, flags, fd, + offset >> (PAGE_SHIFT - page_shift_offset)); +} + +#ifdef CONFIG_64BIT +SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, + unsigned long, fd, off_t, offset) +{ + return riscv_sys_mmap(addr, len, prot, flags, fd, offset, 0); +} +#else +SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, + unsigned long, fd, off_t, offset) +{ + /* + * Note that the shift for mmap2 is constant (12), + * regardless of PAGE_SIZE + */ + return riscv_sys_mmap(addr, len, prot, flags, fd, offset, 12); +} +#endif /* !CONFIG_64BIT */ + +/* + * Allows the instruction cache to be flushed from userspace. Despite RISC-V + * having a direct 'fence.i' instruction available to userspace (which we + * can't trap!), that's not actually viable when running on Linux because the + * kernel might schedule a process on another hart. There is no way for + * userspace to handle this without invoking the kernel (as it doesn't know the + * thread->hart mappings), so we've defined a RISC-V specific system call to + * flush the instruction cache. + * + * sys_riscv_flush_icache() is defined to flush the instruction cache over an + * address range, with the flush applying to either all threads or just the + * caller. We don't currently do anything with the address range, that's just + * in there for forwards compatibility. + */ +SYSCALL_DEFINE3(riscv_flush_icache, uintptr_t, start, uintptr_t, end, + uintptr_t, flags) +{ + /* Check the reserved flags. */ + if (unlikely(flags & ~SYS_RISCV_FLUSH_ICACHE_ALL)) + return -EINVAL; + + flush_icache_mm(current->mm, flags & SYS_RISCV_FLUSH_ICACHE_LOCAL); + + return 0; +} diff --git a/arch/riscv/kernel/syscall_table.c b/arch/riscv/kernel/syscall_table.c new file mode 100644 index 000000000..f1ead9df9 --- /dev/null +++ b/arch/riscv/kernel/syscall_table.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2009 Arnd Bergmann <arnd@arndb.de> + * Copyright (C) 2012 Regents of the University of California + */ + +#include <linux/linkage.h> +#include <linux/syscalls.h> +#include <asm-generic/syscalls.h> +#include <asm/vdso.h> +#include <asm/syscall.h> + +#undef __SYSCALL +#define __SYSCALL(nr, call) [nr] = (call), + +void *sys_call_table[__NR_syscalls] = { + [0 ... __NR_syscalls - 1] = sys_ni_syscall, +#include <asm/unistd.h> +}; diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c new file mode 100644 index 000000000..303ae47df --- /dev/null +++ b/arch/riscv/kernel/time.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#include <linux/of_clk.h> +#include <linux/clockchips.h> +#include <linux/clocksource.h> +#include <linux/delay.h> +#include <asm/sbi.h> +#include <asm/processor.h> + +unsigned long riscv_timebase; +EXPORT_SYMBOL_GPL(riscv_timebase); + +void __init time_init(void) +{ + struct device_node *cpu; + u32 prop; + + cpu = of_find_node_by_path("/cpus"); + if (!cpu || of_property_read_u32(cpu, "timebase-frequency", &prop)) + panic(KERN_WARNING "RISC-V system with no 'timebase-frequency' in DTS\n"); + of_node_put(cpu); + riscv_timebase = prop; + + lpj_fine = riscv_timebase / HZ; + + of_clk_init(NULL); + timer_probe(); + + tick_setup_hrtimer_broadcast(); +} + +void clocksource_arch_init(struct clocksource *cs) +{ +#ifdef CONFIG_GENERIC_GETTIMEOFDAY + cs->vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER; +#else + cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; +#endif +} diff --git a/arch/riscv/kernel/trace_irq.c b/arch/riscv/kernel/trace_irq.c new file mode 100644 index 000000000..095ac976d --- /dev/null +++ b/arch/riscv/kernel/trace_irq.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Changbin Du <changbin.du@gmail.com> + */ + +#include <linux/irqflags.h> +#include <linux/kprobes.h> +#include "trace_irq.h" + +/* + * trace_hardirqs_on/off require the caller to setup frame pointer properly. + * Otherwise, CALLER_ADDR1 might trigger an pagging exception in kernel. + * Here we add one extra level so they can be safely called by low + * level entry code which $fp is used for other purpose. + */ + +void __trace_hardirqs_on(void) +{ + trace_hardirqs_on(); +} +NOKPROBE_SYMBOL(__trace_hardirqs_on); + +void __trace_hardirqs_off(void) +{ + trace_hardirqs_off(); +} +NOKPROBE_SYMBOL(__trace_hardirqs_off); diff --git a/arch/riscv/kernel/trace_irq.h b/arch/riscv/kernel/trace_irq.h new file mode 100644 index 000000000..99fe67377 --- /dev/null +++ b/arch/riscv/kernel/trace_irq.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 Changbin Du <changbin.du@gmail.com> + */ +#ifndef __TRACE_IRQ_H +#define __TRACE_IRQ_H + +void __trace_hardirqs_on(void); +void __trace_hardirqs_off(void); + +#endif /* __TRACE_IRQ_H */ diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c new file mode 100644 index 000000000..227253fde --- /dev/null +++ b/arch/riscv/kernel/traps.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Regents of the University of California + */ + +#include <linux/cpu.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/sched.h> +#include <linux/sched/debug.h> +#include <linux/sched/signal.h> +#include <linux/signal.h> +#include <linux/kdebug.h> +#include <linux/uaccess.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/irq.h> +#include <linux/kexec.h> + +#include <asm/processor.h> +#include <asm/ptrace.h> +#include <asm/csr.h> + +int show_unhandled_signals = 1; + +extern asmlinkage void handle_exception(void); + +static DEFINE_SPINLOCK(die_lock); + +void die(struct pt_regs *regs, const char *str) +{ + static int die_counter; + int ret; + long cause; + unsigned long flags; + + oops_enter(); + + spin_lock_irqsave(&die_lock, flags); + console_verbose(); + bust_spinlocks(1); + + pr_emerg("%s [#%d]\n", str, ++die_counter); + print_modules(); + if (regs) + show_regs(regs); + + cause = regs ? regs->cause : -1; + ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV); + + if (kexec_should_crash(current)) + crash_kexec(regs); + + bust_spinlocks(0); + add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); + spin_unlock_irqrestore(&die_lock, flags); + oops_exit(); + + if (in_interrupt()) + panic("Fatal exception in interrupt"); + if (panic_on_oops) + panic("Fatal exception"); + if (ret != NOTIFY_STOP) + make_task_dead(SIGSEGV); +} + +void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) +{ + struct task_struct *tsk = current; + + if (show_unhandled_signals && unhandled_signal(tsk, signo) + && printk_ratelimit()) { + pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT, + tsk->comm, task_pid_nr(tsk), signo, code, addr); + print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); + pr_cont("\n"); + show_regs(regs); + } + + force_sig_fault(signo, code, (void __user *)addr); +} + +static void do_trap_error(struct pt_regs *regs, int signo, int code, + unsigned long addr, const char *str) +{ + if (user_mode(regs)) { + do_trap(regs, signo, code, addr); + } else { + if (!fixup_exception(regs)) + die(regs, str); + } +} + +#define DO_ERROR_INFO(name, signo, code, str) \ +asmlinkage __visible void name(struct pt_regs *regs) \ +{ \ + do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ +} + +DO_ERROR_INFO(do_trap_unknown, + SIGILL, ILL_ILLTRP, "unknown exception"); +DO_ERROR_INFO(do_trap_insn_misaligned, + SIGBUS, BUS_ADRALN, "instruction address misaligned"); +DO_ERROR_INFO(do_trap_insn_fault, + SIGSEGV, SEGV_ACCERR, "instruction access fault"); +DO_ERROR_INFO(do_trap_insn_illegal, + SIGILL, ILL_ILLOPC, "illegal instruction"); +DO_ERROR_INFO(do_trap_load_fault, + SIGSEGV, SEGV_ACCERR, "load access fault"); +#ifndef CONFIG_RISCV_M_MODE +DO_ERROR_INFO(do_trap_load_misaligned, + SIGBUS, BUS_ADRALN, "Oops - load address misaligned"); +DO_ERROR_INFO(do_trap_store_misaligned, + SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned"); +#else +int handle_misaligned_load(struct pt_regs *regs); +int handle_misaligned_store(struct pt_regs *regs); + +asmlinkage void do_trap_load_misaligned(struct pt_regs *regs) +{ + if (!handle_misaligned_load(regs)) + return; + do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, + "Oops - load address misaligned"); +} + +asmlinkage void do_trap_store_misaligned(struct pt_regs *regs) +{ + if (!handle_misaligned_store(regs)) + return; + do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, + "Oops - store (or AMO) address misaligned"); +} +#endif +DO_ERROR_INFO(do_trap_store_fault, + SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault"); +DO_ERROR_INFO(do_trap_ecall_u, + SIGILL, ILL_ILLTRP, "environment call from U-mode"); +DO_ERROR_INFO(do_trap_ecall_s, + SIGILL, ILL_ILLTRP, "environment call from S-mode"); +DO_ERROR_INFO(do_trap_ecall_m, + SIGILL, ILL_ILLTRP, "environment call from M-mode"); + +static inline unsigned long get_break_insn_length(unsigned long pc) +{ + bug_insn_t insn; + + if (get_kernel_nofault(insn, (bug_insn_t *)pc)) + return 0; + + return GET_INSN_LENGTH(insn); +} + +asmlinkage __visible void do_trap_break(struct pt_regs *regs) +{ + if (user_mode(regs)) + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); +#ifdef CONFIG_KGDB + else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP) + == NOTIFY_STOP) + return; +#endif + else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN) + regs->epc += get_break_insn_length(regs->epc); + else + die(regs, "Kernel BUG"); +} + +#ifdef CONFIG_GENERIC_BUG +int is_valid_bugaddr(unsigned long pc) +{ + bug_insn_t insn; + + if (pc < VMALLOC_START) + return 0; + if (get_kernel_nofault(insn, (bug_insn_t *)pc)) + return 0; + if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) + return (insn == __BUG_INSN_32); + else + return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16); +} +#endif /* CONFIG_GENERIC_BUG */ + +/* stvec & scratch is already set from head.S */ +void trap_init(void) +{ +} diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c new file mode 100644 index 000000000..b246c3dc6 --- /dev/null +++ b/arch/riscv/kernel/traps_misaligned.c @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/irq.h> + +#include <asm/processor.h> +#include <asm/ptrace.h> +#include <asm/csr.h> + +#define INSN_MATCH_LB 0x3 +#define INSN_MASK_LB 0x707f +#define INSN_MATCH_LH 0x1003 +#define INSN_MASK_LH 0x707f +#define INSN_MATCH_LW 0x2003 +#define INSN_MASK_LW 0x707f +#define INSN_MATCH_LD 0x3003 +#define INSN_MASK_LD 0x707f +#define INSN_MATCH_LBU 0x4003 +#define INSN_MASK_LBU 0x707f +#define INSN_MATCH_LHU 0x5003 +#define INSN_MASK_LHU 0x707f +#define INSN_MATCH_LWU 0x6003 +#define INSN_MASK_LWU 0x707f +#define INSN_MATCH_SB 0x23 +#define INSN_MASK_SB 0x707f +#define INSN_MATCH_SH 0x1023 +#define INSN_MASK_SH 0x707f +#define INSN_MATCH_SW 0x2023 +#define INSN_MASK_SW 0x707f +#define INSN_MATCH_SD 0x3023 +#define INSN_MASK_SD 0x707f + +#define INSN_MATCH_FLW 0x2007 +#define INSN_MASK_FLW 0x707f +#define INSN_MATCH_FLD 0x3007 +#define INSN_MASK_FLD 0x707f +#define INSN_MATCH_FLQ 0x4007 +#define INSN_MASK_FLQ 0x707f +#define INSN_MATCH_FSW 0x2027 +#define INSN_MASK_FSW 0x707f +#define INSN_MATCH_FSD 0x3027 +#define INSN_MASK_FSD 0x707f +#define INSN_MATCH_FSQ 0x4027 +#define INSN_MASK_FSQ 0x707f + +#define INSN_MATCH_C_LD 0x6000 +#define INSN_MASK_C_LD 0xe003 +#define INSN_MATCH_C_SD 0xe000 +#define INSN_MASK_C_SD 0xe003 +#define INSN_MATCH_C_LW 0x4000 +#define INSN_MASK_C_LW 0xe003 +#define INSN_MATCH_C_SW 0xc000 +#define INSN_MASK_C_SW 0xe003 +#define INSN_MATCH_C_LDSP 0x6002 +#define INSN_MASK_C_LDSP 0xe003 +#define INSN_MATCH_C_SDSP 0xe002 +#define INSN_MASK_C_SDSP 0xe003 +#define INSN_MATCH_C_LWSP 0x4002 +#define INSN_MASK_C_LWSP 0xe003 +#define INSN_MATCH_C_SWSP 0xc002 +#define INSN_MASK_C_SWSP 0xe003 + +#define INSN_MATCH_C_FLD 0x2000 +#define INSN_MASK_C_FLD 0xe003 +#define INSN_MATCH_C_FLW 0x6000 +#define INSN_MASK_C_FLW 0xe003 +#define INSN_MATCH_C_FSD 0xa000 +#define INSN_MASK_C_FSD 0xe003 +#define INSN_MATCH_C_FSW 0xe000 +#define INSN_MASK_C_FSW 0xe003 +#define INSN_MATCH_C_FLDSP 0x2002 +#define INSN_MASK_C_FLDSP 0xe003 +#define INSN_MATCH_C_FSDSP 0xa002 +#define INSN_MASK_C_FSDSP 0xe003 +#define INSN_MATCH_C_FLWSP 0x6002 +#define INSN_MASK_C_FLWSP 0xe003 +#define INSN_MATCH_C_FSWSP 0xe002 +#define INSN_MASK_C_FSWSP 0xe003 + +#define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4) + +#if defined(CONFIG_64BIT) +#define LOG_REGBYTES 3 +#define XLEN 64 +#else +#define LOG_REGBYTES 2 +#define XLEN 32 +#endif +#define REGBYTES (1 << LOG_REGBYTES) +#define XLEN_MINUS_16 ((XLEN) - 16) + +#define SH_RD 7 +#define SH_RS1 15 +#define SH_RS2 20 +#define SH_RS2C 2 + +#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) +#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ + (RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 5, 1) << 6)) +#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 5, 2) << 6)) +#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ + (RV_X(x, 12, 1) << 5) | \ + (RV_X(x, 2, 2) << 6)) +#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ + (RV_X(x, 12, 1) << 5) | \ + (RV_X(x, 2, 3) << 6)) +#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ + (RV_X(x, 7, 2) << 6)) +#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 7, 3) << 6)) +#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) +#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) +#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) + +#define SHIFT_RIGHT(x, y) \ + ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) + +#define REG_MASK \ + ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) + +#define REG_OFFSET(insn, pos) \ + (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) + +#define REG_PTR(insn, pos, regs) \ + (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)) + +#define GET_RM(insn) (((insn) >> 12) & 7) + +#define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) +#define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) +#define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) +#define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) +#define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) +#define GET_SP(regs) (*REG_PTR(2, 0, regs)) +#define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) +#define IMM_I(insn) ((s32)(insn) >> 20) +#define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ + (s32)(((insn) >> 7) & 0x1f)) +#define MASK_FUNCT3 0x7000 + +#define GET_PRECISION(insn) (((insn) >> 25) & 3) +#define GET_RM(insn) (((insn) >> 12) & 7) +#define PRECISION_S 0 +#define PRECISION_D 1 + +#define STR(x) XSTR(x) +#define XSTR(x) #x + +#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn) \ +static inline type load_##type(const type *addr) \ +{ \ + type val; \ + asm (#insn " %0, %1" \ + : "=&r" (val) : "m" (*addr)); \ + return val; \ +} + +#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn) \ +static inline void store_##type(type *addr, type val) \ +{ \ + asm volatile (#insn " %0, %1\n" \ + : : "r" (val), "m" (*addr)); \ +} + +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u8, lbu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u16, lhu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s8, lb) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s16, lh) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s32, lw) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(u8, sb) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(u16, sh) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(u32, sw) +#if defined(CONFIG_64BIT) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u32, lwu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u64, ld) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(u64, sd) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong, ld) +#else +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u32, lw) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong, lw) + +static inline u64 load_u64(const u64 *addr) +{ + return load_u32((u32 *)addr) + + ((u64)load_u32((u32 *)addr + 1) << 32); +} + +static inline void store_u64(u64 *addr, u64 val) +{ + store_u32((u32 *)addr, val); + store_u32((u32 *)addr + 1, val >> 32); +} +#endif + +static inline ulong get_insn(ulong mepc) +{ + register ulong __mepc asm ("a2") = mepc; + ulong val, rvc_mask = 3, tmp; + + asm ("and %[tmp], %[addr], 2\n" + "bnez %[tmp], 1f\n" +#if defined(CONFIG_64BIT) + STR(LWU) " %[insn], (%[addr])\n" +#else + STR(LW) " %[insn], (%[addr])\n" +#endif + "and %[tmp], %[insn], %[rvc_mask]\n" + "beq %[tmp], %[rvc_mask], 2f\n" + "sll %[insn], %[insn], %[xlen_minus_16]\n" + "srl %[insn], %[insn], %[xlen_minus_16]\n" + "j 2f\n" + "1:\n" + "lhu %[insn], (%[addr])\n" + "and %[tmp], %[insn], %[rvc_mask]\n" + "bne %[tmp], %[rvc_mask], 2f\n" + "lhu %[tmp], 2(%[addr])\n" + "sll %[tmp], %[tmp], 16\n" + "add %[insn], %[insn], %[tmp]\n" + "2:" + : [insn] "=&r" (val), [tmp] "=&r" (tmp) + : [addr] "r" (__mepc), [rvc_mask] "r" (rvc_mask), + [xlen_minus_16] "i" (XLEN_MINUS_16)); + + return val; +} + +union reg_data { + u8 data_bytes[8]; + ulong data_ulong; + u64 data_u64; +}; + +int handle_misaligned_load(struct pt_regs *regs) +{ + union reg_data val; + unsigned long epc = regs->epc; + unsigned long insn = get_insn(epc); + unsigned long addr = csr_read(mtval); + int i, fp = 0, shift = 0, len = 0; + + regs->epc = 0; + + if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { + len = 4; + shift = 8 * (sizeof(unsigned long) - len); +#if defined(CONFIG_64BIT) + } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { + len = 8; + shift = 8 * (sizeof(unsigned long) - len); + } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { + len = 4; +#endif + } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { + fp = 1; + len = 8; + } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { + fp = 1; + len = 4; + } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { + len = 2; + shift = 8 * (sizeof(unsigned long) - len); + } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { + len = 2; +#if defined(CONFIG_64BIT) + } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { + len = 8; + shift = 8 * (sizeof(unsigned long) - len); + insn = RVC_RS2S(insn) << SH_RD; + } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && + ((insn >> SH_RD) & 0x1f)) { + len = 8; + shift = 8 * (sizeof(unsigned long) - len); +#endif + } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { + len = 4; + shift = 8 * (sizeof(unsigned long) - len); + insn = RVC_RS2S(insn) << SH_RD; + } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && + ((insn >> SH_RD) & 0x1f)) { + len = 4; + shift = 8 * (sizeof(unsigned long) - len); + } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { + fp = 1; + len = 8; + insn = RVC_RS2S(insn) << SH_RD; + } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { + fp = 1; + len = 8; +#if defined(CONFIG_32BIT) + } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { + fp = 1; + len = 4; + insn = RVC_RS2S(insn) << SH_RD; + } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { + fp = 1; + len = 4; +#endif + } else { + regs->epc = epc; + return -1; + } + + val.data_u64 = 0; + for (i = 0; i < len; i++) + val.data_bytes[i] = load_u8((void *)(addr + i)); + + if (fp) + return -1; + SET_RD(insn, regs, val.data_ulong << shift >> shift); + + regs->epc = epc + INSN_LEN(insn); + + return 0; +} + +int handle_misaligned_store(struct pt_regs *regs) +{ + union reg_data val; + unsigned long epc = regs->epc; + unsigned long insn = get_insn(epc); + unsigned long addr = csr_read(mtval); + int i, len = 0; + + regs->epc = 0; + + val.data_ulong = GET_RS2(insn, regs); + + if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { + len = 4; +#if defined(CONFIG_64BIT) + } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { + len = 8; +#endif + } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { + len = 2; +#if defined(CONFIG_64BIT) + } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { + len = 8; + val.data_ulong = GET_RS2S(insn, regs); + } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { + len = 8; + val.data_ulong = GET_RS2C(insn, regs); +#endif + } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { + len = 4; + val.data_ulong = GET_RS2S(insn, regs); + } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { + len = 4; + val.data_ulong = GET_RS2C(insn, regs); + } else { + regs->epc = epc; + return -1; + } + + for (i = 0; i < len; i++) + store_u8((void *)(addr + i), val.data_bytes[i]); + + regs->epc = epc + INSN_LEN(insn); + + return 0; +} diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c new file mode 100644 index 000000000..73d45931a --- /dev/null +++ b/arch/riscv/kernel/vdso.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp. + * <benh@kernel.crashing.org> + * Copyright (C) 2012 ARM Limited + * Copyright (C) 2015 Regents of the University of California + */ + +#include <linux/elf.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/binfmts.h> +#include <linux/err.h> +#include <asm/page.h> +#ifdef CONFIG_GENERIC_TIME_VSYSCALL +#include <vdso/datapage.h> +#else +#include <asm/vdso.h> +#endif + +extern char vdso_start[], vdso_end[]; + +static unsigned int vdso_pages; +static struct page **vdso_pagelist; + +/* + * The vDSO data page. + */ +static union { + struct vdso_data data; + u8 page[PAGE_SIZE]; +} vdso_data_store __page_aligned_data; +struct vdso_data *vdso_data = &vdso_data_store.data; + +static int __init vdso_init(void) +{ + unsigned int i; + + vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; + vdso_pagelist = + kcalloc(vdso_pages + 1, sizeof(struct page *), GFP_KERNEL); + if (unlikely(vdso_pagelist == NULL)) { + pr_err("vdso: pagelist allocation failed\n"); + return -ENOMEM; + } + + for (i = 0; i < vdso_pages; i++) { + struct page *pg; + + pg = virt_to_page(vdso_start + (i << PAGE_SHIFT)); + vdso_pagelist[i] = pg; + } + vdso_pagelist[i] = virt_to_page(vdso_data); + + return 0; +} +arch_initcall(vdso_init); + +int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp) +{ + struct mm_struct *mm = current->mm; + unsigned long vdso_base, vdso_len; + int ret; + + vdso_len = (vdso_pages + 1) << PAGE_SHIFT; + + if (mmap_write_lock_killable(mm)) + return -EINTR; + + vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0); + if (IS_ERR_VALUE(vdso_base)) { + ret = vdso_base; + goto end; + } + + /* + * Put vDSO base into mm struct. We need to do this before calling + * install_special_mapping or the perf counter mmap tracking code + * will fail to recognise it as a vDSO (since arch_vma_name fails). + */ + mm->context.vdso = (void *)vdso_base; + + ret = + install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, + (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), + vdso_pagelist); + + if (unlikely(ret)) { + mm->context.vdso = NULL; + goto end; + } + + vdso_base += (vdso_pages << PAGE_SHIFT); + ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, + (VM_READ | VM_MAYREAD), &vdso_pagelist[vdso_pages]); + + if (unlikely(ret)) + mm->context.vdso = NULL; +end: + mmap_write_unlock(mm); + return ret; +} + +const char *arch_vma_name(struct vm_area_struct *vma) +{ + if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso)) + return "[vdso]"; + if (vma->vm_mm && (vma->vm_start == + (long)vma->vm_mm->context.vdso + PAGE_SIZE)) + return "[vdso_data]"; + return NULL; +} diff --git a/arch/riscv/kernel/vdso/.gitignore b/arch/riscv/kernel/vdso/.gitignore new file mode 100644 index 000000000..3a19def86 --- /dev/null +++ b/arch/riscv/kernel/vdso/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +vdso.lds +*.tmp +vdso-syms.S diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile new file mode 100644 index 000000000..f4ac7ff56 --- /dev/null +++ b/arch/riscv/kernel/vdso/Makefile @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copied from arch/tile/kernel/vdso/Makefile + +# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before +# the inclusion of generic Makefile. +ARCH_REL_TYPE_ABS := R_RISCV_32|R_RISCV_64|R_RISCV_JUMP_SLOT +include $(srctree)/lib/vdso/Makefile +# Symbols present in the vdso +vdso-syms = rt_sigreturn +ifdef CONFIG_64BIT +vdso-syms += vgettimeofday +endif +vdso-syms += getcpu +vdso-syms += flush_icache + +# Files to link into the vdso +obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o + +ccflags-y := -fno-stack-protector +ccflags-y += -DDISABLE_BRANCH_PROFILING + +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) +endif + +# Build rules +targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-syms.S +obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) + +obj-y += vdso.o vdso-syms.o +CPPFLAGS_vdso.lds += -P -C -U$(ARCH) +ifneq ($(filter vgettimeofday, $(vdso-syms)),) +CPPFLAGS_vdso.lds += -DHAS_VGETTIMEOFDAY +endif + +# Disable -pg to prevent insert call site +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) + +# Disable profiling and instrumentation for VDSO code +GCOV_PROFILE := n +KCOV_INSTRUMENT := n +KASAN_SANITIZE := n + +# Force dependency +$(obj)/vdso.o: $(obj)/vdso.so + +# link rule for the .so file, .lds has to be first +$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE + $(call if_changed,vdsold) +LDFLAGS_vdso.so.dbg = -shared -s -soname=linux-vdso.so.1 \ + --build-id=sha1 --hash-style=both --eh-frame-hdr + +# We also create a special relocatable object that should mirror the symbol +# table and layout of the linked DSO. With ld --just-symbols we can then +# refer to these symbols in the kernel code rather than hand-coded addresses. +$(obj)/vdso-syms.S: $(obj)/vdso.so FORCE + $(call if_changed,so2s) + +# strip rule for the .so file +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +# actual build commands +# The DSO images are built using a special linker script +# Make sure only to export the intended __vdso_xxx symbol offsets. +quiet_cmd_vdsold = VDSOLD $@ + cmd_vdsold = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \ + $(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ + rm $@.tmp + +# Extracts symbol offsets from the VDSO, converting them into an assembly file +# that contains the same symbols at the same offsets. +quiet_cmd_so2s = SO2S $@ + cmd_so2s = $(NM) -D $< | $(srctree)/$(src)/so2s.sh > $@ + +# install commands for the unstripped file +quiet_cmd_vdso_install = INSTALL $@ + cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ + +vdso.so: $(obj)/vdso.so.dbg + @mkdir -p $(MODLIB)/vdso + $(call cmd,vdso_install) + +vdso_install: vdso.so diff --git a/arch/riscv/kernel/vdso/flush_icache.S b/arch/riscv/kernel/vdso/flush_icache.S new file mode 100644 index 000000000..82f97d67c --- /dev/null +++ b/arch/riscv/kernel/vdso/flush_icache.S @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 SiFive + */ + +#include <linux/linkage.h> +#include <asm/unistd.h> + + .text +/* int __vdso_flush_icache(void *start, void *end, unsigned long flags); */ +ENTRY(__vdso_flush_icache) + .cfi_startproc +#ifdef CONFIG_SMP + li a7, __NR_riscv_flush_icache + ecall +#else + fence.i + li a0, 0 +#endif + ret + .cfi_endproc +ENDPROC(__vdso_flush_icache) diff --git a/arch/riscv/kernel/vdso/getcpu.S b/arch/riscv/kernel/vdso/getcpu.S new file mode 100644 index 000000000..bb0c05e2f --- /dev/null +++ b/arch/riscv/kernel/vdso/getcpu.S @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 SiFive + */ + +#include <linux/linkage.h> +#include <asm/unistd.h> + + .text +/* int __vdso_getcpu(unsigned *cpu, unsigned *node, void *unused); */ +ENTRY(__vdso_getcpu) + .cfi_startproc + /* For now, just do the syscall. */ + li a7, __NR_getcpu + ecall + ret + .cfi_endproc +ENDPROC(__vdso_getcpu) diff --git a/arch/riscv/kernel/vdso/note.S b/arch/riscv/kernel/vdso/note.S new file mode 100644 index 000000000..2a956c942 --- /dev/null +++ b/arch/riscv/kernel/vdso/note.S @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include <linux/elfnote.h> +#include <linux/version.h> + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S new file mode 100644 index 000000000..0573705ea --- /dev/null +++ b/arch/riscv/kernel/vdso/rt_sigreturn.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2014 Regents of the University of California + */ + +#include <linux/linkage.h> +#include <asm/unistd.h> + + .text +ENTRY(__vdso_rt_sigreturn) + .cfi_startproc + .cfi_signal_frame + li a7, __NR_rt_sigreturn + scall + .cfi_endproc +ENDPROC(__vdso_rt_sigreturn) diff --git a/arch/riscv/kernel/vdso/so2s.sh b/arch/riscv/kernel/vdso/so2s.sh new file mode 100755 index 000000000..e64cb6d94 --- /dev/null +++ b/arch/riscv/kernel/vdso/so2s.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2020 Palmer Dabbelt <palmerdabbelt@google.com> + +sed 's!\([0-9a-f]*\) T \([a-z0-9_]*\)\(@@LINUX_4.15\)*!.global \2\n.set \2,0x\1!' \ +| grep '^\.' diff --git a/arch/riscv/kernel/vdso/vdso.S b/arch/riscv/kernel/vdso/vdso.S new file mode 100644 index 000000000..df222245b --- /dev/null +++ b/arch/riscv/kernel/vdso/vdso.S @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2014 Regents of the University of California + */ + +#include <linux/init.h> +#include <linux/linkage.h> +#include <asm/page.h> + + __PAGE_ALIGNED_DATA + + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin "arch/riscv/kernel/vdso/vdso.so" + .balign PAGE_SIZE +vdso_end: + + .previous diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S new file mode 100644 index 000000000..b3e58402c --- /dev/null +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ +#include <asm/page.h> + +OUTPUT_ARCH(riscv) + +SECTIONS +{ + PROVIDE(_vdso_data = . + PAGE_SIZE); + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + .dynamic : { *(.dynamic) } :text :dynamic + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } + + /* + * This linker script is used both with -r and with -shared. + * For the layouts to match, we need to skip more than enough + * space for the dynamic symbol table, etc. If this amount is + * insufficient, ld -shared will error; simply increase it here. + */ + . = 0x800; + .text : { *(.text .text.*) } :text + + .data : { + *(.got.plt) *(.got) + *(.data .data.* .gnu.linkonce.d.*) + *(.dynbss) + *(.bss .bss.* .gnu.linkonce.b.*) + } +} + +/* + * We must supply the ELF program headers explicitly to get just one + * PT_LOAD segment, and set the flags explicitly to make segments read-only. + */ +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +/* + * This controls what symbols we export from the DSO. + */ +VERSION +{ + LINUX_4.15 { + global: + __vdso_rt_sigreturn; +#ifdef HAS_VGETTIMEOFDAY + __vdso_gettimeofday; + __vdso_clock_gettime; + __vdso_clock_getres; +#endif + __vdso_getcpu; + __vdso_flush_icache; + local: *; + }; +} diff --git a/arch/riscv/kernel/vdso/vgettimeofday.c b/arch/riscv/kernel/vdso/vgettimeofday.c new file mode 100644 index 000000000..cc0d80699 --- /dev/null +++ b/arch/riscv/kernel/vdso/vgettimeofday.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copied from arch/arm64/kernel/vdso/vgettimeofday.c + * + * Copyright (C) 2018 ARM Ltd. + * Copyright (C) 2020 SiFive + */ + +#include <linux/time.h> +#include <linux/types.h> + +extern +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +extern +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +extern +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res); +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres(clock_id, res); +} diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S new file mode 100644 index 000000000..3ffbd6cbd --- /dev/null +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#define LOAD_OFFSET PAGE_OFFSET +#include <asm/vmlinux.lds.h> +#include <asm/page.h> +#include <asm/cache.h> +#include <asm/thread_info.h> +#include <asm/set_memory.h> +#include "image-vars.h" + +#include <linux/sizes.h> +OUTPUT_ARCH(riscv) +ENTRY(_start) + +jiffies = jiffies_64; + +PECOFF_SECTION_ALIGNMENT = 0x1000; +PECOFF_FILE_ALIGNMENT = 0x200; + +SECTIONS +{ + /* Beginning of code and text segment */ + . = LOAD_OFFSET; + _start = .; + HEAD_TEXT_SECTION + . = ALIGN(PAGE_SIZE); + + __init_begin = .; + INIT_TEXT_SECTION(PAGE_SIZE) + . = ALIGN(8); + __soc_early_init_table : { + __soc_early_init_table_start = .; + KEEP(*(__soc_early_init_table)) + __soc_early_init_table_end = .; + } + __soc_builtin_dtb_table : { + __soc_builtin_dtb_table_start = .; + KEEP(*(__soc_builtin_dtb_table)) + __soc_builtin_dtb_table_end = .; + } + /* we have to discard exit text and such at runtime, not link time */ + .exit.text : + { + EXIT_TEXT + } + .exit.data : + { + EXIT_DATA + } + PERCPU_SECTION(L1_CACHE_BYTES) + __init_end = .; + + . = ALIGN(SECTION_ALIGN); + .text : { + _text = .; + _stext = .; + TEXT_TEXT + SCHED_TEXT + CPUIDLE_TEXT + LOCK_TEXT + KPROBES_TEXT + ENTRY_TEXT + IRQENTRY_TEXT + SOFTIRQENTRY_TEXT + *(.fixup) + _etext = .; + } + +#ifdef CONFIG_EFI + . = ALIGN(PECOFF_SECTION_ALIGNMENT); + __pecoff_text_end = .; +#endif + + INIT_DATA_SECTION(16) + + /* Start of data section */ + _sdata = .; + RO_DATA(SECTION_ALIGN) + .srodata : { + *(.srodata*) + } + + EXCEPTION_TABLE(0x10) + + . = ALIGN(SECTION_ALIGN); + _data = .; + + RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) + .sdata : { + __global_pointer$ = . + 0x800; + *(.sdata*) + } + +#ifdef CONFIG_EFI + .pecoff_edata_padding : { BYTE(0); . = ALIGN(PECOFF_FILE_ALIGNMENT); } + __pecoff_data_raw_size = ABSOLUTE(. - __pecoff_text_end); +#endif + + /* End of data section */ + _edata = .; + + BSS_SECTION(PAGE_SIZE, PAGE_SIZE, 0) + + .rel.dyn : { + *(.rel.dyn*) + } + +#ifdef CONFIG_EFI + . = ALIGN(PECOFF_SECTION_ALIGNMENT); + __pecoff_data_virt_size = ABSOLUTE(. - __pecoff_text_end); +#endif + _end = .; + + STABS_DEBUG + DWARF_DEBUG + ELF_DETAILS + + DISCARDS +} |