summaryrefslogtreecommitdiffstats
path: root/arch/riscv/kernel/vector.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:27:49 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:27:49 +0000
commitace9429bb58fd418f0c81d4c2835699bddf6bde6 (patch)
treeb2d64bc10158fdd5497876388cd68142ca374ed3 /arch/riscv/kernel/vector.c
parentInitial commit. (diff)
downloadlinux-ace9429bb58fd418f0c81d4c2835699bddf6bde6.tar.xz
linux-ace9429bb58fd418f0c81d4c2835699bddf6bde6.zip
Adding upstream version 6.6.15.upstream/6.6.15
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'arch/riscv/kernel/vector.c')
-rw-r--r--arch/riscv/kernel/vector.c277
1 files changed, 277 insertions, 0 deletions
diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c
new file mode 100644
index 0000000000..8d92fb6c52
--- /dev/null
+++ b/arch/riscv/kernel/vector.c
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2023 SiFive
+ * Author: Andy Chiu <andy.chiu@sifive.com>
+ */
+#include <linux/export.h>
+#include <linux/sched/signal.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/uaccess.h>
+#include <linux/prctl.h>
+
+#include <asm/thread_info.h>
+#include <asm/processor.h>
+#include <asm/insn.h>
+#include <asm/vector.h>
+#include <asm/csr.h>
+#include <asm/elf.h>
+#include <asm/ptrace.h>
+#include <asm/bug.h>
+
+static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
+
+unsigned long riscv_v_vsize __read_mostly;
+EXPORT_SYMBOL_GPL(riscv_v_vsize);
+
+int riscv_v_setup_vsize(void)
+{
+ unsigned long this_vsize;
+
+ /* There are 32 vector registers with vlenb length. */
+ riscv_v_enable();
+ this_vsize = csr_read(CSR_VLENB) * 32;
+ riscv_v_disable();
+
+ if (!riscv_v_vsize) {
+ riscv_v_vsize = this_vsize;
+ return 0;
+ }
+
+ if (riscv_v_vsize != this_vsize) {
+ WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static bool insn_is_vector(u32 insn_buf)
+{
+ u32 opcode = insn_buf & __INSN_OPCODE_MASK;
+ u32 width, csr;
+
+ /*
+ * All V-related instructions, including CSR operations are 4-Byte. So,
+ * do not handle if the instruction length is not 4-Byte.
+ */
+ if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
+ return false;
+
+ switch (opcode) {
+ case RVV_OPCODE_VECTOR:
+ return true;
+ case RVV_OPCODE_VL:
+ case RVV_OPCODE_VS:
+ width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
+ if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
+ width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
+ return true;
+
+ break;
+ case RVG_OPCODE_SYSTEM:
+ csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
+ if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
+ (csr >= CSR_VL && csr <= CSR_VLENB))
+ return true;
+ }
+
+ return false;
+}
+
+static int riscv_v_thread_zalloc(void)
+{
+ void *datap;
+
+ datap = kzalloc(riscv_v_vsize, GFP_KERNEL);
+ if (!datap)
+ return -ENOMEM;
+
+ current->thread.vstate.datap = datap;
+ memset(&current->thread.vstate, 0, offsetof(struct __riscv_v_ext_state,
+ datap));
+ return 0;
+}
+
+#define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
+#define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
+#define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
+#define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
+static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
+{
+ return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
+}
+
+static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
+{
+ return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
+}
+
+static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
+{
+ return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
+}
+
+static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
+ bool inherit)
+{
+ unsigned long ctrl;
+
+ ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
+ ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
+ if (inherit)
+ ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
+ tsk->thread.vstate_ctrl = ctrl;
+}
+
+bool riscv_v_vstate_ctrl_user_allowed(void)
+{
+ return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
+}
+EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
+
+bool riscv_v_first_use_handler(struct pt_regs *regs)
+{
+ u32 __user *epc = (u32 __user *)regs->epc;
+ u32 insn = (u32)regs->badaddr;
+
+ /* Do not handle if V is not supported, or disabled */
+ if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V))
+ return false;
+
+ /* If V has been enabled then it is not the first-use trap */
+ if (riscv_v_vstate_query(regs))
+ return false;
+
+ /* Get the instruction */
+ if (!insn) {
+ if (__get_user(insn, epc))
+ return false;
+ }
+
+ /* Filter out non-V instructions */
+ if (!insn_is_vector(insn))
+ return false;
+
+ /* Sanity check. datap should be null by the time of the first-use trap */
+ WARN_ON(current->thread.vstate.datap);
+
+ /*
+ * Now we sure that this is a V instruction. And it executes in the
+ * context where VS has been off. So, try to allocate the user's V
+ * context and resume execution.
+ */
+ if (riscv_v_thread_zalloc()) {
+ force_sig(SIGBUS);
+ return true;
+ }
+ riscv_v_vstate_on(regs);
+ riscv_v_vstate_restore(current, regs);
+ return true;
+}
+
+void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
+{
+ bool inherit;
+ int cur, next;
+
+ if (!has_vector())
+ return;
+
+ next = riscv_v_ctrl_get_next(tsk);
+ if (!next) {
+ if (READ_ONCE(riscv_v_implicit_uacc))
+ cur = PR_RISCV_V_VSTATE_CTRL_ON;
+ else
+ cur = PR_RISCV_V_VSTATE_CTRL_OFF;
+ } else {
+ cur = next;
+ }
+ /* Clear next mask if inherit-bit is not set */
+ inherit = riscv_v_ctrl_test_inherit(tsk);
+ if (!inherit)
+ next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
+
+ riscv_v_ctrl_set(tsk, cur, next, inherit);
+}
+
+long riscv_v_vstate_ctrl_get_current(void)
+{
+ if (!has_vector())
+ return -EINVAL;
+
+ return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
+}
+
+long riscv_v_vstate_ctrl_set_current(unsigned long arg)
+{
+ bool inherit;
+ int cur, next;
+
+ if (!has_vector())
+ return -EINVAL;
+
+ if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
+ return -EINVAL;
+
+ cur = VSTATE_CTRL_GET_CUR(arg);
+ switch (cur) {
+ case PR_RISCV_V_VSTATE_CTRL_OFF:
+ /* Do not allow user to turn off V if current is not off */
+ if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
+ return -EPERM;
+
+ break;
+ case PR_RISCV_V_VSTATE_CTRL_ON:
+ break;
+ case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
+ cur = riscv_v_ctrl_get_cur(current);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ next = VSTATE_CTRL_GET_NEXT(arg);
+ inherit = VSTATE_CTRL_GET_INHERIT(arg);
+ switch (next) {
+ case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
+ case PR_RISCV_V_VSTATE_CTRL_OFF:
+ case PR_RISCV_V_VSTATE_CTRL_ON:
+ riscv_v_ctrl_set(current, cur, next, inherit);
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+#ifdef CONFIG_SYSCTL
+
+static struct ctl_table riscv_v_default_vstate_table[] = {
+ {
+ .procname = "riscv_v_default_allow",
+ .data = &riscv_v_implicit_uacc,
+ .maxlen = sizeof(riscv_v_implicit_uacc),
+ .mode = 0644,
+ .proc_handler = proc_dobool,
+ },
+ { }
+};
+
+static int __init riscv_v_sysctl_init(void)
+{
+ if (has_vector())
+ if (!register_sysctl("abi", riscv_v_default_vstate_table))
+ return -EINVAL;
+ return 0;
+}
+
+#else /* ! CONFIG_SYSCTL */
+static int __init riscv_v_sysctl_init(void) { return 0; }
+#endif /* ! CONFIG_SYSCTL */
+
+static int riscv_v_init(void)
+{
+ return riscv_v_sysctl_init();
+}
+core_initcall(riscv_v_init);