summaryrefslogtreecommitdiffstats
path: root/arch/s390/lib
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--arch/s390/lib/Makefile25
-rw-r--r--arch/s390/lib/delay.c45
-rw-r--r--arch/s390/lib/error-inject.c14
-rw-r--r--arch/s390/lib/expoline/Makefile3
-rw-r--r--arch/s390/lib/expoline/expoline.S12
-rw-r--r--arch/s390/lib/find.c76
-rw-r--r--arch/s390/lib/mem.S197
-rw-r--r--arch/s390/lib/probes.c161
-rw-r--r--arch/s390/lib/spinlock.c324
-rw-r--r--arch/s390/lib/string.c348
-rw-r--r--arch/s390/lib/test_kprobes.c75
-rw-r--r--arch/s390/lib/test_kprobes.h10
-rw-r--r--arch/s390/lib/test_kprobes_asm.S45
-rw-r--r--arch/s390/lib/test_modules.c32
-rw-r--r--arch/s390/lib/test_modules.h53
-rw-r--r--arch/s390/lib/test_modules_helpers.c13
-rw-r--r--arch/s390/lib/test_unwind.c522
-rw-r--r--arch/s390/lib/tishift.S63
-rw-r--r--arch/s390/lib/uaccess.c185
-rw-r--r--arch/s390/lib/xor.c140
20 files changed, 2343 insertions, 0 deletions
diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
new file mode 100644
index 000000000..7c50eca85
--- /dev/null
+++ b/arch/s390/lib/Makefile
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for s390-specific library files..
+#
+
+lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o
+obj-y += mem.o xor.o
+lib-$(CONFIG_KPROBES) += probes.o
+lib-$(CONFIG_UPROBES) += probes.o
+obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o
+test_kprobes_s390-objs += test_kprobes_asm.o test_kprobes.o
+
+# Instrumenting memory accesses to __user data (in different address space)
+# produce false positives
+KASAN_SANITIZE_uaccess.o := n
+
+obj-$(CONFIG_S390_UNWIND_SELFTEST) += test_unwind.o
+CFLAGS_test_unwind.o += -fno-optimize-sibling-calls
+
+obj-$(CONFIG_S390_MODULES_SANITY_TEST) += test_modules.o
+obj-$(CONFIG_S390_MODULES_SANITY_TEST_HELPERS) += test_modules_helpers.o
+
+lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
+
+obj-$(CONFIG_EXPOLINE_EXTERN) += expoline/
diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c
new file mode 100644
index 000000000..be14c58cb
--- /dev/null
+++ b/arch/s390/lib/delay.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Precise Delay Loops for S390
+ *
+ * Copyright IBM Corp. 1999, 2008
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
+ */
+
+#include <linux/processor.h>
+#include <linux/delay.h>
+#include <asm/div64.h>
+#include <asm/timex.h>
+
+void __delay(unsigned long loops)
+{
+ /*
+ * Loop 'loops' times. Callers must not assume a specific
+ * amount of time passes before this function returns.
+ */
+ asm volatile("0: brct %0,0b" : : "d" ((loops/2) + 1));
+}
+EXPORT_SYMBOL(__delay);
+
+static void delay_loop(unsigned long delta)
+{
+ unsigned long end;
+
+ end = get_tod_clock_monotonic() + delta;
+ while (!tod_after(get_tod_clock_monotonic(), end))
+ cpu_relax();
+}
+
+void __udelay(unsigned long usecs)
+{
+ delay_loop(usecs << 12);
+}
+EXPORT_SYMBOL(__udelay);
+
+void __ndelay(unsigned long nsecs)
+{
+ nsecs <<= 9;
+ do_div(nsecs, 125);
+ delay_loop(nsecs);
+}
+EXPORT_SYMBOL(__ndelay);
diff --git a/arch/s390/lib/error-inject.c b/arch/s390/lib/error-inject.c
new file mode 100644
index 000000000..8c9d4da87
--- /dev/null
+++ b/arch/s390/lib/error-inject.c
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0+
+#include <asm/ptrace.h>
+#include <linux/error-injection.h>
+#include <linux/kprobes.h>
+
+void override_function_with_return(struct pt_regs *regs)
+{
+ /*
+ * Emulate 'br 14'. 'regs' is captured by kprobes on entry to some
+ * kernel function.
+ */
+ regs->psw.addr = regs->gprs[14];
+}
+NOKPROBE_SYMBOL(override_function_with_return);
diff --git a/arch/s390/lib/expoline/Makefile b/arch/s390/lib/expoline/Makefile
new file mode 100644
index 000000000..854631d9c
--- /dev/null
+++ b/arch/s390/lib/expoline/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-y += expoline.o
diff --git a/arch/s390/lib/expoline/expoline.S b/arch/s390/lib/expoline/expoline.S
new file mode 100644
index 000000000..92ed8409a
--- /dev/null
+++ b/arch/s390/lib/expoline/expoline.S
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <asm/nospec-insn.h>
+#include <linux/linkage.h>
+
+.macro GEN_ALL_BR_THUNK_EXTERN
+ .irp r1,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+ GEN_BR_THUNK_EXTERN %r\r1
+ .endr
+.endm
+
+GEN_ALL_BR_THUNK_EXTERN
diff --git a/arch/s390/lib/find.c b/arch/s390/lib/find.c
new file mode 100644
index 000000000..96a8a2e2d
--- /dev/null
+++ b/arch/s390/lib/find.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * MSB0 numbered special bitops handling.
+ *
+ * The bits are numbered:
+ * |0..............63|64............127|128...........191|192...........255|
+ *
+ * The reason for this bit numbering is the fact that the hardware sets bits
+ * in a bitmap starting at bit 0 (MSB) and we don't want to scan the bitmap
+ * from the 'wrong end'.
+ */
+
+#include <linux/compiler.h>
+#include <linux/bitops.h>
+#include <linux/export.h>
+
+unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size)
+{
+ const unsigned long *p = addr;
+ unsigned long result = 0;
+ unsigned long tmp;
+
+ while (size & ~(BITS_PER_LONG - 1)) {
+ if ((tmp = *(p++)))
+ goto found;
+ result += BITS_PER_LONG;
+ size -= BITS_PER_LONG;
+ }
+ if (!size)
+ return result;
+ tmp = (*p) & (~0UL << (BITS_PER_LONG - size));
+ if (!tmp) /* Are any bits set? */
+ return result + size; /* Nope. */
+found:
+ return result + (__fls(tmp) ^ (BITS_PER_LONG - 1));
+}
+EXPORT_SYMBOL(find_first_bit_inv);
+
+unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
+ unsigned long offset)
+{
+ const unsigned long *p = addr + (offset / BITS_PER_LONG);
+ unsigned long result = offset & ~(BITS_PER_LONG - 1);
+ unsigned long tmp;
+
+ if (offset >= size)
+ return size;
+ size -= result;
+ offset %= BITS_PER_LONG;
+ if (offset) {
+ tmp = *(p++);
+ tmp &= (~0UL >> offset);
+ if (size < BITS_PER_LONG)
+ goto found_first;
+ if (tmp)
+ goto found_middle;
+ size -= BITS_PER_LONG;
+ result += BITS_PER_LONG;
+ }
+ while (size & ~(BITS_PER_LONG-1)) {
+ if ((tmp = *(p++)))
+ goto found_middle;
+ result += BITS_PER_LONG;
+ size -= BITS_PER_LONG;
+ }
+ if (!size)
+ return result;
+ tmp = *p;
+found_first:
+ tmp &= (~0UL << (BITS_PER_LONG - size));
+ if (!tmp) /* Are any bits set? */
+ return result + size; /* Nope. */
+found_middle:
+ return result + (__fls(tmp) ^ (BITS_PER_LONG - 1));
+}
+EXPORT_SYMBOL(find_next_bit_inv);
diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S
new file mode 100644
index 000000000..08f60a42b
--- /dev/null
+++ b/arch/s390/lib/mem.S
@@ -0,0 +1,197 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * String handling functions.
+ *
+ * Copyright IBM Corp. 2012
+ */
+
+#include <linux/export.h>
+#include <linux/linkage.h>
+#include <asm/nospec-insn.h>
+
+ GEN_BR_THUNK %r14
+
+/*
+ * void *memmove(void *dest, const void *src, size_t n)
+ */
+SYM_FUNC_START(__memmove)
+ ltgr %r4,%r4
+ lgr %r1,%r2
+ jz .Lmemmove_exit
+ aghi %r4,-1
+ clgr %r2,%r3
+ jnh .Lmemmove_forward
+ la %r5,1(%r4,%r3)
+ clgr %r2,%r5
+ jl .Lmemmove_reverse
+.Lmemmove_forward:
+ srlg %r0,%r4,8
+ ltgr %r0,%r0
+ jz .Lmemmove_forward_remainder
+.Lmemmove_forward_loop:
+ mvc 0(256,%r1),0(%r3)
+ la %r1,256(%r1)
+ la %r3,256(%r3)
+ brctg %r0,.Lmemmove_forward_loop
+.Lmemmove_forward_remainder:
+ larl %r5,.Lmemmove_mvc
+ ex %r4,0(%r5)
+.Lmemmove_exit:
+ BR_EX %r14
+.Lmemmove_reverse:
+ ic %r0,0(%r4,%r3)
+ stc %r0,0(%r4,%r1)
+ brctg %r4,.Lmemmove_reverse
+ ic %r0,0(%r4,%r3)
+ stc %r0,0(%r4,%r1)
+ BR_EX %r14
+.Lmemmove_mvc:
+ mvc 0(1,%r1),0(%r3)
+SYM_FUNC_END(__memmove)
+EXPORT_SYMBOL(__memmove)
+
+SYM_FUNC_ALIAS(memmove, __memmove)
+EXPORT_SYMBOL(memmove)
+
+/*
+ * memset implementation
+ *
+ * This code corresponds to the C construct below. We do distinguish
+ * between clearing (c == 0) and setting a memory array (c != 0) simply
+ * because nearly all memset invocations in the kernel clear memory and
+ * the xc instruction is preferred in such cases.
+ *
+ * void *memset(void *s, int c, size_t n)
+ * {
+ * if (likely(c == 0))
+ * return __builtin_memset(s, 0, n);
+ * return __builtin_memset(s, c, n);
+ * }
+ */
+SYM_FUNC_START(__memset)
+ ltgr %r4,%r4
+ jz .Lmemset_exit
+ ltgr %r3,%r3
+ jnz .Lmemset_fill
+ aghi %r4,-1
+ srlg %r3,%r4,8
+ ltgr %r3,%r3
+ lgr %r1,%r2
+ jz .Lmemset_clear_remainder
+.Lmemset_clear_loop:
+ xc 0(256,%r1),0(%r1)
+ la %r1,256(%r1)
+ brctg %r3,.Lmemset_clear_loop
+.Lmemset_clear_remainder:
+ larl %r3,.Lmemset_xc
+ ex %r4,0(%r3)
+.Lmemset_exit:
+ BR_EX %r14
+.Lmemset_fill:
+ cghi %r4,1
+ lgr %r1,%r2
+ je .Lmemset_fill_exit
+ aghi %r4,-2
+ srlg %r5,%r4,8
+ ltgr %r5,%r5
+ jz .Lmemset_fill_remainder
+.Lmemset_fill_loop:
+ stc %r3,0(%r1)
+ mvc 1(255,%r1),0(%r1)
+ la %r1,256(%r1)
+ brctg %r5,.Lmemset_fill_loop
+.Lmemset_fill_remainder:
+ stc %r3,0(%r1)
+ larl %r5,.Lmemset_mvc
+ ex %r4,0(%r5)
+ BR_EX %r14
+.Lmemset_fill_exit:
+ stc %r3,0(%r1)
+ BR_EX %r14
+.Lmemset_xc:
+ xc 0(1,%r1),0(%r1)
+.Lmemset_mvc:
+ mvc 1(1,%r1),0(%r1)
+SYM_FUNC_END(__memset)
+EXPORT_SYMBOL(__memset)
+
+SYM_FUNC_ALIAS(memset, __memset)
+EXPORT_SYMBOL(memset)
+
+/*
+ * memcpy implementation
+ *
+ * void *memcpy(void *dest, const void *src, size_t n)
+ */
+SYM_FUNC_START(__memcpy)
+ ltgr %r4,%r4
+ jz .Lmemcpy_exit
+ aghi %r4,-1
+ srlg %r5,%r4,8
+ ltgr %r5,%r5
+ lgr %r1,%r2
+ jnz .Lmemcpy_loop
+.Lmemcpy_remainder:
+ larl %r5,.Lmemcpy_mvc
+ ex %r4,0(%r5)
+.Lmemcpy_exit:
+ BR_EX %r14
+.Lmemcpy_loop:
+ mvc 0(256,%r1),0(%r3)
+ la %r1,256(%r1)
+ la %r3,256(%r3)
+ brctg %r5,.Lmemcpy_loop
+ j .Lmemcpy_remainder
+.Lmemcpy_mvc:
+ mvc 0(1,%r1),0(%r3)
+SYM_FUNC_END(__memcpy)
+EXPORT_SYMBOL(__memcpy)
+
+SYM_FUNC_ALIAS(memcpy, __memcpy)
+EXPORT_SYMBOL(memcpy)
+
+/*
+ * __memset16/32/64
+ *
+ * void *__memset16(uint16_t *s, uint16_t v, size_t count)
+ * void *__memset32(uint32_t *s, uint32_t v, size_t count)
+ * void *__memset64(uint64_t *s, uint64_t v, size_t count)
+ */
+.macro __MEMSET bits,bytes,insn
+SYM_FUNC_START(__memset\bits)
+ ltgr %r4,%r4
+ jz .L__memset_exit\bits
+ cghi %r4,\bytes
+ je .L__memset_store\bits
+ aghi %r4,-(\bytes+1)
+ srlg %r5,%r4,8
+ ltgr %r5,%r5
+ lgr %r1,%r2
+ jz .L__memset_remainder\bits
+.L__memset_loop\bits:
+ \insn %r3,0(%r1)
+ mvc \bytes(256-\bytes,%r1),0(%r1)
+ la %r1,256(%r1)
+ brctg %r5,.L__memset_loop\bits
+.L__memset_remainder\bits:
+ \insn %r3,0(%r1)
+ larl %r5,.L__memset_mvc\bits
+ ex %r4,0(%r5)
+ BR_EX %r14
+.L__memset_store\bits:
+ \insn %r3,0(%r2)
+.L__memset_exit\bits:
+ BR_EX %r14
+.L__memset_mvc\bits:
+ mvc \bytes(1,%r1),0(%r1)
+SYM_FUNC_END(__memset\bits)
+.endm
+
+__MEMSET 16,2,sth
+EXPORT_SYMBOL(__memset16)
+
+__MEMSET 32,4,st
+EXPORT_SYMBOL(__memset32)
+
+__MEMSET 64,8,stg
+EXPORT_SYMBOL(__memset64)
diff --git a/arch/s390/lib/probes.c b/arch/s390/lib/probes.c
new file mode 100644
index 000000000..1e184a034
--- /dev/null
+++ b/arch/s390/lib/probes.c
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Common helper functions for kprobes and uprobes
+ *
+ * Copyright IBM Corp. 2014
+ */
+
+#include <linux/errno.h>
+#include <asm/kprobes.h>
+#include <asm/dis.h>
+
+int probe_is_prohibited_opcode(u16 *insn)
+{
+ if (!is_known_insn((unsigned char *)insn))
+ return -EINVAL;
+ switch (insn[0] >> 8) {
+ case 0x0c: /* bassm */
+ case 0x0b: /* bsm */
+ case 0x83: /* diag */
+ case 0x44: /* ex */
+ case 0xac: /* stnsm */
+ case 0xad: /* stosm */
+ return -EINVAL;
+ case 0xc6:
+ switch (insn[0] & 0x0f) {
+ case 0x00: /* exrl */
+ return -EINVAL;
+ }
+ }
+ switch (insn[0]) {
+ case 0x0101: /* pr */
+ case 0xb25a: /* bsa */
+ case 0xb240: /* bakr */
+ case 0xb258: /* bsg */
+ case 0xb218: /* pc */
+ case 0xb228: /* pt */
+ case 0xb98d: /* epsw */
+ case 0xe560: /* tbegin */
+ case 0xe561: /* tbeginc */
+ case 0xb2f8: /* tend */
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int probe_get_fixup_type(u16 *insn)
+{
+ /* default fixup method */
+ int fixup = FIXUP_PSW_NORMAL;
+
+ switch (insn[0] >> 8) {
+ case 0x05: /* balr */
+ case 0x0d: /* basr */
+ fixup = FIXUP_RETURN_REGISTER;
+ /* if r2 = 0, no branch will be taken */
+ if ((insn[0] & 0x0f) == 0)
+ fixup |= FIXUP_BRANCH_NOT_TAKEN;
+ break;
+ case 0x06: /* bctr */
+ case 0x07: /* bcr */
+ fixup = FIXUP_BRANCH_NOT_TAKEN;
+ break;
+ case 0x45: /* bal */
+ case 0x4d: /* bas */
+ fixup = FIXUP_RETURN_REGISTER;
+ break;
+ case 0x47: /* bc */
+ case 0x46: /* bct */
+ case 0x86: /* bxh */
+ case 0x87: /* bxle */
+ fixup = FIXUP_BRANCH_NOT_TAKEN;
+ break;
+ case 0x82: /* lpsw */
+ fixup = FIXUP_NOT_REQUIRED;
+ break;
+ case 0xb2: /* lpswe */
+ if ((insn[0] & 0xff) == 0xb2)
+ fixup = FIXUP_NOT_REQUIRED;
+ break;
+ case 0xa7: /* bras */
+ if ((insn[0] & 0x0f) == 0x05)
+ fixup |= FIXUP_RETURN_REGISTER;
+ break;
+ case 0xc0:
+ if ((insn[0] & 0x0f) == 0x05) /* brasl */
+ fixup |= FIXUP_RETURN_REGISTER;
+ break;
+ case 0xeb:
+ switch (insn[2] & 0xff) {
+ case 0x44: /* bxhg */
+ case 0x45: /* bxleg */
+ fixup = FIXUP_BRANCH_NOT_TAKEN;
+ break;
+ }
+ break;
+ case 0xe3: /* bctg */
+ if ((insn[2] & 0xff) == 0x46)
+ fixup = FIXUP_BRANCH_NOT_TAKEN;
+ break;
+ case 0xec:
+ switch (insn[2] & 0xff) {
+ case 0xe5: /* clgrb */
+ case 0xe6: /* cgrb */
+ case 0xf6: /* crb */
+ case 0xf7: /* clrb */
+ case 0xfc: /* cgib */
+ case 0xfd: /* cglib */
+ case 0xfe: /* cib */
+ case 0xff: /* clib */
+ fixup = FIXUP_BRANCH_NOT_TAKEN;
+ break;
+ }
+ break;
+ }
+ return fixup;
+}
+
+int probe_is_insn_relative_long(u16 *insn)
+{
+ /* Check if we have a RIL-b or RIL-c format instruction which
+ * we need to modify in order to avoid instruction emulation. */
+ switch (insn[0] >> 8) {
+ case 0xc0:
+ if ((insn[0] & 0x0f) == 0x00) /* larl */
+ return true;
+ break;
+ case 0xc4:
+ switch (insn[0] & 0x0f) {
+ case 0x02: /* llhrl */
+ case 0x04: /* lghrl */
+ case 0x05: /* lhrl */
+ case 0x06: /* llghrl */
+ case 0x07: /* sthrl */
+ case 0x08: /* lgrl */
+ case 0x0b: /* stgrl */
+ case 0x0c: /* lgfrl */
+ case 0x0d: /* lrl */
+ case 0x0e: /* llgfrl */
+ case 0x0f: /* strl */
+ return true;
+ }
+ break;
+ case 0xc6:
+ switch (insn[0] & 0x0f) {
+ case 0x02: /* pfdrl */
+ case 0x04: /* cghrl */
+ case 0x05: /* chrl */
+ case 0x06: /* clghrl */
+ case 0x07: /* clhrl */
+ case 0x08: /* cgrl */
+ case 0x0a: /* clgrl */
+ case 0x0c: /* cgfrl */
+ case 0x0d: /* crl */
+ case 0x0e: /* clgfrl */
+ case 0x0f: /* clrl */
+ return true;
+ }
+ break;
+ }
+ return false;
+}
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
new file mode 100644
index 000000000..81c53440b
--- /dev/null
+++ b/arch/s390/lib/spinlock.c
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Out of line spinlock code.
+ *
+ * Copyright IBM Corp. 2004, 2006
+ * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
+ */
+
+#include <linux/types.h>
+#include <linux/export.h>
+#include <linux/spinlock.h>
+#include <linux/jiffies.h>
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/percpu.h>
+#include <linux/io.h>
+#include <asm/alternative.h>
+
+int spin_retry = -1;
+
+static int __init spin_retry_init(void)
+{
+ if (spin_retry < 0)
+ spin_retry = 1000;
+ return 0;
+}
+early_initcall(spin_retry_init);
+
+/*
+ * spin_retry= parameter
+ */
+static int __init spin_retry_setup(char *str)
+{
+ spin_retry = simple_strtoul(str, &str, 0);
+ return 1;
+}
+__setup("spin_retry=", spin_retry_setup);
+
+struct spin_wait {
+ struct spin_wait *next, *prev;
+ int node_id;
+} __aligned(32);
+
+static DEFINE_PER_CPU_ALIGNED(struct spin_wait, spin_wait[4]);
+
+#define _Q_LOCK_CPU_OFFSET 0
+#define _Q_LOCK_STEAL_OFFSET 16
+#define _Q_TAIL_IDX_OFFSET 18
+#define _Q_TAIL_CPU_OFFSET 20
+
+#define _Q_LOCK_CPU_MASK 0x0000ffff
+#define _Q_LOCK_STEAL_ADD 0x00010000
+#define _Q_LOCK_STEAL_MASK 0x00030000
+#define _Q_TAIL_IDX_MASK 0x000c0000
+#define _Q_TAIL_CPU_MASK 0xfff00000
+
+#define _Q_LOCK_MASK (_Q_LOCK_CPU_MASK | _Q_LOCK_STEAL_MASK)
+#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
+
+void arch_spin_lock_setup(int cpu)
+{
+ struct spin_wait *node;
+ int ix;
+
+ node = per_cpu_ptr(&spin_wait[0], cpu);
+ for (ix = 0; ix < 4; ix++, node++) {
+ memset(node, 0, sizeof(*node));
+ node->node_id = ((cpu + 1) << _Q_TAIL_CPU_OFFSET) +
+ (ix << _Q_TAIL_IDX_OFFSET);
+ }
+}
+
+static inline int arch_load_niai4(int *lock)
+{
+ int owner;
+
+ asm_inline volatile(
+ ALTERNATIVE("nop", ".insn rre,0xb2fa0000,4,0", 49) /* NIAI 4 */
+ " l %0,%1\n"
+ : "=d" (owner) : "Q" (*lock) : "memory");
+ return owner;
+}
+
+static inline int arch_cmpxchg_niai8(int *lock, int old, int new)
+{
+ int expected = old;
+
+ asm_inline volatile(
+ ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", 49) /* NIAI 8 */
+ " cs %0,%3,%1\n"
+ : "=d" (old), "=Q" (*lock)
+ : "0" (old), "d" (new), "Q" (*lock)
+ : "cc", "memory");
+ return expected == old;
+}
+
+static inline struct spin_wait *arch_spin_decode_tail(int lock)
+{
+ int ix, cpu;
+
+ ix = (lock & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
+ cpu = (lock & _Q_TAIL_CPU_MASK) >> _Q_TAIL_CPU_OFFSET;
+ return per_cpu_ptr(&spin_wait[ix], cpu - 1);
+}
+
+static inline int arch_spin_yield_target(int lock, struct spin_wait *node)
+{
+ if (lock & _Q_LOCK_CPU_MASK)
+ return lock & _Q_LOCK_CPU_MASK;
+ if (node == NULL || node->prev == NULL)
+ return 0; /* 0 -> no target cpu */
+ while (node->prev)
+ node = node->prev;
+ return node->node_id >> _Q_TAIL_CPU_OFFSET;
+}
+
+static inline void arch_spin_lock_queued(arch_spinlock_t *lp)
+{
+ struct spin_wait *node, *next;
+ int lockval, ix, node_id, tail_id, old, new, owner, count;
+
+ ix = S390_lowcore.spinlock_index++;
+ barrier();
+ lockval = SPINLOCK_LOCKVAL; /* cpu + 1 */
+ node = this_cpu_ptr(&spin_wait[ix]);
+ node->prev = node->next = NULL;
+ node_id = node->node_id;
+
+ /* Enqueue the node for this CPU in the spinlock wait queue */
+ while (1) {
+ old = READ_ONCE(lp->lock);
+ if ((old & _Q_LOCK_CPU_MASK) == 0 &&
+ (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) {
+ /*
+ * The lock is free but there may be waiters.
+ * With no waiters simply take the lock, if there
+ * are waiters try to steal the lock. The lock may
+ * be stolen three times before the next queued
+ * waiter will get the lock.
+ */
+ new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval;
+ if (__atomic_cmpxchg_bool(&lp->lock, old, new))
+ /* Got the lock */
+ goto out;
+ /* lock passing in progress */
+ continue;
+ }
+ /* Make the node of this CPU the new tail. */
+ new = node_id | (old & _Q_LOCK_MASK);
+ if (__atomic_cmpxchg_bool(&lp->lock, old, new))
+ break;
+ }
+ /* Set the 'next' pointer of the tail node in the queue */
+ tail_id = old & _Q_TAIL_MASK;
+ if (tail_id != 0) {
+ node->prev = arch_spin_decode_tail(tail_id);
+ WRITE_ONCE(node->prev->next, node);
+ }
+
+ /* Pass the virtual CPU to the lock holder if it is not running */
+ owner = arch_spin_yield_target(old, node);
+ if (owner && arch_vcpu_is_preempted(owner - 1))
+ smp_yield_cpu(owner - 1);
+
+ /* Spin on the CPU local node->prev pointer */
+ if (tail_id != 0) {
+ count = spin_retry;
+ while (READ_ONCE(node->prev) != NULL) {
+ if (count-- >= 0)
+ continue;
+ count = spin_retry;
+ /* Query running state of lock holder again. */
+ owner = arch_spin_yield_target(old, node);
+ if (owner && arch_vcpu_is_preempted(owner - 1))
+ smp_yield_cpu(owner - 1);
+ }
+ }
+
+ /* Spin on the lock value in the spinlock_t */
+ count = spin_retry;
+ while (1) {
+ old = READ_ONCE(lp->lock);
+ owner = old & _Q_LOCK_CPU_MASK;
+ if (!owner) {
+ tail_id = old & _Q_TAIL_MASK;
+ new = ((tail_id != node_id) ? tail_id : 0) | lockval;
+ if (__atomic_cmpxchg_bool(&lp->lock, old, new))
+ /* Got the lock */
+ break;
+ continue;
+ }
+ if (count-- >= 0)
+ continue;
+ count = spin_retry;
+ if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(owner - 1))
+ smp_yield_cpu(owner - 1);
+ }
+
+ /* Pass lock_spin job to next CPU in the queue */
+ if (node_id && tail_id != node_id) {
+ /* Wait until the next CPU has set up the 'next' pointer */
+ while ((next = READ_ONCE(node->next)) == NULL)
+ ;
+ next->prev = NULL;
+ }
+
+ out:
+ S390_lowcore.spinlock_index--;
+}
+
+static inline void arch_spin_lock_classic(arch_spinlock_t *lp)
+{
+ int lockval, old, new, owner, count;
+
+ lockval = SPINLOCK_LOCKVAL; /* cpu + 1 */
+
+ /* Pass the virtual CPU to the lock holder if it is not running */
+ owner = arch_spin_yield_target(READ_ONCE(lp->lock), NULL);
+ if (owner && arch_vcpu_is_preempted(owner - 1))
+ smp_yield_cpu(owner - 1);
+
+ count = spin_retry;
+ while (1) {
+ old = arch_load_niai4(&lp->lock);
+ owner = old & _Q_LOCK_CPU_MASK;
+ /* Try to get the lock if it is free. */
+ if (!owner) {
+ new = (old & _Q_TAIL_MASK) | lockval;
+ if (arch_cmpxchg_niai8(&lp->lock, old, new)) {
+ /* Got the lock */
+ return;
+ }
+ continue;
+ }
+ if (count-- >= 0)
+ continue;
+ count = spin_retry;
+ if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(owner - 1))
+ smp_yield_cpu(owner - 1);
+ }
+}
+
+void arch_spin_lock_wait(arch_spinlock_t *lp)
+{
+ if (test_cpu_flag(CIF_DEDICATED_CPU))
+ arch_spin_lock_queued(lp);
+ else
+ arch_spin_lock_classic(lp);
+}
+EXPORT_SYMBOL(arch_spin_lock_wait);
+
+int arch_spin_trylock_retry(arch_spinlock_t *lp)
+{
+ int cpu = SPINLOCK_LOCKVAL;
+ int owner, count;
+
+ for (count = spin_retry; count > 0; count--) {
+ owner = READ_ONCE(lp->lock);
+ /* Try to get the lock if it is free. */
+ if (!owner) {
+ if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
+ return 1;
+ }
+ }
+ return 0;
+}
+EXPORT_SYMBOL(arch_spin_trylock_retry);
+
+void arch_read_lock_wait(arch_rwlock_t *rw)
+{
+ if (unlikely(in_interrupt())) {
+ while (READ_ONCE(rw->cnts) & 0x10000)
+ barrier();
+ return;
+ }
+
+ /* Remove this reader again to allow recursive read locking */
+ __atomic_add_const(-1, &rw->cnts);
+ /* Put the reader into the wait queue */
+ arch_spin_lock(&rw->wait);
+ /* Now add this reader to the count value again */
+ __atomic_add_const(1, &rw->cnts);
+ /* Loop until the writer is done */
+ while (READ_ONCE(rw->cnts) & 0x10000)
+ barrier();
+ arch_spin_unlock(&rw->wait);
+}
+EXPORT_SYMBOL(arch_read_lock_wait);
+
+void arch_write_lock_wait(arch_rwlock_t *rw)
+{
+ int old;
+
+ /* Add this CPU to the write waiters */
+ __atomic_add(0x20000, &rw->cnts);
+
+ /* Put the writer into the wait queue */
+ arch_spin_lock(&rw->wait);
+
+ while (1) {
+ old = READ_ONCE(rw->cnts);
+ if ((old & 0x1ffff) == 0 &&
+ __atomic_cmpxchg_bool(&rw->cnts, old, old | 0x10000))
+ /* Got the lock */
+ break;
+ barrier();
+ }
+
+ arch_spin_unlock(&rw->wait);
+}
+EXPORT_SYMBOL(arch_write_lock_wait);
+
+void arch_spin_relax(arch_spinlock_t *lp)
+{
+ int cpu;
+
+ cpu = READ_ONCE(lp->lock) & _Q_LOCK_CPU_MASK;
+ if (!cpu)
+ return;
+ if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted(cpu - 1))
+ return;
+ smp_yield_cpu(cpu - 1);
+}
+EXPORT_SYMBOL(arch_spin_relax);
diff --git a/arch/s390/lib/string.c b/arch/s390/lib/string.c
new file mode 100644
index 000000000..7d8741818
--- /dev/null
+++ b/arch/s390/lib/string.c
@@ -0,0 +1,348 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Optimized string functions
+ *
+ * S390 version
+ * Copyright IBM Corp. 2004
+ * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
+ */
+
+#define IN_ARCH_STRING_C 1
+#ifndef __NO_FORTIFY
+# define __NO_FORTIFY
+#endif
+
+#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/export.h>
+
+/*
+ * Helper functions to find the end of a string
+ */
+static inline char *__strend(const char *s)
+{
+ unsigned long e = 0;
+
+ asm volatile(
+ " lghi 0,0\n"
+ "0: srst %[e],%[s]\n"
+ " jo 0b\n"
+ : [e] "+&a" (e), [s] "+&a" (s)
+ :
+ : "cc", "memory", "0");
+ return (char *)e;
+}
+
+static inline char *__strnend(const char *s, size_t n)
+{
+ const char *p = s + n;
+
+ asm volatile(
+ " lghi 0,0\n"
+ "0: srst %[p],%[s]\n"
+ " jo 0b\n"
+ : [p] "+&d" (p), [s] "+&a" (s)
+ :
+ : "cc", "memory", "0");
+ return (char *)p;
+}
+
+/**
+ * strlen - Find the length of a string
+ * @s: The string to be sized
+ *
+ * returns the length of @s
+ */
+#ifdef __HAVE_ARCH_STRLEN
+size_t strlen(const char *s)
+{
+ return __strend(s) - s;
+}
+EXPORT_SYMBOL(strlen);
+#endif
+
+/**
+ * strnlen - Find the length of a length-limited string
+ * @s: The string to be sized
+ * @n: The maximum number of bytes to search
+ *
+ * returns the minimum of the length of @s and @n
+ */
+#ifdef __HAVE_ARCH_STRNLEN
+size_t strnlen(const char *s, size_t n)
+{
+ return __strnend(s, n) - s;
+}
+EXPORT_SYMBOL(strnlen);
+#endif
+
+/**
+ * strcpy - Copy a %NUL terminated string
+ * @dest: Where to copy the string to
+ * @src: Where to copy the string from
+ *
+ * returns a pointer to @dest
+ */
+#ifdef __HAVE_ARCH_STRCPY
+char *strcpy(char *dest, const char *src)
+{
+ char *ret = dest;
+
+ asm volatile(
+ " lghi 0,0\n"
+ "0: mvst %[dest],%[src]\n"
+ " jo 0b\n"
+ : [dest] "+&a" (dest), [src] "+&a" (src)
+ :
+ : "cc", "memory", "0");
+ return ret;
+}
+EXPORT_SYMBOL(strcpy);
+#endif
+
+/**
+ * strncpy - Copy a length-limited, %NUL-terminated string
+ * @dest: Where to copy the string to
+ * @src: Where to copy the string from
+ * @n: The maximum number of bytes to copy
+ *
+ * The result is not %NUL-terminated if the source exceeds
+ * @n bytes.
+ */
+#ifdef __HAVE_ARCH_STRNCPY
+char *strncpy(char *dest, const char *src, size_t n)
+{
+ size_t len = __strnend(src, n) - src;
+ memset(dest + len, 0, n - len);
+ memcpy(dest, src, len);
+ return dest;
+}
+EXPORT_SYMBOL(strncpy);
+#endif
+
+/**
+ * strcat - Append one %NUL-terminated string to another
+ * @dest: The string to be appended to
+ * @src: The string to append to it
+ *
+ * returns a pointer to @dest
+ */
+#ifdef __HAVE_ARCH_STRCAT
+char *strcat(char *dest, const char *src)
+{
+ unsigned long dummy = 0;
+ char *ret = dest;
+
+ asm volatile(
+ " lghi 0,0\n"
+ "0: srst %[dummy],%[dest]\n"
+ " jo 0b\n"
+ "1: mvst %[dummy],%[src]\n"
+ " jo 1b\n"
+ : [dummy] "+&a" (dummy), [dest] "+&a" (dest), [src] "+&a" (src)
+ :
+ : "cc", "memory", "0");
+ return ret;
+}
+EXPORT_SYMBOL(strcat);
+#endif
+
+/**
+ * strlcat - Append a length-limited, %NUL-terminated string to another
+ * @dest: The string to be appended to
+ * @src: The string to append to it
+ * @n: The size of the destination buffer.
+ */
+#ifdef __HAVE_ARCH_STRLCAT
+size_t strlcat(char *dest, const char *src, size_t n)
+{
+ size_t dsize = __strend(dest) - dest;
+ size_t len = __strend(src) - src;
+ size_t res = dsize + len;
+
+ if (dsize < n) {
+ dest += dsize;
+ n -= dsize;
+ if (len >= n)
+ len = n - 1;
+ dest[len] = '\0';
+ memcpy(dest, src, len);
+ }
+ return res;
+}
+EXPORT_SYMBOL(strlcat);
+#endif
+
+/**
+ * strncat - Append a length-limited, %NUL-terminated string to another
+ * @dest: The string to be appended to
+ * @src: The string to append to it
+ * @n: The maximum numbers of bytes to copy
+ *
+ * returns a pointer to @dest
+ *
+ * Note that in contrast to strncpy, strncat ensures the result is
+ * terminated.
+ */
+#ifdef __HAVE_ARCH_STRNCAT
+char *strncat(char *dest, const char *src, size_t n)
+{
+ size_t len = __strnend(src, n) - src;
+ char *p = __strend(dest);
+
+ p[len] = '\0';
+ memcpy(p, src, len);
+ return dest;
+}
+EXPORT_SYMBOL(strncat);
+#endif
+
+/**
+ * strcmp - Compare two strings
+ * @s1: One string
+ * @s2: Another string
+ *
+ * returns 0 if @s1 and @s2 are equal,
+ * < 0 if @s1 is less than @s2
+ * > 0 if @s1 is greater than @s2
+ */
+#ifdef __HAVE_ARCH_STRCMP
+int strcmp(const char *s1, const char *s2)
+{
+ int ret = 0;
+
+ asm volatile(
+ " lghi 0,0\n"
+ "0: clst %[s1],%[s2]\n"
+ " jo 0b\n"
+ " je 1f\n"
+ " ic %[ret],0(%[s1])\n"
+ " ic 0,0(%[s2])\n"
+ " sr %[ret],0\n"
+ "1:"
+ : [ret] "+&d" (ret), [s1] "+&a" (s1), [s2] "+&a" (s2)
+ :
+ : "cc", "memory", "0");
+ return ret;
+}
+EXPORT_SYMBOL(strcmp);
+#endif
+
+static inline int clcle(const char *s1, unsigned long l1,
+ const char *s2, unsigned long l2)
+{
+ union register_pair r1 = { .even = (unsigned long)s1, .odd = l1, };
+ union register_pair r3 = { .even = (unsigned long)s2, .odd = l2, };
+ int cc;
+
+ asm volatile(
+ "0: clcle %[r1],%[r3],0\n"
+ " jo 0b\n"
+ " ipm %[cc]\n"
+ " srl %[cc],28\n"
+ : [cc] "=&d" (cc), [r1] "+&d" (r1.pair), [r3] "+&d" (r3.pair)
+ :
+ : "cc", "memory");
+ return cc;
+}
+
+/**
+ * strstr - Find the first substring in a %NUL terminated string
+ * @s1: The string to be searched
+ * @s2: The string to search for
+ */
+#ifdef __HAVE_ARCH_STRSTR
+char *strstr(const char *s1, const char *s2)
+{
+ int l1, l2;
+
+ l2 = __strend(s2) - s2;
+ if (!l2)
+ return (char *) s1;
+ l1 = __strend(s1) - s1;
+ while (l1-- >= l2) {
+ int cc;
+
+ cc = clcle(s1, l2, s2, l2);
+ if (!cc)
+ return (char *) s1;
+ s1++;
+ }
+ return NULL;
+}
+EXPORT_SYMBOL(strstr);
+#endif
+
+/**
+ * memchr - Find a character in an area of memory.
+ * @s: The memory area
+ * @c: The byte to search for
+ * @n: The size of the area.
+ *
+ * returns the address of the first occurrence of @c, or %NULL
+ * if @c is not found
+ */
+#ifdef __HAVE_ARCH_MEMCHR
+void *memchr(const void *s, int c, size_t n)
+{
+ const void *ret = s + n;
+
+ asm volatile(
+ " lgr 0,%[c]\n"
+ "0: srst %[ret],%[s]\n"
+ " jo 0b\n"
+ " jl 1f\n"
+ " la %[ret],0\n"
+ "1:"
+ : [ret] "+&a" (ret), [s] "+&a" (s)
+ : [c] "d" (c)
+ : "cc", "memory", "0");
+ return (void *) ret;
+}
+EXPORT_SYMBOL(memchr);
+#endif
+
+/**
+ * memcmp - Compare two areas of memory
+ * @s1: One area of memory
+ * @s2: Another area of memory
+ * @n: The size of the area.
+ */
+#ifdef __HAVE_ARCH_MEMCMP
+int memcmp(const void *s1, const void *s2, size_t n)
+{
+ int ret;
+
+ ret = clcle(s1, n, s2, n);
+ if (ret)
+ ret = ret == 1 ? -1 : 1;
+ return ret;
+}
+EXPORT_SYMBOL(memcmp);
+#endif
+
+/**
+ * memscan - Find a character in an area of memory.
+ * @s: The memory area
+ * @c: The byte to search for
+ * @n: The size of the area.
+ *
+ * returns the address of the first occurrence of @c, or 1 byte past
+ * the area if @c is not found
+ */
+#ifdef __HAVE_ARCH_MEMSCAN
+void *memscan(void *s, int c, size_t n)
+{
+ const void *ret = s + n;
+
+ asm volatile(
+ " lgr 0,%[c]\n"
+ "0: srst %[ret],%[s]\n"
+ " jo 0b\n"
+ : [ret] "+&a" (ret), [s] "+&a" (s)
+ : [c] "d" (c)
+ : "cc", "memory", "0");
+ return (void *)ret;
+}
+EXPORT_SYMBOL(memscan);
+#endif
diff --git a/arch/s390/lib/test_kprobes.c b/arch/s390/lib/test_kprobes.c
new file mode 100644
index 000000000..9e62d6281
--- /dev/null
+++ b/arch/s390/lib/test_kprobes.c
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/random.h>
+#include <kunit/test.h>
+#include "test_kprobes.h"
+
+static struct kprobe kp;
+
+static void setup_kprobe(struct kunit *test, struct kprobe *kp,
+ const char *symbol, int offset)
+{
+ kp->offset = offset;
+ kp->addr = NULL;
+ kp->symbol_name = symbol;
+}
+
+static void test_kprobe_offset(struct kunit *test, struct kprobe *kp,
+ const char *target, int offset)
+{
+ int ret;
+
+ setup_kprobe(test, kp, target, 0);
+ ret = register_kprobe(kp);
+ if (!ret)
+ unregister_kprobe(kp);
+ KUNIT_EXPECT_EQ(test, 0, ret);
+ setup_kprobe(test, kp, target, offset);
+ ret = register_kprobe(kp);
+ KUNIT_EXPECT_EQ(test, -EINVAL, ret);
+ if (!ret)
+ unregister_kprobe(kp);
+}
+
+static void test_kprobe_odd(struct kunit *test)
+{
+ test_kprobe_offset(test, &kp, "kprobes_target_odd",
+ kprobes_target_odd_offs);
+}
+
+static void test_kprobe_in_insn4(struct kunit *test)
+{
+ test_kprobe_offset(test, &kp, "kprobes_target_in_insn4",
+ kprobes_target_in_insn4_offs);
+}
+
+static void test_kprobe_in_insn6_lo(struct kunit *test)
+{
+ test_kprobe_offset(test, &kp, "kprobes_target_in_insn6_lo",
+ kprobes_target_in_insn6_lo_offs);
+}
+
+static void test_kprobe_in_insn6_hi(struct kunit *test)
+{
+ test_kprobe_offset(test, &kp, "kprobes_target_in_insn6_hi",
+ kprobes_target_in_insn6_hi_offs);
+}
+
+static struct kunit_case kprobes_testcases[] = {
+ KUNIT_CASE(test_kprobe_odd),
+ KUNIT_CASE(test_kprobe_in_insn4),
+ KUNIT_CASE(test_kprobe_in_insn6_lo),
+ KUNIT_CASE(test_kprobe_in_insn6_hi),
+ {}
+};
+
+static struct kunit_suite kprobes_test_suite = {
+ .name = "kprobes_test_s390",
+ .test_cases = kprobes_testcases,
+};
+
+kunit_test_suites(&kprobes_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/arch/s390/lib/test_kprobes.h b/arch/s390/lib/test_kprobes.h
new file mode 100644
index 000000000..2b4c9bc33
--- /dev/null
+++ b/arch/s390/lib/test_kprobes.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef TEST_KPROBES_H
+#define TEST_KPROBES_H
+
+extern unsigned long kprobes_target_odd_offs;
+extern unsigned long kprobes_target_in_insn4_offs;
+extern unsigned long kprobes_target_in_insn6_lo_offs;
+extern unsigned long kprobes_target_in_insn6_hi_offs;
+
+#endif
diff --git a/arch/s390/lib/test_kprobes_asm.S b/arch/s390/lib/test_kprobes_asm.S
new file mode 100644
index 000000000..ade7a3042
--- /dev/null
+++ b/arch/s390/lib/test_kprobes_asm.S
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#include <linux/linkage.h>
+#include <asm/ftrace.h>
+
+#define KPROBES_TARGET_START(name) \
+ SYM_FUNC_START(name); \
+ FTRACE_GEN_NOP_ASM(name)
+
+#define KPROBES_TARGET_END(name) \
+ SYM_FUNC_END(name); \
+ SYM_DATA(name##_offs, .quad 1b - name)
+
+KPROBES_TARGET_START(kprobes_target_in_insn4)
+ .word 0x4700 // bc 0,0
+1: .word 0x0000
+ br %r14
+KPROBES_TARGET_END(kprobes_target_in_insn4)
+
+KPROBES_TARGET_START(kprobes_target_in_insn6_lo)
+ .word 0xe310 // ly 1,0
+1: .word 0x0000
+ .word 0x0058
+ br %r14
+KPROBES_TARGET_END(kprobes_target_in_insn6_lo)
+
+KPROBES_TARGET_START(kprobes_target_in_insn6_hi)
+ .word 0xe310 // ly 1,0
+ .word 0x0000
+1: .word 0x0058
+ br %r14
+KPROBES_TARGET_END(kprobes_target_in_insn6_hi)
+
+KPROBES_TARGET_START(kprobes_target_bp)
+ nop
+ .word 0x0000
+ nop
+1: br %r14
+KPROBES_TARGET_END(kprobes_target_bp)
+
+KPROBES_TARGET_START(kprobes_target_odd)
+ .byte 0x07
+1: .byte 0x07
+ br %r14
+KPROBES_TARGET_END(kprobes_target_odd)
diff --git a/arch/s390/lib/test_modules.c b/arch/s390/lib/test_modules.c
new file mode 100644
index 000000000..9894009fc
--- /dev/null
+++ b/arch/s390/lib/test_modules.c
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <kunit/test.h>
+#include <linux/module.h>
+
+#include "test_modules.h"
+
+/*
+ * Test that modules with many relocations are loaded properly.
+ */
+static void test_modules_many_vmlinux_relocs(struct kunit *test)
+{
+ int result = 0;
+
+#define CALL_RETURN(i) result += test_modules_return_ ## i()
+ REPEAT_10000(CALL_RETURN);
+ KUNIT_ASSERT_EQ(test, result, 49995000);
+}
+
+static struct kunit_case modules_testcases[] = {
+ KUNIT_CASE(test_modules_many_vmlinux_relocs),
+ {}
+};
+
+static struct kunit_suite modules_test_suite = {
+ .name = "modules_test_s390",
+ .test_cases = modules_testcases,
+};
+
+kunit_test_suites(&modules_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/arch/s390/lib/test_modules.h b/arch/s390/lib/test_modules.h
new file mode 100644
index 000000000..6371fcf17
--- /dev/null
+++ b/arch/s390/lib/test_modules.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef TEST_MODULES_H
+#define TEST_MODULES_H
+
+#define __REPEAT_10000_3(f, x) \
+ f(x ## 0); \
+ f(x ## 1); \
+ f(x ## 2); \
+ f(x ## 3); \
+ f(x ## 4); \
+ f(x ## 5); \
+ f(x ## 6); \
+ f(x ## 7); \
+ f(x ## 8); \
+ f(x ## 9)
+#define __REPEAT_10000_2(f, x) \
+ __REPEAT_10000_3(f, x ## 0); \
+ __REPEAT_10000_3(f, x ## 1); \
+ __REPEAT_10000_3(f, x ## 2); \
+ __REPEAT_10000_3(f, x ## 3); \
+ __REPEAT_10000_3(f, x ## 4); \
+ __REPEAT_10000_3(f, x ## 5); \
+ __REPEAT_10000_3(f, x ## 6); \
+ __REPEAT_10000_3(f, x ## 7); \
+ __REPEAT_10000_3(f, x ## 8); \
+ __REPEAT_10000_3(f, x ## 9)
+#define __REPEAT_10000_1(f, x) \
+ __REPEAT_10000_2(f, x ## 0); \
+ __REPEAT_10000_2(f, x ## 1); \
+ __REPEAT_10000_2(f, x ## 2); \
+ __REPEAT_10000_2(f, x ## 3); \
+ __REPEAT_10000_2(f, x ## 4); \
+ __REPEAT_10000_2(f, x ## 5); \
+ __REPEAT_10000_2(f, x ## 6); \
+ __REPEAT_10000_2(f, x ## 7); \
+ __REPEAT_10000_2(f, x ## 8); \
+ __REPEAT_10000_2(f, x ## 9)
+#define REPEAT_10000(f) \
+ __REPEAT_10000_1(f, 0); \
+ __REPEAT_10000_1(f, 1); \
+ __REPEAT_10000_1(f, 2); \
+ __REPEAT_10000_1(f, 3); \
+ __REPEAT_10000_1(f, 4); \
+ __REPEAT_10000_1(f, 5); \
+ __REPEAT_10000_1(f, 6); \
+ __REPEAT_10000_1(f, 7); \
+ __REPEAT_10000_1(f, 8); \
+ __REPEAT_10000_1(f, 9)
+
+#define DECLARE_RETURN(i) int test_modules_return_ ## i(void)
+REPEAT_10000(DECLARE_RETURN);
+
+#endif
diff --git a/arch/s390/lib/test_modules_helpers.c b/arch/s390/lib/test_modules_helpers.c
new file mode 100644
index 000000000..1670349a0
--- /dev/null
+++ b/arch/s390/lib/test_modules_helpers.c
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/export.h>
+
+#include "test_modules.h"
+
+#define DEFINE_RETURN(i) \
+ int test_modules_return_ ## i(void) \
+ { \
+ return 1 ## i - 10000; \
+ } \
+ EXPORT_SYMBOL_GPL(test_modules_return_ ## i)
+REPEAT_10000(DEFINE_RETURN);
diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c
new file mode 100644
index 000000000..7231bf97b
--- /dev/null
+++ b/arch/s390/lib/test_unwind.c
@@ -0,0 +1,522 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Test module for unwind_for_each_frame
+ */
+
+#include <kunit/test.h>
+#include <asm/unwind.h>
+#include <linux/completion.h>
+#include <linux/kallsyms.h>
+#include <linux/kthread.h>
+#include <linux/ftrace.h>
+#include <linux/module.h>
+#include <linux/timer.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/kprobes.h>
+#include <linux/wait.h>
+#include <asm/irq.h>
+
+static struct kunit *current_test;
+
+#define BT_BUF_SIZE (PAGE_SIZE * 4)
+
+static bool force_bt;
+module_param_named(backtrace, force_bt, bool, 0444);
+MODULE_PARM_DESC(backtrace, "print backtraces for all tests");
+
+/*
+ * To avoid printk line limit split backtrace by lines
+ */
+static void print_backtrace(char *bt)
+{
+ char *p;
+
+ while (true) {
+ p = strsep(&bt, "\n");
+ if (!p)
+ break;
+ kunit_err(current_test, "%s\n", p);
+ }
+}
+
+/*
+ * Calls unwind_for_each_frame(task, regs, sp) and verifies that the result
+ * contains unwindme_func2 followed by unwindme_func1.
+ */
+static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs,
+ unsigned long sp)
+{
+ int frame_count, prev_is_func2, seen_func2_func1, seen_arch_rethook_trampoline;
+ const int max_frames = 128;
+ struct unwind_state state;
+ size_t bt_pos = 0;
+ int ret = 0;
+ char *bt;
+
+ bt = kmalloc(BT_BUF_SIZE, GFP_ATOMIC);
+ if (!bt) {
+ kunit_err(current_test, "failed to allocate backtrace buffer\n");
+ return -ENOMEM;
+ }
+ /* Unwind. */
+ frame_count = 0;
+ prev_is_func2 = 0;
+ seen_func2_func1 = 0;
+ seen_arch_rethook_trampoline = 0;
+ unwind_for_each_frame(&state, task, regs, sp) {
+ unsigned long addr = unwind_get_return_address(&state);
+ char sym[KSYM_SYMBOL_LEN];
+
+ if (frame_count++ == max_frames)
+ break;
+ if (state.reliable && !addr) {
+ kunit_err(current_test, "unwind state reliable but addr is 0\n");
+ ret = -EINVAL;
+ break;
+ }
+ sprint_symbol(sym, addr);
+ if (bt_pos < BT_BUF_SIZE) {
+ bt_pos += snprintf(bt + bt_pos, BT_BUF_SIZE - bt_pos,
+ state.reliable ? " [%-7s%px] %pSR\n" :
+ "([%-7s%px] %pSR)\n",
+ stack_type_name(state.stack_info.type),
+ (void *)state.sp, (void *)state.ip);
+ if (bt_pos >= BT_BUF_SIZE)
+ kunit_err(current_test, "backtrace buffer is too small\n");
+ }
+ frame_count += 1;
+ if (prev_is_func2 && str_has_prefix(sym, "unwindme_func1"))
+ seen_func2_func1 = 1;
+ prev_is_func2 = str_has_prefix(sym, "unwindme_func2");
+ if (str_has_prefix(sym, "arch_rethook_trampoline+0x0/"))
+ seen_arch_rethook_trampoline = 1;
+ }
+
+ /* Check the results. */
+ if (unwind_error(&state)) {
+ kunit_err(current_test, "unwind error\n");
+ ret = -EINVAL;
+ }
+ if (!seen_func2_func1) {
+ kunit_err(current_test, "unwindme_func2 and unwindme_func1 not found\n");
+ ret = -EINVAL;
+ }
+ if (frame_count == max_frames) {
+ kunit_err(current_test, "Maximum number of frames exceeded\n");
+ ret = -EINVAL;
+ }
+ if (seen_arch_rethook_trampoline) {
+ kunit_err(current_test, "arch_rethook_trampoline+0x0 in unwinding results\n");
+ ret = -EINVAL;
+ }
+ if (ret || force_bt)
+ print_backtrace(bt);
+ kfree(bt);
+ return ret;
+}
+
+/* State of the task being unwound. */
+struct unwindme {
+ int flags;
+ int ret;
+ struct task_struct *task;
+ struct completion task_ready;
+ wait_queue_head_t task_wq;
+ unsigned long sp;
+};
+
+static struct unwindme *unwindme;
+
+/* Values of unwindme.flags. */
+#define UWM_DEFAULT 0x0
+#define UWM_THREAD 0x1 /* Unwind a separate task. */
+#define UWM_REGS 0x2 /* Pass regs to test_unwind(). */
+#define UWM_SP 0x4 /* Pass sp to test_unwind(). */
+#define UWM_CALLER 0x8 /* Unwind starting from caller. */
+#define UWM_SWITCH_STACK 0x10 /* Use call_on_stack. */
+#define UWM_IRQ 0x20 /* Unwind from irq context. */
+#define UWM_PGM 0x40 /* Unwind from program check handler */
+#define UWM_KPROBE_ON_FTRACE 0x80 /* Unwind from kprobe handler called via ftrace. */
+#define UWM_FTRACE 0x100 /* Unwind from ftrace handler. */
+#define UWM_KRETPROBE 0x200 /* Unwind through kretprobed function. */
+#define UWM_KRETPROBE_HANDLER 0x400 /* Unwind from kretprobe handler. */
+
+static __always_inline struct pt_regs fake_pt_regs(void)
+{
+ struct pt_regs regs;
+
+ memset(&regs, 0, sizeof(regs));
+ regs.gprs[15] = current_stack_pointer;
+
+ asm volatile(
+ "basr %[psw_addr],0\n"
+ : [psw_addr] "=d" (regs.psw.addr));
+ return regs;
+}
+
+static int kretprobe_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
+{
+ struct unwindme *u = unwindme;
+
+ if (!(u->flags & UWM_KRETPROBE_HANDLER))
+ return 0;
+
+ u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? regs : NULL,
+ (u->flags & UWM_SP) ? u->sp : 0);
+
+ return 0;
+}
+
+static noinline notrace int test_unwind_kretprobed_func(struct unwindme *u)
+{
+ struct pt_regs regs;
+
+ if (!(u->flags & UWM_KRETPROBE))
+ return 0;
+
+ regs = fake_pt_regs();
+ return test_unwind(NULL, (u->flags & UWM_REGS) ? &regs : NULL,
+ (u->flags & UWM_SP) ? u->sp : 0);
+}
+
+static noinline int test_unwind_kretprobed_func_caller(struct unwindme *u)
+{
+ return test_unwind_kretprobed_func(u);
+}
+
+static int test_unwind_kretprobe(struct unwindme *u)
+{
+ int ret;
+ struct kretprobe my_kretprobe;
+
+ if (!IS_ENABLED(CONFIG_KPROBES))
+ kunit_skip(current_test, "requires CONFIG_KPROBES");
+
+ u->ret = -1; /* make sure kprobe is called */
+ unwindme = u;
+
+ memset(&my_kretprobe, 0, sizeof(my_kretprobe));
+ my_kretprobe.handler = kretprobe_ret_handler;
+ my_kretprobe.maxactive = 1;
+ my_kretprobe.kp.addr = (kprobe_opcode_t *)test_unwind_kretprobed_func;
+
+ ret = register_kretprobe(&my_kretprobe);
+
+ if (ret < 0) {
+ kunit_err(current_test, "register_kretprobe failed %d\n", ret);
+ return -EINVAL;
+ }
+
+ ret = test_unwind_kretprobed_func_caller(u);
+ unregister_kretprobe(&my_kretprobe);
+ unwindme = NULL;
+ if (u->flags & UWM_KRETPROBE_HANDLER)
+ ret = u->ret;
+ return ret;
+}
+
+static int kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs)
+{
+ struct unwindme *u = unwindme;
+
+ u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? regs : NULL,
+ (u->flags & UWM_SP) ? u->sp : 0);
+ return 0;
+}
+
+extern const char test_unwind_kprobed_insn[];
+
+static noinline void test_unwind_kprobed_func(void)
+{
+ asm volatile(
+ " nopr %%r7\n"
+ "test_unwind_kprobed_insn:\n"
+ " nopr %%r7\n"
+ :);
+}
+
+static int test_unwind_kprobe(struct unwindme *u)
+{
+ struct kprobe kp;
+ int ret;
+
+ if (!IS_ENABLED(CONFIG_KPROBES))
+ kunit_skip(current_test, "requires CONFIG_KPROBES");
+ if (!IS_ENABLED(CONFIG_KPROBES_ON_FTRACE) && u->flags & UWM_KPROBE_ON_FTRACE)
+ kunit_skip(current_test, "requires CONFIG_KPROBES_ON_FTRACE");
+
+ u->ret = -1; /* make sure kprobe is called */
+ unwindme = u;
+ memset(&kp, 0, sizeof(kp));
+ kp.pre_handler = kprobe_pre_handler;
+ kp.addr = u->flags & UWM_KPROBE_ON_FTRACE ?
+ (kprobe_opcode_t *)test_unwind_kprobed_func :
+ (kprobe_opcode_t *)test_unwind_kprobed_insn;
+ ret = register_kprobe(&kp);
+ if (ret < 0) {
+ kunit_err(current_test, "register_kprobe failed %d\n", ret);
+ return -EINVAL;
+ }
+
+ test_unwind_kprobed_func();
+ unregister_kprobe(&kp);
+ unwindme = NULL;
+ return u->ret;
+}
+
+static void notrace __used test_unwind_ftrace_handler(unsigned long ip,
+ unsigned long parent_ip,
+ struct ftrace_ops *fops,
+ struct ftrace_regs *fregs)
+{
+ struct unwindme *u = (struct unwindme *)fregs->regs.gprs[2];
+
+ u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? &fregs->regs : NULL,
+ (u->flags & UWM_SP) ? u->sp : 0);
+}
+
+static noinline int test_unwind_ftraced_func(struct unwindme *u)
+{
+ return READ_ONCE(u)->ret;
+}
+
+static int test_unwind_ftrace(struct unwindme *u)
+{
+ int ret;
+#ifdef CONFIG_DYNAMIC_FTRACE
+ struct ftrace_ops *fops;
+
+ fops = kunit_kzalloc(current_test, sizeof(*fops), GFP_KERNEL);
+ fops->func = test_unwind_ftrace_handler;
+ fops->flags = FTRACE_OPS_FL_DYNAMIC |
+ FTRACE_OPS_FL_RECURSION |
+ FTRACE_OPS_FL_SAVE_REGS |
+ FTRACE_OPS_FL_PERMANENT;
+#else
+ kunit_skip(current_test, "requires CONFIG_DYNAMIC_FTRACE");
+#endif
+
+ ret = ftrace_set_filter_ip(fops, (unsigned long)test_unwind_ftraced_func, 0, 0);
+ if (ret) {
+ kunit_err(current_test, "failed to set ftrace filter (%d)\n", ret);
+ return -1;
+ }
+
+ ret = register_ftrace_function(fops);
+ if (!ret) {
+ ret = test_unwind_ftraced_func(u);
+ unregister_ftrace_function(fops);
+ } else {
+ kunit_err(current_test, "failed to register ftrace handler (%d)\n", ret);
+ }
+
+ ftrace_set_filter_ip(fops, (unsigned long)test_unwind_ftraced_func, 1, 0);
+ return ret;
+}
+
+/* This function may or may not appear in the backtrace. */
+static noinline int unwindme_func4(struct unwindme *u)
+{
+ if (!(u->flags & UWM_CALLER))
+ u->sp = current_frame_address();
+ if (u->flags & UWM_THREAD) {
+ complete(&u->task_ready);
+ wait_event(u->task_wq, kthread_should_park());
+ kthread_parkme();
+ return 0;
+ } else if (u->flags & (UWM_PGM | UWM_KPROBE_ON_FTRACE)) {
+ return test_unwind_kprobe(u);
+ } else if (u->flags & (UWM_KRETPROBE | UWM_KRETPROBE_HANDLER)) {
+ return test_unwind_kretprobe(u);
+ } else if (u->flags & UWM_FTRACE) {
+ return test_unwind_ftrace(u);
+ } else {
+ struct pt_regs regs = fake_pt_regs();
+
+ return test_unwind(NULL,
+ (u->flags & UWM_REGS) ? &regs : NULL,
+ (u->flags & UWM_SP) ? u->sp : 0);
+ }
+}
+
+/* This function may or may not appear in the backtrace. */
+static noinline int unwindme_func3(struct unwindme *u)
+{
+ u->sp = current_frame_address();
+ return unwindme_func4(u);
+}
+
+/* This function must appear in the backtrace. */
+static noinline int unwindme_func2(struct unwindme *u)
+{
+ unsigned long flags;
+ int rc;
+
+ if (u->flags & UWM_SWITCH_STACK) {
+ local_irq_save(flags);
+ local_mcck_disable();
+ rc = call_on_stack(1, S390_lowcore.nodat_stack,
+ int, unwindme_func3, struct unwindme *, u);
+ local_mcck_enable();
+ local_irq_restore(flags);
+ return rc;
+ } else {
+ return unwindme_func3(u);
+ }
+}
+
+/* This function must follow unwindme_func2 in the backtrace. */
+static noinline int unwindme_func1(void *u)
+{
+ return unwindme_func2((struct unwindme *)u);
+}
+
+static void unwindme_timer_fn(struct timer_list *unused)
+{
+ struct unwindme *u = READ_ONCE(unwindme);
+
+ if (u) {
+ unwindme = NULL;
+ u->task = NULL;
+ u->ret = unwindme_func1(u);
+ complete(&u->task_ready);
+ }
+}
+
+static struct timer_list unwind_timer;
+
+static int test_unwind_irq(struct unwindme *u)
+{
+ unwindme = u;
+ init_completion(&u->task_ready);
+ timer_setup(&unwind_timer, unwindme_timer_fn, 0);
+ mod_timer(&unwind_timer, jiffies + 1);
+ wait_for_completion(&u->task_ready);
+ return u->ret;
+}
+
+/* Spawns a task and passes it to test_unwind(). */
+static int test_unwind_task(struct unwindme *u)
+{
+ struct task_struct *task;
+ int ret;
+
+ /* Initialize thread-related fields. */
+ init_completion(&u->task_ready);
+ init_waitqueue_head(&u->task_wq);
+
+ /*
+ * Start the task and wait until it reaches unwindme_func4() and sleeps
+ * in (task_ready, unwind_done] range.
+ */
+ task = kthread_run(unwindme_func1, u, "%s", __func__);
+ if (IS_ERR(task)) {
+ kunit_err(current_test, "kthread_run() failed\n");
+ return PTR_ERR(task);
+ }
+ /*
+ * Make sure task reaches unwindme_func4 before parking it,
+ * we might park it before kthread function has been executed otherwise
+ */
+ wait_for_completion(&u->task_ready);
+ kthread_park(task);
+ /* Unwind. */
+ ret = test_unwind(task, NULL, (u->flags & UWM_SP) ? u->sp : 0);
+ kthread_stop(task);
+ return ret;
+}
+
+struct test_params {
+ int flags;
+ char *name;
+};
+
+/*
+ * Create required parameter list for tests
+ */
+#define TEST_WITH_FLAGS(f) { .flags = f, .name = #f }
+static const struct test_params param_list[] = {
+ TEST_WITH_FLAGS(UWM_DEFAULT),
+ TEST_WITH_FLAGS(UWM_SP),
+ TEST_WITH_FLAGS(UWM_REGS),
+ TEST_WITH_FLAGS(UWM_SWITCH_STACK),
+ TEST_WITH_FLAGS(UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_CALLER | UWM_SP),
+ TEST_WITH_FLAGS(UWM_CALLER | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK),
+ TEST_WITH_FLAGS(UWM_THREAD),
+ TEST_WITH_FLAGS(UWM_THREAD | UWM_SP),
+ TEST_WITH_FLAGS(UWM_THREAD | UWM_CALLER | UWM_SP),
+ TEST_WITH_FLAGS(UWM_IRQ),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_SWITCH_STACK),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_SP),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK),
+ TEST_WITH_FLAGS(UWM_PGM),
+ TEST_WITH_FLAGS(UWM_PGM | UWM_SP),
+ TEST_WITH_FLAGS(UWM_PGM | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_PGM | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE),
+ TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_SP),
+ TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_FTRACE),
+ TEST_WITH_FLAGS(UWM_FTRACE | UWM_SP),
+ TEST_WITH_FLAGS(UWM_FTRACE | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_FTRACE | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KRETPROBE),
+ TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP),
+ TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER),
+ TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_SP),
+ TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_SP | UWM_REGS),
+};
+
+/*
+ * Parameter description generator: required for KUNIT_ARRAY_PARAM()
+ */
+static void get_desc(const struct test_params *params, char *desc)
+{
+ strscpy(desc, params->name, KUNIT_PARAM_DESC_SIZE);
+}
+
+/*
+ * Create test_unwind_gen_params
+ */
+KUNIT_ARRAY_PARAM(test_unwind, param_list, get_desc);
+
+static void test_unwind_flags(struct kunit *test)
+{
+ struct unwindme u;
+ const struct test_params *params;
+
+ current_test = test;
+ params = (const struct test_params *)test->param_value;
+ u.flags = params->flags;
+ if (u.flags & UWM_THREAD)
+ KUNIT_EXPECT_EQ(test, 0, test_unwind_task(&u));
+ else if (u.flags & UWM_IRQ)
+ KUNIT_EXPECT_EQ(test, 0, test_unwind_irq(&u));
+ else
+ KUNIT_EXPECT_EQ(test, 0, unwindme_func1(&u));
+}
+
+static struct kunit_case unwind_test_cases[] = {
+ KUNIT_CASE_PARAM(test_unwind_flags, test_unwind_gen_params),
+ {}
+};
+
+static struct kunit_suite test_unwind_suite = {
+ .name = "test_unwind",
+ .test_cases = unwind_test_cases,
+};
+
+kunit_test_suites(&test_unwind_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/arch/s390/lib/tishift.S b/arch/s390/lib/tishift.S
new file mode 100644
index 000000000..96214f51f
--- /dev/null
+++ b/arch/s390/lib/tishift.S
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/export.h>
+#include <linux/linkage.h>
+#include <asm/nospec-insn.h>
+
+ .section .noinstr.text, "ax"
+
+ GEN_BR_THUNK %r14
+
+SYM_FUNC_START(__ashlti3)
+ lmg %r0,%r1,0(%r3)
+ cije %r4,0,1f
+ lhi %r3,64
+ sr %r3,%r4
+ jnh 0f
+ srlg %r3,%r1,0(%r3)
+ sllg %r0,%r0,0(%r4)
+ sllg %r1,%r1,0(%r4)
+ ogr %r0,%r3
+ j 1f
+0: sllg %r0,%r1,-64(%r4)
+ lghi %r1,0
+1: stmg %r0,%r1,0(%r2)
+ BR_EX %r14
+SYM_FUNC_END(__ashlti3)
+EXPORT_SYMBOL(__ashlti3)
+
+SYM_FUNC_START(__ashrti3)
+ lmg %r0,%r1,0(%r3)
+ cije %r4,0,1f
+ lhi %r3,64
+ sr %r3,%r4
+ jnh 0f
+ sllg %r3,%r0,0(%r3)
+ srlg %r1,%r1,0(%r4)
+ srag %r0,%r0,0(%r4)
+ ogr %r1,%r3
+ j 1f
+0: srag %r1,%r0,-64(%r4)
+ srag %r0,%r0,63
+1: stmg %r0,%r1,0(%r2)
+ BR_EX %r14
+SYM_FUNC_END(__ashrti3)
+EXPORT_SYMBOL(__ashrti3)
+
+SYM_FUNC_START(__lshrti3)
+ lmg %r0,%r1,0(%r3)
+ cije %r4,0,1f
+ lhi %r3,64
+ sr %r3,%r4
+ jnh 0f
+ sllg %r3,%r0,0(%r3)
+ srlg %r1,%r1,0(%r4)
+ srlg %r0,%r0,0(%r4)
+ ogr %r1,%r3
+ j 1f
+0: srlg %r1,%r0,-64(%r4)
+ lghi %r0,0
+1: stmg %r0,%r1,0(%r2)
+ BR_EX %r14
+SYM_FUNC_END(__lshrti3)
+EXPORT_SYMBOL(__lshrti3)
diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
new file mode 100644
index 000000000..e4a13d7ca
--- /dev/null
+++ b/arch/s390/lib/uaccess.c
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Standard user space access functions based on mvcp/mvcs and doing
+ * interesting things in the secondary space mode.
+ *
+ * Copyright IBM Corp. 2006,2014
+ * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
+ * Gerald Schaefer (gerald.schaefer@de.ibm.com)
+ */
+
+#include <linux/uaccess.h>
+#include <linux/export.h>
+#include <linux/mm.h>
+#include <asm/asm-extable.h>
+
+#ifdef CONFIG_DEBUG_ENTRY
+void debug_user_asce(int exit)
+{
+ unsigned long cr1, cr7;
+
+ __ctl_store(cr1, 1, 1);
+ __ctl_store(cr7, 7, 7);
+ if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce)
+ return;
+ panic("incorrect ASCE on kernel %s\n"
+ "cr1: %016lx cr7: %016lx\n"
+ "kernel: %016llx user: %016llx\n",
+ exit ? "exit" : "entry", cr1, cr7,
+ S390_lowcore.kernel_asce, S390_lowcore.user_asce);
+}
+#endif /*CONFIG_DEBUG_ENTRY */
+
+static unsigned long raw_copy_from_user_key(void *to, const void __user *from,
+ unsigned long size, unsigned long key)
+{
+ unsigned long rem;
+ union oac spec = {
+ .oac2.key = key,
+ .oac2.as = PSW_BITS_AS_SECONDARY,
+ .oac2.k = 1,
+ .oac2.a = 1,
+ };
+
+ asm volatile(
+ " lr 0,%[spec]\n"
+ "0: mvcos 0(%[to]),0(%[from]),%[size]\n"
+ "1: jz 5f\n"
+ " algr %[size],%[val]\n"
+ " slgr %[from],%[val]\n"
+ " slgr %[to],%[val]\n"
+ " j 0b\n"
+ "2: la %[rem],4095(%[from])\n" /* rem = from + 4095 */
+ " nr %[rem],%[val]\n" /* rem = (from + 4095) & -4096 */
+ " slgr %[rem],%[from]\n"
+ " clgr %[size],%[rem]\n" /* copy crosses next page boundary? */
+ " jnh 6f\n"
+ "3: mvcos 0(%[to]),0(%[from]),%[rem]\n"
+ "4: slgr %[size],%[rem]\n"
+ " j 6f\n"
+ "5: slgr %[size],%[size]\n"
+ "6:\n"
+ EX_TABLE(0b, 2b)
+ EX_TABLE(1b, 2b)
+ EX_TABLE(3b, 6b)
+ EX_TABLE(4b, 6b)
+ : [size] "+&a" (size), [from] "+&a" (from), [to] "+&a" (to), [rem] "=&a" (rem)
+ : [val] "a" (-4096UL), [spec] "d" (spec.val)
+ : "cc", "memory", "0");
+ return size;
+}
+
+unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ return raw_copy_from_user_key(to, from, n, 0);
+}
+EXPORT_SYMBOL(raw_copy_from_user);
+
+unsigned long _copy_from_user_key(void *to, const void __user *from,
+ unsigned long n, unsigned long key)
+{
+ unsigned long res = n;
+
+ might_fault();
+ if (!should_fail_usercopy()) {
+ instrument_copy_from_user_before(to, from, n);
+ res = raw_copy_from_user_key(to, from, n, key);
+ instrument_copy_from_user_after(to, from, n, res);
+ }
+ if (unlikely(res))
+ memset(to + (n - res), 0, res);
+ return res;
+}
+EXPORT_SYMBOL(_copy_from_user_key);
+
+static unsigned long raw_copy_to_user_key(void __user *to, const void *from,
+ unsigned long size, unsigned long key)
+{
+ unsigned long rem;
+ union oac spec = {
+ .oac1.key = key,
+ .oac1.as = PSW_BITS_AS_SECONDARY,
+ .oac1.k = 1,
+ .oac1.a = 1,
+ };
+
+ asm volatile(
+ " lr 0,%[spec]\n"
+ "0: mvcos 0(%[to]),0(%[from]),%[size]\n"
+ "1: jz 5f\n"
+ " algr %[size],%[val]\n"
+ " slgr %[to],%[val]\n"
+ " slgr %[from],%[val]\n"
+ " j 0b\n"
+ "2: la %[rem],4095(%[to])\n" /* rem = to + 4095 */
+ " nr %[rem],%[val]\n" /* rem = (to + 4095) & -4096 */
+ " slgr %[rem],%[to]\n"
+ " clgr %[size],%[rem]\n" /* copy crosses next page boundary? */
+ " jnh 6f\n"
+ "3: mvcos 0(%[to]),0(%[from]),%[rem]\n"
+ "4: slgr %[size],%[rem]\n"
+ " j 6f\n"
+ "5: slgr %[size],%[size]\n"
+ "6:\n"
+ EX_TABLE(0b, 2b)
+ EX_TABLE(1b, 2b)
+ EX_TABLE(3b, 6b)
+ EX_TABLE(4b, 6b)
+ : [size] "+&a" (size), [to] "+&a" (to), [from] "+&a" (from), [rem] "=&a" (rem)
+ : [val] "a" (-4096UL), [spec] "d" (spec.val)
+ : "cc", "memory", "0");
+ return size;
+}
+
+unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ return raw_copy_to_user_key(to, from, n, 0);
+}
+EXPORT_SYMBOL(raw_copy_to_user);
+
+unsigned long _copy_to_user_key(void __user *to, const void *from,
+ unsigned long n, unsigned long key)
+{
+ might_fault();
+ if (should_fail_usercopy())
+ return n;
+ instrument_copy_to_user(to, from, n);
+ return raw_copy_to_user_key(to, from, n, key);
+}
+EXPORT_SYMBOL(_copy_to_user_key);
+
+unsigned long __clear_user(void __user *to, unsigned long size)
+{
+ unsigned long rem;
+ union oac spec = {
+ .oac1.as = PSW_BITS_AS_SECONDARY,
+ .oac1.a = 1,
+ };
+
+ asm volatile(
+ " lr 0,%[spec]\n"
+ "0: mvcos 0(%[to]),0(%[zeropg]),%[size]\n"
+ "1: jz 5f\n"
+ " algr %[size],%[val]\n"
+ " slgr %[to],%[val]\n"
+ " j 0b\n"
+ "2: la %[rem],4095(%[to])\n" /* rem = to + 4095 */
+ " nr %[rem],%[val]\n" /* rem = (to + 4095) & -4096 */
+ " slgr %[rem],%[to]\n"
+ " clgr %[size],%[rem]\n" /* copy crosses next page boundary? */
+ " jnh 6f\n"
+ "3: mvcos 0(%[to]),0(%[zeropg]),%[rem]\n"
+ "4: slgr %[size],%[rem]\n"
+ " j 6f\n"
+ "5: slgr %[size],%[size]\n"
+ "6:\n"
+ EX_TABLE(0b, 2b)
+ EX_TABLE(1b, 2b)
+ EX_TABLE(3b, 6b)
+ EX_TABLE(4b, 6b)
+ : [size] "+&a" (size), [to] "+&a" (to), [rem] "=&a" (rem)
+ : [val] "a" (-4096UL), [zeropg] "a" (empty_zero_page), [spec] "d" (spec.val)
+ : "cc", "memory", "0");
+ return size;
+}
+EXPORT_SYMBOL(__clear_user);
diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c
new file mode 100644
index 000000000..fb924a804
--- /dev/null
+++ b/arch/s390/lib/xor.c
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Optimized xor_block operation for RAID4/5
+ *
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+
+#include <linux/types.h>
+#include <linux/export.h>
+#include <linux/raid/xor.h>
+#include <asm/xor.h>
+
+static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ asm volatile(
+ " larl 1,2f\n"
+ " aghi %0,-1\n"
+ " jm 3f\n"
+ " srlg 0,%0,8\n"
+ " ltgr 0,0\n"
+ " jz 1f\n"
+ "0: xc 0(256,%1),0(%2)\n"
+ " la %1,256(%1)\n"
+ " la %2,256(%2)\n"
+ " brctg 0,0b\n"
+ "1: ex %0,0(1)\n"
+ " j 3f\n"
+ "2: xc 0(1,%1),0(%2)\n"
+ "3:\n"
+ : : "d" (bytes), "a" (p1), "a" (p2)
+ : "0", "1", "cc", "memory");
+}
+
+static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ asm volatile(
+ " larl 1,2f\n"
+ " aghi %0,-1\n"
+ " jm 3f\n"
+ " srlg 0,%0,8\n"
+ " ltgr 0,0\n"
+ " jz 1f\n"
+ "0: xc 0(256,%1),0(%2)\n"
+ " xc 0(256,%1),0(%3)\n"
+ " la %1,256(%1)\n"
+ " la %2,256(%2)\n"
+ " la %3,256(%3)\n"
+ " brctg 0,0b\n"
+ "1: ex %0,0(1)\n"
+ " ex %0,6(1)\n"
+ " j 3f\n"
+ "2: xc 0(1,%1),0(%2)\n"
+ " xc 0(1,%1),0(%3)\n"
+ "3:\n"
+ : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3)
+ : : "0", "1", "cc", "memory");
+}
+
+static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ asm volatile(
+ " larl 1,2f\n"
+ " aghi %0,-1\n"
+ " jm 3f\n"
+ " srlg 0,%0,8\n"
+ " ltgr 0,0\n"
+ " jz 1f\n"
+ "0: xc 0(256,%1),0(%2)\n"
+ " xc 0(256,%1),0(%3)\n"
+ " xc 0(256,%1),0(%4)\n"
+ " la %1,256(%1)\n"
+ " la %2,256(%2)\n"
+ " la %3,256(%3)\n"
+ " la %4,256(%4)\n"
+ " brctg 0,0b\n"
+ "1: ex %0,0(1)\n"
+ " ex %0,6(1)\n"
+ " ex %0,12(1)\n"
+ " j 3f\n"
+ "2: xc 0(1,%1),0(%2)\n"
+ " xc 0(1,%1),0(%3)\n"
+ " xc 0(1,%1),0(%4)\n"
+ "3:\n"
+ : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4)
+ : : "0", "1", "cc", "memory");
+}
+
+static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ asm volatile(
+ " larl 1,2f\n"
+ " aghi %0,-1\n"
+ " jm 3f\n"
+ " srlg 0,%0,8\n"
+ " ltgr 0,0\n"
+ " jz 1f\n"
+ "0: xc 0(256,%1),0(%2)\n"
+ " xc 0(256,%1),0(%3)\n"
+ " xc 0(256,%1),0(%4)\n"
+ " xc 0(256,%1),0(%5)\n"
+ " la %1,256(%1)\n"
+ " la %2,256(%2)\n"
+ " la %3,256(%3)\n"
+ " la %4,256(%4)\n"
+ " la %5,256(%5)\n"
+ " brctg 0,0b\n"
+ "1: ex %0,0(1)\n"
+ " ex %0,6(1)\n"
+ " ex %0,12(1)\n"
+ " ex %0,18(1)\n"
+ " j 3f\n"
+ "2: xc 0(1,%1),0(%2)\n"
+ " xc 0(1,%1),0(%3)\n"
+ " xc 0(1,%1),0(%4)\n"
+ " xc 0(1,%1),0(%5)\n"
+ "3:\n"
+ : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4),
+ "+a" (p5)
+ : : "0", "1", "cc", "memory");
+}
+
+struct xor_block_template xor_block_xc = {
+ .name = "xc",
+ .do_2 = xor_xc_2,
+ .do_3 = xor_xc_3,
+ .do_4 = xor_xc_4,
+ .do_5 = xor_xc_5,
+};
+EXPORT_SYMBOL(xor_block_xc);
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-16 19:50:57 +0000