summaryrefslogtreecommitdiffstats
path: root/arch/s390/kernel/entry.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/s390/kernel/entry.S')
-rw-r--r--arch/s390/kernel/entry.S93
1 files changed, 38 insertions, 55 deletions
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 26c08ee877..6a1e0fbbaa 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -24,7 +24,7 @@
#include <asm/page.h>
#include <asm/sigp.h>
#include <asm/irq.h>
-#include <asm/vx-insn.h>
+#include <asm/fpu-insn.h>
#include <asm/setup.h>
#include <asm/nmi.h>
#include <asm/nospec-insn.h>
@@ -119,33 +119,11 @@ _LPP_OFFSET = __LC_LPP
.endm
#if IS_ENABLED(CONFIG_KVM)
- /*
- * The OUTSIDE macro jumps to the provided label in case the value
- * in the provided register is outside of the provided range. The
- * macro is useful for checking whether a PSW stored in a register
- * pair points inside or outside of a block of instructions.
- * @reg: register to check
- * @start: start of the range
- * @end: end of the range
- * @outside_label: jump here if @reg is outside of [@start..@end)
- */
- .macro OUTSIDE reg,start,end,outside_label
- lgr %r14,\reg
- larl %r13,\start
- slgr %r14,%r13
- clgfrl %r14,.Lrange_size\@
- jhe \outside_label
- .section .rodata, "a"
- .balign 4
-.Lrange_size\@:
- .long \end - \start
- .previous
- .endm
-
- .macro SIEEXIT
- lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer
+ .macro SIEEXIT sie_control
+ lg %r9,\sie_control # get control block pointer
ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce
+ ni __LC_CPU_FLAGS+7,255-_CIF_SIE
larl %r9,sie_exit # skip forward to sie_exit
.endm
#endif
@@ -171,13 +149,13 @@ _LPP_OFFSET = __LC_LPP
nop 0
/*
- * Scheduler resume function, called by switch_to
- * gpr2 = (task_struct *) prev
- * gpr3 = (task_struct *) next
+ * Scheduler resume function, called by __switch_to
+ * gpr2 = (task_struct *)prev
+ * gpr3 = (task_struct *)next
* Returns:
* gpr2 = prev
*/
-SYM_FUNC_START(__switch_to)
+SYM_FUNC_START(__switch_to_asm)
stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
lghi %r4,__TASK_stack
lghi %r1,__TASK_thread
@@ -193,7 +171,7 @@ SYM_FUNC_START(__switch_to)
lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
BR_EX %r14
-SYM_FUNC_END(__switch_to)
+SYM_FUNC_END(__switch_to_asm)
#if IS_ENABLED(CONFIG_KVM)
/*
@@ -214,14 +192,13 @@ SYM_FUNC_START(__sie64a)
lg %r14,__LC_GMAP # get gmap pointer
ltgr %r14,%r14
jz .Lsie_gmap
+ oi __LC_CPU_FLAGS+7,_CIF_SIE
lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
.Lsie_gmap:
lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
tm __SIE_PROG20+3(%r14),3 # last exit...
jnz .Lsie_skip
- TSTMSK __LC_CPU_FLAGS,_CIF_FPU
- jo .Lsie_skip # exit if fp/vx regs changed
lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr
BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
.Lsie_entry:
@@ -236,7 +213,7 @@ SYM_FUNC_START(__sie64a)
lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce
-.Lsie_done:
+ ni __LC_CPU_FLAGS+7,255-_CIF_SIE
# some program checks are suppressing. C code (e.g. do_protection_exception)
# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
@@ -339,20 +316,13 @@ SYM_CODE_START(pgm_check_handler)
stpt __LC_SYS_ENTER_TIMER
BPOFF
stmg %r8,%r15,__LC_SAVE_AREA_SYNC
- lghi %r10,0
+ lgr %r10,%r15
lmg %r8,%r9,__LC_PGM_OLD_PSW
tmhh %r8,0x0001 # coming from user space?
jno .Lpgm_skip_asce
lctlg %c1,%c1,__LC_KERNEL_ASCE
j 3f # -> fault in user space
.Lpgm_skip_asce:
-#if IS_ENABLED(CONFIG_KVM)
- # cleanup critical section for program checks in __sie64a
- OUTSIDE %r9,.Lsie_gmap,.Lsie_done,1f
- BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
- SIEEXIT
- lghi %r10,_PIF_GUEST_FAULT
-#endif
1: tmhh %r8,0x4000 # PER bit set in old PSW ?
jnz 2f # -> enabled, can't be a double fault
tm __LC_PGM_ILC+3,0x80 # check for per exception
@@ -363,13 +333,21 @@ SYM_CODE_START(pgm_check_handler)
CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
3: lg %r15,__LC_KERNEL_STACK
4: la %r11,STACK_FRAME_OVERHEAD(%r15)
- stg %r10,__PT_FLAGS(%r11)
+ xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
stmg %r0,%r7,__PT_R0(%r11)
mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK
- stmg %r8,%r9,__PT_PSW(%r11)
-
+ stctg %c1,%c1,__PT_CR1(%r11)
+#if IS_ENABLED(CONFIG_KVM)
+ ltg %r12,__LC_GMAP
+ jz 5f
+ clc __GMAP_ASCE(8,%r12), __PT_CR1(%r11)
+ jne 5f
+ BPENTER __SF_SIE_FLAGS(%r10),_TIF_ISOLATE_BP_GUEST
+ SIEEXIT __SF_SIE_CONTROL(%r10)
+#endif
+5: stmg %r8,%r9,__PT_PSW(%r11)
# clear user controlled registers to prevent speculative use
xgr %r0,%r0
xgr %r1,%r1
@@ -418,9 +396,10 @@ SYM_CODE_START(\name)
tmhh %r8,0x0001 # interrupting from user ?
jnz 1f
#if IS_ENABLED(CONFIG_KVM)
- OUTSIDE %r9,.Lsie_gmap,.Lsie_done,0f
+ TSTMSK __LC_CPU_FLAGS,_CIF_SIE
+ jz 0f
BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
- SIEEXIT
+ SIEEXIT __SF_SIE_CONTROL(%r15)
#endif
0: CHECK_STACK __LC_SAVE_AREA_ASYNC
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
@@ -489,16 +468,11 @@ SYM_FUNC_END(psw_idle)
*/
SYM_CODE_START(mcck_int_handler)
BPOFF
- la %r1,4095 # validate r1
- spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer
- LBEAR __LC_LAST_BREAK_SAVE_AREA-4095(%r1) # validate bear
- lmg %r0,%r15,__LC_GPREGS_SAVE_AREA # validate gprs
lmg %r8,%r9,__LC_MCK_OLD_PSW
TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
jo .Lmcck_panic # yes -> rest of mcck code invalid
TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID
jno .Lmcck_panic # control registers invalid -> panic
- lctlg %c0,%c15,__LC_CREGS_SAVE_AREA # validate ctl regs
ptlb
lghi %r14,__LC_CPU_TIMER_SAVE_AREA
mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
@@ -520,11 +494,20 @@ SYM_CODE_START(mcck_int_handler)
TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
jno .Lmcck_panic
#if IS_ENABLED(CONFIG_KVM)
- OUTSIDE %r9,.Lsie_gmap,.Lsie_done,.Lmcck_user
- OUTSIDE %r9,.Lsie_entry,.Lsie_leave,4f
+ TSTMSK __LC_CPU_FLAGS,_CIF_SIE
+ jz .Lmcck_user
+ # Need to compare the address instead of a CIF_SIE* flag.
+ # Otherwise there would be a race between setting the flag
+ # and entering SIE (or leaving and clearing the flag). This
+ # would cause machine checks targeted at the guest to be
+ # handled by the host.
+ larl %r14,.Lsie_entry
+ clgrjl %r9,%r14, 4f
+ larl %r14,.Lsie_leave
+ clgrjhe %r9,%r14, 4f
oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
4: BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
- SIEEXIT
+ SIEEXIT __SF_SIE_CONTROL(%r15)
#endif
.Lmcck_user:
lg %r15,__LC_MCCK_STACK