diff options
Diffstat (limited to 'arch/s390/kernel/entry.S')
| -rw-r--r-- | arch/s390/kernel/entry.S | 93 |
1 files changed, 38 insertions, 55 deletions
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 26c08ee877..6a1e0fbbaa 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -24,7 +24,7 @@ #include <asm/page.h> #include <asm/sigp.h> #include <asm/irq.h> -#include <asm/vx-insn.h> +#include <asm/fpu-insn.h> #include <asm/setup.h> #include <asm/nmi.h> #include <asm/nospec-insn.h> @@ -119,33 +119,11 @@ _LPP_OFFSET = __LC_LPP .endm #if IS_ENABLED(CONFIG_KVM) - /* - * The OUTSIDE macro jumps to the provided label in case the value - * in the provided register is outside of the provided range. The - * macro is useful for checking whether a PSW stored in a register - * pair points inside or outside of a block of instructions. - * @reg: register to check - * @start: start of the range - * @end: end of the range - * @outside_label: jump here if @reg is outside of [@start..@end) - */ - .macro OUTSIDE reg,start,end,outside_label - lgr %r14,\reg - larl %r13,\start - slgr %r14,%r13 - clgfrl %r14,.Lrange_size\@ - jhe \outside_label - .section .rodata, "a" - .balign 4 -.Lrange_size\@: - .long \end - \start - .previous - .endm - - .macro SIEEXIT - lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer + .macro SIEEXIT sie_control + lg %r9,\sie_control # get control block pointer ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce + ni __LC_CPU_FLAGS+7,255-_CIF_SIE larl %r9,sie_exit # skip forward to sie_exit .endm #endif @@ -171,13 +149,13 @@ _LPP_OFFSET = __LC_LPP nop 0 /* - * Scheduler resume function, called by switch_to - * gpr2 = (task_struct *) prev - * gpr3 = (task_struct *) next + * Scheduler resume function, called by __switch_to + * gpr2 = (task_struct *)prev + * gpr3 = (task_struct *)next * Returns: * gpr2 = prev */ -SYM_FUNC_START(__switch_to) +SYM_FUNC_START(__switch_to_asm) stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task lghi %r4,__TASK_stack lghi %r1,__TASK_thread @@ -193,7 +171,7 @@ SYM_FUNC_START(__switch_to) lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40 BR_EX %r14 -SYM_FUNC_END(__switch_to) +SYM_FUNC_END(__switch_to_asm) #if IS_ENABLED(CONFIG_KVM) /* @@ -214,14 +192,13 @@ SYM_FUNC_START(__sie64a) lg %r14,__LC_GMAP # get gmap pointer ltgr %r14,%r14 jz .Lsie_gmap + oi __LC_CPU_FLAGS+7,_CIF_SIE lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce .Lsie_gmap: lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now tm __SIE_PROG20+3(%r14),3 # last exit... jnz .Lsie_skip - TSTMSK __LC_CPU_FLAGS,_CIF_FPU - jo .Lsie_skip # exit if fp/vx regs changed lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST .Lsie_entry: @@ -236,7 +213,7 @@ SYM_FUNC_START(__sie64a) lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce -.Lsie_done: + ni __LC_CPU_FLAGS+7,255-_CIF_SIE # some program checks are suppressing. C code (e.g. do_protection_exception) # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There # are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable. @@ -339,20 +316,13 @@ SYM_CODE_START(pgm_check_handler) stpt __LC_SYS_ENTER_TIMER BPOFF stmg %r8,%r15,__LC_SAVE_AREA_SYNC - lghi %r10,0 + lgr %r10,%r15 lmg %r8,%r9,__LC_PGM_OLD_PSW tmhh %r8,0x0001 # coming from user space? jno .Lpgm_skip_asce lctlg %c1,%c1,__LC_KERNEL_ASCE j 3f # -> fault in user space .Lpgm_skip_asce: -#if IS_ENABLED(CONFIG_KVM) - # cleanup critical section for program checks in __sie64a - OUTSIDE %r9,.Lsie_gmap,.Lsie_done,1f - BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST - SIEEXIT - lghi %r10,_PIF_GUEST_FAULT -#endif 1: tmhh %r8,0x4000 # PER bit set in old PSW ? jnz 2f # -> enabled, can't be a double fault tm __LC_PGM_ILC+3,0x80 # check for per exception @@ -363,13 +333,21 @@ SYM_CODE_START(pgm_check_handler) CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f 3: lg %r15,__LC_KERNEL_STACK 4: la %r11,STACK_FRAME_OVERHEAD(%r15) - stg %r10,__PT_FLAGS(%r11) + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) stmg %r0,%r7,__PT_R0(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK - stmg %r8,%r9,__PT_PSW(%r11) - + stctg %c1,%c1,__PT_CR1(%r11) +#if IS_ENABLED(CONFIG_KVM) + ltg %r12,__LC_GMAP + jz 5f + clc __GMAP_ASCE(8,%r12), __PT_CR1(%r11) + jne 5f + BPENTER __SF_SIE_FLAGS(%r10),_TIF_ISOLATE_BP_GUEST + SIEEXIT __SF_SIE_CONTROL(%r10) +#endif +5: stmg %r8,%r9,__PT_PSW(%r11) # clear user controlled registers to prevent speculative use xgr %r0,%r0 xgr %r1,%r1 @@ -418,9 +396,10 @@ SYM_CODE_START(\name) tmhh %r8,0x0001 # interrupting from user ? jnz 1f #if IS_ENABLED(CONFIG_KVM) - OUTSIDE %r9,.Lsie_gmap,.Lsie_done,0f + TSTMSK __LC_CPU_FLAGS,_CIF_SIE + jz 0f BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST - SIEEXIT + SIEEXIT __SF_SIE_CONTROL(%r15) #endif 0: CHECK_STACK __LC_SAVE_AREA_ASYNC aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) @@ -489,16 +468,11 @@ SYM_FUNC_END(psw_idle) */ SYM_CODE_START(mcck_int_handler) BPOFF - la %r1,4095 # validate r1 - spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer - LBEAR __LC_LAST_BREAK_SAVE_AREA-4095(%r1) # validate bear - lmg %r0,%r15,__LC_GPREGS_SAVE_AREA # validate gprs lmg %r8,%r9,__LC_MCK_OLD_PSW TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE jo .Lmcck_panic # yes -> rest of mcck code invalid TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID jno .Lmcck_panic # control registers invalid -> panic - lctlg %c0,%c15,__LC_CREGS_SAVE_AREA # validate ctl regs ptlb lghi %r14,__LC_CPU_TIMER_SAVE_AREA mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) @@ -520,11 +494,20 @@ SYM_CODE_START(mcck_int_handler) TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID jno .Lmcck_panic #if IS_ENABLED(CONFIG_KVM) - OUTSIDE %r9,.Lsie_gmap,.Lsie_done,.Lmcck_user - OUTSIDE %r9,.Lsie_entry,.Lsie_leave,4f + TSTMSK __LC_CPU_FLAGS,_CIF_SIE + jz .Lmcck_user + # Need to compare the address instead of a CIF_SIE* flag. + # Otherwise there would be a race between setting the flag + # and entering SIE (or leaving and clearing the flag). This + # would cause machine checks targeted at the guest to be + # handled by the host. + larl %r14,.Lsie_entry + clgrjl %r9,%r14, 4f + larl %r14,.Lsie_leave + clgrjhe %r9,%r14, 4f oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST 4: BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST - SIEEXIT + SIEEXIT __SF_SIE_CONTROL(%r15) #endif .Lmcck_user: lg %r15,__LC_MCCK_STACK |