1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
|
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2017 Arm Ltd.
#define pr_fmt(fmt) "sdei: " fmt
#include <linux/arm-smccc.h>
#include <linux/arm_sdei.h>
#include <linux/hardirq.h>
#include <linux/irqflags.h>
#include <linux/sched/task_stack.h>
#include <linux/uaccess.h>
#include <asm/alternative.h>
#include <asm/exception.h>
#include <asm/kprobes.h>
#include <asm/mmu.h>
#include <asm/ptrace.h>
#include <asm/sections.h>
#include <asm/stacktrace.h>
#include <asm/sysreg.h>
#include <asm/vmap_stack.h>
unsigned long sdei_exit_mode;
/*
* VMAP'd stacks checking for stack overflow on exception using sp as a scratch
* register, meaning SDEI has to switch to its own stack. We need two stacks as
* a critical event may interrupt a normal event that has just taken a
* synchronous exception, and is using sp as scratch register. For a critical
* event interrupting a normal event, we can't reliably tell if we were on the
* sdei stack.
* For now, we allocate stacks when the driver is probed.
*/
DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
#ifdef CONFIG_VMAP_STACK
DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
#endif
DEFINE_PER_CPU(struct sdei_registered_event *, sdei_active_normal_event);
DEFINE_PER_CPU(struct sdei_registered_event *, sdei_active_critical_event);
static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
{
unsigned long *p;
p = per_cpu(*ptr, cpu);
if (p) {
per_cpu(*ptr, cpu) = NULL;
vfree(p);
}
}
static void free_sdei_stacks(void)
{
int cpu;
for_each_possible_cpu(cpu) {
_free_sdei_stack(&sdei_stack_normal_ptr, cpu);
_free_sdei_stack(&sdei_stack_critical_ptr, cpu);
}
}
static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
{
unsigned long *p;
p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
if (!p)
return -ENOMEM;
per_cpu(*ptr, cpu) = p;
return 0;
}
static int init_sdei_stacks(void)
{
int cpu;
int err = 0;
for_each_possible_cpu(cpu) {
err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
if (err)
break;
err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
if (err)
break;
}
if (err)
free_sdei_stacks();
return err;
}
static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
{
unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
unsigned long high = low + SDEI_STACK_SIZE;
return on_stack(sp, low, high, STACK_TYPE_SDEI_NORMAL, info);
}
static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info)
{
unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
unsigned long high = low + SDEI_STACK_SIZE;
return on_stack(sp, low, high, STACK_TYPE_SDEI_CRITICAL, info);
}
bool _on_sdei_stack(unsigned long sp, struct stack_info *info)
{
if (!IS_ENABLED(CONFIG_VMAP_STACK))
return false;
if (on_sdei_critical_stack(sp, info))
return true;
if (on_sdei_normal_stack(sp, info))
return true;
return false;
}
unsigned long sdei_arch_get_entry_point(int conduit)
{
/*
* SDEI works between adjacent exception levels. If we booted at EL1 we
* assume a hypervisor is marshalling events. If we booted at EL2 and
* dropped to EL1 because we don't support VHE, then we can't support
* SDEI.
*/
if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
pr_err("Not supported on this hardware/boot configuration\n");
return 0;
}
if (IS_ENABLED(CONFIG_VMAP_STACK)) {
if (init_sdei_stacks())
return 0;
}
sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
if (arm64_kernel_unmapped_at_el0()) {
unsigned long offset;
offset = (unsigned long)__sdei_asm_entry_trampoline -
(unsigned long)__entry_tramp_text_start;
return TRAMP_VALIAS + offset;
} else
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
return (unsigned long)__sdei_asm_handler;
}
/*
* __sdei_handler() returns one of:
* SDEI_EV_HANDLED - success, return to the interrupted context.
* SDEI_EV_FAILED - failure, return this error code to firmare.
* virtual-address - success, return to this address.
*/
static __kprobes unsigned long _sdei_handler(struct pt_regs *regs,
struct sdei_registered_event *arg)
{
u32 mode;
int i, err = 0;
int clobbered_registers = 4;
u64 elr = read_sysreg(elr_el1);
u32 kernel_mode = read_sysreg(CurrentEL) | 1; /* +SPSel */
unsigned long vbar = read_sysreg(vbar_el1);
if (arm64_kernel_unmapped_at_el0())
clobbered_registers++;
/* Retrieve the missing registers values */
for (i = 0; i < clobbered_registers; i++) {
/* from within the handler, this call always succeeds */
sdei_api_event_context(i, ®s->regs[i]);
}
/*
* We didn't take an exception to get here, set PAN. UAO will be cleared
* by sdei_event_handler()s force_uaccess_begin() call.
*/
__uaccess_enable_hw_pan();
err = sdei_event_handler(regs, arg);
if (err)
return SDEI_EV_FAILED;
if (elr != read_sysreg(elr_el1)) {
/*
* We took a synchronous exception from the SDEI handler.
* This could deadlock, and if you interrupt KVM it will
* hyp-panic instead.
*/
pr_warn("unsafe: exception during handler\n");
}
mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);
/*
* If we interrupted the kernel with interrupts masked, we always go
* back to wherever we came from.
*/
if (mode == kernel_mode && !interrupts_enabled(regs))
return SDEI_EV_HANDLED;
/*
* Otherwise, we pretend this was an IRQ. This lets user space tasks
* receive signals before we return to them, and KVM to invoke it's
* world switch to do the same.
*
* See DDI0487B.a Table D1-7 'Vector offsets from vector table base
* address'.
*/
if (mode == kernel_mode)
return vbar + 0x280;
else if (mode & PSR_MODE32_BIT)
return vbar + 0x680;
return vbar + 0x480;
}
asmlinkage noinstr unsigned long
__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
{
unsigned long ret;
arm64_enter_nmi(regs);
ret = _sdei_handler(regs, arg);
arm64_exit_nmi(regs);
return ret;
}
|
