1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
|
/*
* Copyright (c) 2017-2022, ARM Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <arch.h>
#include <asm_macros.S>
#include <context.h>
#include <services/arm_arch_svc.h>
.globl wa_cve_2017_5715_bpiall_vbar
#define EMIT_BPIALL 0xee070fd5
#define EMIT_SMC 0xe1600070
#define ESR_EL3_A64_SMC0 0x5e000000
.macro apply_cve_2017_5715_wa _from_vector
/*
* Save register state to enable a call to AArch32 S-EL1 and return
* Identify the original calling vector in w2 (==_from_vector)
* Use w3-w6 for additional register state preservation while in S-EL1
*/
/* Save GP regs */
stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
stp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6]
stp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8]
stp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10]
stp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12]
stp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14]
stp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16]
stp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18]
stp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20]
stp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22]
stp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24]
stp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26]
stp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28]
/* Identify the original exception vector */
mov w2, \_from_vector
/* Preserve 32-bit system registers in GP registers through the workaround */
mrs x3, esr_el3
mrs x4, spsr_el3
mrs x5, scr_el3
mrs x6, sctlr_el1
/*
* Preserve LR and ELR_EL3 registers in the GP regs context.
* Temporarily use the CTX_GPREG_SP_EL0 slot to preserve ELR_EL3
* through the workaround. This is OK because at this point the
* current state for this context's SP_EL0 is in the live system
* register, which is unmodified by the workaround.
*/
mrs x7, elr_el3
stp x30, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
/*
* Load system registers for entry to S-EL1.
*/
/* Mask all interrupts and set AArch32 Supervisor mode */
movz w8, SPSR_MODE32(MODE32_svc, SPSR_T_ARM, SPSR_E_LITTLE, SPSR_AIF_MASK)
/* Switch EL3 exception vectors while the workaround is executing. */
adr x9, wa_cve_2017_5715_bpiall_ret_vbar
/* Setup SCTLR_EL1 with MMU off and I$ on */
ldr x10, stub_sel1_sctlr
/* Land at the S-EL1 workaround stub */
adr x11, aarch32_stub
/*
* Setting SCR_EL3 to all zeroes means that the NS, RW
* and SMD bits are configured as expected.
*/
msr scr_el3, xzr
msr spsr_el3, x8
msr vbar_el3, x9
msr sctlr_el1, x10
msr elr_el3, x11
eret
.endm
/* ---------------------------------------------------------------------
* This vector table is used at runtime to enter the workaround at
* AArch32 S-EL1 for Sync/IRQ/FIQ/SError exceptions. If the workaround
* is not enabled, the existing runtime exception vector table is used.
* ---------------------------------------------------------------------
*/
vector_base wa_cve_2017_5715_bpiall_vbar
/* ---------------------------------------------------------------------
* Current EL with SP_EL0 : 0x0 - 0x200
* ---------------------------------------------------------------------
*/
vector_entry bpiall_sync_exception_sp_el0
b sync_exception_sp_el0
nop /* to force 8 byte alignment for the following stub */
/*
* Since each vector table entry is 128 bytes, we can store the
* stub context in the unused space to minimize memory footprint.
*/
stub_sel1_sctlr:
.quad SCTLR_AARCH32_EL1_RES1 | SCTLR_I_BIT
aarch32_stub:
.word EMIT_BPIALL
.word EMIT_SMC
end_vector_entry bpiall_sync_exception_sp_el0
vector_entry bpiall_irq_sp_el0
b irq_sp_el0
end_vector_entry bpiall_irq_sp_el0
vector_entry bpiall_fiq_sp_el0
b fiq_sp_el0
end_vector_entry bpiall_fiq_sp_el0
vector_entry bpiall_serror_sp_el0
b serror_sp_el0
end_vector_entry bpiall_serror_sp_el0
/* ---------------------------------------------------------------------
* Current EL with SP_ELx: 0x200 - 0x400
* ---------------------------------------------------------------------
*/
vector_entry bpiall_sync_exception_sp_elx
b sync_exception_sp_elx
end_vector_entry bpiall_sync_exception_sp_elx
vector_entry bpiall_irq_sp_elx
b irq_sp_elx
end_vector_entry bpiall_irq_sp_elx
vector_entry bpiall_fiq_sp_elx
b fiq_sp_elx
end_vector_entry bpiall_fiq_sp_elx
vector_entry bpiall_serror_sp_elx
b serror_sp_elx
end_vector_entry bpiall_serror_sp_elx
/* ---------------------------------------------------------------------
* Lower EL using AArch64 : 0x400 - 0x600
* ---------------------------------------------------------------------
*/
vector_entry bpiall_sync_exception_aarch64
apply_cve_2017_5715_wa 1
end_vector_entry bpiall_sync_exception_aarch64
vector_entry bpiall_irq_aarch64
apply_cve_2017_5715_wa 2
end_vector_entry bpiall_irq_aarch64
vector_entry bpiall_fiq_aarch64
apply_cve_2017_5715_wa 4
end_vector_entry bpiall_fiq_aarch64
vector_entry bpiall_serror_aarch64
apply_cve_2017_5715_wa 8
end_vector_entry bpiall_serror_aarch64
/* ---------------------------------------------------------------------
* Lower EL using AArch32 : 0x600 - 0x800
* ---------------------------------------------------------------------
*/
vector_entry bpiall_sync_exception_aarch32
apply_cve_2017_5715_wa 1
end_vector_entry bpiall_sync_exception_aarch32
vector_entry bpiall_irq_aarch32
apply_cve_2017_5715_wa 2
end_vector_entry bpiall_irq_aarch32
vector_entry bpiall_fiq_aarch32
apply_cve_2017_5715_wa 4
end_vector_entry bpiall_fiq_aarch32
vector_entry bpiall_serror_aarch32
apply_cve_2017_5715_wa 8
end_vector_entry bpiall_serror_aarch32
/* ---------------------------------------------------------------------
* This vector table is used while the workaround is executing. It
* installs a simple SMC handler to allow the Sync/IRQ/FIQ/SError
* workaround stubs to enter EL3 from S-EL1. It restores the previous
* EL3 state before proceeding with the normal runtime exception vector.
* ---------------------------------------------------------------------
*/
vector_base wa_cve_2017_5715_bpiall_ret_vbar
/* ---------------------------------------------------------------------
* Current EL with SP_EL0 : 0x0 - 0x200 (UNUSED)
* ---------------------------------------------------------------------
*/
vector_entry bpiall_ret_sync_exception_sp_el0
b report_unhandled_exception
end_vector_entry bpiall_ret_sync_exception_sp_el0
vector_entry bpiall_ret_irq_sp_el0
b report_unhandled_interrupt
end_vector_entry bpiall_ret_irq_sp_el0
vector_entry bpiall_ret_fiq_sp_el0
b report_unhandled_interrupt
end_vector_entry bpiall_ret_fiq_sp_el0
vector_entry bpiall_ret_serror_sp_el0
b report_unhandled_exception
end_vector_entry bpiall_ret_serror_sp_el0
/* ---------------------------------------------------------------------
* Current EL with SP_ELx: 0x200 - 0x400 (UNUSED)
* ---------------------------------------------------------------------
*/
vector_entry bpiall_ret_sync_exception_sp_elx
b report_unhandled_exception
end_vector_entry bpiall_ret_sync_exception_sp_elx
vector_entry bpiall_ret_irq_sp_elx
b report_unhandled_interrupt
end_vector_entry bpiall_ret_irq_sp_elx
vector_entry bpiall_ret_fiq_sp_elx
b report_unhandled_interrupt
end_vector_entry bpiall_ret_fiq_sp_elx
vector_entry bpiall_ret_serror_sp_elx
b report_unhandled_exception
end_vector_entry bpiall_ret_serror_sp_elx
/* ---------------------------------------------------------------------
* Lower EL using AArch64 : 0x400 - 0x600 (UNUSED)
* ---------------------------------------------------------------------
*/
vector_entry bpiall_ret_sync_exception_aarch64
b report_unhandled_exception
end_vector_entry bpiall_ret_sync_exception_aarch64
vector_entry bpiall_ret_irq_aarch64
b report_unhandled_interrupt
end_vector_entry bpiall_ret_irq_aarch64
vector_entry bpiall_ret_fiq_aarch64
b report_unhandled_interrupt
end_vector_entry bpiall_ret_fiq_aarch64
vector_entry bpiall_ret_serror_aarch64
b report_unhandled_exception
end_vector_entry bpiall_ret_serror_aarch64
/* ---------------------------------------------------------------------
* Lower EL using AArch32 : 0x600 - 0x800
* ---------------------------------------------------------------------
*/
vector_entry bpiall_ret_sync_exception_aarch32
/*
* w2 indicates which SEL1 stub was run and thus which original vector was used
* w3-w6 contain saved system register state (esr_el3 in w3)
* Restore LR and ELR_EL3 register state from the GP regs context
*/
ldp x30, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
/* Apply the restored system register state */
msr esr_el3, x3
msr spsr_el3, x4
msr scr_el3, x5
msr sctlr_el1, x6
msr elr_el3, x7
/*
* Workaround is complete, so swap VBAR_EL3 to point
* to workaround entry table in preparation for subsequent
* Sync/IRQ/FIQ/SError exceptions.
*/
adr x0, wa_cve_2017_5715_bpiall_vbar
msr vbar_el3, x0
/*
* Restore all GP regs except x2 and x3 (esr). The value in x2
* indicates the type of the original exception.
*/
ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
ldp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6]
ldp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8]
ldp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10]
ldp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12]
ldp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14]
ldp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16]
ldp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18]
ldp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20]
ldp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22]
ldp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24]
ldp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26]
ldp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28]
/* Fast path Sync exceptions. Static predictor will fall through. */
tbz w2, #0, workaround_not_sync
/*
* Check if SMC is coming from A64 state on #0
* with W0 = SMCCC_ARCH_WORKAROUND_1 or W0 = SMCCC_ARCH_WORKAROUND_3
*
* This sequence evaluates as:
* (W0==SMCCC_ARCH_WORKAROUND_1) || (W0==SMCCC_ARCH_WORKAROUND_3) ?
* (ESR_EL3==SMC#0) : (NE)
* allowing use of a single branch operation
*/
orr w2, wzr, #SMCCC_ARCH_WORKAROUND_1
cmp w0, w2
orr w2, wzr, #SMCCC_ARCH_WORKAROUND_3
ccmp w0, w2, #4, ne
mov_imm w2, ESR_EL3_A64_SMC0
ccmp w3, w2, #0, eq
/* Static predictor will predict a fall through */
bne 1f
eret
1:
/* restore x2 and x3 and continue sync exception handling */
b bpiall_ret_sync_exception_aarch32_tail
end_vector_entry bpiall_ret_sync_exception_aarch32
vector_entry bpiall_ret_irq_aarch32
b report_unhandled_interrupt
/*
* Post-workaround fan-out for non-sync exceptions
*/
workaround_not_sync:
tbnz w2, #3, bpiall_ret_serror
tbnz w2, #2, bpiall_ret_fiq
/* IRQ */
ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
b irq_aarch64
bpiall_ret_fiq:
ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
b fiq_aarch64
bpiall_ret_serror:
ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
b serror_aarch64
end_vector_entry bpiall_ret_irq_aarch32
vector_entry bpiall_ret_fiq_aarch32
b report_unhandled_interrupt
end_vector_entry bpiall_ret_fiq_aarch32
vector_entry bpiall_ret_serror_aarch32
b report_unhandled_exception
end_vector_entry bpiall_ret_serror_aarch32
/*
* Part of bpiall_ret_sync_exception_aarch32 to save vector space
*/
func bpiall_ret_sync_exception_aarch32_tail
ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
b sync_exception_aarch64
endfunc bpiall_ret_sync_exception_aarch32_tail
|
