summaryrefslogtreecommitdiffstats
path: root/bl31/ehf.c
blob: b328380d1c7902290501766282f2546232cf828f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
/*
 * Copyright (c) 2017-2022, ARM Limited and Contributors. All rights reserved.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */

/*
 * Exception handlers at EL3, their priority levels, and management.
 */

#include <assert.h>
#include <stdbool.h>

#include <bl31/ehf.h>
#include <bl31/interrupt_mgmt.h>
#include <context.h>
#include <common/debug.h>
#include <drivers/arm/gic_common.h>
#include <lib/el3_runtime/context_mgmt.h>
#include <lib/el3_runtime/cpu_data.h>
#include <lib/el3_runtime/pubsub_events.h>
#include <plat/common/platform.h>

/* Output EHF logs as verbose */
#define EHF_LOG(...)	VERBOSE("EHF: " __VA_ARGS__)

#define EHF_INVALID_IDX	(-1)

/* For a valid handler, return the actual function pointer; otherwise, 0. */
#define RAW_HANDLER(h) \
	((ehf_handler_t) ((((h) & EHF_PRI_VALID_) != 0U) ? \
		((h) & ~EHF_PRI_VALID_) : 0U))

#define PRI_BIT(idx)	(((ehf_pri_bits_t) 1u) << (idx))

/*
 * Convert index into secure priority using the platform-defined priority bits
 * field.
 */
#define IDX_TO_PRI(idx) \
	((((unsigned) idx) << (7u - exception_data.pri_bits)) & 0x7fU)

/* Check whether a given index is valid */
#define IS_IDX_VALID(idx) \
	((exception_data.ehf_priorities[idx].ehf_handler & EHF_PRI_VALID_) != 0U)

/* Returns whether given priority is in secure priority range */
#define IS_PRI_SECURE(pri)	(((pri) & 0x80U) == 0U)

/* To be defined by the platform */
extern const ehf_priorities_t exception_data;

/* Translate priority to the index in the priority array */
static unsigned int pri_to_idx(unsigned int priority)
{
	unsigned int idx;

	idx = EHF_PRI_TO_IDX(priority, exception_data.pri_bits);
	assert(idx < exception_data.num_priorities);
	assert(IS_IDX_VALID(idx));

	return idx;
}

/* Return whether there are outstanding priority activation */
static bool has_valid_pri_activations(pe_exc_data_t *pe_data)
{
	return pe_data->active_pri_bits != 0U;
}

static pe_exc_data_t *this_cpu_data(void)
{
	return &get_cpu_data(ehf_data);
}

/*
 * Return the current priority index of this CPU. If no priority is active,
 * return EHF_INVALID_IDX.
 */
static int get_pe_highest_active_idx(pe_exc_data_t *pe_data)
{
	if (!has_valid_pri_activations(pe_data))
		return EHF_INVALID_IDX;

	/* Current priority is the right-most bit */
	return (int) __builtin_ctz(pe_data->active_pri_bits);
}

/*
 * Mark priority active by setting the corresponding bit in active_pri_bits and
 * programming the priority mask.
 *
 * This API is to be used as part of delegating to lower ELs other than for
 * interrupts; e.g. while handling synchronous exceptions.
 *
 * This API is expected to be invoked before restoring context (Secure or
 * Non-secure) in preparation for the respective dispatch.
 */
void ehf_activate_priority(unsigned int priority)
{
	int cur_pri_idx;
	unsigned int old_mask, run_pri, idx;
	pe_exc_data_t *pe_data = this_cpu_data();

	/*
	 * Query interrupt controller for the running priority, or idle priority
	 * if no interrupts are being handled. The requested priority must be
	 * less (higher priority) than the active running priority.
	 */
	run_pri = plat_ic_get_running_priority();
	if (priority >= run_pri) {
		ERROR("Running priority higher (0x%x) than requested (0x%x)\n",
				run_pri, priority);
		panic();
	}

	/*
	 * If there were priority activations already, the requested priority
	 * must be less (higher priority) than the current highest priority
	 * activation so far.
	 */
	cur_pri_idx = get_pe_highest_active_idx(pe_data);
	idx = pri_to_idx(priority);
	if ((cur_pri_idx != EHF_INVALID_IDX) &&
			(idx >= ((unsigned int) cur_pri_idx))) {
		ERROR("Activation priority mismatch: req=0x%x current=0x%x\n",
				priority, IDX_TO_PRI(cur_pri_idx));
		panic();
	}

	/* Set the bit corresponding to the requested priority */
	pe_data->active_pri_bits |= PRI_BIT(idx);

	/*
	 * Program priority mask for the activated level. Check that the new
	 * priority mask is setting a higher priority level than the existing
	 * mask.
	 */
	old_mask = plat_ic_set_priority_mask(priority);
	if (priority >= old_mask) {
		ERROR("Requested priority (0x%x) lower than Priority Mask (0x%x)\n",
				priority, old_mask);
		panic();
	}

	/*
	 * If this is the first activation, save the priority mask. This will be
	 * restored after the last deactivation.
	 */
	if (cur_pri_idx == EHF_INVALID_IDX)
		pe_data->init_pri_mask = (uint8_t) old_mask;

	EHF_LOG("activate prio=%d\n", get_pe_highest_active_idx(pe_data));
}

/*
 * Mark priority inactive by clearing the corresponding bit in active_pri_bits,
 * and programming the priority mask.
 *
 * This API is expected to be used as part of delegating to to lower ELs other
 * than for interrupts; e.g. while handling synchronous exceptions.
 *
 * This API is expected to be invoked after saving context (Secure or
 * Non-secure), having concluded the respective dispatch.
 */
void ehf_deactivate_priority(unsigned int priority)
{
	int cur_pri_idx;
	pe_exc_data_t *pe_data = this_cpu_data();
	unsigned int old_mask, run_pri, idx;

	/*
	 * Query interrupt controller for the running priority, or idle priority
	 * if no interrupts are being handled. The requested priority must be
	 * less (higher priority) than the active running priority.
	 */
	run_pri = plat_ic_get_running_priority();
	if (priority >= run_pri) {
		ERROR("Running priority higher (0x%x) than requested (0x%x)\n",
				run_pri, priority);
		panic();
	}

	/*
	 * Deactivation is allowed only when there are priority activations, and
	 * the deactivation priority level must match the current activated
	 * priority.
	 */
	cur_pri_idx = get_pe_highest_active_idx(pe_data);
	idx = pri_to_idx(priority);
	if ((cur_pri_idx == EHF_INVALID_IDX) ||
			(idx != ((unsigned int) cur_pri_idx))) {
		ERROR("Deactivation priority mismatch: req=0x%x current=0x%x\n",
				priority, IDX_TO_PRI(cur_pri_idx));
		panic();
	}

	/* Clear bit corresponding to highest priority */
	pe_data->active_pri_bits &= (pe_data->active_pri_bits - 1u);

	/*
	 * Restore priority mask corresponding to the next priority, or the
	 * one stashed earlier if there are no more to deactivate.
	 */
	cur_pri_idx = get_pe_highest_active_idx(pe_data);
	if (cur_pri_idx == EHF_INVALID_IDX)
		old_mask = plat_ic_set_priority_mask(pe_data->init_pri_mask);
	else
		old_mask = plat_ic_set_priority_mask(priority);

	if (old_mask > priority) {
		ERROR("Deactivation priority (0x%x) lower than Priority Mask (0x%x)\n",
				priority, old_mask);
		panic();
	}

	EHF_LOG("deactivate prio=%d\n", get_pe_highest_active_idx(pe_data));
}

/*
 * After leaving Non-secure world, stash current Non-secure Priority Mask, and
 * set Priority Mask to the highest Non-secure priority so that Non-secure
 * interrupts cannot preempt Secure execution.
 *
 * If the current running priority is in the secure range, or if there are
 * outstanding priority activations, this function does nothing.
 *
 * This function subscribes to the 'cm_exited_normal_world' event published by
 * the Context Management Library.
 */
static void *ehf_exited_normal_world(const void *arg)
{
	unsigned int run_pri;
	pe_exc_data_t *pe_data = this_cpu_data();

	/* If the running priority is in the secure range, do nothing */
	run_pri = plat_ic_get_running_priority();
	if (IS_PRI_SECURE(run_pri))
		return NULL;

	/* Do nothing if there are explicit activations */
	if (has_valid_pri_activations(pe_data))
		return NULL;

	assert(pe_data->ns_pri_mask == 0u);

	pe_data->ns_pri_mask =
		(uint8_t) plat_ic_set_priority_mask(GIC_HIGHEST_NS_PRIORITY);

	/* The previous Priority Mask is not expected to be in secure range */
	if (IS_PRI_SECURE(pe_data->ns_pri_mask)) {
		ERROR("Priority Mask (0x%x) already in secure range\n",
				pe_data->ns_pri_mask);
		panic();
	}

	EHF_LOG("Priority Mask: 0x%x => 0x%x\n", pe_data->ns_pri_mask,
			GIC_HIGHEST_NS_PRIORITY);

	return NULL;
}

/*
 * Conclude Secure execution and prepare for return to Non-secure world. Restore
 * the Non-secure Priority Mask previously stashed upon leaving Non-secure
 * world.
 *
 * If there the current running priority is in the secure range, or if there are
 * outstanding priority activations, this function does nothing.
 *
 * This function subscribes to the 'cm_entering_normal_world' event published by
 * the Context Management Library.
 */
static void *ehf_entering_normal_world(const void *arg)
{
	unsigned int old_pmr, run_pri;
	pe_exc_data_t *pe_data = this_cpu_data();

	/* If the running priority is in the secure range, do nothing */
	run_pri = plat_ic_get_running_priority();
	if (IS_PRI_SECURE(run_pri))
		return NULL;

	/*
	 * If there are explicit activations, do nothing. The Priority Mask will
	 * be restored upon the last deactivation.
	 */
	if (has_valid_pri_activations(pe_data))
		return NULL;

	/* Do nothing if we don't have a valid Priority Mask to restore */
	if (pe_data->ns_pri_mask == 0U)
		return NULL;

	old_pmr = plat_ic_set_priority_mask(pe_data->ns_pri_mask);

	/*
	 * When exiting secure world, the current Priority Mask must be
	 * GIC_HIGHEST_NS_PRIORITY (as set during entry), or the Non-secure
	 * priority mask set upon calling ehf_allow_ns_preemption()
	 */
	if ((old_pmr != GIC_HIGHEST_NS_PRIORITY) &&
			(old_pmr != pe_data->ns_pri_mask)) {
		ERROR("Invalid Priority Mask (0x%x) restored\n", old_pmr);
		panic();
	}

	EHF_LOG("Priority Mask: 0x%x => 0x%x\n", old_pmr, pe_data->ns_pri_mask);

	pe_data->ns_pri_mask = 0;

	return NULL;
}

/*
 * Program Priority Mask to the original Non-secure priority such that
 * Non-secure interrupts may preempt Secure execution (for example, during
 * Yielding SMC calls). The 'preempt_ret_code' parameter indicates the Yielding
 * SMC's return value in case the call was preempted.
 *
 * This API is expected to be invoked before delegating a yielding SMC to Secure
 * EL1. I.e. within the window of secure execution after Non-secure context is
 * saved (after entry into EL3) and Secure context is restored (before entering
 * Secure EL1).
 */
void ehf_allow_ns_preemption(uint64_t preempt_ret_code)
{
	cpu_context_t *ns_ctx;
	unsigned int old_pmr __unused;
	pe_exc_data_t *pe_data = this_cpu_data();

	/*
	 * We should have been notified earlier of entering secure world, and
	 * therefore have stashed the Non-secure priority mask.
	 */
	assert(pe_data->ns_pri_mask != 0U);

	/* Make sure no priority levels are active when requesting this */
	if (has_valid_pri_activations(pe_data)) {
		ERROR("PE %lx has priority activations: 0x%x\n",
				read_mpidr_el1(), pe_data->active_pri_bits);
		panic();
	}

	/*
	 * Program preempted return code to x0 right away so that, if the
	 * Yielding SMC was indeed preempted before a dispatcher gets a chance
	 * to populate it, the caller would find the correct return value.
	 */
	ns_ctx = cm_get_context(NON_SECURE);
	assert(ns_ctx != NULL);
	write_ctx_reg(get_gpregs_ctx(ns_ctx), CTX_GPREG_X0, preempt_ret_code);

	old_pmr = plat_ic_set_priority_mask(pe_data->ns_pri_mask);

	EHF_LOG("Priority Mask: 0x%x => 0x%x\n", old_pmr, pe_data->ns_pri_mask);

	pe_data->ns_pri_mask = 0;
}

/*
 * Return whether Secure execution has explicitly allowed Non-secure interrupts
 * to preempt itself (for example, during Yielding SMC calls).
 */
unsigned int ehf_is_ns_preemption_allowed(void)
{
	unsigned int run_pri;
	pe_exc_data_t *pe_data = this_cpu_data();

	/* If running priority is in secure range, return false */
	run_pri = plat_ic_get_running_priority();
	if (IS_PRI_SECURE(run_pri))
		return 0;

	/*
	 * If Non-secure preemption was permitted by calling
	 * ehf_allow_ns_preemption() earlier:
	 *
	 * - There wouldn't have been priority activations;
	 * - We would have cleared the stashed the Non-secure Priority Mask.
	 */
	if (has_valid_pri_activations(pe_data))
		return 0;
	if (pe_data->ns_pri_mask != 0U)
		return 0;

	return 1;
}

/*
 * Top-level EL3 interrupt handler.
 */
static uint64_t ehf_el3_interrupt_handler(uint32_t id, uint32_t flags,
		void *handle, void *cookie)
{
	int ret = 0;
	uint32_t intr_raw;
	unsigned int intr, pri, idx;
	ehf_handler_t handler;

	/*
	 * Top-level interrupt type handler from Interrupt Management Framework
	 * doesn't acknowledge the interrupt; so the interrupt ID must be
	 * invalid.
	 */
	assert(id == INTR_ID_UNAVAILABLE);

	/*
	 * Acknowledge interrupt. Proceed with handling only for valid interrupt
	 * IDs. This situation may arise because of Interrupt Management
	 * Framework identifying an EL3 interrupt, but before it's been
	 * acknowledged here, the interrupt was either deasserted, or there was
	 * a higher-priority interrupt of another type.
	 */
	intr_raw = plat_ic_acknowledge_interrupt();
	intr = plat_ic_get_interrupt_id(intr_raw);
	if (intr == INTR_ID_UNAVAILABLE)
		return 0;

	/* Having acknowledged the interrupt, get the running priority */
	pri = plat_ic_get_running_priority();

	/* Check EL3 interrupt priority is in secure range */
	assert(IS_PRI_SECURE(pri));

	/*
	 * Translate the priority to a descriptor index. We do this by masking
	 * and shifting the running priority value (platform-supplied).
	 */
	idx = pri_to_idx(pri);

	/* Validate priority */
	assert(pri == IDX_TO_PRI(idx));

	handler = (ehf_handler_t) RAW_HANDLER(
			exception_data.ehf_priorities[idx].ehf_handler);
	if (handler == NULL) {
		ERROR("No EL3 exception handler for priority 0x%x\n",
				IDX_TO_PRI(idx));
		panic();
	}

	/*
	 * Call registered handler. Pass the raw interrupt value to registered
	 * handlers.
	 */
	ret = handler(intr_raw, flags, handle, cookie);

	return (uint64_t) ret;
}

/*
 * Initialize the EL3 exception handling.
 */
void __init ehf_init(void)
{
	unsigned int flags = 0;
	int ret __unused;

	/* Ensure EL3 interrupts are supported */
	assert(plat_ic_has_interrupt_type(INTR_TYPE_EL3) != 0);

	/*
	 * Make sure that priority water mark has enough bits to represent the
	 * whole priority array.
	 */
	assert(exception_data.num_priorities <= (sizeof(ehf_pri_bits_t) * 8U));

	assert(exception_data.ehf_priorities != NULL);

	/*
	 * Bit 7 of GIC priority must be 0 for secure interrupts. This means
	 * platforms must use at least 1 of the remaining 7 bits.
	 */
	assert((exception_data.pri_bits >= 1U) ||
			(exception_data.pri_bits < 8U));

	/* Route EL3 interrupts when in Non-secure. */
	set_interrupt_rm_flag(flags, NON_SECURE);

	/*
	 * Route EL3 interrupts when in secure, only when SPMC is not present
	 * in S-EL2.
	 */
#if !(defined(SPD_spmd) && (SPMD_SPM_AT_SEL2 == 1))
	set_interrupt_rm_flag(flags, SECURE);
#endif /* !(defined(SPD_spmd) && (SPMD_SPM_AT_SEL2 == 1)) */

	/* Register handler for EL3 interrupts */
	ret = register_interrupt_type_handler(INTR_TYPE_EL3,
			ehf_el3_interrupt_handler, flags);
	assert(ret == 0);
}

/*
 * Register a handler at the supplied priority. Registration is allowed only if
 * a handler hasn't been registered before, or one wasn't provided at build
 * time. The priority for which the handler is being registered must also accord
 * with the platform-supplied data.
 */
void ehf_register_priority_handler(unsigned int pri, ehf_handler_t handler)
{
	unsigned int idx;

	/* Sanity check for handler */
	assert(handler != NULL);

	/* Handler ought to be 4-byte aligned */
	assert((((uintptr_t) handler) & 3U) == 0U);

	/* Ensure we register for valid priority */
	idx = pri_to_idx(pri);
	assert(idx < exception_data.num_priorities);
	assert(IDX_TO_PRI(idx) == pri);

	/* Return failure if a handler was already registered */
	if (exception_data.ehf_priorities[idx].ehf_handler != EHF_NO_HANDLER_) {
		ERROR("Handler already registered for priority 0x%x\n", pri);
		panic();
	}

	/*
	 * Install handler, and retain the valid bit. We assume that the handler
	 * is 4-byte aligned, which is usually the case.
	 */
	exception_data.ehf_priorities[idx].ehf_handler =
		(((uintptr_t) handler) | EHF_PRI_VALID_);

	EHF_LOG("register pri=0x%x handler=%p\n", pri, handler);
}

SUBSCRIBE_TO_EVENT(cm_entering_normal_world, ehf_entering_normal_world);
SUBSCRIBE_TO_EVENT(cm_exited_normal_world, ehf_exited_normal_world);