1 /*
2 * Copyright (c) 2016-2025, Arm Limited and Contributors. All rights reserved.
3 * Copyright (c) 2020, NVIDIA Corporation. All rights reserved.
4 *
5 * SPDX-License-Identifier: BSD-3-Clause
6 */
7
8 #include <assert.h>
9 #include <inttypes.h>
10 #include <lib/xlat_tables/xlat_tables_v2.h>
11 #include <stdbool.h>
12 #include <stdint.h>
13 #include <string.h>
14
15 #include <arch_helpers.h>
16 #include <bl31/bl31.h>
17 #include <bl31/interrupt_mgmt.h>
18 #include <common/bl_common.h>
19 #include <common/debug.h>
20 #include <common/runtime_svc.h>
21 #include <lib/el3_runtime/context_mgmt.h>
22 #include <lib/smccc.h>
23 #include <plat/common/platform.h>
24 #include <tools_share/uuid.h>
25
26 #include "sm_err.h"
27 #include "smcall.h"
28
29 /* Trusty UID: RFC-4122 compliant UUID version 4 */
30 DEFINE_SVC_UUID2(trusty_uuid,
31 0x40ee25f0, 0xa2bc, 0x304c, 0x8c, 0x4c,
32 0xa1, 0x73, 0xc5, 0x7d, 0x8a, 0xf1);
33
34 /* macro to check if Hypervisor is enabled in the HCR_EL2 register */
35 #define HYP_ENABLE_FLAG 0x286001U
36
37 /* length of Trusty's input parameters (in bytes) */
38 #define TRUSTY_PARAMS_LEN_BYTES (4096U * 2)
39
40 struct trusty_stack {
41 uint8_t space[PLATFORM_STACK_SIZE] __aligned(16);
42 uint32_t end;
43 };
44
45 struct trusty_cpu_ctx {
46 cpu_context_t cpu_ctx;
47 void *saved_sp;
48 uint32_t saved_security_state;
49 int32_t fiq_handler_active;
50 uint64_t fiq_handler_pc;
51 uint64_t fiq_handler_cpsr;
52 uint64_t fiq_handler_sp;
53 uint64_t fiq_pc;
54 uint64_t fiq_cpsr;
55 uint64_t fiq_sp_el1;
56 gp_regs_t fiq_gpregs;
57 struct trusty_stack secure_stack;
58 };
59
60 struct smc_args {
61 uint64_t r0;
62 uint64_t r1;
63 uint64_t r2;
64 uint64_t r3;
65 uint64_t r4;
66 uint64_t r5;
67 uint64_t r6;
68 uint64_t r7;
69 };
70
71 static struct trusty_cpu_ctx trusty_cpu_ctx[PLATFORM_CORE_COUNT];
72
73 struct smc_args trusty_init_context_stack(void **sp, void *new_stack);
74 struct smc_args trusty_context_switch_helper(void **sp, void *smc_params);
75
76 static uint32_t current_vmid;
77
get_trusty_ctx(void)78 static struct trusty_cpu_ctx *get_trusty_ctx(void)
79 {
80 return &trusty_cpu_ctx[plat_my_core_pos()];
81 }
82
is_hypervisor_mode(void)83 static bool is_hypervisor_mode(void)
84 {
85 uint64_t hcr = read_hcr();
86
87 return ((hcr & HYP_ENABLE_FLAG) != 0U) ? true : false;
88 }
89
trusty_context_switch(uint32_t security_state,uint64_t r0,uint64_t r1,uint64_t r2,uint64_t r3)90 static struct smc_args trusty_context_switch(uint32_t security_state, uint64_t r0,
91 uint64_t r1, uint64_t r2, uint64_t r3)
92 {
93 struct smc_args args, ret_args;
94 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
95 struct trusty_cpu_ctx *ctx_smc;
96
97 assert(ctx->saved_security_state != security_state);
98
99 args.r7 = 0;
100 if (is_hypervisor_mode()) {
101 /* According to the ARM DEN0028A spec, VMID is stored in x7 */
102 ctx_smc = cm_get_context(NON_SECURE);
103 assert(ctx_smc != NULL);
104 args.r7 = SMC_GET_GP(ctx_smc, CTX_GPREG_X7);
105 }
106 /* r4, r5, r6 reserved for future use. */
107 args.r6 = 0;
108 args.r5 = 0;
109 args.r4 = 0;
110 args.r3 = r3;
111 args.r2 = r2;
112 args.r1 = r1;
113 args.r0 = r0;
114
115 /*
116 * To avoid the additional overhead in PSCI flow, skip FP context
117 * saving/restoring in case of CPU suspend and resume, assuming that
118 * when it's needed the PSCI caller has preserved FP context before
119 * going here.
120 */
121 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME) {
122 simd_ctx_save(security_state, false);
123 }
124
125 cm_el1_sysregs_context_save(security_state);
126
127 ctx->saved_security_state = security_state;
128 ret_args = trusty_context_switch_helper(&ctx->saved_sp, &args);
129
130 assert(ctx->saved_security_state == ((security_state == 0U) ? 1U : 0U));
131
132 cm_el1_sysregs_context_restore(security_state);
133 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME) {
134 simd_ctx_restore(security_state);
135 }
136
137 cm_set_next_eret_context(security_state);
138
139 return ret_args;
140 }
141
trusty_fiq_handler(uint32_t id,uint32_t flags,void * handle,void * cookie)142 static uint64_t trusty_fiq_handler(uint32_t id,
143 uint32_t flags,
144 void *handle,
145 void *cookie)
146 {
147 struct smc_args ret;
148 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
149
150 assert(!is_caller_secure(flags));
151
152 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_ENTER, 0, 0, 0);
153 if (ret.r0 != 0U) {
154 SMC_RET0(handle);
155 }
156
157 if (ctx->fiq_handler_active != 0) {
158 INFO("%s: fiq handler already active\n", __func__);
159 SMC_RET0(handle);
160 }
161
162 ctx->fiq_handler_active = 1;
163 (void)memcpy(&ctx->fiq_gpregs, get_gpregs_ctx(handle), sizeof(ctx->fiq_gpregs));
164 ctx->fiq_pc = SMC_GET_EL3(handle, CTX_ELR_EL3);
165 ctx->fiq_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3);
166 ctx->fiq_sp_el1 = read_el1_ctx_common(get_el1_sysregs_ctx(handle), sp_el1);
167
168 write_el1_ctx_common(get_el1_sysregs_ctx(handle), sp_el1, ctx->fiq_handler_sp);
169 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_handler_pc, (uint32_t)ctx->fiq_handler_cpsr);
170
171 SMC_RET0(handle);
172 }
173
trusty_set_fiq_handler(void * handle,uint64_t cpu,uint64_t handler,uint64_t stack)174 static uint64_t trusty_set_fiq_handler(void *handle, uint64_t cpu,
175 uint64_t handler, uint64_t stack)
176 {
177 struct trusty_cpu_ctx *ctx;
178
179 if (cpu >= (uint64_t)PLATFORM_CORE_COUNT) {
180 ERROR("%s: cpu %" PRId64 " >= %d\n", __func__, cpu, PLATFORM_CORE_COUNT);
181 return (uint64_t)SM_ERR_INVALID_PARAMETERS;
182 }
183
184 ctx = &trusty_cpu_ctx[cpu];
185 ctx->fiq_handler_pc = handler;
186 ctx->fiq_handler_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3);
187 ctx->fiq_handler_sp = stack;
188
189 SMC_RET1(handle, 0);
190 }
191
trusty_get_fiq_regs(void * handle)192 static uint64_t trusty_get_fiq_regs(void *handle)
193 {
194 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
195 uint64_t sp_el0 = read_ctx_reg(&ctx->fiq_gpregs, CTX_GPREG_SP_EL0);
196
197 SMC_RET4(handle, ctx->fiq_pc, ctx->fiq_cpsr, sp_el0, ctx->fiq_sp_el1);
198 }
199
trusty_fiq_exit(void * handle,uint64_t x1,uint64_t x2,uint64_t x3)200 static uint64_t trusty_fiq_exit(void *handle, uint64_t x1, uint64_t x2, uint64_t x3)
201 {
202 struct smc_args ret;
203 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
204
205 if (ctx->fiq_handler_active == 0) {
206 NOTICE("%s: fiq handler not active\n", __func__);
207 SMC_RET1(handle, (uint64_t)SM_ERR_INVALID_PARAMETERS);
208 }
209
210 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_EXIT, 0, 0, 0);
211 if (ret.r0 != 1U) {
212 INFO("%s(%p) SMC_FC_FIQ_EXIT returned unexpected value, %" PRId64 "\n",
213 __func__, handle, ret.r0);
214 }
215
216 /*
217 * Restore register state to state recorded on fiq entry.
218 *
219 * x0, sp_el1, pc and cpsr need to be restored because el1 cannot
220 * restore them.
221 *
222 * x1-x4 and x8-x17 need to be restored here because smc_handler64
223 * corrupts them (el1 code also restored them).
224 */
225 (void)memcpy(get_gpregs_ctx(handle), &ctx->fiq_gpregs, sizeof(ctx->fiq_gpregs));
226 ctx->fiq_handler_active = 0;
227 write_el1_ctx_common(get_el1_sysregs_ctx(handle), sp_el1, ctx->fiq_sp_el1);
228 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_pc, (uint32_t)ctx->fiq_cpsr);
229
230 SMC_RET0(handle);
231 }
232
trusty_smc_handler(uint32_t smc_fid,u_register_t x1,u_register_t x2,u_register_t x3,u_register_t x4,void * cookie,void * handle,u_register_t flags)233 static uintptr_t trusty_smc_handler(uint32_t smc_fid,
234 u_register_t x1,
235 u_register_t x2,
236 u_register_t x3,
237 u_register_t x4,
238 void *cookie,
239 void *handle,
240 u_register_t flags)
241 {
242 struct smc_args ret;
243 uint32_t vmid = 0U;
244 entry_point_info_t *ep_info = bl31_plat_get_next_image_ep_info(SECURE);
245
246 /*
247 * Return success for SET_ROT_PARAMS if Trusty is not present, as
248 * Verified Boot is not even supported and returning success here
249 * would not compromise the boot process.
250 */
251 if ((ep_info == NULL) && (smc_fid == SMC_YC_SET_ROT_PARAMS)) {
252 SMC_RET1(handle, 0);
253 } else if (ep_info == NULL) {
254 SMC_RET1(handle, SMC_UNK);
255 } else {
256 ; /* do nothing */
257 }
258
259 if (is_caller_secure(flags)) {
260 if (smc_fid == SMC_YC_NS_RETURN) {
261 ret = trusty_context_switch(SECURE, x1, 0, 0, 0);
262 SMC_RET8(handle, ret.r0, ret.r1, ret.r2, ret.r3,
263 ret.r4, ret.r5, ret.r6, ret.r7);
264 }
265 INFO("%s (0x%x, 0x%lx, 0x%lx, 0x%lx, 0x%lx, %p, %p, 0x%lx) \
266 cpu %d, unknown smc\n",
267 __func__, smc_fid, x1, x2, x3, x4, cookie, handle, flags,
268 plat_my_core_pos());
269 SMC_RET1(handle, SMC_UNK);
270 } else {
271 switch (smc_fid) {
272 case SMC_FC64_GET_UUID:
273 case SMC_FC_GET_UUID:
274 /* provide the UUID for the service to the client */
275 SMC_UUID_RET(handle, trusty_uuid);
276 break;
277 case SMC_FC64_SET_FIQ_HANDLER:
278 return trusty_set_fiq_handler(handle, x1, x2, x3);
279 case SMC_FC64_GET_FIQ_REGS:
280 return trusty_get_fiq_regs(handle);
281 case SMC_FC_FIQ_EXIT:
282 return trusty_fiq_exit(handle, x1, x2, x3);
283 default:
284 /* Not all OENs greater than SMC_ENTITY_SECURE_MONITOR are supported */
285 if (SMC_ENTITY(smc_fid) > SMC_ENTITY_SECURE_MONITOR) {
286 VERBOSE("%s: unsupported SMC FID (0x%x)\n", __func__, smc_fid);
287 SMC_RET1(handle, SMC_UNK);
288 }
289
290 if (is_hypervisor_mode())
291 vmid = SMC_GET_GP(handle, CTX_GPREG_X7);
292
293 if ((current_vmid != 0) && (current_vmid != vmid)) {
294 /* This message will cause SMC mechanism
295 * abnormal in multi-guest environment.
296 * Change it to WARN in case you need it.
297 */
298 VERBOSE("Previous SMC not finished.\n");
299 SMC_RET1(handle, SM_ERR_BUSY);
300 }
301 current_vmid = vmid;
302 ret = trusty_context_switch(NON_SECURE, smc_fid, x1,
303 x2, x3);
304 current_vmid = 0;
305 SMC_RET1(handle, ret.r0);
306 }
307 }
308 }
309
trusty_init(void)310 static int32_t trusty_init(void)
311 {
312 entry_point_info_t *ep_info;
313 struct smc_args zero_args = {0};
314 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
315 uint32_t cpu = plat_my_core_pos();
316 uint64_t reg_width = GET_RW(read_ctx_reg(get_el3state_ctx(&ctx->cpu_ctx),
317 CTX_SPSR_EL3));
318
319 /*
320 * Get information about the Trusty image. Its absence is a critical
321 * failure.
322 */
323 ep_info = bl31_plat_get_next_image_ep_info(SECURE);
324 assert(ep_info != NULL);
325
326 simd_ctx_save(NON_SECURE, false);
327 cm_el1_sysregs_context_save(NON_SECURE);
328
329 cm_set_context(&ctx->cpu_ctx, SECURE);
330 cm_init_my_context(ep_info);
331
332 /*
333 * Adjust secondary cpu entry point for 32 bit images to the
334 * end of exception vectors
335 */
336 if ((cpu != 0U) && (reg_width == MODE_RW_32)) {
337 INFO("trusty: cpu %d, adjust entry point to 0x%lx\n",
338 cpu, ep_info->pc + (1U << 5));
339 cm_set_elr_el3(SECURE, ep_info->pc + (1U << 5));
340 }
341
342 cm_el1_sysregs_context_restore(SECURE);
343 simd_ctx_restore(SECURE);
344 cm_set_next_eret_context(SECURE);
345
346 ctx->saved_security_state = ~0U; /* initial saved state is invalid */
347 (void)trusty_init_context_stack(&ctx->saved_sp, &ctx->secure_stack.end);
348
349 (void)trusty_context_switch_helper(&ctx->saved_sp, &zero_args);
350
351 cm_el1_sysregs_context_restore(NON_SECURE);
352 simd_ctx_restore(NON_SECURE);
353 cm_set_next_eret_context(NON_SECURE);
354
355 return 1;
356 }
357
trusty_cpu_suspend(uint32_t off)358 static void trusty_cpu_suspend(uint32_t off)
359 {
360 struct smc_args ret;
361
362 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_SUSPEND, off, 0, 0);
363 if (ret.r0 != 0U) {
364 INFO("%s: cpu %d, SMC_FC_CPU_SUSPEND returned unexpected value, %" PRId64 "\n",
365 __func__, plat_my_core_pos(), ret.r0);
366 }
367 }
368
trusty_cpu_resume(uint32_t on)369 static void trusty_cpu_resume(uint32_t on)
370 {
371 struct smc_args ret;
372
373 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_RESUME, on, 0, 0);
374 if (ret.r0 != 0U) {
375 INFO("%s: cpu %d, SMC_FC_CPU_RESUME returned unexpected value, %" PRId64 "\n",
376 __func__, plat_my_core_pos(), ret.r0);
377 }
378 }
379
trusty_cpu_off_handler(u_register_t max_off_lvl)380 static int32_t trusty_cpu_off_handler(u_register_t max_off_lvl)
381 {
382 trusty_cpu_suspend(max_off_lvl);
383
384 return 0;
385 }
386
trusty_cpu_on_finish_handler(u_register_t max_off_lvl)387 static void trusty_cpu_on_finish_handler(u_register_t max_off_lvl)
388 {
389 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
390
391 if (ctx->saved_sp == NULL) {
392 (void)trusty_init();
393 } else {
394 trusty_cpu_resume(max_off_lvl);
395 }
396 }
397
trusty_cpu_suspend_handler(u_register_t max_off_lvl)398 static void trusty_cpu_suspend_handler(u_register_t max_off_lvl)
399 {
400 /* Save NS context in case we need to return to it */
401 cm_el1_sysregs_context_save(NON_SECURE);
402
403 trusty_cpu_suspend(max_off_lvl);
404 }
405
trusty_cpu_suspend_finish_handler(u_register_t max_off_lvl,bool abandon)406 static void trusty_cpu_suspend_finish_handler(u_register_t max_off_lvl, bool abandon)
407 {
408 trusty_cpu_resume(max_off_lvl);
409
410 /* We're returning back to NS so we need to put back its context */
411 if (abandon) {
412 cm_el1_sysregs_context_restore(NON_SECURE);
413 }
414
415 }
416
417 static const spd_pm_ops_t trusty_pm = {
418 .svc_off = trusty_cpu_off_handler,
419 .svc_suspend = trusty_cpu_suspend_handler,
420 .svc_on_finish = trusty_cpu_on_finish_handler,
421 .svc_suspend_finish = trusty_cpu_suspend_finish_handler,
422 };
423
424 void plat_trusty_set_boot_args(aapcs64_params_t *args);
425
426 #if !defined(TSP_SEC_MEM_SIZE) && defined(BL32_MEM_SIZE)
427 #define TSP_SEC_MEM_SIZE BL32_MEM_SIZE
428 #endif
429
430 #ifdef TSP_SEC_MEM_SIZE
431 #pragma weak plat_trusty_set_boot_args
plat_trusty_set_boot_args(aapcs64_params_t * args)432 void plat_trusty_set_boot_args(aapcs64_params_t *args)
433 {
434 args->arg0 = TSP_SEC_MEM_SIZE;
435 }
436 #endif
437
trusty_setup(void)438 static int32_t trusty_setup(void)
439 {
440 entry_point_info_t *ep_info;
441 uint32_t instr;
442 uint32_t flags;
443 int32_t ret;
444 bool aarch32 = false;
445
446 /* Get trusty's entry point info */
447 ep_info = bl31_plat_get_next_image_ep_info(SECURE);
448 if (ep_info == NULL) {
449 VERBOSE("Trusty image missing.\n");
450 return -1;
451 }
452
453 /* memmap first page of trusty's code memory before peeking */
454 ret = mmap_add_dynamic_region(ep_info->pc, /* PA */
455 ep_info->pc, /* VA */
456 PAGE_SIZE, /* size */
457 MT_SECURE | MT_RW_DATA); /* attrs */
458 assert(ret == 0);
459
460 /* peek into trusty's code to see if we have a 32-bit or 64-bit image */
461 instr = *(uint32_t *)ep_info->pc;
462
463 if (instr >> 24 == 0xeaU) {
464 INFO("trusty: Found 32 bit image\n");
465 aarch32 = true;
466 } else if (instr >> 8 == 0xd53810U || instr >> 16 == 0x9400U) {
467 INFO("trusty: Found 64 bit image\n");
468 } else {
469 ERROR("trusty: Found unknown image, 0x%x\n", instr);
470 return -1;
471 }
472
473 /* unmap trusty's memory page */
474 (void)mmap_remove_dynamic_region(ep_info->pc, PAGE_SIZE);
475
476 SET_PARAM_HEAD(ep_info, PARAM_EP, VERSION_1, SECURE | EP_ST_ENABLE);
477 if (!aarch32)
478 ep_info->spsr = SPSR_64(MODE_EL1, MODE_SP_ELX,
479 DISABLE_ALL_EXCEPTIONS);
480 else
481 ep_info->spsr = SPSR_MODE32(MODE32_svc, SPSR_T_ARM,
482 SPSR_E_LITTLE,
483 DAIF_FIQ_BIT |
484 DAIF_IRQ_BIT |
485 DAIF_ABT_BIT);
486 (void)memset(&ep_info->args, 0, sizeof(ep_info->args));
487 plat_trusty_set_boot_args(&ep_info->args);
488
489 /* register init handler */
490 bl31_register_bl32_init(trusty_init);
491
492 /* register power management hooks */
493 psci_register_spd_pm_hook(&trusty_pm);
494
495 /* register interrupt handler */
496 flags = 0;
497 set_interrupt_rm_flag(flags, NON_SECURE);
498 ret = register_interrupt_type_handler(INTR_TYPE_S_EL1,
499 trusty_fiq_handler,
500 flags);
501 if (ret != 0) {
502 VERBOSE("trusty: failed to register fiq handler, ret = %d\n", ret);
503 }
504
505 if (aarch32) {
506 entry_point_info_t *ns_ep_info;
507 uint32_t spsr;
508
509 ns_ep_info = bl31_plat_get_next_image_ep_info(NON_SECURE);
510 if (ns_ep_info == NULL) {
511 NOTICE("Trusty: non-secure image missing.\n");
512 return -1;
513 }
514 spsr = ns_ep_info->spsr;
515 if (GET_RW(spsr) == MODE_RW_64 && GET_EL(spsr) == MODE_EL2) {
516 spsr &= ~(MODE_EL_MASK << MODE_EL_SHIFT);
517 spsr |= MODE_EL1 << MODE_EL_SHIFT;
518 }
519 if (GET_RW(spsr) == MODE_RW_32 && GET_M32(spsr) == MODE32_hyp) {
520 spsr &= ~(MODE32_MASK << MODE32_SHIFT);
521 spsr |= MODE32_svc << MODE32_SHIFT;
522 }
523 if (spsr != ns_ep_info->spsr) {
524 NOTICE("Trusty: Switch bl33 from EL2 to EL1 (spsr 0x%x -> 0x%x)\n",
525 ns_ep_info->spsr, spsr);
526 ns_ep_info->spsr = spsr;
527 }
528 }
529
530 return 0;
531 }
532
533 /* Define a SPD runtime service descriptor for fast SMC calls */
534 DECLARE_RT_SVC(
535 trusty_fast,
536
537 OEN_TOS_START,
538 OEN_TOS_END,
539 SMC_TYPE_FAST,
540 trusty_setup,
541 trusty_smc_handler
542 );
543
544 /* Define a SPD runtime service descriptor for yielding SMC calls */
545 DECLARE_RT_SVC(
546 trusty_std,
547
548 OEN_TAP_START,
549 SMC_ENTITY_SECURE_MONITOR,
550 SMC_TYPE_YIELD,
551 NULL,
552 trusty_smc_handler
553 );
554