1 /* 2 * Copyright (c) 2016-2025, Arm Limited and Contributors. All rights reserved. 3 * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. 4 * 5 * SPDX-License-Identifier: BSD-3-Clause 6 */ 7 8 #include <assert.h> 9 #include <inttypes.h> 10 #include <lib/xlat_tables/xlat_tables_v2.h> 11 #include <stdbool.h> 12 #include <stdint.h> 13 #include <string.h> 14 15 #include <arch_helpers.h> 16 #include <bl31/bl31.h> 17 #include <bl31/interrupt_mgmt.h> 18 #include <common/bl_common.h> 19 #include <common/debug.h> 20 #include <common/runtime_svc.h> 21 #include <lib/el3_runtime/context_mgmt.h> 22 #include <lib/smccc.h> 23 #include <plat/common/platform.h> 24 #include <tools_share/uuid.h> 25 26 #include "sm_err.h" 27 #include "smcall.h" 28 29 /* Trusty UID: RFC-4122 compliant UUID version 4 */ 30 DEFINE_SVC_UUID2(trusty_uuid, 31 0x40ee25f0, 0xa2bc, 0x304c, 0x8c, 0x4c, 32 0xa1, 0x73, 0xc5, 0x7d, 0x8a, 0xf1); 33 34 /* macro to check if Hypervisor is enabled in the HCR_EL2 register */ 35 #define HYP_ENABLE_FLAG 0x286001U 36 37 /* length of Trusty's input parameters (in bytes) */ 38 #define TRUSTY_PARAMS_LEN_BYTES (4096U * 2) 39 40 struct trusty_stack { 41 uint8_t space[PLATFORM_STACK_SIZE] __aligned(16); 42 uint32_t end; 43 }; 44 45 struct trusty_cpu_ctx { 46 cpu_context_t cpu_ctx; 47 void *saved_sp; 48 uint32_t saved_security_state; 49 int32_t fiq_handler_active; 50 uint64_t fiq_handler_pc; 51 uint64_t fiq_handler_cpsr; 52 uint64_t fiq_handler_sp; 53 uint64_t fiq_pc; 54 uint64_t fiq_cpsr; 55 uint64_t fiq_sp_el1; 56 gp_regs_t fiq_gpregs; 57 struct trusty_stack secure_stack; 58 }; 59 60 struct smc_args { 61 uint64_t r0; 62 uint64_t r1; 63 uint64_t r2; 64 uint64_t r3; 65 uint64_t r4; 66 uint64_t r5; 67 uint64_t r6; 68 uint64_t r7; 69 }; 70 71 static struct trusty_cpu_ctx trusty_cpu_ctx[PLATFORM_CORE_COUNT]; 72 73 struct smc_args trusty_init_context_stack(void **sp, void *new_stack); 74 struct smc_args trusty_context_switch_helper(void **sp, void *smc_params); 75 76 static uint32_t current_vmid; 77 78 static struct trusty_cpu_ctx *get_trusty_ctx(void) 79 { 80 return &trusty_cpu_ctx[plat_my_core_pos()]; 81 } 82 83 static bool is_hypervisor_mode(void) 84 { 85 uint64_t hcr = read_hcr(); 86 87 return ((hcr & HYP_ENABLE_FLAG) != 0U) ? true : false; 88 } 89 90 static struct smc_args trusty_context_switch(uint32_t security_state, uint64_t r0, 91 uint64_t r1, uint64_t r2, uint64_t r3) 92 { 93 struct smc_args args, ret_args; 94 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 95 struct trusty_cpu_ctx *ctx_smc; 96 97 assert(ctx->saved_security_state != security_state); 98 99 args.r7 = 0; 100 if (is_hypervisor_mode()) { 101 /* According to the ARM DEN0028A spec, VMID is stored in x7 */ 102 ctx_smc = cm_get_context(NON_SECURE); 103 assert(ctx_smc != NULL); 104 args.r7 = SMC_GET_GP(ctx_smc, CTX_GPREG_X7); 105 } 106 /* r4, r5, r6 reserved for future use. */ 107 args.r6 = 0; 108 args.r5 = 0; 109 args.r4 = 0; 110 args.r3 = r3; 111 args.r2 = r2; 112 args.r1 = r1; 113 args.r0 = r0; 114 115 /* 116 * To avoid the additional overhead in PSCI flow, skip FP context 117 * saving/restoring in case of CPU suspend and resume, assuming that 118 * when it's needed the PSCI caller has preserved FP context before 119 * going here. 120 */ 121 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME) { 122 simd_ctx_save(security_state, false); 123 } 124 125 cm_el1_sysregs_context_save(security_state); 126 127 ctx->saved_security_state = security_state; 128 ret_args = trusty_context_switch_helper(&ctx->saved_sp, &args); 129 130 assert(ctx->saved_security_state == ((security_state == 0U) ? 1U : 0U)); 131 132 cm_el1_sysregs_context_restore(security_state); 133 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME) { 134 simd_ctx_restore(security_state); 135 } 136 137 cm_set_next_eret_context(security_state); 138 139 return ret_args; 140 } 141 142 static uint64_t trusty_fiq_handler(uint32_t id, 143 uint32_t flags, 144 void *handle, 145 void *cookie) 146 { 147 struct smc_args ret; 148 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 149 150 assert(!is_caller_secure(flags)); 151 152 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_ENTER, 0, 0, 0); 153 if (ret.r0 != 0U) { 154 SMC_RET0(handle); 155 } 156 157 if (ctx->fiq_handler_active != 0) { 158 INFO("%s: fiq handler already active\n", __func__); 159 SMC_RET0(handle); 160 } 161 162 ctx->fiq_handler_active = 1; 163 (void)memcpy(&ctx->fiq_gpregs, get_gpregs_ctx(handle), sizeof(ctx->fiq_gpregs)); 164 ctx->fiq_pc = SMC_GET_EL3(handle, CTX_ELR_EL3); 165 ctx->fiq_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3); 166 ctx->fiq_sp_el1 = read_el1_ctx_common(get_el1_sysregs_ctx(handle), sp_el1); 167 168 write_el1_ctx_common(get_el1_sysregs_ctx(handle), sp_el1, ctx->fiq_handler_sp); 169 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_handler_pc, (uint32_t)ctx->fiq_handler_cpsr); 170 171 SMC_RET0(handle); 172 } 173 174 static uint64_t trusty_set_fiq_handler(void *handle, uint64_t cpu, 175 uint64_t handler, uint64_t stack) 176 { 177 struct trusty_cpu_ctx *ctx; 178 179 if (cpu >= (uint64_t)PLATFORM_CORE_COUNT) { 180 ERROR("%s: cpu %" PRId64 " >= %d\n", __func__, cpu, PLATFORM_CORE_COUNT); 181 return (uint64_t)SM_ERR_INVALID_PARAMETERS; 182 } 183 184 ctx = &trusty_cpu_ctx[cpu]; 185 ctx->fiq_handler_pc = handler; 186 ctx->fiq_handler_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3); 187 ctx->fiq_handler_sp = stack; 188 189 SMC_RET1(handle, 0); 190 } 191 192 static uint64_t trusty_get_fiq_regs(void *handle) 193 { 194 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 195 uint64_t sp_el0 = read_ctx_reg(&ctx->fiq_gpregs, CTX_GPREG_SP_EL0); 196 197 SMC_RET4(handle, ctx->fiq_pc, ctx->fiq_cpsr, sp_el0, ctx->fiq_sp_el1); 198 } 199 200 static uint64_t trusty_fiq_exit(void *handle, uint64_t x1, uint64_t x2, uint64_t x3) 201 { 202 struct smc_args ret; 203 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 204 205 if (ctx->fiq_handler_active == 0) { 206 NOTICE("%s: fiq handler not active\n", __func__); 207 SMC_RET1(handle, (uint64_t)SM_ERR_INVALID_PARAMETERS); 208 } 209 210 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_EXIT, 0, 0, 0); 211 if (ret.r0 != 1U) { 212 INFO("%s(%p) SMC_FC_FIQ_EXIT returned unexpected value, %" PRId64 "\n", 213 __func__, handle, ret.r0); 214 } 215 216 /* 217 * Restore register state to state recorded on fiq entry. 218 * 219 * x0, sp_el1, pc and cpsr need to be restored because el1 cannot 220 * restore them. 221 * 222 * x1-x4 and x8-x17 need to be restored here because smc_handler64 223 * corrupts them (el1 code also restored them). 224 */ 225 (void)memcpy(get_gpregs_ctx(handle), &ctx->fiq_gpregs, sizeof(ctx->fiq_gpregs)); 226 ctx->fiq_handler_active = 0; 227 write_el1_ctx_common(get_el1_sysregs_ctx(handle), sp_el1, ctx->fiq_sp_el1); 228 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_pc, (uint32_t)ctx->fiq_cpsr); 229 230 SMC_RET0(handle); 231 } 232 233 static uintptr_t trusty_smc_handler(uint32_t smc_fid, 234 u_register_t x1, 235 u_register_t x2, 236 u_register_t x3, 237 u_register_t x4, 238 void *cookie, 239 void *handle, 240 u_register_t flags) 241 { 242 struct smc_args ret; 243 uint32_t vmid = 0U; 244 entry_point_info_t *ep_info = bl31_plat_get_next_image_ep_info(SECURE); 245 246 /* 247 * Return success for SET_ROT_PARAMS if Trusty is not present, as 248 * Verified Boot is not even supported and returning success here 249 * would not compromise the boot process. 250 */ 251 if ((ep_info == NULL) && (smc_fid == SMC_YC_SET_ROT_PARAMS)) { 252 SMC_RET1(handle, 0); 253 } else if (ep_info == NULL) { 254 SMC_RET1(handle, SMC_UNK); 255 } else { 256 ; /* do nothing */ 257 } 258 259 if (is_caller_secure(flags)) { 260 if (smc_fid == SMC_YC_NS_RETURN) { 261 ret = trusty_context_switch(SECURE, x1, 0, 0, 0); 262 SMC_RET8(handle, ret.r0, ret.r1, ret.r2, ret.r3, 263 ret.r4, ret.r5, ret.r6, ret.r7); 264 } 265 INFO("%s (0x%x, 0x%lx, 0x%lx, 0x%lx, 0x%lx, %p, %p, 0x%lx) \ 266 cpu %d, unknown smc\n", 267 __func__, smc_fid, x1, x2, x3, x4, cookie, handle, flags, 268 plat_my_core_pos()); 269 SMC_RET1(handle, SMC_UNK); 270 } else { 271 switch (smc_fid) { 272 case SMC_FC64_GET_UUID: 273 case SMC_FC_GET_UUID: 274 /* provide the UUID for the service to the client */ 275 SMC_UUID_RET(handle, trusty_uuid); 276 break; 277 case SMC_FC64_SET_FIQ_HANDLER: 278 return trusty_set_fiq_handler(handle, x1, x2, x3); 279 case SMC_FC64_GET_FIQ_REGS: 280 return trusty_get_fiq_regs(handle); 281 case SMC_FC_FIQ_EXIT: 282 return trusty_fiq_exit(handle, x1, x2, x3); 283 default: 284 /* Not all OENs greater than SMC_ENTITY_SECURE_MONITOR are supported */ 285 if (SMC_ENTITY(smc_fid) > SMC_ENTITY_SECURE_MONITOR) { 286 VERBOSE("%s: unsupported SMC FID (0x%x)\n", __func__, smc_fid); 287 SMC_RET1(handle, SMC_UNK); 288 } 289 290 if (is_hypervisor_mode()) 291 vmid = SMC_GET_GP(handle, CTX_GPREG_X7); 292 293 if ((current_vmid != 0) && (current_vmid != vmid)) { 294 /* This message will cause SMC mechanism 295 * abnormal in multi-guest environment. 296 * Change it to WARN in case you need it. 297 */ 298 VERBOSE("Previous SMC not finished.\n"); 299 SMC_RET1(handle, SM_ERR_BUSY); 300 } 301 current_vmid = vmid; 302 ret = trusty_context_switch(NON_SECURE, smc_fid, x1, 303 x2, x3); 304 current_vmid = 0; 305 SMC_RET1(handle, ret.r0); 306 } 307 } 308 } 309 310 static int32_t trusty_init(void) 311 { 312 entry_point_info_t *ep_info; 313 struct smc_args zero_args = {0}; 314 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 315 uint32_t cpu = plat_my_core_pos(); 316 uint64_t reg_width = GET_RW(read_ctx_reg(get_el3state_ctx(&ctx->cpu_ctx), 317 CTX_SPSR_EL3)); 318 319 /* 320 * Get information about the Trusty image. Its absence is a critical 321 * failure. 322 */ 323 ep_info = bl31_plat_get_next_image_ep_info(SECURE); 324 assert(ep_info != NULL); 325 326 simd_ctx_save(NON_SECURE, false); 327 cm_el1_sysregs_context_save(NON_SECURE); 328 329 cm_set_context(&ctx->cpu_ctx, SECURE); 330 cm_init_my_context(ep_info); 331 332 /* 333 * Adjust secondary cpu entry point for 32 bit images to the 334 * end of exception vectors 335 */ 336 if ((cpu != 0U) && (reg_width == MODE_RW_32)) { 337 INFO("trusty: cpu %d, adjust entry point to 0x%lx\n", 338 cpu, ep_info->pc + (1U << 5)); 339 cm_set_elr_el3(SECURE, ep_info->pc + (1U << 5)); 340 } 341 342 cm_el1_sysregs_context_restore(SECURE); 343 simd_ctx_restore(SECURE); 344 cm_set_next_eret_context(SECURE); 345 346 ctx->saved_security_state = ~0U; /* initial saved state is invalid */ 347 (void)trusty_init_context_stack(&ctx->saved_sp, &ctx->secure_stack.end); 348 349 (void)trusty_context_switch_helper(&ctx->saved_sp, &zero_args); 350 351 cm_el1_sysregs_context_restore(NON_SECURE); 352 simd_ctx_restore(NON_SECURE); 353 cm_set_next_eret_context(NON_SECURE); 354 355 return 1; 356 } 357 358 static void trusty_cpu_suspend(uint32_t off) 359 { 360 struct smc_args ret; 361 362 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_SUSPEND, off, 0, 0); 363 if (ret.r0 != 0U) { 364 INFO("%s: cpu %d, SMC_FC_CPU_SUSPEND returned unexpected value, %" PRId64 "\n", 365 __func__, plat_my_core_pos(), ret.r0); 366 } 367 } 368 369 static void trusty_cpu_resume(uint32_t on) 370 { 371 struct smc_args ret; 372 373 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_RESUME, on, 0, 0); 374 if (ret.r0 != 0U) { 375 INFO("%s: cpu %d, SMC_FC_CPU_RESUME returned unexpected value, %" PRId64 "\n", 376 __func__, plat_my_core_pos(), ret.r0); 377 } 378 } 379 380 static int32_t trusty_cpu_off_handler(u_register_t max_off_lvl) 381 { 382 trusty_cpu_suspend(max_off_lvl); 383 384 return 0; 385 } 386 387 static void trusty_cpu_on_finish_handler(u_register_t max_off_lvl) 388 { 389 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 390 391 if (ctx->saved_sp == NULL) { 392 (void)trusty_init(); 393 } else { 394 trusty_cpu_resume(max_off_lvl); 395 } 396 } 397 398 static void trusty_cpu_suspend_handler(u_register_t max_off_lvl) 399 { 400 /* Save NS context in case we need to return to it */ 401 cm_el1_sysregs_context_save(NON_SECURE); 402 403 trusty_cpu_suspend(max_off_lvl); 404 } 405 406 static void trusty_cpu_suspend_finish_handler(u_register_t max_off_lvl, bool abandon) 407 { 408 trusty_cpu_resume(max_off_lvl); 409 410 /* We're returning back to NS so we need to put back its context */ 411 if (abandon) { 412 cm_el1_sysregs_context_restore(NON_SECURE); 413 } 414 415 } 416 417 static const spd_pm_ops_t trusty_pm = { 418 .svc_off = trusty_cpu_off_handler, 419 .svc_suspend = trusty_cpu_suspend_handler, 420 .svc_on_finish = trusty_cpu_on_finish_handler, 421 .svc_suspend_finish = trusty_cpu_suspend_finish_handler, 422 }; 423 424 void plat_trusty_set_boot_args(aapcs64_params_t *args); 425 426 #if !defined(TSP_SEC_MEM_SIZE) && defined(BL32_MEM_SIZE) 427 #define TSP_SEC_MEM_SIZE BL32_MEM_SIZE 428 #endif 429 430 #ifdef TSP_SEC_MEM_SIZE 431 #pragma weak plat_trusty_set_boot_args 432 void plat_trusty_set_boot_args(aapcs64_params_t *args) 433 { 434 args->arg0 = TSP_SEC_MEM_SIZE; 435 } 436 #endif 437 438 static int32_t trusty_setup(void) 439 { 440 entry_point_info_t *ep_info; 441 uint32_t instr; 442 uint32_t flags; 443 int32_t ret; 444 bool aarch32 = false; 445 446 /* Get trusty's entry point info */ 447 ep_info = bl31_plat_get_next_image_ep_info(SECURE); 448 if (ep_info == NULL) { 449 VERBOSE("Trusty image missing.\n"); 450 return -1; 451 } 452 453 /* memmap first page of trusty's code memory before peeking */ 454 ret = mmap_add_dynamic_region(ep_info->pc, /* PA */ 455 ep_info->pc, /* VA */ 456 PAGE_SIZE, /* size */ 457 MT_SECURE | MT_RW_DATA); /* attrs */ 458 assert(ret == 0); 459 460 /* peek into trusty's code to see if we have a 32-bit or 64-bit image */ 461 instr = *(uint32_t *)ep_info->pc; 462 463 if (instr >> 24 == 0xeaU) { 464 INFO("trusty: Found 32 bit image\n"); 465 aarch32 = true; 466 } else if (instr >> 8 == 0xd53810U || instr >> 16 == 0x9400U) { 467 INFO("trusty: Found 64 bit image\n"); 468 } else { 469 ERROR("trusty: Found unknown image, 0x%x\n", instr); 470 return -1; 471 } 472 473 /* unmap trusty's memory page */ 474 (void)mmap_remove_dynamic_region(ep_info->pc, PAGE_SIZE); 475 476 SET_PARAM_HEAD(ep_info, PARAM_EP, VERSION_1, SECURE | EP_ST_ENABLE); 477 if (!aarch32) 478 ep_info->spsr = SPSR_64(MODE_EL1, MODE_SP_ELX, 479 DISABLE_ALL_EXCEPTIONS); 480 else 481 ep_info->spsr = SPSR_MODE32(MODE32_svc, SPSR_T_ARM, 482 SPSR_E_LITTLE, 483 DAIF_FIQ_BIT | 484 DAIF_IRQ_BIT | 485 DAIF_ABT_BIT); 486 (void)memset(&ep_info->args, 0, sizeof(ep_info->args)); 487 plat_trusty_set_boot_args(&ep_info->args); 488 489 /* register init handler */ 490 bl31_register_bl32_init(trusty_init); 491 492 /* register power management hooks */ 493 psci_register_spd_pm_hook(&trusty_pm); 494 495 /* register interrupt handler */ 496 flags = 0; 497 set_interrupt_rm_flag(flags, NON_SECURE); 498 ret = register_interrupt_type_handler(INTR_TYPE_S_EL1, 499 trusty_fiq_handler, 500 flags); 501 if (ret != 0) { 502 VERBOSE("trusty: failed to register fiq handler, ret = %d\n", ret); 503 } 504 505 if (aarch32) { 506 entry_point_info_t *ns_ep_info; 507 uint32_t spsr; 508 509 ns_ep_info = bl31_plat_get_next_image_ep_info(NON_SECURE); 510 if (ns_ep_info == NULL) { 511 NOTICE("Trusty: non-secure image missing.\n"); 512 return -1; 513 } 514 spsr = ns_ep_info->spsr; 515 if (GET_RW(spsr) == MODE_RW_64 && GET_EL(spsr) == MODE_EL2) { 516 spsr &= ~(MODE_EL_MASK << MODE_EL_SHIFT); 517 spsr |= MODE_EL1 << MODE_EL_SHIFT; 518 } 519 if (GET_RW(spsr) == MODE_RW_32 && GET_M32(spsr) == MODE32_hyp) { 520 spsr &= ~(MODE32_MASK << MODE32_SHIFT); 521 spsr |= MODE32_svc << MODE32_SHIFT; 522 } 523 if (spsr != ns_ep_info->spsr) { 524 NOTICE("Trusty: Switch bl33 from EL2 to EL1 (spsr 0x%x -> 0x%x)\n", 525 ns_ep_info->spsr, spsr); 526 ns_ep_info->spsr = spsr; 527 } 528 } 529 530 return 0; 531 } 532 533 /* Define a SPD runtime service descriptor for fast SMC calls */ 534 DECLARE_RT_SVC( 535 trusty_fast, 536 537 OEN_TOS_START, 538 OEN_TOS_END, 539 SMC_TYPE_FAST, 540 trusty_setup, 541 trusty_smc_handler 542 ); 543 544 /* Define a SPD runtime service descriptor for yielding SMC calls */ 545 DECLARE_RT_SVC( 546 trusty_std, 547 548 OEN_TAP_START, 549 SMC_ENTITY_SECURE_MONITOR, 550 SMC_TYPE_YIELD, 551 NULL, 552 trusty_smc_handler 553 ); 554