1 /* 2 * Copyright (c) 2016-2019, ARM Limited and Contributors. All rights reserved. 3 * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. 4 * 5 * SPDX-License-Identifier: BSD-3-Clause 6 */ 7 8 #include <assert.h> 9 #include <lib/xlat_tables/xlat_tables_v2.h> 10 #include <stdbool.h> 11 #include <string.h> 12 13 #include <arch_helpers.h> 14 #include <bl31/bl31.h> 15 #include <bl31/interrupt_mgmt.h> 16 #include <common/bl_common.h> 17 #include <common/debug.h> 18 #include <common/runtime_svc.h> 19 #include <lib/el3_runtime/context_mgmt.h> 20 #include <lib/smccc.h> 21 #include <plat/common/platform.h> 22 #include <tools_share/uuid.h> 23 24 #include "sm_err.h" 25 #include "smcall.h" 26 27 /* Trusty UID: RFC-4122 compliant UUID version 4 */ 28 DEFINE_SVC_UUID2(trusty_uuid, 29 0x40ee25f0, 0xa2bc, 0x304c, 0x8c, 0x4c, 30 0xa1, 0x73, 0xc5, 0x7d, 0x8a, 0xf1); 31 32 /* macro to check if Hypervisor is enabled in the HCR_EL2 register */ 33 #define HYP_ENABLE_FLAG 0x286001U 34 35 /* length of Trusty's input parameters (in bytes) */ 36 #define TRUSTY_PARAMS_LEN_BYTES (4096U * 2) 37 38 struct trusty_stack { 39 uint8_t space[PLATFORM_STACK_SIZE] __aligned(16); 40 uint32_t end; 41 }; 42 43 struct trusty_cpu_ctx { 44 cpu_context_t cpu_ctx; 45 void *saved_sp; 46 uint32_t saved_security_state; 47 int32_t fiq_handler_active; 48 uint64_t fiq_handler_pc; 49 uint64_t fiq_handler_cpsr; 50 uint64_t fiq_handler_sp; 51 uint64_t fiq_pc; 52 uint64_t fiq_cpsr; 53 uint64_t fiq_sp_el1; 54 gp_regs_t fiq_gpregs; 55 struct trusty_stack secure_stack; 56 }; 57 58 struct smc_args { 59 uint64_t r0; 60 uint64_t r1; 61 uint64_t r2; 62 uint64_t r3; 63 uint64_t r4; 64 uint64_t r5; 65 uint64_t r6; 66 uint64_t r7; 67 }; 68 69 static struct trusty_cpu_ctx trusty_cpu_ctx[PLATFORM_CORE_COUNT]; 70 71 struct smc_args trusty_init_context_stack(void **sp, void *new_stack); 72 struct smc_args trusty_context_switch_helper(void **sp, void *smc_params); 73 74 static uint32_t current_vmid; 75 76 static struct trusty_cpu_ctx *get_trusty_ctx(void) 77 { 78 return &trusty_cpu_ctx[plat_my_core_pos()]; 79 } 80 81 static bool is_hypervisor_mode(void) 82 { 83 uint64_t hcr = read_hcr(); 84 85 return ((hcr & HYP_ENABLE_FLAG) != 0U) ? true : false; 86 } 87 88 static struct smc_args trusty_context_switch(uint32_t security_state, uint64_t r0, 89 uint64_t r1, uint64_t r2, uint64_t r3) 90 { 91 struct smc_args args, ret_args; 92 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 93 struct trusty_cpu_ctx *ctx_smc; 94 95 assert(ctx->saved_security_state != security_state); 96 97 args.r7 = 0; 98 if (is_hypervisor_mode()) { 99 /* According to the ARM DEN0028A spec, VMID is stored in x7 */ 100 ctx_smc = cm_get_context(NON_SECURE); 101 assert(ctx_smc != NULL); 102 args.r7 = SMC_GET_GP(ctx_smc, CTX_GPREG_X7); 103 } 104 /* r4, r5, r6 reserved for future use. */ 105 args.r6 = 0; 106 args.r5 = 0; 107 args.r4 = 0; 108 args.r3 = r3; 109 args.r2 = r2; 110 args.r1 = r1; 111 args.r0 = r0; 112 113 /* 114 * To avoid the additional overhead in PSCI flow, skip FP context 115 * saving/restoring in case of CPU suspend and resume, assuming that 116 * when it's needed the PSCI caller has preserved FP context before 117 * going here. 118 */ 119 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME) 120 fpregs_context_save(get_fpregs_ctx(cm_get_context(security_state))); 121 cm_el1_sysregs_context_save(security_state); 122 123 ctx->saved_security_state = security_state; 124 ret_args = trusty_context_switch_helper(&ctx->saved_sp, &args); 125 126 assert(ctx->saved_security_state == ((security_state == 0U) ? 1U : 0U)); 127 128 cm_el1_sysregs_context_restore(security_state); 129 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME) 130 fpregs_context_restore(get_fpregs_ctx(cm_get_context(security_state))); 131 132 cm_set_next_eret_context(security_state); 133 134 return ret_args; 135 } 136 137 static uint64_t trusty_fiq_handler(uint32_t id, 138 uint32_t flags, 139 void *handle, 140 void *cookie) 141 { 142 struct smc_args ret; 143 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 144 145 assert(!is_caller_secure(flags)); 146 147 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_ENTER, 0, 0, 0); 148 if (ret.r0 != 0U) { 149 SMC_RET0(handle); 150 } 151 152 if (ctx->fiq_handler_active != 0) { 153 INFO("%s: fiq handler already active\n", __func__); 154 SMC_RET0(handle); 155 } 156 157 ctx->fiq_handler_active = 1; 158 (void)memcpy(&ctx->fiq_gpregs, get_gpregs_ctx(handle), sizeof(ctx->fiq_gpregs)); 159 ctx->fiq_pc = SMC_GET_EL3(handle, CTX_ELR_EL3); 160 ctx->fiq_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3); 161 ctx->fiq_sp_el1 = read_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1); 162 163 write_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1, ctx->fiq_handler_sp); 164 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_handler_pc, (uint32_t)ctx->fiq_handler_cpsr); 165 166 SMC_RET0(handle); 167 } 168 169 static uint64_t trusty_set_fiq_handler(void *handle, uint64_t cpu, 170 uint64_t handler, uint64_t stack) 171 { 172 struct trusty_cpu_ctx *ctx; 173 174 if (cpu >= (uint64_t)PLATFORM_CORE_COUNT) { 175 ERROR("%s: cpu %lld >= %d\n", __func__, cpu, PLATFORM_CORE_COUNT); 176 return (uint64_t)SM_ERR_INVALID_PARAMETERS; 177 } 178 179 ctx = &trusty_cpu_ctx[cpu]; 180 ctx->fiq_handler_pc = handler; 181 ctx->fiq_handler_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3); 182 ctx->fiq_handler_sp = stack; 183 184 SMC_RET1(handle, 0); 185 } 186 187 static uint64_t trusty_get_fiq_regs(void *handle) 188 { 189 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 190 uint64_t sp_el0 = read_ctx_reg(&ctx->fiq_gpregs, CTX_GPREG_SP_EL0); 191 192 SMC_RET4(handle, ctx->fiq_pc, ctx->fiq_cpsr, sp_el0, ctx->fiq_sp_el1); 193 } 194 195 static uint64_t trusty_fiq_exit(void *handle, uint64_t x1, uint64_t x2, uint64_t x3) 196 { 197 struct smc_args ret; 198 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 199 200 if (ctx->fiq_handler_active == 0) { 201 NOTICE("%s: fiq handler not active\n", __func__); 202 SMC_RET1(handle, (uint64_t)SM_ERR_INVALID_PARAMETERS); 203 } 204 205 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_EXIT, 0, 0, 0); 206 if (ret.r0 != 1U) { 207 INFO("%s(%p) SMC_FC_FIQ_EXIT returned unexpected value, %lld\n", 208 __func__, handle, ret.r0); 209 } 210 211 /* 212 * Restore register state to state recorded on fiq entry. 213 * 214 * x0, sp_el1, pc and cpsr need to be restored because el1 cannot 215 * restore them. 216 * 217 * x1-x4 and x8-x17 need to be restored here because smc_handler64 218 * corrupts them (el1 code also restored them). 219 */ 220 (void)memcpy(get_gpregs_ctx(handle), &ctx->fiq_gpregs, sizeof(ctx->fiq_gpregs)); 221 ctx->fiq_handler_active = 0; 222 write_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1, ctx->fiq_sp_el1); 223 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_pc, (uint32_t)ctx->fiq_cpsr); 224 225 SMC_RET0(handle); 226 } 227 228 static uintptr_t trusty_smc_handler(uint32_t smc_fid, 229 u_register_t x1, 230 u_register_t x2, 231 u_register_t x3, 232 u_register_t x4, 233 void *cookie, 234 void *handle, 235 u_register_t flags) 236 { 237 struct smc_args ret; 238 uint32_t vmid = 0U; 239 entry_point_info_t *ep_info = bl31_plat_get_next_image_ep_info(SECURE); 240 241 /* 242 * Return success for SET_ROT_PARAMS if Trusty is not present, as 243 * Verified Boot is not even supported and returning success here 244 * would not compromise the boot process. 245 */ 246 if ((ep_info == NULL) && (smc_fid == SMC_YC_SET_ROT_PARAMS)) { 247 SMC_RET1(handle, 0); 248 } else if (ep_info == NULL) { 249 SMC_RET1(handle, SMC_UNK); 250 } else { 251 ; /* do nothing */ 252 } 253 254 if (is_caller_secure(flags)) { 255 if (smc_fid == SMC_YC_NS_RETURN) { 256 ret = trusty_context_switch(SECURE, x1, 0, 0, 0); 257 SMC_RET8(handle, ret.r0, ret.r1, ret.r2, ret.r3, 258 ret.r4, ret.r5, ret.r6, ret.r7); 259 } 260 INFO("%s (0x%x, 0x%lx, 0x%lx, 0x%lx, 0x%lx, %p, %p, 0x%lx) \ 261 cpu %d, unknown smc\n", 262 __func__, smc_fid, x1, x2, x3, x4, cookie, handle, flags, 263 plat_my_core_pos()); 264 SMC_RET1(handle, SMC_UNK); 265 } else { 266 switch (smc_fid) { 267 case SMC_FC64_GET_UUID: 268 case SMC_FC_GET_UUID: 269 /* provide the UUID for the service to the client */ 270 SMC_UUID_RET(handle, trusty_uuid); 271 break; 272 case SMC_FC64_SET_FIQ_HANDLER: 273 return trusty_set_fiq_handler(handle, x1, x2, x3); 274 case SMC_FC64_GET_FIQ_REGS: 275 return trusty_get_fiq_regs(handle); 276 case SMC_FC_FIQ_EXIT: 277 return trusty_fiq_exit(handle, x1, x2, x3); 278 default: 279 /* Not all OENs greater than SMC_ENTITY_SECURE_MONITOR are supported */ 280 if (SMC_ENTITY(smc_fid) > SMC_ENTITY_SECURE_MONITOR) { 281 VERBOSE("%s: unsupported SMC FID (0x%x)\n", __func__, smc_fid); 282 SMC_RET1(handle, SMC_UNK); 283 } 284 285 if (is_hypervisor_mode()) 286 vmid = SMC_GET_GP(handle, CTX_GPREG_X7); 287 288 if ((current_vmid != 0) && (current_vmid != vmid)) { 289 /* This message will cause SMC mechanism 290 * abnormal in multi-guest environment. 291 * Change it to WARN in case you need it. 292 */ 293 VERBOSE("Previous SMC not finished.\n"); 294 SMC_RET1(handle, SM_ERR_BUSY); 295 } 296 current_vmid = vmid; 297 ret = trusty_context_switch(NON_SECURE, smc_fid, x1, 298 x2, x3); 299 current_vmid = 0; 300 SMC_RET1(handle, ret.r0); 301 } 302 } 303 } 304 305 static int32_t trusty_init(void) 306 { 307 entry_point_info_t *ep_info; 308 struct smc_args zero_args = {0}; 309 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 310 uint32_t cpu = plat_my_core_pos(); 311 uint64_t reg_width = GET_RW(read_ctx_reg(get_el3state_ctx(&ctx->cpu_ctx), 312 CTX_SPSR_EL3)); 313 314 /* 315 * Get information about the Trusty image. Its absence is a critical 316 * failure. 317 */ 318 ep_info = bl31_plat_get_next_image_ep_info(SECURE); 319 assert(ep_info != NULL); 320 321 fpregs_context_save(get_fpregs_ctx(cm_get_context(NON_SECURE))); 322 cm_el1_sysregs_context_save(NON_SECURE); 323 324 cm_set_context(&ctx->cpu_ctx, SECURE); 325 cm_init_my_context(ep_info); 326 327 /* 328 * Adjust secondary cpu entry point for 32 bit images to the 329 * end of exception vectors 330 */ 331 if ((cpu != 0U) && (reg_width == MODE_RW_32)) { 332 INFO("trusty: cpu %d, adjust entry point to 0x%lx\n", 333 cpu, ep_info->pc + (1U << 5)); 334 cm_set_elr_el3(SECURE, ep_info->pc + (1U << 5)); 335 } 336 337 cm_el1_sysregs_context_restore(SECURE); 338 fpregs_context_restore(get_fpregs_ctx(cm_get_context(SECURE))); 339 cm_set_next_eret_context(SECURE); 340 341 ctx->saved_security_state = ~0U; /* initial saved state is invalid */ 342 (void)trusty_init_context_stack(&ctx->saved_sp, &ctx->secure_stack.end); 343 344 (void)trusty_context_switch_helper(&ctx->saved_sp, &zero_args); 345 346 cm_el1_sysregs_context_restore(NON_SECURE); 347 fpregs_context_restore(get_fpregs_ctx(cm_get_context(NON_SECURE))); 348 cm_set_next_eret_context(NON_SECURE); 349 350 return 1; 351 } 352 353 static void trusty_cpu_suspend(uint32_t off) 354 { 355 struct smc_args ret; 356 357 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_SUSPEND, off, 0, 0); 358 if (ret.r0 != 0U) { 359 INFO("%s: cpu %d, SMC_FC_CPU_SUSPEND returned unexpected value, %lld\n", 360 __func__, plat_my_core_pos(), ret.r0); 361 } 362 } 363 364 static void trusty_cpu_resume(uint32_t on) 365 { 366 struct smc_args ret; 367 368 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_RESUME, on, 0, 0); 369 if (ret.r0 != 0U) { 370 INFO("%s: cpu %d, SMC_FC_CPU_RESUME returned unexpected value, %lld\n", 371 __func__, plat_my_core_pos(), ret.r0); 372 } 373 } 374 375 static int32_t trusty_cpu_off_handler(u_register_t max_off_lvl) 376 { 377 trusty_cpu_suspend(max_off_lvl); 378 379 return 0; 380 } 381 382 static void trusty_cpu_on_finish_handler(u_register_t max_off_lvl) 383 { 384 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 385 386 if (ctx->saved_sp == NULL) { 387 (void)trusty_init(); 388 } else { 389 trusty_cpu_resume(max_off_lvl); 390 } 391 } 392 393 static void trusty_cpu_suspend_handler(u_register_t max_off_lvl) 394 { 395 trusty_cpu_suspend(max_off_lvl); 396 } 397 398 static void trusty_cpu_suspend_finish_handler(u_register_t max_off_lvl) 399 { 400 trusty_cpu_resume(max_off_lvl); 401 } 402 403 static const spd_pm_ops_t trusty_pm = { 404 .svc_off = trusty_cpu_off_handler, 405 .svc_suspend = trusty_cpu_suspend_handler, 406 .svc_on_finish = trusty_cpu_on_finish_handler, 407 .svc_suspend_finish = trusty_cpu_suspend_finish_handler, 408 }; 409 410 void plat_trusty_set_boot_args(aapcs64_params_t *args); 411 412 #if !defined(TSP_SEC_MEM_SIZE) && defined(BL32_MEM_SIZE) 413 #define TSP_SEC_MEM_SIZE BL32_MEM_SIZE 414 #endif 415 416 #ifdef TSP_SEC_MEM_SIZE 417 #pragma weak plat_trusty_set_boot_args 418 void plat_trusty_set_boot_args(aapcs64_params_t *args) 419 { 420 args->arg0 = TSP_SEC_MEM_SIZE; 421 } 422 #endif 423 424 static int32_t trusty_setup(void) 425 { 426 entry_point_info_t *ep_info; 427 uint32_t instr; 428 uint32_t flags; 429 int32_t ret; 430 bool aarch32 = false; 431 432 /* Get trusty's entry point info */ 433 ep_info = bl31_plat_get_next_image_ep_info(SECURE); 434 if (ep_info == NULL) { 435 VERBOSE("Trusty image missing.\n"); 436 return -1; 437 } 438 439 /* memmap first page of trusty's code memory before peeking */ 440 ret = mmap_add_dynamic_region(ep_info->pc, /* PA */ 441 ep_info->pc, /* VA */ 442 PAGE_SIZE, /* size */ 443 MT_SECURE | MT_RW_DATA); /* attrs */ 444 assert(ret == 0); 445 446 /* peek into trusty's code to see if we have a 32-bit or 64-bit image */ 447 instr = *(uint32_t *)ep_info->pc; 448 449 if (instr >> 24 == 0xeaU) { 450 INFO("trusty: Found 32 bit image\n"); 451 aarch32 = true; 452 } else if (instr >> 8 == 0xd53810U || instr >> 16 == 0x9400U) { 453 INFO("trusty: Found 64 bit image\n"); 454 } else { 455 ERROR("trusty: Found unknown image, 0x%x\n", instr); 456 return -1; 457 } 458 459 /* unmap trusty's memory page */ 460 (void)mmap_remove_dynamic_region(ep_info->pc, PAGE_SIZE); 461 462 SET_PARAM_HEAD(ep_info, PARAM_EP, VERSION_1, SECURE | EP_ST_ENABLE); 463 if (!aarch32) 464 ep_info->spsr = SPSR_64(MODE_EL1, MODE_SP_ELX, 465 DISABLE_ALL_EXCEPTIONS); 466 else 467 ep_info->spsr = SPSR_MODE32(MODE32_svc, SPSR_T_ARM, 468 SPSR_E_LITTLE, 469 DAIF_FIQ_BIT | 470 DAIF_IRQ_BIT | 471 DAIF_ABT_BIT); 472 (void)memset(&ep_info->args, 0, sizeof(ep_info->args)); 473 plat_trusty_set_boot_args(&ep_info->args); 474 475 /* register init handler */ 476 bl31_register_bl32_init(trusty_init); 477 478 /* register power management hooks */ 479 psci_register_spd_pm_hook(&trusty_pm); 480 481 /* register interrupt handler */ 482 flags = 0; 483 set_interrupt_rm_flag(flags, NON_SECURE); 484 ret = register_interrupt_type_handler(INTR_TYPE_S_EL1, 485 trusty_fiq_handler, 486 flags); 487 if (ret != 0) { 488 VERBOSE("trusty: failed to register fiq handler, ret = %d\n", ret); 489 } 490 491 if (aarch32) { 492 entry_point_info_t *ns_ep_info; 493 uint32_t spsr; 494 495 ns_ep_info = bl31_plat_get_next_image_ep_info(NON_SECURE); 496 if (ns_ep_info == NULL) { 497 NOTICE("Trusty: non-secure image missing.\n"); 498 return -1; 499 } 500 spsr = ns_ep_info->spsr; 501 if (GET_RW(spsr) == MODE_RW_64 && GET_EL(spsr) == MODE_EL2) { 502 spsr &= ~(MODE_EL_MASK << MODE_EL_SHIFT); 503 spsr |= MODE_EL1 << MODE_EL_SHIFT; 504 } 505 if (GET_RW(spsr) == MODE_RW_32 && GET_M32(spsr) == MODE32_hyp) { 506 spsr &= ~(MODE32_MASK << MODE32_SHIFT); 507 spsr |= MODE32_svc << MODE32_SHIFT; 508 } 509 if (spsr != ns_ep_info->spsr) { 510 NOTICE("Trusty: Switch bl33 from EL2 to EL1 (spsr 0x%x -> 0x%x)\n", 511 ns_ep_info->spsr, spsr); 512 ns_ep_info->spsr = spsr; 513 } 514 } 515 516 return 0; 517 } 518 519 /* Define a SPD runtime service descriptor for fast SMC calls */ 520 DECLARE_RT_SVC( 521 trusty_fast, 522 523 OEN_TOS_START, 524 OEN_TOS_END, 525 SMC_TYPE_FAST, 526 trusty_setup, 527 trusty_smc_handler 528 ); 529 530 /* Define a SPD runtime service descriptor for yielding SMC calls */ 531 DECLARE_RT_SVC( 532 trusty_std, 533 534 OEN_TAP_START, 535 SMC_ENTITY_SECURE_MONITOR, 536 SMC_TYPE_YIELD, 537 NULL, 538 trusty_smc_handler 539 ); 540