1 /* 2 * Copyright (c) 2016-2019, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <lib/xlat_tables/xlat_tables_v2.h> 9 #include <stdbool.h> 10 #include <string.h> 11 12 #include <arch_helpers.h> 13 #include <bl31/bl31.h> 14 #include <bl31/interrupt_mgmt.h> 15 #include <common/bl_common.h> 16 #include <common/debug.h> 17 #include <common/runtime_svc.h> 18 #include <lib/el3_runtime/context_mgmt.h> 19 #include <plat/common/platform.h> 20 21 #include "sm_err.h" 22 #include "smcall.h" 23 24 /* macro to check if Hypervisor is enabled in the HCR_EL2 register */ 25 #define HYP_ENABLE_FLAG 0x286001U 26 27 /* length of Trusty's input parameters (in bytes) */ 28 #define TRUSTY_PARAMS_LEN_BYTES (4096U * 2) 29 30 struct trusty_stack { 31 uint8_t space[PLATFORM_STACK_SIZE] __aligned(16); 32 uint32_t end; 33 }; 34 35 struct trusty_cpu_ctx { 36 cpu_context_t cpu_ctx; 37 void *saved_sp; 38 uint32_t saved_security_state; 39 int32_t fiq_handler_active; 40 uint64_t fiq_handler_pc; 41 uint64_t fiq_handler_cpsr; 42 uint64_t fiq_handler_sp; 43 uint64_t fiq_pc; 44 uint64_t fiq_cpsr; 45 uint64_t fiq_sp_el1; 46 gp_regs_t fiq_gpregs; 47 struct trusty_stack secure_stack; 48 }; 49 50 struct smc_args { 51 uint64_t r0; 52 uint64_t r1; 53 uint64_t r2; 54 uint64_t r3; 55 uint64_t r4; 56 uint64_t r5; 57 uint64_t r6; 58 uint64_t r7; 59 }; 60 61 static struct trusty_cpu_ctx trusty_cpu_ctx[PLATFORM_CORE_COUNT]; 62 63 struct smc_args trusty_init_context_stack(void **sp, void *new_stack); 64 struct smc_args trusty_context_switch_helper(void **sp, void *smc_params); 65 66 static uint32_t current_vmid; 67 68 static struct trusty_cpu_ctx *get_trusty_ctx(void) 69 { 70 return &trusty_cpu_ctx[plat_my_core_pos()]; 71 } 72 73 static bool is_hypervisor_mode(void) 74 { 75 uint64_t hcr = read_hcr(); 76 77 return ((hcr & HYP_ENABLE_FLAG) != 0U) ? true : false; 78 } 79 80 static struct smc_args trusty_context_switch(uint32_t security_state, uint64_t r0, 81 uint64_t r1, uint64_t r2, uint64_t r3) 82 { 83 struct smc_args args, ret_args; 84 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 85 struct trusty_cpu_ctx *ctx_smc; 86 87 assert(ctx->saved_security_state != security_state); 88 89 args.r7 = 0; 90 if (is_hypervisor_mode()) { 91 /* According to the ARM DEN0028A spec, VMID is stored in x7 */ 92 ctx_smc = cm_get_context(NON_SECURE); 93 assert(ctx_smc != NULL); 94 args.r7 = SMC_GET_GP(ctx_smc, CTX_GPREG_X7); 95 } 96 /* r4, r5, r6 reserved for future use. */ 97 args.r6 = 0; 98 args.r5 = 0; 99 args.r4 = 0; 100 args.r3 = r3; 101 args.r2 = r2; 102 args.r1 = r1; 103 args.r0 = r0; 104 105 /* 106 * To avoid the additional overhead in PSCI flow, skip FP context 107 * saving/restoring in case of CPU suspend and resume, assuming that 108 * when it's needed the PSCI caller has preserved FP context before 109 * going here. 110 */ 111 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME) 112 fpregs_context_save(get_fpregs_ctx(cm_get_context(security_state))); 113 cm_el1_sysregs_context_save(security_state); 114 115 ctx->saved_security_state = security_state; 116 ret_args = trusty_context_switch_helper(&ctx->saved_sp, &args); 117 118 assert(ctx->saved_security_state == ((security_state == 0U) ? 1U : 0U)); 119 120 cm_el1_sysregs_context_restore(security_state); 121 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME) 122 fpregs_context_restore(get_fpregs_ctx(cm_get_context(security_state))); 123 124 cm_set_next_eret_context(security_state); 125 126 return ret_args; 127 } 128 129 static uint64_t trusty_fiq_handler(uint32_t id, 130 uint32_t flags, 131 void *handle, 132 void *cookie) 133 { 134 struct smc_args ret; 135 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 136 137 assert(!is_caller_secure(flags)); 138 139 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_ENTER, 0, 0, 0); 140 if (ret.r0 != 0U) { 141 SMC_RET0(handle); 142 } 143 144 if (ctx->fiq_handler_active != 0) { 145 INFO("%s: fiq handler already active\n", __func__); 146 SMC_RET0(handle); 147 } 148 149 ctx->fiq_handler_active = 1; 150 (void)memcpy(&ctx->fiq_gpregs, get_gpregs_ctx(handle), sizeof(ctx->fiq_gpregs)); 151 ctx->fiq_pc = SMC_GET_EL3(handle, CTX_ELR_EL3); 152 ctx->fiq_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3); 153 ctx->fiq_sp_el1 = read_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1); 154 155 write_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1, ctx->fiq_handler_sp); 156 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_handler_pc, (uint32_t)ctx->fiq_handler_cpsr); 157 158 SMC_RET0(handle); 159 } 160 161 static uint64_t trusty_set_fiq_handler(void *handle, uint64_t cpu, 162 uint64_t handler, uint64_t stack) 163 { 164 struct trusty_cpu_ctx *ctx; 165 166 if (cpu >= (uint64_t)PLATFORM_CORE_COUNT) { 167 ERROR("%s: cpu %lld >= %d\n", __func__, cpu, PLATFORM_CORE_COUNT); 168 return (uint64_t)SM_ERR_INVALID_PARAMETERS; 169 } 170 171 ctx = &trusty_cpu_ctx[cpu]; 172 ctx->fiq_handler_pc = handler; 173 ctx->fiq_handler_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3); 174 ctx->fiq_handler_sp = stack; 175 176 SMC_RET1(handle, 0); 177 } 178 179 static uint64_t trusty_get_fiq_regs(void *handle) 180 { 181 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 182 uint64_t sp_el0 = read_ctx_reg(&ctx->fiq_gpregs, CTX_GPREG_SP_EL0); 183 184 SMC_RET4(handle, ctx->fiq_pc, ctx->fiq_cpsr, sp_el0, ctx->fiq_sp_el1); 185 } 186 187 static uint64_t trusty_fiq_exit(void *handle, uint64_t x1, uint64_t x2, uint64_t x3) 188 { 189 struct smc_args ret; 190 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 191 192 if (ctx->fiq_handler_active == 0) { 193 NOTICE("%s: fiq handler not active\n", __func__); 194 SMC_RET1(handle, (uint64_t)SM_ERR_INVALID_PARAMETERS); 195 } 196 197 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_EXIT, 0, 0, 0); 198 if (ret.r0 != 1U) { 199 INFO("%s(%p) SMC_FC_FIQ_EXIT returned unexpected value, %lld\n", 200 __func__, handle, ret.r0); 201 } 202 203 /* 204 * Restore register state to state recorded on fiq entry. 205 * 206 * x0, sp_el1, pc and cpsr need to be restored because el1 cannot 207 * restore them. 208 * 209 * x1-x4 and x8-x17 need to be restored here because smc_handler64 210 * corrupts them (el1 code also restored them). 211 */ 212 (void)memcpy(get_gpregs_ctx(handle), &ctx->fiq_gpregs, sizeof(ctx->fiq_gpregs)); 213 ctx->fiq_handler_active = 0; 214 write_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1, ctx->fiq_sp_el1); 215 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_pc, (uint32_t)ctx->fiq_cpsr); 216 217 SMC_RET0(handle); 218 } 219 220 static uintptr_t trusty_smc_handler(uint32_t smc_fid, 221 u_register_t x1, 222 u_register_t x2, 223 u_register_t x3, 224 u_register_t x4, 225 void *cookie, 226 void *handle, 227 u_register_t flags) 228 { 229 struct smc_args ret; 230 uint32_t vmid = 0U; 231 entry_point_info_t *ep_info = bl31_plat_get_next_image_ep_info(SECURE); 232 233 /* 234 * Return success for SET_ROT_PARAMS if Trusty is not present, as 235 * Verified Boot is not even supported and returning success here 236 * would not compromise the boot process. 237 */ 238 if ((ep_info == NULL) && (smc_fid == SMC_YC_SET_ROT_PARAMS)) { 239 SMC_RET1(handle, 0); 240 } else if (ep_info == NULL) { 241 SMC_RET1(handle, SMC_UNK); 242 } else { 243 ; /* do nothing */ 244 } 245 246 if (is_caller_secure(flags)) { 247 if (smc_fid == SMC_YC_NS_RETURN) { 248 ret = trusty_context_switch(SECURE, x1, 0, 0, 0); 249 SMC_RET8(handle, ret.r0, ret.r1, ret.r2, ret.r3, 250 ret.r4, ret.r5, ret.r6, ret.r7); 251 } 252 INFO("%s (0x%x, 0x%lx, 0x%lx, 0x%lx, 0x%lx, %p, %p, 0x%lx) \ 253 cpu %d, unknown smc\n", 254 __func__, smc_fid, x1, x2, x3, x4, cookie, handle, flags, 255 plat_my_core_pos()); 256 SMC_RET1(handle, SMC_UNK); 257 } else { 258 switch (smc_fid) { 259 case SMC_FC64_SET_FIQ_HANDLER: 260 return trusty_set_fiq_handler(handle, x1, x2, x3); 261 case SMC_FC64_GET_FIQ_REGS: 262 return trusty_get_fiq_regs(handle); 263 case SMC_FC_FIQ_EXIT: 264 return trusty_fiq_exit(handle, x1, x2, x3); 265 default: 266 if (is_hypervisor_mode()) 267 vmid = SMC_GET_GP(handle, CTX_GPREG_X7); 268 269 if ((current_vmid != 0) && (current_vmid != vmid)) { 270 /* This message will cause SMC mechanism 271 * abnormal in multi-guest environment. 272 * Change it to WARN in case you need it. 273 */ 274 VERBOSE("Previous SMC not finished.\n"); 275 SMC_RET1(handle, SM_ERR_BUSY); 276 } 277 current_vmid = vmid; 278 ret = trusty_context_switch(NON_SECURE, smc_fid, x1, 279 x2, x3); 280 current_vmid = 0; 281 SMC_RET1(handle, ret.r0); 282 } 283 } 284 } 285 286 static int32_t trusty_init(void) 287 { 288 entry_point_info_t *ep_info; 289 struct smc_args zero_args = {0}; 290 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 291 uint32_t cpu = plat_my_core_pos(); 292 uint64_t reg_width = GET_RW(read_ctx_reg(get_el3state_ctx(&ctx->cpu_ctx), 293 CTX_SPSR_EL3)); 294 295 /* 296 * Get information about the Trusty image. Its absence is a critical 297 * failure. 298 */ 299 ep_info = bl31_plat_get_next_image_ep_info(SECURE); 300 assert(ep_info != NULL); 301 302 fpregs_context_save(get_fpregs_ctx(cm_get_context(NON_SECURE))); 303 cm_el1_sysregs_context_save(NON_SECURE); 304 305 cm_set_context(&ctx->cpu_ctx, SECURE); 306 cm_init_my_context(ep_info); 307 308 /* 309 * Adjust secondary cpu entry point for 32 bit images to the 310 * end of exception vectors 311 */ 312 if ((cpu != 0U) && (reg_width == MODE_RW_32)) { 313 INFO("trusty: cpu %d, adjust entry point to 0x%lx\n", 314 cpu, ep_info->pc + (1U << 5)); 315 cm_set_elr_el3(SECURE, ep_info->pc + (1U << 5)); 316 } 317 318 cm_el1_sysregs_context_restore(SECURE); 319 fpregs_context_restore(get_fpregs_ctx(cm_get_context(SECURE))); 320 cm_set_next_eret_context(SECURE); 321 322 ctx->saved_security_state = ~0U; /* initial saved state is invalid */ 323 (void)trusty_init_context_stack(&ctx->saved_sp, &ctx->secure_stack.end); 324 325 (void)trusty_context_switch_helper(&ctx->saved_sp, &zero_args); 326 327 cm_el1_sysregs_context_restore(NON_SECURE); 328 fpregs_context_restore(get_fpregs_ctx(cm_get_context(NON_SECURE))); 329 cm_set_next_eret_context(NON_SECURE); 330 331 return 1; 332 } 333 334 static void trusty_cpu_suspend(uint32_t off) 335 { 336 struct smc_args ret; 337 338 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_SUSPEND, off, 0, 0); 339 if (ret.r0 != 0U) { 340 INFO("%s: cpu %d, SMC_FC_CPU_SUSPEND returned unexpected value, %lld\n", 341 __func__, plat_my_core_pos(), ret.r0); 342 } 343 } 344 345 static void trusty_cpu_resume(uint32_t on) 346 { 347 struct smc_args ret; 348 349 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_RESUME, on, 0, 0); 350 if (ret.r0 != 0U) { 351 INFO("%s: cpu %d, SMC_FC_CPU_RESUME returned unexpected value, %lld\n", 352 __func__, plat_my_core_pos(), ret.r0); 353 } 354 } 355 356 static int32_t trusty_cpu_off_handler(u_register_t max_off_lvl) 357 { 358 trusty_cpu_suspend(max_off_lvl); 359 360 return 0; 361 } 362 363 static void trusty_cpu_on_finish_handler(u_register_t max_off_lvl) 364 { 365 struct trusty_cpu_ctx *ctx = get_trusty_ctx(); 366 367 if (ctx->saved_sp == NULL) { 368 (void)trusty_init(); 369 } else { 370 trusty_cpu_resume(max_off_lvl); 371 } 372 } 373 374 static void trusty_cpu_suspend_handler(u_register_t max_off_lvl) 375 { 376 trusty_cpu_suspend(max_off_lvl); 377 } 378 379 static void trusty_cpu_suspend_finish_handler(u_register_t max_off_lvl) 380 { 381 trusty_cpu_resume(max_off_lvl); 382 } 383 384 static const spd_pm_ops_t trusty_pm = { 385 .svc_off = trusty_cpu_off_handler, 386 .svc_suspend = trusty_cpu_suspend_handler, 387 .svc_on_finish = trusty_cpu_on_finish_handler, 388 .svc_suspend_finish = trusty_cpu_suspend_finish_handler, 389 }; 390 391 void plat_trusty_set_boot_args(aapcs64_params_t *args); 392 393 #if !defined(TSP_SEC_MEM_SIZE) && defined(BL32_MEM_SIZE) 394 #define TSP_SEC_MEM_SIZE BL32_MEM_SIZE 395 #endif 396 397 #ifdef TSP_SEC_MEM_SIZE 398 #pragma weak plat_trusty_set_boot_args 399 void plat_trusty_set_boot_args(aapcs64_params_t *args) 400 { 401 args->arg0 = TSP_SEC_MEM_SIZE; 402 } 403 #endif 404 405 static int32_t trusty_setup(void) 406 { 407 entry_point_info_t *ep_info; 408 uint32_t instr; 409 uint32_t flags; 410 int32_t ret; 411 bool aarch32 = false; 412 413 /* Get trusty's entry point info */ 414 ep_info = bl31_plat_get_next_image_ep_info(SECURE); 415 if (ep_info == NULL) { 416 INFO("Trusty image missing.\n"); 417 return -1; 418 } 419 420 /* memmap first page of trusty's code memory before peeking */ 421 ret = mmap_add_dynamic_region(ep_info->pc, /* PA */ 422 ep_info->pc, /* VA */ 423 PAGE_SIZE, /* size */ 424 MT_SECURE | MT_RW_DATA); /* attrs */ 425 assert(ret == 0); 426 427 /* peek into trusty's code to see if we have a 32-bit or 64-bit image */ 428 instr = *(uint32_t *)ep_info->pc; 429 430 if (instr >> 24 == 0xeaU) { 431 INFO("trusty: Found 32 bit image\n"); 432 aarch32 = true; 433 } else if (instr >> 8 == 0xd53810U || instr >> 16 == 0x9400U) { 434 INFO("trusty: Found 64 bit image\n"); 435 } else { 436 ERROR("trusty: Found unknown image, 0x%x\n", instr); 437 return -1; 438 } 439 440 /* unmap trusty's memory page */ 441 (void)mmap_remove_dynamic_region(ep_info->pc, PAGE_SIZE); 442 443 SET_PARAM_HEAD(ep_info, PARAM_EP, VERSION_1, SECURE | EP_ST_ENABLE); 444 if (!aarch32) 445 ep_info->spsr = SPSR_64(MODE_EL1, MODE_SP_ELX, 446 DISABLE_ALL_EXCEPTIONS); 447 else 448 ep_info->spsr = SPSR_MODE32(MODE32_svc, SPSR_T_ARM, 449 SPSR_E_LITTLE, 450 DAIF_FIQ_BIT | 451 DAIF_IRQ_BIT | 452 DAIF_ABT_BIT); 453 (void)memset(&ep_info->args, 0, sizeof(ep_info->args)); 454 plat_trusty_set_boot_args(&ep_info->args); 455 456 /* register init handler */ 457 bl31_register_bl32_init(trusty_init); 458 459 /* register power management hooks */ 460 psci_register_spd_pm_hook(&trusty_pm); 461 462 /* register interrupt handler */ 463 flags = 0; 464 set_interrupt_rm_flag(flags, NON_SECURE); 465 ret = register_interrupt_type_handler(INTR_TYPE_S_EL1, 466 trusty_fiq_handler, 467 flags); 468 if (ret != 0) { 469 ERROR("trusty: failed to register fiq handler, ret = %d\n", ret); 470 } 471 472 if (aarch32) { 473 entry_point_info_t *ns_ep_info; 474 uint32_t spsr; 475 476 ns_ep_info = bl31_plat_get_next_image_ep_info(NON_SECURE); 477 if (ns_ep_info == NULL) { 478 NOTICE("Trusty: non-secure image missing.\n"); 479 return -1; 480 } 481 spsr = ns_ep_info->spsr; 482 if (GET_RW(spsr) == MODE_RW_64 && GET_EL(spsr) == MODE_EL2) { 483 spsr &= ~(MODE_EL_MASK << MODE_EL_SHIFT); 484 spsr |= MODE_EL1 << MODE_EL_SHIFT; 485 } 486 if (GET_RW(spsr) == MODE_RW_32 && GET_M32(spsr) == MODE32_hyp) { 487 spsr &= ~(MODE32_MASK << MODE32_SHIFT); 488 spsr |= MODE32_svc << MODE32_SHIFT; 489 } 490 if (spsr != ns_ep_info->spsr) { 491 NOTICE("Trusty: Switch bl33 from EL2 to EL1 (spsr 0x%x -> 0x%x)\n", 492 ns_ep_info->spsr, spsr); 493 ns_ep_info->spsr = spsr; 494 } 495 } 496 497 return 0; 498 } 499 500 /* Define a SPD runtime service descriptor for fast SMC calls */ 501 DECLARE_RT_SVC( 502 trusty_fast, 503 504 OEN_TOS_START, 505 SMC_ENTITY_SECURE_MONITOR, 506 SMC_TYPE_FAST, 507 trusty_setup, 508 trusty_smc_handler 509 ); 510 511 /* Define a SPD runtime service descriptor for yielding SMC calls */ 512 DECLARE_RT_SVC( 513 trusty_std, 514 515 OEN_TAP_START, 516 SMC_ENTITY_SECURE_MONITOR, 517 SMC_TYPE_YIELD, 518 NULL, 519 trusty_smc_handler 520 ); 521