1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2015-2022, Linaro Limited 4 */ 5 6 #include <arm.h> 7 #include <kernel/abort.h> 8 #include <kernel/linker.h> 9 #include <kernel/misc.h> 10 #include <kernel/panic.h> 11 #include <kernel/tee_ta_manager.h> 12 #include <kernel/thread_private.h> 13 #include <kernel/user_mode_ctx.h> 14 #include <memtag.h> 15 #include <mm/core_mmu.h> 16 #include <mm/mobj.h> 17 #include <mm/tee_pager.h> 18 #include <tee/tee_svc.h> 19 #include <trace.h> 20 #include <unw/unwind.h> 21 22 enum fault_type { 23 FAULT_TYPE_USER_MODE_PANIC, 24 FAULT_TYPE_USER_MODE_VFP, 25 FAULT_TYPE_PAGEABLE, 26 FAULT_TYPE_IGNORE, 27 }; 28 29 #ifdef CFG_UNWIND 30 31 #ifdef ARM32 32 /* 33 * Kernel or user mode unwind (32-bit execution state). 34 */ 35 static void __print_stack_unwind(struct abort_info *ai) 36 { 37 struct unwind_state_arm32 state = { }; 38 uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; 39 uint32_t sp = 0; 40 uint32_t lr = 0; 41 42 assert(!abort_is_user_exception(ai)); 43 44 if (mode == CPSR_MODE_SYS) { 45 sp = ai->regs->usr_sp; 46 lr = ai->regs->usr_lr; 47 } else { 48 sp = read_mode_sp(mode); 49 lr = read_mode_lr(mode); 50 } 51 52 memset(&state, 0, sizeof(state)); 53 state.registers[0] = ai->regs->r0; 54 state.registers[1] = ai->regs->r1; 55 state.registers[2] = ai->regs->r2; 56 state.registers[3] = ai->regs->r3; 57 state.registers[4] = ai->regs->r4; 58 state.registers[5] = ai->regs->r5; 59 state.registers[6] = ai->regs->r6; 60 state.registers[7] = ai->regs->r7; 61 state.registers[8] = ai->regs->r8; 62 state.registers[9] = ai->regs->r9; 63 state.registers[10] = ai->regs->r10; 64 state.registers[11] = ai->regs->r11; 65 state.registers[13] = sp; 66 state.registers[14] = lr; 67 state.registers[15] = ai->pc; 68 69 print_stack_arm32(&state, thread_stack_start(), thread_stack_size()); 70 } 71 #endif /* ARM32 */ 72 73 #ifdef ARM64 74 /* Kernel mode unwind (64-bit execution state) */ 75 static void __print_stack_unwind(struct abort_info *ai) 76 { 77 struct unwind_state_arm64 state = { 78 .pc = ai->regs->elr, 79 .fp = ai->regs->x29, 80 }; 81 82 print_stack_arm64(&state, thread_stack_start(), thread_stack_size()); 83 } 84 #endif /*ARM64*/ 85 86 #else /* CFG_UNWIND */ 87 static void __print_stack_unwind(struct abort_info *ai __unused) 88 { 89 } 90 #endif /* CFG_UNWIND */ 91 92 static __maybe_unused const char *abort_type_to_str(uint32_t abort_type) 93 { 94 if (abort_type == ABORT_TYPE_DATA) 95 return "data"; 96 if (abort_type == ABORT_TYPE_PREFETCH) 97 return "prefetch"; 98 return "undef"; 99 } 100 101 static __maybe_unused const char *fault_to_str(uint32_t abort_type, 102 uint32_t fault_descr) 103 { 104 /* fault_descr is only valid for data or prefetch abort */ 105 if (abort_type != ABORT_TYPE_DATA && abort_type != ABORT_TYPE_PREFETCH) 106 return ""; 107 108 switch (core_mmu_get_fault_type(fault_descr)) { 109 case CORE_MMU_FAULT_ALIGNMENT: 110 return " (alignment fault)"; 111 case CORE_MMU_FAULT_TRANSLATION: 112 return " (translation fault)"; 113 case CORE_MMU_FAULT_READ_PERMISSION: 114 return " (read permission fault)"; 115 case CORE_MMU_FAULT_WRITE_PERMISSION: 116 return " (write permission fault)"; 117 case CORE_MMU_FAULT_TAG_CHECK: 118 return " (tag check fault)"; 119 default: 120 return ""; 121 } 122 } 123 124 static __maybe_unused void 125 __print_abort_info(struct abort_info *ai __maybe_unused, 126 const char *ctx __maybe_unused) 127 { 128 __maybe_unused size_t core_pos = 0; 129 #ifdef ARM32 130 uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; 131 __maybe_unused uint32_t sp = 0; 132 __maybe_unused uint32_t lr = 0; 133 134 if (mode == CPSR_MODE_USR || mode == CPSR_MODE_SYS) { 135 sp = ai->regs->usr_sp; 136 lr = ai->regs->usr_lr; 137 core_pos = thread_get_tsd()->abort_core; 138 } else { 139 sp = read_mode_sp(mode); 140 lr = read_mode_lr(mode); 141 core_pos = get_core_pos(); 142 } 143 #endif /*ARM32*/ 144 #ifdef ARM64 145 if (abort_is_user_exception(ai)) 146 core_pos = thread_get_tsd()->abort_core; 147 else 148 core_pos = get_core_pos(); 149 #endif /*ARM64*/ 150 151 EMSG_RAW(""); 152 if (IS_ENABLED(CFG_MEMTAG)) 153 EMSG_RAW("%s %s-abort at address 0x%" PRIxVA 154 " [tagged 0x%" PRIxVA "]%s", ctx, 155 abort_type_to_str(ai->abort_type), 156 memtag_strip_tag_vaddr((void *)ai->va), ai->va, 157 fault_to_str(ai->abort_type, ai->fault_descr)); 158 else 159 EMSG_RAW("%s %s-abort at address 0x%" PRIxVA "%s", ctx, 160 abort_type_to_str(ai->abort_type), ai->va, 161 fault_to_str(ai->abort_type, ai->fault_descr)); 162 #ifdef ARM32 163 EMSG_RAW(" fsr 0x%08x ttbr0 0x%08x ttbr1 0x%08x cidr 0x%X", 164 ai->fault_descr, read_ttbr0(), read_ttbr1(), 165 read_contextidr()); 166 EMSG_RAW(" cpu #%zu cpsr 0x%08x", 167 core_pos, ai->regs->spsr); 168 EMSG_RAW(" r0 0x%08x r4 0x%08x r8 0x%08x r12 0x%08x", 169 ai->regs->r0, ai->regs->r4, ai->regs->r8, ai->regs->ip); 170 EMSG_RAW(" r1 0x%08x r5 0x%08x r9 0x%08x sp 0x%08x", 171 ai->regs->r1, ai->regs->r5, ai->regs->r9, sp); 172 EMSG_RAW(" r2 0x%08x r6 0x%08x r10 0x%08x lr 0x%08x", 173 ai->regs->r2, ai->regs->r6, ai->regs->r10, lr); 174 EMSG_RAW(" r3 0x%08x r7 0x%08x r11 0x%08x pc 0x%08x", 175 ai->regs->r3, ai->regs->r7, ai->regs->r11, ai->pc); 176 #endif /*ARM32*/ 177 #ifdef ARM64 178 EMSG_RAW(" esr 0x%08x ttbr0 0x%08" PRIx64 " ttbr1 0x%08" PRIx64 179 " cidr 0x%X", ai->fault_descr, read_ttbr0_el1(), 180 read_ttbr1_el1(), read_contextidr_el1()); 181 EMSG_RAW(" cpu #%zu cpsr 0x%08x", 182 core_pos, (uint32_t)ai->regs->spsr); 183 EMSG_RAW(" x0 %016" PRIx64 " x1 %016" PRIx64, 184 ai->regs->x0, ai->regs->x1); 185 EMSG_RAW(" x2 %016" PRIx64 " x3 %016" PRIx64, 186 ai->regs->x2, ai->regs->x3); 187 EMSG_RAW(" x4 %016" PRIx64 " x5 %016" PRIx64, 188 ai->regs->x4, ai->regs->x5); 189 EMSG_RAW(" x6 %016" PRIx64 " x7 %016" PRIx64, 190 ai->regs->x6, ai->regs->x7); 191 EMSG_RAW(" x8 %016" PRIx64 " x9 %016" PRIx64, 192 ai->regs->x8, ai->regs->x9); 193 EMSG_RAW(" x10 %016" PRIx64 " x11 %016" PRIx64, 194 ai->regs->x10, ai->regs->x11); 195 EMSG_RAW(" x12 %016" PRIx64 " x13 %016" PRIx64, 196 ai->regs->x12, ai->regs->x13); 197 EMSG_RAW(" x14 %016" PRIx64 " x15 %016" PRIx64, 198 ai->regs->x14, ai->regs->x15); 199 EMSG_RAW(" x16 %016" PRIx64 " x17 %016" PRIx64, 200 ai->regs->x16, ai->regs->x17); 201 EMSG_RAW(" x18 %016" PRIx64 " x19 %016" PRIx64, 202 ai->regs->x18, ai->regs->x19); 203 EMSG_RAW(" x20 %016" PRIx64 " x21 %016" PRIx64, 204 ai->regs->x20, ai->regs->x21); 205 EMSG_RAW(" x22 %016" PRIx64 " x23 %016" PRIx64, 206 ai->regs->x22, ai->regs->x23); 207 EMSG_RAW(" x24 %016" PRIx64 " x25 %016" PRIx64, 208 ai->regs->x24, ai->regs->x25); 209 EMSG_RAW(" x26 %016" PRIx64 " x27 %016" PRIx64, 210 ai->regs->x26, ai->regs->x27); 211 EMSG_RAW(" x28 %016" PRIx64 " x29 %016" PRIx64, 212 ai->regs->x28, ai->regs->x29); 213 EMSG_RAW(" x30 %016" PRIx64 " elr %016" PRIx64, 214 ai->regs->x30, ai->regs->elr); 215 EMSG_RAW(" sp_el0 %016" PRIx64, ai->regs->sp_el0); 216 #endif /*ARM64*/ 217 } 218 219 /* 220 * Print abort info and (optionally) stack dump to the console 221 * @ai kernel-mode abort info. 222 * @stack_dump true to show a stack trace 223 */ 224 static void __abort_print(struct abort_info *ai, bool stack_dump) 225 { 226 assert(!abort_is_user_exception(ai)); 227 228 __print_abort_info(ai, "Core"); 229 230 if (stack_dump) { 231 trace_printf_helper_raw(TRACE_ERROR, true, 232 "TEE load address @ %#"PRIxVA, 233 VCORE_START_VA); 234 __print_stack_unwind(ai); 235 } 236 } 237 238 void abort_print(struct abort_info *ai) 239 { 240 __abort_print(ai, false); 241 } 242 243 void abort_print_error(struct abort_info *ai) 244 { 245 __abort_print(ai, true); 246 } 247 248 /* This function must be called from a normal thread */ 249 void abort_print_current_ts(void) 250 { 251 struct thread_specific_data *tsd = thread_get_tsd(); 252 struct abort_info ai = { }; 253 struct ts_session *s = ts_get_current_session(); 254 255 ai.abort_type = tsd->abort_type; 256 ai.fault_descr = tsd->abort_descr; 257 ai.va = tsd->abort_va; 258 ai.pc = tsd->abort_regs.elr; 259 ai.regs = &tsd->abort_regs; 260 261 if (ai.abort_type != ABORT_TYPE_USER_MODE_PANIC) 262 __print_abort_info(&ai, "User mode"); 263 264 s->ctx->ops->dump_state(s->ctx); 265 266 #if defined(CFG_FTRACE_SUPPORT) 267 if (s->ctx->ops->dump_ftrace) { 268 s->fbuf = NULL; 269 s->ctx->ops->dump_ftrace(s->ctx); 270 } 271 #endif 272 } 273 274 static void save_abort_info_in_tsd(struct abort_info *ai) 275 { 276 struct thread_specific_data *tsd = thread_get_tsd(); 277 278 tsd->abort_type = ai->abort_type; 279 tsd->abort_descr = ai->fault_descr; 280 tsd->abort_va = ai->va; 281 tsd->abort_regs = *ai->regs; 282 tsd->abort_core = get_core_pos(); 283 } 284 285 #ifdef ARM32 286 static void set_abort_info(uint32_t abort_type, struct thread_abort_regs *regs, 287 struct abort_info *ai) 288 { 289 switch (abort_type) { 290 case ABORT_TYPE_DATA: 291 ai->fault_descr = read_dfsr(); 292 ai->va = read_dfar(); 293 break; 294 case ABORT_TYPE_PREFETCH: 295 ai->fault_descr = read_ifsr(); 296 ai->va = read_ifar(); 297 break; 298 default: 299 ai->fault_descr = 0; 300 ai->va = regs->elr; 301 break; 302 } 303 ai->abort_type = abort_type; 304 ai->pc = regs->elr; 305 ai->regs = regs; 306 } 307 #endif /*ARM32*/ 308 309 #ifdef ARM64 310 static void set_abort_info(uint32_t abort_type __unused, 311 struct thread_abort_regs *regs, struct abort_info *ai) 312 { 313 ai->fault_descr = read_esr_el1(); 314 switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { 315 case ESR_EC_IABT_EL0: 316 case ESR_EC_IABT_EL1: 317 ai->abort_type = ABORT_TYPE_PREFETCH; 318 ai->va = read_far_el1(); 319 break; 320 case ESR_EC_DABT_EL0: 321 case ESR_EC_DABT_EL1: 322 case ESR_EC_SP_ALIGN: 323 ai->abort_type = ABORT_TYPE_DATA; 324 ai->va = read_far_el1(); 325 break; 326 default: 327 ai->abort_type = ABORT_TYPE_UNDEF; 328 ai->va = regs->elr; 329 } 330 ai->pc = regs->elr; 331 ai->regs = regs; 332 } 333 #endif /*ARM64*/ 334 335 #ifdef ARM32 336 static void handle_user_mode_panic(struct abort_info *ai) 337 { 338 /* 339 * It was a user exception, stop user execution and return 340 * to TEE Core. 341 */ 342 ai->regs->r0 = TEE_ERROR_TARGET_DEAD; 343 ai->regs->r1 = true; 344 ai->regs->r2 = 0xdeadbeef; 345 ai->regs->elr = (uint32_t)thread_unwind_user_mode; 346 ai->regs->spsr &= CPSR_FIA; 347 ai->regs->spsr &= ~CPSR_MODE_MASK; 348 ai->regs->spsr |= CPSR_MODE_SVC; 349 /* Select Thumb or ARM mode */ 350 if (ai->regs->elr & 1) 351 ai->regs->spsr |= CPSR_T; 352 else 353 ai->regs->spsr &= ~CPSR_T; 354 } 355 #endif /*ARM32*/ 356 357 #ifdef ARM64 358 static void handle_user_mode_panic(struct abort_info *ai) 359 { 360 struct thread_ctx *tc __maybe_unused = NULL; 361 uint32_t daif = 0; 362 uint32_t pan_bit = 0; 363 364 /* 365 * It was a user exception, stop user execution and return 366 * to TEE Core. 367 */ 368 ai->regs->x0 = TEE_ERROR_TARGET_DEAD; 369 ai->regs->x1 = true; 370 ai->regs->x2 = 0xdeadbeef; 371 ai->regs->elr = (vaddr_t)thread_unwind_user_mode; 372 ai->regs->sp_el0 = thread_get_saved_thread_sp(); 373 374 #if defined(CFG_CORE_PAUTH) 375 /* 376 * We're going to return to the privileged core thread, update the 377 * APIA key to match the key used by the thread. 378 */ 379 tc = threads + thread_get_id(); 380 ai->regs->apiakey_hi = tc->keys.apia_hi; 381 ai->regs->apiakey_lo = tc->keys.apia_lo; 382 #endif 383 384 if (IS_ENABLED(CFG_PAN) && feat_pan_implemented() && read_pan()) 385 pan_bit = SPSR_64_PAN; 386 daif = (ai->regs->spsr >> SPSR_32_AIF_SHIFT) & SPSR_32_AIF_MASK; 387 /* XXX what about DAIF_D? */ 388 ai->regs->spsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, daif) | 389 pan_bit; 390 } 391 #endif /*ARM64*/ 392 393 #ifdef CFG_WITH_VFP 394 static void handle_user_mode_vfp(void) 395 { 396 struct ts_session *s = ts_get_current_session(); 397 398 thread_user_enable_vfp(&to_user_mode_ctx(s->ctx)->vfp); 399 } 400 #endif /*CFG_WITH_VFP*/ 401 402 #ifdef CFG_WITH_USER_TA 403 #ifdef ARM32 404 /* Returns true if the exception originated from user mode */ 405 bool abort_is_user_exception(struct abort_info *ai) 406 { 407 return (ai->regs->spsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; 408 } 409 #endif /*ARM32*/ 410 411 #ifdef ARM64 412 /* Returns true if the exception originated from user mode */ 413 bool abort_is_user_exception(struct abort_info *ai) 414 { 415 uint32_t spsr = ai->regs->spsr; 416 417 if (spsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) 418 return true; 419 if (((spsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == 420 SPSR_64_MODE_EL0) 421 return true; 422 return false; 423 } 424 #endif /*ARM64*/ 425 #else /*CFG_WITH_USER_TA*/ 426 bool abort_is_user_exception(struct abort_info *ai __unused) 427 { 428 return false; 429 } 430 #endif /*CFG_WITH_USER_TA*/ 431 432 #if defined(CFG_WITH_VFP) && defined(CFG_WITH_USER_TA) 433 #ifdef ARM32 434 static bool is_vfp_fault(struct abort_info *ai) 435 { 436 if ((ai->abort_type != ABORT_TYPE_UNDEF) || vfp_is_enabled()) 437 return false; 438 439 /* 440 * Not entirely accurate, but if it's a truly undefined instruction 441 * we'll end up in this function again, except this time 442 * vfp_is_enabled() so we'll return false. 443 */ 444 return true; 445 } 446 #endif /*ARM32*/ 447 448 #ifdef ARM64 449 static bool is_vfp_fault(struct abort_info *ai) 450 { 451 switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { 452 case ESR_EC_FP_ASIMD: 453 case ESR_EC_AARCH32_FP: 454 case ESR_EC_AARCH64_FP: 455 return true; 456 default: 457 return false; 458 } 459 } 460 #endif /*ARM64*/ 461 #else /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ 462 static bool is_vfp_fault(struct abort_info *ai __unused) 463 { 464 return false; 465 } 466 #endif /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ 467 468 bool abort_is_write_fault(struct abort_info *ai) 469 { 470 #ifdef ARM32 471 unsigned int write_not_read = 11; 472 #endif 473 #ifdef ARM64 474 unsigned int write_not_read = 6; 475 #endif 476 477 return ai->abort_type == ABORT_TYPE_DATA && 478 (ai->fault_descr & BIT(write_not_read)); 479 } 480 481 static enum fault_type get_fault_type(struct abort_info *ai) 482 { 483 if (abort_is_user_exception(ai)) { 484 if (is_vfp_fault(ai)) 485 return FAULT_TYPE_USER_MODE_VFP; 486 #ifndef CFG_WITH_PAGER 487 return FAULT_TYPE_USER_MODE_PANIC; 488 #endif 489 } 490 491 if (thread_is_from_abort_mode()) { 492 abort_print_error(ai); 493 panic("[abort] abort in abort handler (trap CPU)"); 494 } 495 496 if (ai->abort_type == ABORT_TYPE_UNDEF) { 497 if (abort_is_user_exception(ai)) 498 return FAULT_TYPE_USER_MODE_PANIC; 499 abort_print_error(ai); 500 panic("[abort] undefined abort (trap CPU)"); 501 } 502 503 switch (core_mmu_get_fault_type(ai->fault_descr)) { 504 case CORE_MMU_FAULT_ALIGNMENT: 505 if (abort_is_user_exception(ai)) 506 return FAULT_TYPE_USER_MODE_PANIC; 507 abort_print_error(ai); 508 panic("[abort] alignement fault! (trap CPU)"); 509 break; 510 511 case CORE_MMU_FAULT_ACCESS_BIT: 512 if (abort_is_user_exception(ai)) 513 return FAULT_TYPE_USER_MODE_PANIC; 514 abort_print_error(ai); 515 panic("[abort] access bit fault! (trap CPU)"); 516 break; 517 518 case CORE_MMU_FAULT_DEBUG_EVENT: 519 if (!abort_is_user_exception(ai)) 520 abort_print(ai); 521 DMSG("[abort] Ignoring debug event!"); 522 return FAULT_TYPE_IGNORE; 523 524 case CORE_MMU_FAULT_TRANSLATION: 525 case CORE_MMU_FAULT_WRITE_PERMISSION: 526 case CORE_MMU_FAULT_READ_PERMISSION: 527 return FAULT_TYPE_PAGEABLE; 528 529 case CORE_MMU_FAULT_ASYNC_EXTERNAL: 530 if (!abort_is_user_exception(ai)) 531 abort_print(ai); 532 DMSG("[abort] Ignoring async external abort!"); 533 return FAULT_TYPE_IGNORE; 534 535 case CORE_MMU_FAULT_TAG_CHECK: 536 if (abort_is_user_exception(ai)) 537 return FAULT_TYPE_USER_MODE_PANIC; 538 abort_print_error(ai); 539 panic("[abort] Tag check fault! (trap CPU)"); 540 break; 541 542 case CORE_MMU_FAULT_OTHER: 543 default: 544 if (!abort_is_user_exception(ai)) 545 abort_print(ai); 546 DMSG("[abort] Unhandled fault!"); 547 return FAULT_TYPE_IGNORE; 548 } 549 } 550 551 void abort_handler(uint32_t abort_type, struct thread_abort_regs *regs) 552 { 553 struct abort_info ai; 554 bool handled; 555 556 set_abort_info(abort_type, regs, &ai); 557 558 switch (get_fault_type(&ai)) { 559 case FAULT_TYPE_IGNORE: 560 break; 561 case FAULT_TYPE_USER_MODE_PANIC: 562 DMSG("[abort] abort in User mode (TA will panic)"); 563 save_abort_info_in_tsd(&ai); 564 vfp_disable(); 565 handle_user_mode_panic(&ai); 566 break; 567 #ifdef CFG_WITH_VFP 568 case FAULT_TYPE_USER_MODE_VFP: 569 handle_user_mode_vfp(); 570 break; 571 #endif 572 case FAULT_TYPE_PAGEABLE: 573 default: 574 if (thread_get_id_may_fail() < 0) { 575 abort_print_error(&ai); 576 panic("abort outside thread context"); 577 } 578 thread_kernel_save_vfp(); 579 handled = tee_pager_handle_fault(&ai); 580 thread_kernel_restore_vfp(); 581 if (!handled) { 582 if (!abort_is_user_exception(&ai)) { 583 abort_print_error(&ai); 584 panic("unhandled pageable abort"); 585 } 586 DMSG("[abort] abort in User mode (TA will panic)"); 587 save_abort_info_in_tsd(&ai); 588 vfp_disable(); 589 handle_user_mode_panic(&ai); 590 } 591 break; 592 } 593 } 594