1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2015, Linaro Limited 4 */ 5 6 #include <arm.h> 7 #include <kernel/abort.h> 8 #include <kernel/linker.h> 9 #include <kernel/misc.h> 10 #include <kernel/panic.h> 11 #include <kernel/tee_ta_manager.h> 12 #include <kernel/unwind.h> 13 #include <kernel/user_ta.h> 14 #include <mm/core_mmu.h> 15 #include <mm/mobj.h> 16 #include <mm/tee_pager.h> 17 #include <tee/tee_svc.h> 18 #include <trace.h> 19 20 #include "thread_private.h" 21 22 enum fault_type { 23 FAULT_TYPE_USER_TA_PANIC, 24 FAULT_TYPE_USER_TA_VFP, 25 FAULT_TYPE_PAGEABLE, 26 FAULT_TYPE_IGNORE, 27 }; 28 29 #ifdef CFG_UNWIND 30 31 #ifdef ARM32 32 static void get_current_ta_exidx_stack(vaddr_t *exidx, size_t *exidx_sz, 33 vaddr_t *stack, size_t *stack_size) 34 { 35 *exidx = 0; 36 *exidx_sz = 0; 37 *stack = 0; 38 *stack_size = 0; 39 } 40 41 /* 42 * Kernel or user mode unwind (32-bit execution state). 43 */ 44 static void __print_stack_unwind(struct abort_info *ai) 45 { 46 struct unwind_state_arm32 state; 47 vaddr_t exidx; 48 size_t exidx_sz; 49 uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; 50 uint32_t sp; 51 uint32_t lr; 52 vaddr_t stack; 53 size_t stack_size; 54 bool kernel_stack; 55 56 if (abort_is_user_exception(ai)) { 57 get_current_ta_exidx_stack(&exidx, &exidx_sz, &stack, 58 &stack_size); 59 if (!exidx) { 60 EMSG_RAW("Call stack not available"); 61 return; 62 } 63 kernel_stack = false; 64 } else { 65 exidx = (vaddr_t)__exidx_start; 66 exidx_sz = (vaddr_t)__exidx_end - (vaddr_t)__exidx_start; 67 /* Kernel stack */ 68 stack = thread_stack_start(); 69 stack_size = thread_stack_size(); 70 kernel_stack = true; 71 } 72 73 if (mode == CPSR_MODE_USR || mode == CPSR_MODE_SYS) { 74 sp = ai->regs->usr_sp; 75 lr = ai->regs->usr_lr; 76 } else { 77 sp = read_mode_sp(mode); 78 lr = read_mode_lr(mode); 79 } 80 81 memset(&state, 0, sizeof(state)); 82 state.registers[0] = ai->regs->r0; 83 state.registers[1] = ai->regs->r1; 84 state.registers[2] = ai->regs->r2; 85 state.registers[3] = ai->regs->r3; 86 state.registers[4] = ai->regs->r4; 87 state.registers[5] = ai->regs->r5; 88 state.registers[6] = ai->regs->r6; 89 state.registers[7] = ai->regs->r7; 90 state.registers[8] = ai->regs->r8; 91 state.registers[9] = ai->regs->r9; 92 state.registers[10] = ai->regs->r10; 93 state.registers[11] = ai->regs->r11; 94 state.registers[13] = sp; 95 state.registers[14] = lr; 96 state.registers[15] = ai->pc; 97 98 print_stack_arm32(TRACE_ERROR, &state, exidx, exidx_sz, kernel_stack, 99 stack, stack_size); 100 } 101 #endif /* ARM32 */ 102 103 #ifdef ARM64 104 /* Kernel or user mode unwind (64-bit execution state) */ 105 static void __print_stack_unwind(struct abort_info *ai) 106 { 107 struct unwind_state_arm64 state = { }; 108 bool kernel_stack = false; 109 uaddr_t stack = 0; 110 size_t stack_size = 0; 111 112 if (abort_is_user_exception(ai)) { 113 /* User stack */ 114 stack = 0; 115 stack_size = 0; 116 } else { 117 /* Kernel stack */ 118 stack = thread_stack_start(); 119 stack_size = thread_stack_size(); 120 kernel_stack = true; 121 } 122 123 state.pc = ai->regs->elr; 124 state.fp = ai->regs->x29; 125 126 print_stack_arm64(TRACE_ERROR, &state, kernel_stack, stack, stack_size); 127 } 128 #endif /*ARM64*/ 129 130 #else /* CFG_UNWIND */ 131 static void __print_stack_unwind(struct abort_info *ai __unused) 132 { 133 } 134 #endif /* CFG_UNWIND */ 135 136 static __maybe_unused const char *abort_type_to_str(uint32_t abort_type) 137 { 138 if (abort_type == ABORT_TYPE_DATA) 139 return "data"; 140 if (abort_type == ABORT_TYPE_PREFETCH) 141 return "prefetch"; 142 return "undef"; 143 } 144 145 static __maybe_unused const char *fault_to_str(uint32_t abort_type, 146 uint32_t fault_descr) 147 { 148 /* fault_descr is only valid for data or prefetch abort */ 149 if (abort_type != ABORT_TYPE_DATA && abort_type != ABORT_TYPE_PREFETCH) 150 return ""; 151 152 switch (core_mmu_get_fault_type(fault_descr)) { 153 case CORE_MMU_FAULT_ALIGNMENT: 154 return " (alignment fault)"; 155 case CORE_MMU_FAULT_TRANSLATION: 156 return " (translation fault)"; 157 case CORE_MMU_FAULT_READ_PERMISSION: 158 return " (read permission fault)"; 159 case CORE_MMU_FAULT_WRITE_PERMISSION: 160 return " (write permission fault)"; 161 default: 162 return ""; 163 } 164 } 165 166 static __maybe_unused void 167 __print_abort_info(struct abort_info *ai __maybe_unused, 168 const char *ctx __maybe_unused) 169 { 170 __maybe_unused size_t core_pos = 0; 171 #ifdef ARM32 172 uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; 173 __maybe_unused uint32_t sp = 0; 174 __maybe_unused uint32_t lr = 0; 175 176 if (mode == CPSR_MODE_USR || mode == CPSR_MODE_SYS) { 177 sp = ai->regs->usr_sp; 178 lr = ai->regs->usr_lr; 179 core_pos = thread_get_tsd()->abort_core; 180 } else { 181 sp = read_mode_sp(mode); 182 lr = read_mode_lr(mode); 183 core_pos = get_core_pos(); 184 } 185 #endif /*ARM32*/ 186 #ifdef ARM64 187 if (abort_is_user_exception(ai)) 188 core_pos = thread_get_tsd()->abort_core; 189 else 190 core_pos = get_core_pos(); 191 #endif /*ARM64*/ 192 193 EMSG_RAW(""); 194 EMSG_RAW("%s %s-abort at address 0x%" PRIxVA "%s", 195 ctx, abort_type_to_str(ai->abort_type), ai->va, 196 fault_to_str(ai->abort_type, ai->fault_descr)); 197 #ifdef ARM32 198 EMSG_RAW(" fsr 0x%08x ttbr0 0x%08x ttbr1 0x%08x cidr 0x%X", 199 ai->fault_descr, read_ttbr0(), read_ttbr1(), 200 read_contextidr()); 201 EMSG_RAW(" cpu #%zu cpsr 0x%08x", 202 core_pos, ai->regs->spsr); 203 EMSG_RAW(" r0 0x%08x r4 0x%08x r8 0x%08x r12 0x%08x", 204 ai->regs->r0, ai->regs->r4, ai->regs->r8, ai->regs->ip); 205 EMSG_RAW(" r1 0x%08x r5 0x%08x r9 0x%08x sp 0x%08x", 206 ai->regs->r1, ai->regs->r5, ai->regs->r9, sp); 207 EMSG_RAW(" r2 0x%08x r6 0x%08x r10 0x%08x lr 0x%08x", 208 ai->regs->r2, ai->regs->r6, ai->regs->r10, lr); 209 EMSG_RAW(" r3 0x%08x r7 0x%08x r11 0x%08x pc 0x%08x", 210 ai->regs->r3, ai->regs->r7, ai->regs->r11, ai->pc); 211 #endif /*ARM32*/ 212 #ifdef ARM64 213 EMSG_RAW(" esr 0x%08x ttbr0 0x%08" PRIx64 " ttbr1 0x%08" PRIx64 214 " cidr 0x%X", ai->fault_descr, read_ttbr0_el1(), 215 read_ttbr1_el1(), read_contextidr_el1()); 216 EMSG_RAW(" cpu #%zu cpsr 0x%08x", 217 core_pos, (uint32_t)ai->regs->spsr); 218 EMSG_RAW(" x0 %016" PRIx64 " x1 %016" PRIx64, 219 ai->regs->x0, ai->regs->x1); 220 EMSG_RAW(" x2 %016" PRIx64 " x3 %016" PRIx64, 221 ai->regs->x2, ai->regs->x3); 222 EMSG_RAW(" x4 %016" PRIx64 " x5 %016" PRIx64, 223 ai->regs->x4, ai->regs->x5); 224 EMSG_RAW(" x6 %016" PRIx64 " x7 %016" PRIx64, 225 ai->regs->x6, ai->regs->x7); 226 EMSG_RAW(" x8 %016" PRIx64 " x9 %016" PRIx64, 227 ai->regs->x8, ai->regs->x9); 228 EMSG_RAW(" x10 %016" PRIx64 " x11 %016" PRIx64, 229 ai->regs->x10, ai->regs->x11); 230 EMSG_RAW(" x12 %016" PRIx64 " x13 %016" PRIx64, 231 ai->regs->x12, ai->regs->x13); 232 EMSG_RAW(" x14 %016" PRIx64 " x15 %016" PRIx64, 233 ai->regs->x14, ai->regs->x15); 234 EMSG_RAW(" x16 %016" PRIx64 " x17 %016" PRIx64, 235 ai->regs->x16, ai->regs->x17); 236 EMSG_RAW(" x18 %016" PRIx64 " x19 %016" PRIx64, 237 ai->regs->x18, ai->regs->x19); 238 EMSG_RAW(" x20 %016" PRIx64 " x21 %016" PRIx64, 239 ai->regs->x20, ai->regs->x21); 240 EMSG_RAW(" x22 %016" PRIx64 " x23 %016" PRIx64, 241 ai->regs->x22, ai->regs->x23); 242 EMSG_RAW(" x24 %016" PRIx64 " x25 %016" PRIx64, 243 ai->regs->x24, ai->regs->x25); 244 EMSG_RAW(" x26 %016" PRIx64 " x27 %016" PRIx64, 245 ai->regs->x26, ai->regs->x27); 246 EMSG_RAW(" x28 %016" PRIx64 " x29 %016" PRIx64, 247 ai->regs->x28, ai->regs->x29); 248 EMSG_RAW(" x30 %016" PRIx64 " elr %016" PRIx64, 249 ai->regs->x30, ai->regs->elr); 250 EMSG_RAW(" sp_el0 %016" PRIx64, ai->regs->sp_el0); 251 #endif /*ARM64*/ 252 } 253 254 /* 255 * Print abort info and (optionally) stack dump to the console 256 * @ai kernel-mode abort info. 257 * @stack_dump true to show a stack trace 258 */ 259 static void __abort_print(struct abort_info *ai, bool stack_dump) 260 { 261 assert(!abort_is_user_exception(ai)); 262 263 __print_abort_info(ai, "Core"); 264 265 if (stack_dump) 266 __print_stack_unwind(ai); 267 } 268 269 void abort_print(struct abort_info *ai) 270 { 271 __abort_print(ai, false); 272 } 273 274 void abort_print_error(struct abort_info *ai) 275 { 276 __abort_print(ai, true); 277 } 278 279 /* This function must be called from a normal thread */ 280 void abort_print_current_ta(void) 281 { 282 struct thread_specific_data *tsd = thread_get_tsd(); 283 struct abort_info ai = { }; 284 struct tee_ta_session *s = NULL; 285 286 if (tee_ta_get_current_session(&s) != TEE_SUCCESS) 287 panic(); 288 289 ai.abort_type = tsd->abort_type; 290 ai.fault_descr = tsd->abort_descr; 291 ai.va = tsd->abort_va; 292 ai.pc = tsd->abort_regs.elr; 293 ai.regs = &tsd->abort_regs; 294 295 if (ai.abort_type != ABORT_TYPE_TA_PANIC) 296 __print_abort_info(&ai, "User TA"); 297 298 s->ctx->ops->dump_state(s->ctx); 299 300 if (s->ctx->ops->dump_ftrace) 301 s->ctx->ops->dump_ftrace(s->ctx); 302 } 303 304 static void save_abort_info_in_tsd(struct abort_info *ai) 305 { 306 struct thread_specific_data *tsd = thread_get_tsd(); 307 308 tsd->abort_type = ai->abort_type; 309 tsd->abort_descr = ai->fault_descr; 310 tsd->abort_va = ai->va; 311 tsd->abort_regs = *ai->regs; 312 tsd->abort_core = get_core_pos(); 313 } 314 315 #ifdef ARM32 316 static void set_abort_info(uint32_t abort_type, struct thread_abort_regs *regs, 317 struct abort_info *ai) 318 { 319 switch (abort_type) { 320 case ABORT_TYPE_DATA: 321 ai->fault_descr = read_dfsr(); 322 ai->va = read_dfar(); 323 break; 324 case ABORT_TYPE_PREFETCH: 325 ai->fault_descr = read_ifsr(); 326 ai->va = read_ifar(); 327 break; 328 default: 329 ai->fault_descr = 0; 330 ai->va = regs->elr; 331 break; 332 } 333 ai->abort_type = abort_type; 334 ai->pc = regs->elr; 335 ai->regs = regs; 336 } 337 #endif /*ARM32*/ 338 339 #ifdef ARM64 340 static void set_abort_info(uint32_t abort_type __unused, 341 struct thread_abort_regs *regs, struct abort_info *ai) 342 { 343 ai->fault_descr = read_esr_el1(); 344 switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { 345 case ESR_EC_IABT_EL0: 346 case ESR_EC_IABT_EL1: 347 ai->abort_type = ABORT_TYPE_PREFETCH; 348 ai->va = read_far_el1(); 349 break; 350 case ESR_EC_DABT_EL0: 351 case ESR_EC_DABT_EL1: 352 case ESR_EC_SP_ALIGN: 353 ai->abort_type = ABORT_TYPE_DATA; 354 ai->va = read_far_el1(); 355 break; 356 default: 357 ai->abort_type = ABORT_TYPE_UNDEF; 358 ai->va = regs->elr; 359 } 360 ai->pc = regs->elr; 361 ai->regs = regs; 362 } 363 #endif /*ARM64*/ 364 365 #ifdef ARM32 366 static void handle_user_ta_panic(struct abort_info *ai) 367 { 368 /* 369 * It was a user exception, stop user execution and return 370 * to TEE Core. 371 */ 372 ai->regs->r0 = TEE_ERROR_TARGET_DEAD; 373 ai->regs->r1 = true; 374 ai->regs->r2 = 0xdeadbeef; 375 ai->regs->elr = (uint32_t)thread_unwind_user_mode; 376 ai->regs->spsr &= CPSR_FIA; 377 ai->regs->spsr &= ~CPSR_MODE_MASK; 378 ai->regs->spsr |= CPSR_MODE_SVC; 379 /* Select Thumb or ARM mode */ 380 if (ai->regs->elr & 1) 381 ai->regs->spsr |= CPSR_T; 382 else 383 ai->regs->spsr &= ~CPSR_T; 384 } 385 #endif /*ARM32*/ 386 387 #ifdef ARM64 388 static void handle_user_ta_panic(struct abort_info *ai) 389 { 390 uint32_t daif; 391 392 /* 393 * It was a user exception, stop user execution and return 394 * to TEE Core. 395 */ 396 ai->regs->x0 = TEE_ERROR_TARGET_DEAD; 397 ai->regs->x1 = true; 398 ai->regs->x2 = 0xdeadbeef; 399 ai->regs->elr = (vaddr_t)thread_unwind_user_mode; 400 ai->regs->sp_el0 = thread_get_saved_thread_sp(); 401 402 daif = (ai->regs->spsr >> SPSR_32_AIF_SHIFT) & SPSR_32_AIF_MASK; 403 /* XXX what about DAIF_D? */ 404 ai->regs->spsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, daif); 405 } 406 #endif /*ARM64*/ 407 408 #ifdef CFG_WITH_VFP 409 static void handle_user_ta_vfp(void) 410 { 411 struct tee_ta_session *s; 412 413 if (tee_ta_get_current_session(&s) != TEE_SUCCESS) 414 panic(); 415 416 thread_user_enable_vfp(&to_user_ta_ctx(s->ctx)->vfp); 417 } 418 #endif /*CFG_WITH_VFP*/ 419 420 #ifdef CFG_WITH_USER_TA 421 #ifdef ARM32 422 /* Returns true if the exception originated from user mode */ 423 bool abort_is_user_exception(struct abort_info *ai) 424 { 425 return (ai->regs->spsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; 426 } 427 #endif /*ARM32*/ 428 429 #ifdef ARM64 430 /* Returns true if the exception originated from user mode */ 431 bool abort_is_user_exception(struct abort_info *ai) 432 { 433 uint32_t spsr = ai->regs->spsr; 434 435 if (spsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) 436 return true; 437 if (((spsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == 438 SPSR_64_MODE_EL0) 439 return true; 440 return false; 441 } 442 #endif /*ARM64*/ 443 #else /*CFG_WITH_USER_TA*/ 444 bool abort_is_user_exception(struct abort_info *ai __unused) 445 { 446 return false; 447 } 448 #endif /*CFG_WITH_USER_TA*/ 449 450 #if defined(CFG_WITH_VFP) && defined(CFG_WITH_USER_TA) 451 #ifdef ARM32 452 static bool is_vfp_fault(struct abort_info *ai) 453 { 454 if ((ai->abort_type != ABORT_TYPE_UNDEF) || vfp_is_enabled()) 455 return false; 456 457 /* 458 * Not entirely accurate, but if it's a truly undefined instruction 459 * we'll end up in this function again, except this time 460 * vfp_is_enabled() so we'll return false. 461 */ 462 return true; 463 } 464 #endif /*ARM32*/ 465 466 #ifdef ARM64 467 static bool is_vfp_fault(struct abort_info *ai) 468 { 469 switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { 470 case ESR_EC_FP_ASIMD: 471 case ESR_EC_AARCH32_FP: 472 case ESR_EC_AARCH64_FP: 473 return true; 474 default: 475 return false; 476 } 477 } 478 #endif /*ARM64*/ 479 #else /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ 480 static bool is_vfp_fault(struct abort_info *ai __unused) 481 { 482 return false; 483 } 484 #endif /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ 485 486 static enum fault_type get_fault_type(struct abort_info *ai) 487 { 488 if (abort_is_user_exception(ai)) { 489 if (is_vfp_fault(ai)) 490 return FAULT_TYPE_USER_TA_VFP; 491 #ifndef CFG_WITH_PAGER 492 return FAULT_TYPE_USER_TA_PANIC; 493 #endif 494 } 495 496 if (thread_is_from_abort_mode()) { 497 abort_print_error(ai); 498 panic("[abort] abort in abort handler (trap CPU)"); 499 } 500 501 if (ai->abort_type == ABORT_TYPE_UNDEF) { 502 if (abort_is_user_exception(ai)) 503 return FAULT_TYPE_USER_TA_PANIC; 504 abort_print_error(ai); 505 panic("[abort] undefined abort (trap CPU)"); 506 } 507 508 switch (core_mmu_get_fault_type(ai->fault_descr)) { 509 case CORE_MMU_FAULT_ALIGNMENT: 510 if (abort_is_user_exception(ai)) 511 return FAULT_TYPE_USER_TA_PANIC; 512 abort_print_error(ai); 513 panic("[abort] alignement fault! (trap CPU)"); 514 break; 515 516 case CORE_MMU_FAULT_ACCESS_BIT: 517 if (abort_is_user_exception(ai)) 518 return FAULT_TYPE_USER_TA_PANIC; 519 abort_print_error(ai); 520 panic("[abort] access bit fault! (trap CPU)"); 521 break; 522 523 case CORE_MMU_FAULT_DEBUG_EVENT: 524 if (!abort_is_user_exception(ai)) 525 abort_print(ai); 526 DMSG("[abort] Ignoring debug event!"); 527 return FAULT_TYPE_IGNORE; 528 529 case CORE_MMU_FAULT_TRANSLATION: 530 case CORE_MMU_FAULT_WRITE_PERMISSION: 531 case CORE_MMU_FAULT_READ_PERMISSION: 532 return FAULT_TYPE_PAGEABLE; 533 534 case CORE_MMU_FAULT_ASYNC_EXTERNAL: 535 if (!abort_is_user_exception(ai)) 536 abort_print(ai); 537 DMSG("[abort] Ignoring async external abort!"); 538 return FAULT_TYPE_IGNORE; 539 540 case CORE_MMU_FAULT_OTHER: 541 default: 542 if (!abort_is_user_exception(ai)) 543 abort_print(ai); 544 DMSG("[abort] Unhandled fault!"); 545 return FAULT_TYPE_IGNORE; 546 } 547 } 548 549 void abort_handler(uint32_t abort_type, struct thread_abort_regs *regs) 550 { 551 struct abort_info ai; 552 bool handled; 553 554 set_abort_info(abort_type, regs, &ai); 555 556 switch (get_fault_type(&ai)) { 557 case FAULT_TYPE_IGNORE: 558 break; 559 case FAULT_TYPE_USER_TA_PANIC: 560 DMSG("[abort] abort in User mode (TA will panic)"); 561 save_abort_info_in_tsd(&ai); 562 vfp_disable(); 563 handle_user_ta_panic(&ai); 564 break; 565 #ifdef CFG_WITH_VFP 566 case FAULT_TYPE_USER_TA_VFP: 567 handle_user_ta_vfp(); 568 break; 569 #endif 570 case FAULT_TYPE_PAGEABLE: 571 default: 572 if (thread_get_id_may_fail() < 0) { 573 abort_print_error(&ai); 574 panic("abort outside thread context"); 575 } 576 thread_kernel_save_vfp(); 577 handled = tee_pager_handle_fault(&ai); 578 thread_kernel_restore_vfp(); 579 if (!handled) { 580 if (!abort_is_user_exception(&ai)) { 581 abort_print_error(&ai); 582 panic("unhandled pageable abort"); 583 } 584 DMSG("[abort] abort in User mode (TA will panic)"); 585 save_abort_info_in_tsd(&ai); 586 vfp_disable(); 587 handle_user_ta_panic(&ai); 588 } 589 break; 590 } 591 } 592