1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2015, Linaro Limited 4 */ 5 6 #include <arm.h> 7 #include <kernel/abort.h> 8 #include <kernel/linker.h> 9 #include <kernel/misc.h> 10 #include <kernel/panic.h> 11 #include <kernel/tee_ta_manager.h> 12 #include <kernel/unwind.h> 13 #include <kernel/user_mode_ctx.h> 14 #include <mm/core_mmu.h> 15 #include <mm/mobj.h> 16 #include <mm/tee_pager.h> 17 #include <tee/tee_svc.h> 18 #include <trace.h> 19 20 #include "thread_private.h" 21 22 enum fault_type { 23 FAULT_TYPE_USER_TA_PANIC, 24 FAULT_TYPE_USER_TA_VFP, 25 FAULT_TYPE_PAGEABLE, 26 FAULT_TYPE_IGNORE, 27 }; 28 29 #ifdef CFG_UNWIND 30 31 #ifdef ARM32 32 /* 33 * Kernel or user mode unwind (32-bit execution state). 34 */ 35 static void __print_stack_unwind(struct abort_info *ai) 36 { 37 struct unwind_state_arm32 state = { }; 38 vaddr_t exidx = (vaddr_t)__exidx_start; 39 size_t exidx_sz = (vaddr_t)__exidx_end - (vaddr_t)__exidx_start; 40 uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; 41 uint32_t sp = 0; 42 uint32_t lr = 0; 43 44 assert(!abort_is_user_exception(ai)); 45 46 if (mode == CPSR_MODE_SYS) { 47 sp = ai->regs->usr_sp; 48 lr = ai->regs->usr_lr; 49 } else { 50 sp = read_mode_sp(mode); 51 lr = read_mode_lr(mode); 52 } 53 54 memset(&state, 0, sizeof(state)); 55 state.registers[0] = ai->regs->r0; 56 state.registers[1] = ai->regs->r1; 57 state.registers[2] = ai->regs->r2; 58 state.registers[3] = ai->regs->r3; 59 state.registers[4] = ai->regs->r4; 60 state.registers[5] = ai->regs->r5; 61 state.registers[6] = ai->regs->r6; 62 state.registers[7] = ai->regs->r7; 63 state.registers[8] = ai->regs->r8; 64 state.registers[9] = ai->regs->r9; 65 state.registers[10] = ai->regs->r10; 66 state.registers[11] = ai->regs->r11; 67 state.registers[13] = sp; 68 state.registers[14] = lr; 69 state.registers[15] = ai->pc; 70 71 print_stack_arm32(&state, exidx, exidx_sz, thread_stack_start(), 72 thread_stack_size()); 73 } 74 #endif /* ARM32 */ 75 76 #ifdef ARM64 77 /* Kernel mode unwind (64-bit execution state) */ 78 static void __print_stack_unwind(struct abort_info *ai) 79 { 80 struct unwind_state_arm64 state = { 81 .pc = ai->regs->elr, 82 .fp = ai->regs->x29, 83 }; 84 85 print_stack_arm64(&state, thread_stack_start(), thread_stack_size()); 86 } 87 #endif /*ARM64*/ 88 89 #else /* CFG_UNWIND */ 90 static void __print_stack_unwind(struct abort_info *ai __unused) 91 { 92 } 93 #endif /* CFG_UNWIND */ 94 95 static __maybe_unused const char *abort_type_to_str(uint32_t abort_type) 96 { 97 if (abort_type == ABORT_TYPE_DATA) 98 return "data"; 99 if (abort_type == ABORT_TYPE_PREFETCH) 100 return "prefetch"; 101 return "undef"; 102 } 103 104 static __maybe_unused const char *fault_to_str(uint32_t abort_type, 105 uint32_t fault_descr) 106 { 107 /* fault_descr is only valid for data or prefetch abort */ 108 if (abort_type != ABORT_TYPE_DATA && abort_type != ABORT_TYPE_PREFETCH) 109 return ""; 110 111 switch (core_mmu_get_fault_type(fault_descr)) { 112 case CORE_MMU_FAULT_ALIGNMENT: 113 return " (alignment fault)"; 114 case CORE_MMU_FAULT_TRANSLATION: 115 return " (translation fault)"; 116 case CORE_MMU_FAULT_READ_PERMISSION: 117 return " (read permission fault)"; 118 case CORE_MMU_FAULT_WRITE_PERMISSION: 119 return " (write permission fault)"; 120 default: 121 return ""; 122 } 123 } 124 125 static __maybe_unused void 126 __print_abort_info(struct abort_info *ai __maybe_unused, 127 const char *ctx __maybe_unused) 128 { 129 __maybe_unused size_t core_pos = 0; 130 #ifdef ARM32 131 uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; 132 __maybe_unused uint32_t sp = 0; 133 __maybe_unused uint32_t lr = 0; 134 135 if (mode == CPSR_MODE_USR || mode == CPSR_MODE_SYS) { 136 sp = ai->regs->usr_sp; 137 lr = ai->regs->usr_lr; 138 core_pos = thread_get_tsd()->abort_core; 139 } else { 140 sp = read_mode_sp(mode); 141 lr = read_mode_lr(mode); 142 core_pos = get_core_pos(); 143 } 144 #endif /*ARM32*/ 145 #ifdef ARM64 146 if (abort_is_user_exception(ai)) 147 core_pos = thread_get_tsd()->abort_core; 148 else 149 core_pos = get_core_pos(); 150 #endif /*ARM64*/ 151 152 EMSG_RAW(""); 153 EMSG_RAW("%s %s-abort at address 0x%" PRIxVA "%s", 154 ctx, abort_type_to_str(ai->abort_type), ai->va, 155 fault_to_str(ai->abort_type, ai->fault_descr)); 156 #ifdef ARM32 157 EMSG_RAW(" fsr 0x%08x ttbr0 0x%08x ttbr1 0x%08x cidr 0x%X", 158 ai->fault_descr, read_ttbr0(), read_ttbr1(), 159 read_contextidr()); 160 EMSG_RAW(" cpu #%zu cpsr 0x%08x", 161 core_pos, ai->regs->spsr); 162 EMSG_RAW(" r0 0x%08x r4 0x%08x r8 0x%08x r12 0x%08x", 163 ai->regs->r0, ai->regs->r4, ai->regs->r8, ai->regs->ip); 164 EMSG_RAW(" r1 0x%08x r5 0x%08x r9 0x%08x sp 0x%08x", 165 ai->regs->r1, ai->regs->r5, ai->regs->r9, sp); 166 EMSG_RAW(" r2 0x%08x r6 0x%08x r10 0x%08x lr 0x%08x", 167 ai->regs->r2, ai->regs->r6, ai->regs->r10, lr); 168 EMSG_RAW(" r3 0x%08x r7 0x%08x r11 0x%08x pc 0x%08x", 169 ai->regs->r3, ai->regs->r7, ai->regs->r11, ai->pc); 170 #endif /*ARM32*/ 171 #ifdef ARM64 172 EMSG_RAW(" esr 0x%08x ttbr0 0x%08" PRIx64 " ttbr1 0x%08" PRIx64 173 " cidr 0x%X", ai->fault_descr, read_ttbr0_el1(), 174 read_ttbr1_el1(), read_contextidr_el1()); 175 EMSG_RAW(" cpu #%zu cpsr 0x%08x", 176 core_pos, (uint32_t)ai->regs->spsr); 177 EMSG_RAW(" x0 %016" PRIx64 " x1 %016" PRIx64, 178 ai->regs->x0, ai->regs->x1); 179 EMSG_RAW(" x2 %016" PRIx64 " x3 %016" PRIx64, 180 ai->regs->x2, ai->regs->x3); 181 EMSG_RAW(" x4 %016" PRIx64 " x5 %016" PRIx64, 182 ai->regs->x4, ai->regs->x5); 183 EMSG_RAW(" x6 %016" PRIx64 " x7 %016" PRIx64, 184 ai->regs->x6, ai->regs->x7); 185 EMSG_RAW(" x8 %016" PRIx64 " x9 %016" PRIx64, 186 ai->regs->x8, ai->regs->x9); 187 EMSG_RAW(" x10 %016" PRIx64 " x11 %016" PRIx64, 188 ai->regs->x10, ai->regs->x11); 189 EMSG_RAW(" x12 %016" PRIx64 " x13 %016" PRIx64, 190 ai->regs->x12, ai->regs->x13); 191 EMSG_RAW(" x14 %016" PRIx64 " x15 %016" PRIx64, 192 ai->regs->x14, ai->regs->x15); 193 EMSG_RAW(" x16 %016" PRIx64 " x17 %016" PRIx64, 194 ai->regs->x16, ai->regs->x17); 195 EMSG_RAW(" x18 %016" PRIx64 " x19 %016" PRIx64, 196 ai->regs->x18, ai->regs->x19); 197 EMSG_RAW(" x20 %016" PRIx64 " x21 %016" PRIx64, 198 ai->regs->x20, ai->regs->x21); 199 EMSG_RAW(" x22 %016" PRIx64 " x23 %016" PRIx64, 200 ai->regs->x22, ai->regs->x23); 201 EMSG_RAW(" x24 %016" PRIx64 " x25 %016" PRIx64, 202 ai->regs->x24, ai->regs->x25); 203 EMSG_RAW(" x26 %016" PRIx64 " x27 %016" PRIx64, 204 ai->regs->x26, ai->regs->x27); 205 EMSG_RAW(" x28 %016" PRIx64 " x29 %016" PRIx64, 206 ai->regs->x28, ai->regs->x29); 207 EMSG_RAW(" x30 %016" PRIx64 " elr %016" PRIx64, 208 ai->regs->x30, ai->regs->elr); 209 EMSG_RAW(" sp_el0 %016" PRIx64, ai->regs->sp_el0); 210 #endif /*ARM64*/ 211 } 212 213 /* 214 * Print abort info and (optionally) stack dump to the console 215 * @ai kernel-mode abort info. 216 * @stack_dump true to show a stack trace 217 */ 218 static void __abort_print(struct abort_info *ai, bool stack_dump) 219 { 220 assert(!abort_is_user_exception(ai)); 221 222 __print_abort_info(ai, "Core"); 223 224 if (stack_dump) 225 __print_stack_unwind(ai); 226 } 227 228 void abort_print(struct abort_info *ai) 229 { 230 __abort_print(ai, false); 231 } 232 233 void abort_print_error(struct abort_info *ai) 234 { 235 __abort_print(ai, true); 236 } 237 238 /* This function must be called from a normal thread */ 239 void abort_print_current_ta(void) 240 { 241 struct thread_specific_data *tsd = thread_get_tsd(); 242 struct abort_info ai = { }; 243 struct tee_ta_session *s = NULL; 244 245 if (tee_ta_get_current_session(&s) != TEE_SUCCESS) 246 panic(); 247 248 ai.abort_type = tsd->abort_type; 249 ai.fault_descr = tsd->abort_descr; 250 ai.va = tsd->abort_va; 251 ai.pc = tsd->abort_regs.elr; 252 ai.regs = &tsd->abort_regs; 253 254 if (ai.abort_type != ABORT_TYPE_TA_PANIC) 255 __print_abort_info(&ai, "User TA"); 256 257 s->ctx->ops->dump_state(s->ctx); 258 259 #if defined(CFG_FTRACE_SUPPORT) 260 if (s->ctx->ops->dump_ftrace) { 261 s->fbuf = NULL; 262 s->ctx->ops->dump_ftrace(s->ctx); 263 } 264 #endif 265 } 266 267 static void save_abort_info_in_tsd(struct abort_info *ai) 268 { 269 struct thread_specific_data *tsd = thread_get_tsd(); 270 271 tsd->abort_type = ai->abort_type; 272 tsd->abort_descr = ai->fault_descr; 273 tsd->abort_va = ai->va; 274 tsd->abort_regs = *ai->regs; 275 tsd->abort_core = get_core_pos(); 276 } 277 278 #ifdef ARM32 279 static void set_abort_info(uint32_t abort_type, struct thread_abort_regs *regs, 280 struct abort_info *ai) 281 { 282 switch (abort_type) { 283 case ABORT_TYPE_DATA: 284 ai->fault_descr = read_dfsr(); 285 ai->va = read_dfar(); 286 break; 287 case ABORT_TYPE_PREFETCH: 288 ai->fault_descr = read_ifsr(); 289 ai->va = read_ifar(); 290 break; 291 default: 292 ai->fault_descr = 0; 293 ai->va = regs->elr; 294 break; 295 } 296 ai->abort_type = abort_type; 297 ai->pc = regs->elr; 298 ai->regs = regs; 299 } 300 #endif /*ARM32*/ 301 302 #ifdef ARM64 303 static void set_abort_info(uint32_t abort_type __unused, 304 struct thread_abort_regs *regs, struct abort_info *ai) 305 { 306 ai->fault_descr = read_esr_el1(); 307 switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { 308 case ESR_EC_IABT_EL0: 309 case ESR_EC_IABT_EL1: 310 ai->abort_type = ABORT_TYPE_PREFETCH; 311 ai->va = read_far_el1(); 312 break; 313 case ESR_EC_DABT_EL0: 314 case ESR_EC_DABT_EL1: 315 case ESR_EC_SP_ALIGN: 316 ai->abort_type = ABORT_TYPE_DATA; 317 ai->va = read_far_el1(); 318 break; 319 default: 320 ai->abort_type = ABORT_TYPE_UNDEF; 321 ai->va = regs->elr; 322 } 323 ai->pc = regs->elr; 324 ai->regs = regs; 325 } 326 #endif /*ARM64*/ 327 328 #ifdef ARM32 329 static void handle_user_ta_panic(struct abort_info *ai) 330 { 331 /* 332 * It was a user exception, stop user execution and return 333 * to TEE Core. 334 */ 335 ai->regs->r0 = TEE_ERROR_TARGET_DEAD; 336 ai->regs->r1 = true; 337 ai->regs->r2 = 0xdeadbeef; 338 ai->regs->elr = (uint32_t)thread_unwind_user_mode; 339 ai->regs->spsr &= CPSR_FIA; 340 ai->regs->spsr &= ~CPSR_MODE_MASK; 341 ai->regs->spsr |= CPSR_MODE_SVC; 342 /* Select Thumb or ARM mode */ 343 if (ai->regs->elr & 1) 344 ai->regs->spsr |= CPSR_T; 345 else 346 ai->regs->spsr &= ~CPSR_T; 347 } 348 #endif /*ARM32*/ 349 350 #ifdef ARM64 351 static void handle_user_ta_panic(struct abort_info *ai) 352 { 353 uint32_t daif; 354 355 /* 356 * It was a user exception, stop user execution and return 357 * to TEE Core. 358 */ 359 ai->regs->x0 = TEE_ERROR_TARGET_DEAD; 360 ai->regs->x1 = true; 361 ai->regs->x2 = 0xdeadbeef; 362 ai->regs->elr = (vaddr_t)thread_unwind_user_mode; 363 ai->regs->sp_el0 = thread_get_saved_thread_sp(); 364 365 daif = (ai->regs->spsr >> SPSR_32_AIF_SHIFT) & SPSR_32_AIF_MASK; 366 /* XXX what about DAIF_D? */ 367 ai->regs->spsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, daif); 368 } 369 #endif /*ARM64*/ 370 371 #ifdef CFG_WITH_VFP 372 static void handle_user_ta_vfp(void) 373 { 374 struct tee_ta_session *s; 375 376 if (tee_ta_get_current_session(&s) != TEE_SUCCESS) 377 panic(); 378 379 thread_user_enable_vfp(&to_user_mode_ctx(s->ctx)->vfp); 380 } 381 #endif /*CFG_WITH_VFP*/ 382 383 #ifdef CFG_WITH_USER_TA 384 #ifdef ARM32 385 /* Returns true if the exception originated from user mode */ 386 bool abort_is_user_exception(struct abort_info *ai) 387 { 388 return (ai->regs->spsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; 389 } 390 #endif /*ARM32*/ 391 392 #ifdef ARM64 393 /* Returns true if the exception originated from user mode */ 394 bool abort_is_user_exception(struct abort_info *ai) 395 { 396 uint32_t spsr = ai->regs->spsr; 397 398 if (spsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) 399 return true; 400 if (((spsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == 401 SPSR_64_MODE_EL0) 402 return true; 403 return false; 404 } 405 #endif /*ARM64*/ 406 #else /*CFG_WITH_USER_TA*/ 407 bool abort_is_user_exception(struct abort_info *ai __unused) 408 { 409 return false; 410 } 411 #endif /*CFG_WITH_USER_TA*/ 412 413 #if defined(CFG_WITH_VFP) && defined(CFG_WITH_USER_TA) 414 #ifdef ARM32 415 static bool is_vfp_fault(struct abort_info *ai) 416 { 417 if ((ai->abort_type != ABORT_TYPE_UNDEF) || vfp_is_enabled()) 418 return false; 419 420 /* 421 * Not entirely accurate, but if it's a truly undefined instruction 422 * we'll end up in this function again, except this time 423 * vfp_is_enabled() so we'll return false. 424 */ 425 return true; 426 } 427 #endif /*ARM32*/ 428 429 #ifdef ARM64 430 static bool is_vfp_fault(struct abort_info *ai) 431 { 432 switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { 433 case ESR_EC_FP_ASIMD: 434 case ESR_EC_AARCH32_FP: 435 case ESR_EC_AARCH64_FP: 436 return true; 437 default: 438 return false; 439 } 440 } 441 #endif /*ARM64*/ 442 #else /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ 443 static bool is_vfp_fault(struct abort_info *ai __unused) 444 { 445 return false; 446 } 447 #endif /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ 448 449 static enum fault_type get_fault_type(struct abort_info *ai) 450 { 451 if (abort_is_user_exception(ai)) { 452 if (is_vfp_fault(ai)) 453 return FAULT_TYPE_USER_TA_VFP; 454 #ifndef CFG_WITH_PAGER 455 return FAULT_TYPE_USER_TA_PANIC; 456 #endif 457 } 458 459 if (thread_is_from_abort_mode()) { 460 abort_print_error(ai); 461 panic("[abort] abort in abort handler (trap CPU)"); 462 } 463 464 if (ai->abort_type == ABORT_TYPE_UNDEF) { 465 if (abort_is_user_exception(ai)) 466 return FAULT_TYPE_USER_TA_PANIC; 467 abort_print_error(ai); 468 panic("[abort] undefined abort (trap CPU)"); 469 } 470 471 switch (core_mmu_get_fault_type(ai->fault_descr)) { 472 case CORE_MMU_FAULT_ALIGNMENT: 473 if (abort_is_user_exception(ai)) 474 return FAULT_TYPE_USER_TA_PANIC; 475 abort_print_error(ai); 476 panic("[abort] alignement fault! (trap CPU)"); 477 break; 478 479 case CORE_MMU_FAULT_ACCESS_BIT: 480 if (abort_is_user_exception(ai)) 481 return FAULT_TYPE_USER_TA_PANIC; 482 abort_print_error(ai); 483 panic("[abort] access bit fault! (trap CPU)"); 484 break; 485 486 case CORE_MMU_FAULT_DEBUG_EVENT: 487 if (!abort_is_user_exception(ai)) 488 abort_print(ai); 489 DMSG("[abort] Ignoring debug event!"); 490 return FAULT_TYPE_IGNORE; 491 492 case CORE_MMU_FAULT_TRANSLATION: 493 case CORE_MMU_FAULT_WRITE_PERMISSION: 494 case CORE_MMU_FAULT_READ_PERMISSION: 495 return FAULT_TYPE_PAGEABLE; 496 497 case CORE_MMU_FAULT_ASYNC_EXTERNAL: 498 if (!abort_is_user_exception(ai)) 499 abort_print(ai); 500 DMSG("[abort] Ignoring async external abort!"); 501 return FAULT_TYPE_IGNORE; 502 503 case CORE_MMU_FAULT_OTHER: 504 default: 505 if (!abort_is_user_exception(ai)) 506 abort_print(ai); 507 DMSG("[abort] Unhandled fault!"); 508 return FAULT_TYPE_IGNORE; 509 } 510 } 511 512 void abort_handler(uint32_t abort_type, struct thread_abort_regs *regs) 513 { 514 struct abort_info ai; 515 bool handled; 516 517 set_abort_info(abort_type, regs, &ai); 518 519 switch (get_fault_type(&ai)) { 520 case FAULT_TYPE_IGNORE: 521 break; 522 case FAULT_TYPE_USER_TA_PANIC: 523 DMSG("[abort] abort in User mode (TA will panic)"); 524 save_abort_info_in_tsd(&ai); 525 vfp_disable(); 526 handle_user_ta_panic(&ai); 527 break; 528 #ifdef CFG_WITH_VFP 529 case FAULT_TYPE_USER_TA_VFP: 530 handle_user_ta_vfp(); 531 break; 532 #endif 533 case FAULT_TYPE_PAGEABLE: 534 default: 535 if (thread_get_id_may_fail() < 0) { 536 abort_print_error(&ai); 537 panic("abort outside thread context"); 538 } 539 thread_kernel_save_vfp(); 540 handled = tee_pager_handle_fault(&ai); 541 thread_kernel_restore_vfp(); 542 if (!handled) { 543 if (!abort_is_user_exception(&ai)) { 544 abort_print_error(&ai); 545 panic("unhandled pageable abort"); 546 } 547 DMSG("[abort] abort in User mode (TA will panic)"); 548 save_abort_info_in_tsd(&ai); 549 vfp_disable(); 550 handle_user_ta_panic(&ai); 551 } 552 break; 553 } 554 } 555