1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2015, Linaro Limited 4 */ 5 6 #include <arm.h> 7 #include <kernel/abort.h> 8 #include <kernel/linker.h> 9 #include <kernel/misc.h> 10 #include <kernel/panic.h> 11 #include <kernel/tee_ta_manager.h> 12 #include <kernel/unwind.h> 13 #include <kernel/user_ta.h> 14 #include <mm/core_mmu.h> 15 #include <mm/mobj.h> 16 #include <mm/tee_pager.h> 17 #include <tee/tee_svc.h> 18 #include <trace.h> 19 20 #include "thread_private.h" 21 22 enum fault_type { 23 FAULT_TYPE_USER_TA_PANIC, 24 FAULT_TYPE_USER_TA_VFP, 25 FAULT_TYPE_PAGEABLE, 26 FAULT_TYPE_IGNORE, 27 }; 28 29 #ifdef CFG_UNWIND 30 31 #ifdef ARM32 32 /* 33 * Kernel or user mode unwind (32-bit execution state). 34 */ 35 static void __print_stack_unwind(struct abort_info *ai) 36 { 37 struct unwind_state_arm32 state = { }; 38 vaddr_t exidx = (vaddr_t)__exidx_start; 39 size_t exidx_sz = (vaddr_t)__exidx_end - (vaddr_t)__exidx_start; 40 uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; 41 uint32_t sp = 0; 42 uint32_t lr = 0; 43 44 assert(!abort_is_user_exception(ai)); 45 46 if (mode == CPSR_MODE_SYS) { 47 sp = ai->regs->usr_sp; 48 lr = ai->regs->usr_lr; 49 } else { 50 sp = read_mode_sp(mode); 51 lr = read_mode_lr(mode); 52 } 53 54 memset(&state, 0, sizeof(state)); 55 state.registers[0] = ai->regs->r0; 56 state.registers[1] = ai->regs->r1; 57 state.registers[2] = ai->regs->r2; 58 state.registers[3] = ai->regs->r3; 59 state.registers[4] = ai->regs->r4; 60 state.registers[5] = ai->regs->r5; 61 state.registers[6] = ai->regs->r6; 62 state.registers[7] = ai->regs->r7; 63 state.registers[8] = ai->regs->r8; 64 state.registers[9] = ai->regs->r9; 65 state.registers[10] = ai->regs->r10; 66 state.registers[11] = ai->regs->r11; 67 state.registers[13] = sp; 68 state.registers[14] = lr; 69 state.registers[15] = ai->pc; 70 71 print_stack_arm32(TRACE_ERROR, &state, exidx, exidx_sz, 72 thread_stack_start(), thread_stack_size()); 73 } 74 #endif /* ARM32 */ 75 76 #ifdef ARM64 77 /* Kernel mode unwind (64-bit execution state) */ 78 static void __print_stack_unwind(struct abort_info *ai) 79 { 80 struct unwind_state_arm64 state = { 81 .pc = ai->regs->elr, 82 .fp = ai->regs->x29, 83 }; 84 85 print_stack_arm64(TRACE_ERROR, &state, thread_stack_start(), 86 thread_stack_size()); 87 } 88 #endif /*ARM64*/ 89 90 #else /* CFG_UNWIND */ 91 static void __print_stack_unwind(struct abort_info *ai __unused) 92 { 93 } 94 #endif /* CFG_UNWIND */ 95 96 static __maybe_unused const char *abort_type_to_str(uint32_t abort_type) 97 { 98 if (abort_type == ABORT_TYPE_DATA) 99 return "data"; 100 if (abort_type == ABORT_TYPE_PREFETCH) 101 return "prefetch"; 102 return "undef"; 103 } 104 105 static __maybe_unused const char *fault_to_str(uint32_t abort_type, 106 uint32_t fault_descr) 107 { 108 /* fault_descr is only valid for data or prefetch abort */ 109 if (abort_type != ABORT_TYPE_DATA && abort_type != ABORT_TYPE_PREFETCH) 110 return ""; 111 112 switch (core_mmu_get_fault_type(fault_descr)) { 113 case CORE_MMU_FAULT_ALIGNMENT: 114 return " (alignment fault)"; 115 case CORE_MMU_FAULT_TRANSLATION: 116 return " (translation fault)"; 117 case CORE_MMU_FAULT_READ_PERMISSION: 118 return " (read permission fault)"; 119 case CORE_MMU_FAULT_WRITE_PERMISSION: 120 return " (write permission fault)"; 121 default: 122 return ""; 123 } 124 } 125 126 static __maybe_unused void 127 __print_abort_info(struct abort_info *ai __maybe_unused, 128 const char *ctx __maybe_unused) 129 { 130 __maybe_unused size_t core_pos = 0; 131 #ifdef ARM32 132 uint32_t mode = ai->regs->spsr & CPSR_MODE_MASK; 133 __maybe_unused uint32_t sp = 0; 134 __maybe_unused uint32_t lr = 0; 135 136 if (mode == CPSR_MODE_USR || mode == CPSR_MODE_SYS) { 137 sp = ai->regs->usr_sp; 138 lr = ai->regs->usr_lr; 139 core_pos = thread_get_tsd()->abort_core; 140 } else { 141 sp = read_mode_sp(mode); 142 lr = read_mode_lr(mode); 143 core_pos = get_core_pos(); 144 } 145 #endif /*ARM32*/ 146 #ifdef ARM64 147 if (abort_is_user_exception(ai)) 148 core_pos = thread_get_tsd()->abort_core; 149 else 150 core_pos = get_core_pos(); 151 #endif /*ARM64*/ 152 153 EMSG_RAW(""); 154 EMSG_RAW("%s %s-abort at address 0x%" PRIxVA "%s", 155 ctx, abort_type_to_str(ai->abort_type), ai->va, 156 fault_to_str(ai->abort_type, ai->fault_descr)); 157 #ifdef ARM32 158 EMSG_RAW(" fsr 0x%08x ttbr0 0x%08x ttbr1 0x%08x cidr 0x%X", 159 ai->fault_descr, read_ttbr0(), read_ttbr1(), 160 read_contextidr()); 161 EMSG_RAW(" cpu #%zu cpsr 0x%08x", 162 core_pos, ai->regs->spsr); 163 EMSG_RAW(" r0 0x%08x r4 0x%08x r8 0x%08x r12 0x%08x", 164 ai->regs->r0, ai->regs->r4, ai->regs->r8, ai->regs->ip); 165 EMSG_RAW(" r1 0x%08x r5 0x%08x r9 0x%08x sp 0x%08x", 166 ai->regs->r1, ai->regs->r5, ai->regs->r9, sp); 167 EMSG_RAW(" r2 0x%08x r6 0x%08x r10 0x%08x lr 0x%08x", 168 ai->regs->r2, ai->regs->r6, ai->regs->r10, lr); 169 EMSG_RAW(" r3 0x%08x r7 0x%08x r11 0x%08x pc 0x%08x", 170 ai->regs->r3, ai->regs->r7, ai->regs->r11, ai->pc); 171 #endif /*ARM32*/ 172 #ifdef ARM64 173 EMSG_RAW(" esr 0x%08x ttbr0 0x%08" PRIx64 " ttbr1 0x%08" PRIx64 174 " cidr 0x%X", ai->fault_descr, read_ttbr0_el1(), 175 read_ttbr1_el1(), read_contextidr_el1()); 176 EMSG_RAW(" cpu #%zu cpsr 0x%08x", 177 core_pos, (uint32_t)ai->regs->spsr); 178 EMSG_RAW(" x0 %016" PRIx64 " x1 %016" PRIx64, 179 ai->regs->x0, ai->regs->x1); 180 EMSG_RAW(" x2 %016" PRIx64 " x3 %016" PRIx64, 181 ai->regs->x2, ai->regs->x3); 182 EMSG_RAW(" x4 %016" PRIx64 " x5 %016" PRIx64, 183 ai->regs->x4, ai->regs->x5); 184 EMSG_RAW(" x6 %016" PRIx64 " x7 %016" PRIx64, 185 ai->regs->x6, ai->regs->x7); 186 EMSG_RAW(" x8 %016" PRIx64 " x9 %016" PRIx64, 187 ai->regs->x8, ai->regs->x9); 188 EMSG_RAW(" x10 %016" PRIx64 " x11 %016" PRIx64, 189 ai->regs->x10, ai->regs->x11); 190 EMSG_RAW(" x12 %016" PRIx64 " x13 %016" PRIx64, 191 ai->regs->x12, ai->regs->x13); 192 EMSG_RAW(" x14 %016" PRIx64 " x15 %016" PRIx64, 193 ai->regs->x14, ai->regs->x15); 194 EMSG_RAW(" x16 %016" PRIx64 " x17 %016" PRIx64, 195 ai->regs->x16, ai->regs->x17); 196 EMSG_RAW(" x18 %016" PRIx64 " x19 %016" PRIx64, 197 ai->regs->x18, ai->regs->x19); 198 EMSG_RAW(" x20 %016" PRIx64 " x21 %016" PRIx64, 199 ai->regs->x20, ai->regs->x21); 200 EMSG_RAW(" x22 %016" PRIx64 " x23 %016" PRIx64, 201 ai->regs->x22, ai->regs->x23); 202 EMSG_RAW(" x24 %016" PRIx64 " x25 %016" PRIx64, 203 ai->regs->x24, ai->regs->x25); 204 EMSG_RAW(" x26 %016" PRIx64 " x27 %016" PRIx64, 205 ai->regs->x26, ai->regs->x27); 206 EMSG_RAW(" x28 %016" PRIx64 " x29 %016" PRIx64, 207 ai->regs->x28, ai->regs->x29); 208 EMSG_RAW(" x30 %016" PRIx64 " elr %016" PRIx64, 209 ai->regs->x30, ai->regs->elr); 210 EMSG_RAW(" sp_el0 %016" PRIx64, ai->regs->sp_el0); 211 #endif /*ARM64*/ 212 } 213 214 /* 215 * Print abort info and (optionally) stack dump to the console 216 * @ai kernel-mode abort info. 217 * @stack_dump true to show a stack trace 218 */ 219 static void __abort_print(struct abort_info *ai, bool stack_dump) 220 { 221 assert(!abort_is_user_exception(ai)); 222 223 __print_abort_info(ai, "Core"); 224 225 if (stack_dump) 226 __print_stack_unwind(ai); 227 } 228 229 void abort_print(struct abort_info *ai) 230 { 231 __abort_print(ai, false); 232 } 233 234 void abort_print_error(struct abort_info *ai) 235 { 236 __abort_print(ai, true); 237 } 238 239 /* This function must be called from a normal thread */ 240 void abort_print_current_ta(void) 241 { 242 struct thread_specific_data *tsd = thread_get_tsd(); 243 struct abort_info ai = { }; 244 struct tee_ta_session *s = NULL; 245 246 if (tee_ta_get_current_session(&s) != TEE_SUCCESS) 247 panic(); 248 249 ai.abort_type = tsd->abort_type; 250 ai.fault_descr = tsd->abort_descr; 251 ai.va = tsd->abort_va; 252 ai.pc = tsd->abort_regs.elr; 253 ai.regs = &tsd->abort_regs; 254 255 if (ai.abort_type != ABORT_TYPE_TA_PANIC) 256 __print_abort_info(&ai, "User TA"); 257 258 s->ctx->ops->dump_state(s->ctx); 259 260 if (s->ctx->ops->dump_ftrace) 261 s->ctx->ops->dump_ftrace(s->ctx); 262 } 263 264 static void save_abort_info_in_tsd(struct abort_info *ai) 265 { 266 struct thread_specific_data *tsd = thread_get_tsd(); 267 268 tsd->abort_type = ai->abort_type; 269 tsd->abort_descr = ai->fault_descr; 270 tsd->abort_va = ai->va; 271 tsd->abort_regs = *ai->regs; 272 tsd->abort_core = get_core_pos(); 273 } 274 275 #ifdef ARM32 276 static void set_abort_info(uint32_t abort_type, struct thread_abort_regs *regs, 277 struct abort_info *ai) 278 { 279 switch (abort_type) { 280 case ABORT_TYPE_DATA: 281 ai->fault_descr = read_dfsr(); 282 ai->va = read_dfar(); 283 break; 284 case ABORT_TYPE_PREFETCH: 285 ai->fault_descr = read_ifsr(); 286 ai->va = read_ifar(); 287 break; 288 default: 289 ai->fault_descr = 0; 290 ai->va = regs->elr; 291 break; 292 } 293 ai->abort_type = abort_type; 294 ai->pc = regs->elr; 295 ai->regs = regs; 296 } 297 #endif /*ARM32*/ 298 299 #ifdef ARM64 300 static void set_abort_info(uint32_t abort_type __unused, 301 struct thread_abort_regs *regs, struct abort_info *ai) 302 { 303 ai->fault_descr = read_esr_el1(); 304 switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { 305 case ESR_EC_IABT_EL0: 306 case ESR_EC_IABT_EL1: 307 ai->abort_type = ABORT_TYPE_PREFETCH; 308 ai->va = read_far_el1(); 309 break; 310 case ESR_EC_DABT_EL0: 311 case ESR_EC_DABT_EL1: 312 case ESR_EC_SP_ALIGN: 313 ai->abort_type = ABORT_TYPE_DATA; 314 ai->va = read_far_el1(); 315 break; 316 default: 317 ai->abort_type = ABORT_TYPE_UNDEF; 318 ai->va = regs->elr; 319 } 320 ai->pc = regs->elr; 321 ai->regs = regs; 322 } 323 #endif /*ARM64*/ 324 325 #ifdef ARM32 326 static void handle_user_ta_panic(struct abort_info *ai) 327 { 328 /* 329 * It was a user exception, stop user execution and return 330 * to TEE Core. 331 */ 332 ai->regs->r0 = TEE_ERROR_TARGET_DEAD; 333 ai->regs->r1 = true; 334 ai->regs->r2 = 0xdeadbeef; 335 ai->regs->elr = (uint32_t)thread_unwind_user_mode; 336 ai->regs->spsr &= CPSR_FIA; 337 ai->regs->spsr &= ~CPSR_MODE_MASK; 338 ai->regs->spsr |= CPSR_MODE_SVC; 339 /* Select Thumb or ARM mode */ 340 if (ai->regs->elr & 1) 341 ai->regs->spsr |= CPSR_T; 342 else 343 ai->regs->spsr &= ~CPSR_T; 344 } 345 #endif /*ARM32*/ 346 347 #ifdef ARM64 348 static void handle_user_ta_panic(struct abort_info *ai) 349 { 350 uint32_t daif; 351 352 /* 353 * It was a user exception, stop user execution and return 354 * to TEE Core. 355 */ 356 ai->regs->x0 = TEE_ERROR_TARGET_DEAD; 357 ai->regs->x1 = true; 358 ai->regs->x2 = 0xdeadbeef; 359 ai->regs->elr = (vaddr_t)thread_unwind_user_mode; 360 ai->regs->sp_el0 = thread_get_saved_thread_sp(); 361 362 daif = (ai->regs->spsr >> SPSR_32_AIF_SHIFT) & SPSR_32_AIF_MASK; 363 /* XXX what about DAIF_D? */ 364 ai->regs->spsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, daif); 365 } 366 #endif /*ARM64*/ 367 368 #ifdef CFG_WITH_VFP 369 static void handle_user_ta_vfp(void) 370 { 371 struct tee_ta_session *s; 372 373 if (tee_ta_get_current_session(&s) != TEE_SUCCESS) 374 panic(); 375 376 thread_user_enable_vfp(&to_user_ta_ctx(s->ctx)->vfp); 377 } 378 #endif /*CFG_WITH_VFP*/ 379 380 #ifdef CFG_WITH_USER_TA 381 #ifdef ARM32 382 /* Returns true if the exception originated from user mode */ 383 bool abort_is_user_exception(struct abort_info *ai) 384 { 385 return (ai->regs->spsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; 386 } 387 #endif /*ARM32*/ 388 389 #ifdef ARM64 390 /* Returns true if the exception originated from user mode */ 391 bool abort_is_user_exception(struct abort_info *ai) 392 { 393 uint32_t spsr = ai->regs->spsr; 394 395 if (spsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) 396 return true; 397 if (((spsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == 398 SPSR_64_MODE_EL0) 399 return true; 400 return false; 401 } 402 #endif /*ARM64*/ 403 #else /*CFG_WITH_USER_TA*/ 404 bool abort_is_user_exception(struct abort_info *ai __unused) 405 { 406 return false; 407 } 408 #endif /*CFG_WITH_USER_TA*/ 409 410 #if defined(CFG_WITH_VFP) && defined(CFG_WITH_USER_TA) 411 #ifdef ARM32 412 static bool is_vfp_fault(struct abort_info *ai) 413 { 414 if ((ai->abort_type != ABORT_TYPE_UNDEF) || vfp_is_enabled()) 415 return false; 416 417 /* 418 * Not entirely accurate, but if it's a truly undefined instruction 419 * we'll end up in this function again, except this time 420 * vfp_is_enabled() so we'll return false. 421 */ 422 return true; 423 } 424 #endif /*ARM32*/ 425 426 #ifdef ARM64 427 static bool is_vfp_fault(struct abort_info *ai) 428 { 429 switch ((ai->fault_descr >> ESR_EC_SHIFT) & ESR_EC_MASK) { 430 case ESR_EC_FP_ASIMD: 431 case ESR_EC_AARCH32_FP: 432 case ESR_EC_AARCH64_FP: 433 return true; 434 default: 435 return false; 436 } 437 } 438 #endif /*ARM64*/ 439 #else /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ 440 static bool is_vfp_fault(struct abort_info *ai __unused) 441 { 442 return false; 443 } 444 #endif /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ 445 446 static enum fault_type get_fault_type(struct abort_info *ai) 447 { 448 if (abort_is_user_exception(ai)) { 449 if (is_vfp_fault(ai)) 450 return FAULT_TYPE_USER_TA_VFP; 451 #ifndef CFG_WITH_PAGER 452 return FAULT_TYPE_USER_TA_PANIC; 453 #endif 454 } 455 456 if (thread_is_from_abort_mode()) { 457 abort_print_error(ai); 458 panic("[abort] abort in abort handler (trap CPU)"); 459 } 460 461 if (ai->abort_type == ABORT_TYPE_UNDEF) { 462 if (abort_is_user_exception(ai)) 463 return FAULT_TYPE_USER_TA_PANIC; 464 abort_print_error(ai); 465 panic("[abort] undefined abort (trap CPU)"); 466 } 467 468 switch (core_mmu_get_fault_type(ai->fault_descr)) { 469 case CORE_MMU_FAULT_ALIGNMENT: 470 if (abort_is_user_exception(ai)) 471 return FAULT_TYPE_USER_TA_PANIC; 472 abort_print_error(ai); 473 panic("[abort] alignement fault! (trap CPU)"); 474 break; 475 476 case CORE_MMU_FAULT_ACCESS_BIT: 477 if (abort_is_user_exception(ai)) 478 return FAULT_TYPE_USER_TA_PANIC; 479 abort_print_error(ai); 480 panic("[abort] access bit fault! (trap CPU)"); 481 break; 482 483 case CORE_MMU_FAULT_DEBUG_EVENT: 484 if (!abort_is_user_exception(ai)) 485 abort_print(ai); 486 DMSG("[abort] Ignoring debug event!"); 487 return FAULT_TYPE_IGNORE; 488 489 case CORE_MMU_FAULT_TRANSLATION: 490 case CORE_MMU_FAULT_WRITE_PERMISSION: 491 case CORE_MMU_FAULT_READ_PERMISSION: 492 return FAULT_TYPE_PAGEABLE; 493 494 case CORE_MMU_FAULT_ASYNC_EXTERNAL: 495 if (!abort_is_user_exception(ai)) 496 abort_print(ai); 497 DMSG("[abort] Ignoring async external abort!"); 498 return FAULT_TYPE_IGNORE; 499 500 case CORE_MMU_FAULT_OTHER: 501 default: 502 if (!abort_is_user_exception(ai)) 503 abort_print(ai); 504 DMSG("[abort] Unhandled fault!"); 505 return FAULT_TYPE_IGNORE; 506 } 507 } 508 509 void abort_handler(uint32_t abort_type, struct thread_abort_regs *regs) 510 { 511 struct abort_info ai; 512 bool handled; 513 514 set_abort_info(abort_type, regs, &ai); 515 516 switch (get_fault_type(&ai)) { 517 case FAULT_TYPE_IGNORE: 518 break; 519 case FAULT_TYPE_USER_TA_PANIC: 520 DMSG("[abort] abort in User mode (TA will panic)"); 521 save_abort_info_in_tsd(&ai); 522 vfp_disable(); 523 handle_user_ta_panic(&ai); 524 break; 525 #ifdef CFG_WITH_VFP 526 case FAULT_TYPE_USER_TA_VFP: 527 handle_user_ta_vfp(); 528 break; 529 #endif 530 case FAULT_TYPE_PAGEABLE: 531 default: 532 if (thread_get_id_may_fail() < 0) { 533 abort_print_error(&ai); 534 panic("abort outside thread context"); 535 } 536 thread_kernel_save_vfp(); 537 handled = tee_pager_handle_fault(&ai); 538 thread_kernel_restore_vfp(); 539 if (!handled) { 540 if (!abort_is_user_exception(&ai)) { 541 abort_print_error(&ai); 542 panic("unhandled pageable abort"); 543 } 544 DMSG("[abort] abort in User mode (TA will panic)"); 545 save_abort_info_in_tsd(&ai); 546 vfp_disable(); 547 handle_user_ta_panic(&ai); 548 } 549 break; 550 } 551 } 552