1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016, Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 */ 6 7 #include <platform_config.h> 8 9 #include <arm.h> 10 #include <assert.h> 11 #include <io.h> 12 #include <keep.h> 13 #include <kernel/asan.h> 14 #include <kernel/linker.h> 15 #include <kernel/lockdep.h> 16 #include <kernel/misc.h> 17 #include <kernel/panic.h> 18 #include <kernel/spinlock.h> 19 #include <kernel/tee_ta_manager.h> 20 #include <kernel/thread_defs.h> 21 #include <kernel/thread.h> 22 #include <kernel/virtualization.h> 23 #include <mm/core_memprot.h> 24 #include <mm/mobj.h> 25 #include <mm/tee_mm.h> 26 #include <mm/tee_mmu.h> 27 #include <mm/tee_pager.h> 28 #include <smccc.h> 29 #include <sm/sm.h> 30 #include <trace.h> 31 #include <util.h> 32 33 #include "thread_private.h" 34 35 #ifdef CFG_WITH_ARM_TRUSTED_FW 36 #define STACK_TMP_OFFS 0 37 #else 38 #define STACK_TMP_OFFS SM_STACK_TMP_RESERVE_SIZE 39 #endif 40 41 42 #ifdef ARM32 43 #ifdef CFG_CORE_SANITIZE_KADDRESS 44 #define STACK_TMP_SIZE (3072 + STACK_TMP_OFFS) 45 #else 46 #define STACK_TMP_SIZE (2048 + STACK_TMP_OFFS) 47 #endif 48 #define STACK_THREAD_SIZE 8192 49 50 #if defined(CFG_CORE_SANITIZE_KADDRESS) || defined(__clang__) 51 #define STACK_ABT_SIZE 3072 52 #else 53 #define STACK_ABT_SIZE 2048 54 #endif 55 56 #endif /*ARM32*/ 57 58 #ifdef ARM64 59 #if defined(__clang__) && !defined(CFG_CC_OPTIMIZE_FOR_SIZE) 60 #define STACK_TMP_SIZE (4096 + STACK_TMP_OFFS) 61 #else 62 #define STACK_TMP_SIZE (2048 + STACK_TMP_OFFS) 63 #endif 64 #define STACK_THREAD_SIZE 8192 65 66 #if TRACE_LEVEL > 0 67 #define STACK_ABT_SIZE 3072 68 #else 69 #define STACK_ABT_SIZE 1024 70 #endif 71 #endif /*ARM64*/ 72 73 struct thread_ctx threads[CFG_NUM_THREADS]; 74 75 struct thread_core_local thread_core_local[CFG_TEE_CORE_NB_CORE] __nex_bss; 76 77 #ifdef CFG_WITH_STACK_CANARIES 78 #ifdef ARM32 79 #define STACK_CANARY_SIZE (4 * sizeof(uint32_t)) 80 #endif 81 #ifdef ARM64 82 #define STACK_CANARY_SIZE (8 * sizeof(uint32_t)) 83 #endif 84 #define START_CANARY_VALUE 0xdededede 85 #define END_CANARY_VALUE 0xabababab 86 #define GET_START_CANARY(name, stack_num) name[stack_num][0] 87 #define GET_END_CANARY(name, stack_num) \ 88 name[stack_num][sizeof(name[stack_num]) / sizeof(uint32_t) - 1] 89 #else 90 #define STACK_CANARY_SIZE 0 91 #endif 92 93 #define DECLARE_STACK(name, num_stacks, stack_size, linkage) \ 94 linkage uint32_t name[num_stacks] \ 95 [ROUNDUP(stack_size + STACK_CANARY_SIZE, STACK_ALIGNMENT) / \ 96 sizeof(uint32_t)] \ 97 __attribute__((section(".nozi_stack." # name), \ 98 aligned(STACK_ALIGNMENT))) 99 100 #define STACK_SIZE(stack) (sizeof(stack) - STACK_CANARY_SIZE / 2) 101 102 #define GET_STACK(stack) \ 103 ((vaddr_t)(stack) + STACK_SIZE(stack)) 104 105 DECLARE_STACK(stack_tmp, CFG_TEE_CORE_NB_CORE, STACK_TMP_SIZE, static); 106 DECLARE_STACK(stack_abt, CFG_TEE_CORE_NB_CORE, STACK_ABT_SIZE, static); 107 #ifndef CFG_WITH_PAGER 108 DECLARE_STACK(stack_thread, CFG_NUM_THREADS, STACK_THREAD_SIZE, static); 109 #endif 110 111 const void *stack_tmp_export __section(".identity_map.stack_tmp_export") = 112 (uint8_t *)stack_tmp + sizeof(stack_tmp[0]) - 113 (STACK_TMP_OFFS + STACK_CANARY_SIZE / 2); 114 const uint32_t stack_tmp_stride __section(".identity_map.stack_tmp_stride") = 115 sizeof(stack_tmp[0]); 116 117 /* 118 * These stack setup info are required by secondary boot cores before they 119 * each locally enable the pager (the mmu). Hence kept in pager sections. 120 */ 121 DECLARE_KEEP_PAGER(stack_tmp_export); 122 DECLARE_KEEP_PAGER(stack_tmp_stride); 123 124 thread_pm_handler_t thread_cpu_on_handler_ptr __nex_bss; 125 thread_pm_handler_t thread_cpu_off_handler_ptr __nex_bss; 126 thread_pm_handler_t thread_cpu_suspend_handler_ptr __nex_bss; 127 thread_pm_handler_t thread_cpu_resume_handler_ptr __nex_bss; 128 thread_pm_handler_t thread_system_off_handler_ptr __nex_bss; 129 thread_pm_handler_t thread_system_reset_handler_ptr __nex_bss; 130 131 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 132 static vaddr_t thread_user_kcode_va __nex_bss; 133 long thread_user_kcode_offset __nex_bss; 134 static size_t thread_user_kcode_size __nex_bss; 135 #endif 136 137 #if defined(CFG_CORE_UNMAP_CORE_AT_EL0) && \ 138 defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) 139 long thread_user_kdata_sp_offset __nex_bss; 140 static uint8_t thread_user_kdata_page[ 141 ROUNDUP(sizeof(thread_core_local), SMALL_PAGE_SIZE)] 142 __aligned(SMALL_PAGE_SIZE) 143 #ifndef CFG_VIRTUALIZATION 144 __section(".nozi.kdata_page"); 145 #else 146 __section(".nex_nozi.kdata_page"); 147 #endif 148 #endif 149 150 static unsigned int thread_global_lock __nex_bss = SPINLOCK_UNLOCK; 151 152 static void init_canaries(void) 153 { 154 #ifdef CFG_WITH_STACK_CANARIES 155 size_t n; 156 #define INIT_CANARY(name) \ 157 for (n = 0; n < ARRAY_SIZE(name); n++) { \ 158 uint32_t *start_canary = &GET_START_CANARY(name, n); \ 159 uint32_t *end_canary = &GET_END_CANARY(name, n); \ 160 \ 161 *start_canary = START_CANARY_VALUE; \ 162 *end_canary = END_CANARY_VALUE; \ 163 DMSG("#Stack canaries for %s[%zu] with top at %p", \ 164 #name, n, (void *)(end_canary - 1)); \ 165 DMSG("watch *%p", (void *)end_canary); \ 166 } 167 168 INIT_CANARY(stack_tmp); 169 INIT_CANARY(stack_abt); 170 #if !defined(CFG_WITH_PAGER) && !defined(CFG_VIRTUALIZATION) 171 INIT_CANARY(stack_thread); 172 #endif 173 #endif/*CFG_WITH_STACK_CANARIES*/ 174 } 175 176 #define CANARY_DIED(stack, loc, n) \ 177 do { \ 178 EMSG_RAW("Dead canary at %s of '%s[%zu]'", #loc, #stack, n); \ 179 panic(); \ 180 } while (0) 181 182 void thread_check_canaries(void) 183 { 184 #ifdef CFG_WITH_STACK_CANARIES 185 size_t n; 186 187 for (n = 0; n < ARRAY_SIZE(stack_tmp); n++) { 188 if (GET_START_CANARY(stack_tmp, n) != START_CANARY_VALUE) 189 CANARY_DIED(stack_tmp, start, n); 190 if (GET_END_CANARY(stack_tmp, n) != END_CANARY_VALUE) 191 CANARY_DIED(stack_tmp, end, n); 192 } 193 194 for (n = 0; n < ARRAY_SIZE(stack_abt); n++) { 195 if (GET_START_CANARY(stack_abt, n) != START_CANARY_VALUE) 196 CANARY_DIED(stack_abt, start, n); 197 if (GET_END_CANARY(stack_abt, n) != END_CANARY_VALUE) 198 CANARY_DIED(stack_abt, end, n); 199 200 } 201 #if !defined(CFG_WITH_PAGER) && !defined(CFG_VIRTUALIZATION) 202 for (n = 0; n < ARRAY_SIZE(stack_thread); n++) { 203 if (GET_START_CANARY(stack_thread, n) != START_CANARY_VALUE) 204 CANARY_DIED(stack_thread, start, n); 205 if (GET_END_CANARY(stack_thread, n) != END_CANARY_VALUE) 206 CANARY_DIED(stack_thread, end, n); 207 } 208 #endif 209 #endif/*CFG_WITH_STACK_CANARIES*/ 210 } 211 212 void thread_lock_global(void) 213 { 214 cpu_spin_lock(&thread_global_lock); 215 } 216 217 void thread_unlock_global(void) 218 { 219 cpu_spin_unlock(&thread_global_lock); 220 } 221 222 #ifdef ARM32 223 uint32_t thread_get_exceptions(void) 224 { 225 uint32_t cpsr = read_cpsr(); 226 227 return (cpsr >> CPSR_F_SHIFT) & THREAD_EXCP_ALL; 228 } 229 230 void thread_set_exceptions(uint32_t exceptions) 231 { 232 uint32_t cpsr = read_cpsr(); 233 234 /* Foreign interrupts must not be unmasked while holding a spinlock */ 235 if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) 236 assert_have_no_spinlock(); 237 238 cpsr &= ~(THREAD_EXCP_ALL << CPSR_F_SHIFT); 239 cpsr |= ((exceptions & THREAD_EXCP_ALL) << CPSR_F_SHIFT); 240 write_cpsr(cpsr); 241 } 242 #endif /*ARM32*/ 243 244 #ifdef ARM64 245 uint32_t thread_get_exceptions(void) 246 { 247 uint32_t daif = read_daif(); 248 249 return (daif >> DAIF_F_SHIFT) & THREAD_EXCP_ALL; 250 } 251 252 void thread_set_exceptions(uint32_t exceptions) 253 { 254 uint32_t daif = read_daif(); 255 256 /* Foreign interrupts must not be unmasked while holding a spinlock */ 257 if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) 258 assert_have_no_spinlock(); 259 260 daif &= ~(THREAD_EXCP_ALL << DAIF_F_SHIFT); 261 daif |= ((exceptions & THREAD_EXCP_ALL) << DAIF_F_SHIFT); 262 write_daif(daif); 263 } 264 #endif /*ARM64*/ 265 266 uint32_t thread_mask_exceptions(uint32_t exceptions) 267 { 268 uint32_t state = thread_get_exceptions(); 269 270 thread_set_exceptions(state | (exceptions & THREAD_EXCP_ALL)); 271 return state; 272 } 273 274 void thread_unmask_exceptions(uint32_t state) 275 { 276 thread_set_exceptions(state & THREAD_EXCP_ALL); 277 } 278 279 280 static struct thread_core_local *get_core_local(unsigned int pos) 281 { 282 /* 283 * Foreign interrupts must be disabled before playing with core_local 284 * since we otherwise may be rescheduled to a different core in the 285 * middle of this function. 286 */ 287 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 288 289 assert(pos < CFG_TEE_CORE_NB_CORE); 290 return &thread_core_local[pos]; 291 } 292 293 struct thread_core_local *thread_get_core_local(void) 294 { 295 unsigned int pos = get_core_pos(); 296 297 return get_core_local(pos); 298 } 299 300 void thread_core_local_set_tmp_stack_flag(void) 301 { 302 thread_get_core_local()->flags |= THREAD_CLF_TMP; 303 } 304 305 static void thread_lazy_save_ns_vfp(void) 306 { 307 #ifdef CFG_WITH_VFP 308 struct thread_ctx *thr = threads + thread_get_id(); 309 310 thr->vfp_state.ns_saved = false; 311 vfp_lazy_save_state_init(&thr->vfp_state.ns); 312 #endif /*CFG_WITH_VFP*/ 313 } 314 315 static void thread_lazy_restore_ns_vfp(void) 316 { 317 #ifdef CFG_WITH_VFP 318 struct thread_ctx *thr = threads + thread_get_id(); 319 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 320 321 assert(!thr->vfp_state.sec_lazy_saved && !thr->vfp_state.sec_saved); 322 323 if (tuv && tuv->lazy_saved && !tuv->saved) { 324 vfp_lazy_save_state_final(&tuv->vfp, false /*!force_save*/); 325 tuv->saved = true; 326 } 327 328 vfp_lazy_restore_state(&thr->vfp_state.ns, thr->vfp_state.ns_saved); 329 thr->vfp_state.ns_saved = false; 330 #endif /*CFG_WITH_VFP*/ 331 } 332 333 #ifdef ARM32 334 static void init_regs(struct thread_ctx *thread, uint32_t a0, uint32_t a1, 335 uint32_t a2, uint32_t a3) 336 { 337 thread->regs.pc = (uint32_t)thread_std_smc_entry; 338 339 /* 340 * Stdcalls starts in SVC mode with masked foreign interrupts, masked 341 * Asynchronous abort and unmasked native interrupts. 342 */ 343 thread->regs.cpsr = read_cpsr() & ARM32_CPSR_E; 344 thread->regs.cpsr |= CPSR_MODE_SVC | CPSR_A | 345 (THREAD_EXCP_FOREIGN_INTR << ARM32_CPSR_F_SHIFT); 346 /* Enable thumb mode if it's a thumb instruction */ 347 if (thread->regs.pc & 1) 348 thread->regs.cpsr |= CPSR_T; 349 /* Reinitialize stack pointer */ 350 thread->regs.svc_sp = thread->stack_va_end; 351 352 /* 353 * Copy arguments into context. This will make the 354 * arguments appear in r0-r7 when thread is started. 355 */ 356 thread->regs.r0 = a0; 357 thread->regs.r1 = a1; 358 thread->regs.r2 = a2; 359 thread->regs.r3 = a3; 360 thread->regs.r4 = 0; 361 thread->regs.r5 = 0; 362 thread->regs.r6 = 0; 363 thread->regs.r7 = 0; 364 } 365 #endif /*ARM32*/ 366 367 #ifdef ARM64 368 static void init_regs(struct thread_ctx *thread, uint32_t a0, uint32_t a1, 369 uint32_t a2, uint32_t a3) 370 { 371 thread->regs.pc = (uint64_t)thread_std_smc_entry; 372 373 /* 374 * Stdcalls starts in SVC mode with masked foreign interrupts, masked 375 * Asynchronous abort and unmasked native interrupts. 376 */ 377 thread->regs.cpsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, 378 THREAD_EXCP_FOREIGN_INTR | DAIFBIT_ABT); 379 /* Reinitialize stack pointer */ 380 thread->regs.sp = thread->stack_va_end; 381 382 /* 383 * Copy arguments into context. This will make the 384 * arguments appear in x0-x7 when thread is started. 385 */ 386 thread->regs.x[0] = a0; 387 thread->regs.x[1] = a1; 388 thread->regs.x[2] = a2; 389 thread->regs.x[3] = a3; 390 thread->regs.x[4] = 0; 391 thread->regs.x[5] = 0; 392 thread->regs.x[6] = 0; 393 thread->regs.x[7] = 0; 394 395 /* Set up frame pointer as per the Aarch64 AAPCS */ 396 thread->regs.x[29] = 0; 397 } 398 #endif /*ARM64*/ 399 400 void thread_init_boot_thread(void) 401 { 402 struct thread_core_local *l = thread_get_core_local(); 403 404 thread_init_threads(); 405 406 l->curr_thread = 0; 407 threads[0].state = THREAD_STATE_ACTIVE; 408 } 409 410 void thread_clr_boot_thread(void) 411 { 412 struct thread_core_local *l = thread_get_core_local(); 413 414 assert(l->curr_thread >= 0 && l->curr_thread < CFG_NUM_THREADS); 415 assert(threads[l->curr_thread].state == THREAD_STATE_ACTIVE); 416 threads[l->curr_thread].state = THREAD_STATE_FREE; 417 l->curr_thread = -1; 418 l->flags &= ~THREAD_CLF_TMP; 419 } 420 421 void thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3) 422 { 423 size_t n; 424 struct thread_core_local *l = thread_get_core_local(); 425 bool found_thread = false; 426 427 assert(l->curr_thread == -1); 428 429 thread_lock_global(); 430 431 for (n = 0; n < CFG_NUM_THREADS; n++) { 432 if (threads[n].state == THREAD_STATE_FREE) { 433 threads[n].state = THREAD_STATE_ACTIVE; 434 found_thread = true; 435 break; 436 } 437 } 438 439 thread_unlock_global(); 440 441 if (!found_thread) 442 return; 443 444 l->curr_thread = n; 445 446 threads[n].flags = 0; 447 init_regs(threads + n, a0, a1, a2, a3); 448 449 thread_lazy_save_ns_vfp(); 450 451 l->flags &= ~THREAD_CLF_TMP; 452 thread_resume(&threads[n].regs); 453 /*NOTREACHED*/ 454 panic(); 455 } 456 457 #ifdef ARM32 458 static void copy_a0_to_a3(struct thread_ctx_regs *regs, uint32_t a0, 459 uint32_t a1, uint32_t a2, uint32_t a3) 460 { 461 /* 462 * Update returned values from RPC, values will appear in 463 * r0-r3 when thread is resumed. 464 */ 465 regs->r0 = a0; 466 regs->r1 = a1; 467 regs->r2 = a2; 468 regs->r3 = a3; 469 } 470 #endif /*ARM32*/ 471 472 #ifdef ARM64 473 static void copy_a0_to_a3(struct thread_ctx_regs *regs, uint32_t a0, 474 uint32_t a1, uint32_t a2, uint32_t a3) 475 { 476 /* 477 * Update returned values from RPC, values will appear in 478 * x0-x3 when thread is resumed. 479 */ 480 regs->x[0] = a0; 481 regs->x[1] = a1; 482 regs->x[2] = a2; 483 regs->x[3] = a3; 484 } 485 #endif /*ARM64*/ 486 487 #ifdef ARM32 488 static bool is_from_user(uint32_t cpsr) 489 { 490 return (cpsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; 491 } 492 #endif 493 494 #ifdef ARM64 495 static bool is_from_user(uint32_t cpsr) 496 { 497 if (cpsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) 498 return true; 499 if (((cpsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == 500 SPSR_64_MODE_EL0) 501 return true; 502 return false; 503 } 504 #endif 505 506 #ifdef CFG_SYSCALL_FTRACE 507 static void __noprof ftrace_suspend(void) 508 { 509 struct tee_ta_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); 510 511 if (!s) 512 return; 513 514 if (s->fbuf) 515 s->fbuf->syscall_trace_suspended = true; 516 } 517 518 static void __noprof ftrace_resume(void) 519 { 520 struct tee_ta_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); 521 522 if (!s) 523 return; 524 525 if (s->fbuf) 526 s->fbuf->syscall_trace_suspended = false; 527 } 528 #else 529 static void __noprof ftrace_suspend(void) 530 { 531 } 532 533 static void __noprof ftrace_resume(void) 534 { 535 } 536 #endif 537 538 static bool is_user_mode(struct thread_ctx_regs *regs) 539 { 540 return is_from_user((uint32_t)regs->cpsr); 541 } 542 543 void thread_resume_from_rpc(uint32_t thread_id, uint32_t a0, uint32_t a1, 544 uint32_t a2, uint32_t a3) 545 { 546 size_t n = thread_id; 547 struct thread_core_local *l = thread_get_core_local(); 548 bool found_thread = false; 549 550 assert(l->curr_thread == -1); 551 552 thread_lock_global(); 553 554 if (n < CFG_NUM_THREADS && threads[n].state == THREAD_STATE_SUSPENDED) { 555 threads[n].state = THREAD_STATE_ACTIVE; 556 found_thread = true; 557 } 558 559 thread_unlock_global(); 560 561 if (!found_thread) 562 return; 563 564 l->curr_thread = n; 565 566 if (threads[n].have_user_map) { 567 core_mmu_set_user_map(&threads[n].user_map); 568 if (threads[n].flags & THREAD_FLAGS_EXIT_ON_FOREIGN_INTR) 569 tee_ta_ftrace_update_times_resume(); 570 } 571 572 if (is_user_mode(&threads[n].regs)) 573 tee_ta_update_session_utime_resume(); 574 575 /* 576 * Return from RPC to request service of a foreign interrupt must not 577 * get parameters from non-secure world. 578 */ 579 if (threads[n].flags & THREAD_FLAGS_COPY_ARGS_ON_RETURN) { 580 copy_a0_to_a3(&threads[n].regs, a0, a1, a2, a3); 581 threads[n].flags &= ~THREAD_FLAGS_COPY_ARGS_ON_RETURN; 582 } 583 584 thread_lazy_save_ns_vfp(); 585 586 if (threads[n].have_user_map) 587 ftrace_resume(); 588 589 l->flags &= ~THREAD_CLF_TMP; 590 thread_resume(&threads[n].regs); 591 /*NOTREACHED*/ 592 panic(); 593 } 594 595 void *thread_get_tmp_sp(void) 596 { 597 struct thread_core_local *l = thread_get_core_local(); 598 599 /* 600 * Called from assembly when switching to the temporary stack, so flags 601 * need updating 602 */ 603 l->flags |= THREAD_CLF_TMP; 604 605 return (void *)l->tmp_stack_va_end; 606 } 607 608 #ifdef ARM64 609 vaddr_t thread_get_saved_thread_sp(void) 610 { 611 struct thread_core_local *l = thread_get_core_local(); 612 int ct = l->curr_thread; 613 614 assert(ct != -1); 615 return threads[ct].kern_sp; 616 } 617 #endif /*ARM64*/ 618 619 vaddr_t thread_stack_start(void) 620 { 621 struct thread_ctx *thr; 622 int ct = thread_get_id_may_fail(); 623 624 if (ct == -1) 625 return 0; 626 627 thr = threads + ct; 628 return thr->stack_va_end - STACK_THREAD_SIZE; 629 } 630 631 size_t thread_stack_size(void) 632 { 633 return STACK_THREAD_SIZE; 634 } 635 636 void get_stack_limits(vaddr_t *start, vaddr_t *end) 637 { 638 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 639 unsigned int pos = get_core_pos(); 640 struct thread_core_local *l = get_core_local(pos); 641 struct thread_ctx *thr = NULL; 642 int ct = -1; 643 644 if (l->flags & THREAD_CLF_TMP) { 645 /* We're using the temporary stack for this core */ 646 *start = (vaddr_t)stack_tmp[pos]; 647 *end = *start + STACK_TMP_SIZE; 648 } else if (l->flags & THREAD_CLF_ABORT) { 649 /* We're using the abort stack for this core */ 650 *start = (vaddr_t)stack_abt[pos]; 651 *end = *start + STACK_ABT_SIZE; 652 } else if (!l->flags) { 653 /* We're using a thread stack */ 654 ct = l->curr_thread; 655 assert(ct >= 0 && ct < CFG_NUM_THREADS); 656 thr = threads + ct; 657 *end = thr->stack_va_end; 658 *start = *end - STACK_THREAD_SIZE; 659 } 660 661 thread_unmask_exceptions(exceptions); 662 } 663 664 bool thread_is_from_abort_mode(void) 665 { 666 struct thread_core_local *l = thread_get_core_local(); 667 668 return (l->flags >> THREAD_CLF_SAVED_SHIFT) & THREAD_CLF_ABORT; 669 } 670 671 #ifdef ARM32 672 bool thread_is_in_normal_mode(void) 673 { 674 return (read_cpsr() & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_SVC; 675 } 676 #endif 677 678 #ifdef ARM64 679 bool thread_is_in_normal_mode(void) 680 { 681 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 682 struct thread_core_local *l = thread_get_core_local(); 683 bool ret; 684 685 /* 686 * If any bit in l->flags is set aside from THREAD_CLF_TMP we're 687 * handling some exception. 688 */ 689 ret = (l->curr_thread != -1) && !(l->flags & ~THREAD_CLF_TMP); 690 thread_unmask_exceptions(exceptions); 691 692 return ret; 693 } 694 #endif 695 696 void thread_state_free(void) 697 { 698 struct thread_core_local *l = thread_get_core_local(); 699 int ct = l->curr_thread; 700 701 assert(ct != -1); 702 703 thread_lazy_restore_ns_vfp(); 704 tee_pager_release_phys( 705 (void *)(threads[ct].stack_va_end - STACK_THREAD_SIZE), 706 STACK_THREAD_SIZE); 707 708 thread_lock_global(); 709 710 assert(threads[ct].state == THREAD_STATE_ACTIVE); 711 threads[ct].state = THREAD_STATE_FREE; 712 threads[ct].flags = 0; 713 l->curr_thread = -1; 714 715 #ifdef CFG_VIRTUALIZATION 716 virt_unset_guest(); 717 #endif 718 thread_unlock_global(); 719 } 720 721 #ifdef CFG_WITH_PAGER 722 static void release_unused_kernel_stack(struct thread_ctx *thr, 723 uint32_t cpsr __maybe_unused) 724 { 725 #ifdef ARM64 726 /* 727 * If we're from user mode then thr->regs.sp is the saved user 728 * stack pointer and thr->kern_sp holds the last kernel stack 729 * pointer. But if we're from kernel mode then thr->kern_sp isn't 730 * up to date so we need to read from thr->regs.sp instead. 731 */ 732 vaddr_t sp = is_from_user(cpsr) ? thr->kern_sp : thr->regs.sp; 733 #else 734 vaddr_t sp = thr->regs.svc_sp; 735 #endif 736 vaddr_t base = thr->stack_va_end - STACK_THREAD_SIZE; 737 size_t len = sp - base; 738 739 tee_pager_release_phys((void *)base, len); 740 } 741 #else 742 static void release_unused_kernel_stack(struct thread_ctx *thr __unused, 743 uint32_t cpsr __unused) 744 { 745 } 746 #endif 747 748 int thread_state_suspend(uint32_t flags, uint32_t cpsr, vaddr_t pc) 749 { 750 struct thread_core_local *l = thread_get_core_local(); 751 int ct = l->curr_thread; 752 753 assert(ct != -1); 754 755 if (core_mmu_user_mapping_is_active()) 756 ftrace_suspend(); 757 758 thread_check_canaries(); 759 760 release_unused_kernel_stack(threads + ct, cpsr); 761 762 if (is_from_user(cpsr)) { 763 thread_user_save_vfp(); 764 tee_ta_update_session_utime_suspend(); 765 tee_ta_gprof_sample_pc(pc); 766 } 767 thread_lazy_restore_ns_vfp(); 768 769 thread_lock_global(); 770 771 assert(threads[ct].state == THREAD_STATE_ACTIVE); 772 threads[ct].flags |= flags; 773 threads[ct].regs.cpsr = cpsr; 774 threads[ct].regs.pc = pc; 775 threads[ct].state = THREAD_STATE_SUSPENDED; 776 777 threads[ct].have_user_map = core_mmu_user_mapping_is_active(); 778 if (threads[ct].have_user_map) { 779 if (threads[ct].flags & THREAD_FLAGS_EXIT_ON_FOREIGN_INTR) 780 tee_ta_ftrace_update_times_suspend(); 781 core_mmu_get_user_map(&threads[ct].user_map); 782 core_mmu_set_user_map(NULL); 783 } 784 785 l->curr_thread = -1; 786 787 #ifdef CFG_VIRTUALIZATION 788 virt_unset_guest(); 789 #endif 790 791 thread_unlock_global(); 792 793 return ct; 794 } 795 796 #ifdef ARM32 797 static void set_tmp_stack(struct thread_core_local *l, vaddr_t sp) 798 { 799 l->tmp_stack_va_end = sp; 800 thread_set_irq_sp(sp); 801 thread_set_fiq_sp(sp); 802 } 803 804 static void set_abt_stack(struct thread_core_local *l, vaddr_t sp) 805 { 806 l->abt_stack_va_end = sp; 807 thread_set_abt_sp((vaddr_t)l); 808 thread_set_und_sp((vaddr_t)l); 809 } 810 #endif /*ARM32*/ 811 812 #ifdef ARM64 813 static void set_tmp_stack(struct thread_core_local *l, vaddr_t sp) 814 { 815 /* 816 * We're already using the tmp stack when this function is called 817 * so there's no need to assign it to any stack pointer. However, 818 * we'll need to restore it at different times so store it here. 819 */ 820 l->tmp_stack_va_end = sp; 821 } 822 823 static void set_abt_stack(struct thread_core_local *l, vaddr_t sp) 824 { 825 l->abt_stack_va_end = sp; 826 } 827 #endif /*ARM64*/ 828 829 bool thread_init_stack(uint32_t thread_id, vaddr_t sp) 830 { 831 if (thread_id >= CFG_NUM_THREADS) 832 return false; 833 threads[thread_id].stack_va_end = sp; 834 return true; 835 } 836 837 int thread_get_id_may_fail(void) 838 { 839 /* 840 * thread_get_core_local() requires foreign interrupts to be disabled 841 */ 842 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 843 struct thread_core_local *l = thread_get_core_local(); 844 int ct = l->curr_thread; 845 846 thread_unmask_exceptions(exceptions); 847 return ct; 848 } 849 850 int thread_get_id(void) 851 { 852 int ct = thread_get_id_may_fail(); 853 854 assert(ct >= 0 && ct < CFG_NUM_THREADS); 855 return ct; 856 } 857 858 static void init_handlers(const struct thread_handlers *handlers) 859 { 860 thread_cpu_on_handler_ptr = handlers->cpu_on; 861 thread_cpu_off_handler_ptr = handlers->cpu_off; 862 thread_cpu_suspend_handler_ptr = handlers->cpu_suspend; 863 thread_cpu_resume_handler_ptr = handlers->cpu_resume; 864 thread_system_off_handler_ptr = handlers->system_off; 865 thread_system_reset_handler_ptr = handlers->system_reset; 866 } 867 868 #ifdef CFG_WITH_PAGER 869 static void init_thread_stacks(void) 870 { 871 size_t n = 0; 872 873 /* 874 * Allocate virtual memory for thread stacks. 875 */ 876 for (n = 0; n < CFG_NUM_THREADS; n++) { 877 tee_mm_entry_t *mm = NULL; 878 vaddr_t sp = 0; 879 size_t num_pages = 0; 880 struct fobj *fobj = NULL; 881 882 /* Find vmem for thread stack and its protection gap */ 883 mm = tee_mm_alloc(&tee_mm_vcore, 884 SMALL_PAGE_SIZE + STACK_THREAD_SIZE); 885 assert(mm); 886 887 /* Claim eventual physical page */ 888 tee_pager_add_pages(tee_mm_get_smem(mm), tee_mm_get_size(mm), 889 true); 890 891 num_pages = tee_mm_get_bytes(mm) / SMALL_PAGE_SIZE - 1; 892 fobj = fobj_locked_paged_alloc(num_pages); 893 894 /* Add the area to the pager */ 895 tee_pager_add_core_area(tee_mm_get_smem(mm) + SMALL_PAGE_SIZE, 896 PAGER_AREA_TYPE_LOCK, fobj); 897 fobj_put(fobj); 898 899 /* init effective stack */ 900 sp = tee_mm_get_smem(mm) + tee_mm_get_bytes(mm); 901 asan_tag_access((void *)tee_mm_get_smem(mm), (void *)sp); 902 if (!thread_init_stack(n, sp)) 903 panic("init stack failed"); 904 } 905 } 906 #else 907 static void init_thread_stacks(void) 908 { 909 size_t n; 910 911 /* Assign the thread stacks */ 912 for (n = 0; n < CFG_NUM_THREADS; n++) { 913 if (!thread_init_stack(n, GET_STACK(stack_thread[n]))) 914 panic("thread_init_stack failed"); 915 } 916 } 917 #endif /*CFG_WITH_PAGER*/ 918 919 static void init_user_kcode(void) 920 { 921 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 922 vaddr_t v = (vaddr_t)thread_excp_vect; 923 vaddr_t ve = (vaddr_t)thread_excp_vect_end; 924 925 thread_user_kcode_va = ROUNDDOWN(v, CORE_MMU_USER_CODE_SIZE); 926 ve = ROUNDUP(ve, CORE_MMU_USER_CODE_SIZE); 927 thread_user_kcode_size = ve - thread_user_kcode_va; 928 929 core_mmu_get_user_va_range(&v, NULL); 930 thread_user_kcode_offset = thread_user_kcode_va - v; 931 932 #if defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) 933 /* 934 * When transitioning to EL0 subtract SP with this much to point to 935 * this special kdata page instead. SP is restored by add this much 936 * while transitioning back to EL1. 937 */ 938 v += thread_user_kcode_size; 939 thread_user_kdata_sp_offset = (vaddr_t)thread_core_local - v; 940 #endif 941 #endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ 942 } 943 944 void thread_init_threads(void) 945 { 946 size_t n = 0; 947 948 init_thread_stacks(); 949 pgt_init(); 950 951 mutex_lockdep_init(); 952 953 for (n = 0; n < CFG_NUM_THREADS; n++) { 954 TAILQ_INIT(&threads[n].tsd.sess_stack); 955 SLIST_INIT(&threads[n].tsd.pgt_cache); 956 } 957 } 958 959 void thread_clr_thread_core_local(void) 960 { 961 size_t n = 0; 962 963 for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) 964 thread_core_local[n].curr_thread = -1; 965 thread_core_local[0].flags |= THREAD_CLF_TMP; 966 } 967 968 void thread_init_primary(const struct thread_handlers *handlers) 969 { 970 init_handlers(handlers); 971 972 /* Initialize canaries around the stacks */ 973 init_canaries(); 974 975 init_user_kcode(); 976 } 977 978 static void init_sec_mon(size_t pos __maybe_unused) 979 { 980 #if !defined(CFG_WITH_ARM_TRUSTED_FW) 981 /* Initialize secure monitor */ 982 sm_init(GET_STACK(stack_tmp[pos])); 983 #endif 984 } 985 986 static uint32_t __maybe_unused get_midr_implementer(uint32_t midr) 987 { 988 return (midr >> MIDR_IMPLEMENTER_SHIFT) & MIDR_IMPLEMENTER_MASK; 989 } 990 991 static uint32_t __maybe_unused get_midr_primary_part(uint32_t midr) 992 { 993 return (midr >> MIDR_PRIMARY_PART_NUM_SHIFT) & 994 MIDR_PRIMARY_PART_NUM_MASK; 995 } 996 997 #ifdef ARM64 998 static bool probe_workaround_available(void) 999 { 1000 int32_t r; 1001 1002 r = thread_smc(SMCCC_VERSION, 0, 0, 0); 1003 if (r < 0) 1004 return false; 1005 if (r < 0x10001) /* compare with version 1.1 */ 1006 return false; 1007 1008 /* Version >= 1.1, so SMCCC_ARCH_FEATURES is available */ 1009 r = thread_smc(SMCCC_ARCH_FEATURES, SMCCC_ARCH_WORKAROUND_1, 0, 0); 1010 return r >= 0; 1011 } 1012 1013 static vaddr_t __maybe_unused select_vector(vaddr_t a) 1014 { 1015 if (probe_workaround_available()) { 1016 DMSG("SMCCC_ARCH_WORKAROUND_1 (%#08" PRIx32 ") available", 1017 SMCCC_ARCH_WORKAROUND_1); 1018 DMSG("SMC Workaround for CVE-2017-5715 used"); 1019 return a; 1020 } 1021 1022 DMSG("SMCCC_ARCH_WORKAROUND_1 (%#08" PRIx32 ") unavailable", 1023 SMCCC_ARCH_WORKAROUND_1); 1024 DMSG("SMC Workaround for CVE-2017-5715 not needed (if ARM-TF is up to date)"); 1025 return (vaddr_t)thread_excp_vect; 1026 } 1027 #else 1028 static vaddr_t __maybe_unused select_vector(vaddr_t a) 1029 { 1030 return a; 1031 } 1032 #endif 1033 1034 static vaddr_t get_excp_vect(void) 1035 { 1036 #ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC 1037 uint32_t midr = read_midr(); 1038 1039 if (get_midr_implementer(midr) != MIDR_IMPLEMENTER_ARM) 1040 return (vaddr_t)thread_excp_vect; 1041 1042 switch (get_midr_primary_part(midr)) { 1043 #ifdef ARM32 1044 case CORTEX_A8_PART_NUM: 1045 case CORTEX_A9_PART_NUM: 1046 case CORTEX_A17_PART_NUM: 1047 #endif 1048 case CORTEX_A57_PART_NUM: 1049 case CORTEX_A72_PART_NUM: 1050 case CORTEX_A73_PART_NUM: 1051 case CORTEX_A75_PART_NUM: 1052 return select_vector((vaddr_t)thread_excp_vect_workaround); 1053 #ifdef ARM32 1054 case CORTEX_A15_PART_NUM: 1055 return select_vector((vaddr_t)thread_excp_vect_workaround_a15); 1056 #endif 1057 default: 1058 return (vaddr_t)thread_excp_vect; 1059 } 1060 #endif /*CFG_CORE_WORKAROUND_SPECTRE_BP_SEC*/ 1061 1062 return (vaddr_t)thread_excp_vect; 1063 } 1064 1065 void thread_init_per_cpu(void) 1066 { 1067 size_t pos = get_core_pos(); 1068 struct thread_core_local *l = thread_get_core_local(); 1069 1070 init_sec_mon(pos); 1071 1072 set_tmp_stack(l, GET_STACK(stack_tmp[pos]) - STACK_TMP_OFFS); 1073 set_abt_stack(l, GET_STACK(stack_abt[pos])); 1074 1075 thread_init_vbar(get_excp_vect()); 1076 1077 #ifdef CFG_FTRACE_SUPPORT 1078 /* 1079 * Enable accesses to frequency register and physical counter 1080 * register in EL0/PL0 required for timestamping during 1081 * function tracing. 1082 */ 1083 write_cntkctl(read_cntkctl() | CNTKCTL_PL0PCTEN); 1084 #endif 1085 } 1086 1087 struct thread_specific_data *thread_get_tsd(void) 1088 { 1089 return &threads[thread_get_id()].tsd; 1090 } 1091 1092 struct thread_ctx_regs *thread_get_ctx_regs(void) 1093 { 1094 struct thread_core_local *l = thread_get_core_local(); 1095 1096 assert(l->curr_thread != -1); 1097 return &threads[l->curr_thread].regs; 1098 } 1099 1100 void thread_set_foreign_intr(bool enable) 1101 { 1102 /* thread_get_core_local() requires foreign interrupts to be disabled */ 1103 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1104 struct thread_core_local *l; 1105 1106 l = thread_get_core_local(); 1107 1108 assert(l->curr_thread != -1); 1109 1110 if (enable) { 1111 threads[l->curr_thread].flags |= 1112 THREAD_FLAGS_FOREIGN_INTR_ENABLE; 1113 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 1114 } else { 1115 /* 1116 * No need to disable foreign interrupts here since they're 1117 * already disabled above. 1118 */ 1119 threads[l->curr_thread].flags &= 1120 ~THREAD_FLAGS_FOREIGN_INTR_ENABLE; 1121 } 1122 } 1123 1124 void thread_restore_foreign_intr(void) 1125 { 1126 /* thread_get_core_local() requires foreign interrupts to be disabled */ 1127 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1128 struct thread_core_local *l; 1129 1130 l = thread_get_core_local(); 1131 1132 assert(l->curr_thread != -1); 1133 1134 if (threads[l->curr_thread].flags & THREAD_FLAGS_FOREIGN_INTR_ENABLE) 1135 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 1136 } 1137 1138 #ifdef CFG_WITH_VFP 1139 uint32_t thread_kernel_enable_vfp(void) 1140 { 1141 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1142 struct thread_ctx *thr = threads + thread_get_id(); 1143 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1144 1145 assert(!vfp_is_enabled()); 1146 1147 if (!thr->vfp_state.ns_saved) { 1148 vfp_lazy_save_state_final(&thr->vfp_state.ns, 1149 true /*force_save*/); 1150 thr->vfp_state.ns_saved = true; 1151 } else if (thr->vfp_state.sec_lazy_saved && 1152 !thr->vfp_state.sec_saved) { 1153 /* 1154 * This happens when we're handling an abort while the 1155 * thread was using the VFP state. 1156 */ 1157 vfp_lazy_save_state_final(&thr->vfp_state.sec, 1158 false /*!force_save*/); 1159 thr->vfp_state.sec_saved = true; 1160 } else if (tuv && tuv->lazy_saved && !tuv->saved) { 1161 /* 1162 * This can happen either during syscall or abort 1163 * processing (while processing a syscall). 1164 */ 1165 vfp_lazy_save_state_final(&tuv->vfp, false /*!force_save*/); 1166 tuv->saved = true; 1167 } 1168 1169 vfp_enable(); 1170 return exceptions; 1171 } 1172 1173 void thread_kernel_disable_vfp(uint32_t state) 1174 { 1175 uint32_t exceptions; 1176 1177 assert(vfp_is_enabled()); 1178 1179 vfp_disable(); 1180 exceptions = thread_get_exceptions(); 1181 assert(exceptions & THREAD_EXCP_FOREIGN_INTR); 1182 exceptions &= ~THREAD_EXCP_FOREIGN_INTR; 1183 exceptions |= state & THREAD_EXCP_FOREIGN_INTR; 1184 thread_set_exceptions(exceptions); 1185 } 1186 1187 void thread_kernel_save_vfp(void) 1188 { 1189 struct thread_ctx *thr = threads + thread_get_id(); 1190 1191 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1192 if (vfp_is_enabled()) { 1193 vfp_lazy_save_state_init(&thr->vfp_state.sec); 1194 thr->vfp_state.sec_lazy_saved = true; 1195 } 1196 } 1197 1198 void thread_kernel_restore_vfp(void) 1199 { 1200 struct thread_ctx *thr = threads + thread_get_id(); 1201 1202 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1203 assert(!vfp_is_enabled()); 1204 if (thr->vfp_state.sec_lazy_saved) { 1205 vfp_lazy_restore_state(&thr->vfp_state.sec, 1206 thr->vfp_state.sec_saved); 1207 thr->vfp_state.sec_saved = false; 1208 thr->vfp_state.sec_lazy_saved = false; 1209 } 1210 } 1211 1212 void thread_user_enable_vfp(struct thread_user_vfp_state *uvfp) 1213 { 1214 struct thread_ctx *thr = threads + thread_get_id(); 1215 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1216 1217 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1218 assert(!vfp_is_enabled()); 1219 1220 if (!thr->vfp_state.ns_saved) { 1221 vfp_lazy_save_state_final(&thr->vfp_state.ns, 1222 true /*force_save*/); 1223 thr->vfp_state.ns_saved = true; 1224 } else if (tuv && uvfp != tuv) { 1225 if (tuv->lazy_saved && !tuv->saved) { 1226 vfp_lazy_save_state_final(&tuv->vfp, 1227 false /*!force_save*/); 1228 tuv->saved = true; 1229 } 1230 } 1231 1232 if (uvfp->lazy_saved) 1233 vfp_lazy_restore_state(&uvfp->vfp, uvfp->saved); 1234 uvfp->lazy_saved = false; 1235 uvfp->saved = false; 1236 1237 thr->vfp_state.uvfp = uvfp; 1238 vfp_enable(); 1239 } 1240 1241 void thread_user_save_vfp(void) 1242 { 1243 struct thread_ctx *thr = threads + thread_get_id(); 1244 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1245 1246 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1247 if (!vfp_is_enabled()) 1248 return; 1249 1250 assert(tuv && !tuv->lazy_saved && !tuv->saved); 1251 vfp_lazy_save_state_init(&tuv->vfp); 1252 tuv->lazy_saved = true; 1253 } 1254 1255 void thread_user_clear_vfp(struct thread_user_vfp_state *uvfp) 1256 { 1257 struct thread_ctx *thr = threads + thread_get_id(); 1258 1259 if (uvfp == thr->vfp_state.uvfp) 1260 thr->vfp_state.uvfp = NULL; 1261 uvfp->lazy_saved = false; 1262 uvfp->saved = false; 1263 } 1264 #endif /*CFG_WITH_VFP*/ 1265 1266 #ifdef ARM32 1267 static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) 1268 { 1269 uint32_t s; 1270 1271 if (!is_32bit) 1272 return false; 1273 1274 s = read_cpsr(); 1275 s &= ~(CPSR_MODE_MASK | CPSR_T | CPSR_IT_MASK1 | CPSR_IT_MASK2); 1276 s |= CPSR_MODE_USR; 1277 if (entry_func & 1) 1278 s |= CPSR_T; 1279 *spsr = s; 1280 return true; 1281 } 1282 #endif 1283 1284 #ifdef ARM64 1285 static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) 1286 { 1287 uint32_t s; 1288 1289 if (is_32bit) { 1290 s = read_daif() & (SPSR_32_AIF_MASK << SPSR_32_AIF_SHIFT); 1291 s |= SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT; 1292 s |= (entry_func & SPSR_32_T_MASK) << SPSR_32_T_SHIFT; 1293 } else { 1294 s = read_daif() & (SPSR_64_DAIF_MASK << SPSR_64_DAIF_SHIFT); 1295 } 1296 1297 *spsr = s; 1298 return true; 1299 } 1300 #endif 1301 1302 static void set_ctx_regs(struct thread_ctx_regs *regs, unsigned long a0, 1303 unsigned long a1, unsigned long a2, unsigned long a3, 1304 unsigned long user_sp, unsigned long entry_func, 1305 uint32_t spsr) 1306 { 1307 /* 1308 * First clear all registers to avoid leaking information from 1309 * other TAs or even the Core itself. 1310 */ 1311 *regs = (struct thread_ctx_regs){ }; 1312 #ifdef ARM32 1313 regs->r0 = a0; 1314 regs->r1 = a1; 1315 regs->r2 = a2; 1316 regs->r3 = a3; 1317 regs->usr_sp = user_sp; 1318 regs->pc = entry_func; 1319 regs->cpsr = spsr; 1320 #endif 1321 #ifdef ARM64 1322 regs->x[0] = a0; 1323 regs->x[1] = a1; 1324 regs->x[2] = a2; 1325 regs->x[3] = a3; 1326 regs->sp = user_sp; 1327 regs->pc = entry_func; 1328 regs->cpsr = spsr; 1329 regs->x[13] = user_sp; /* Used when running TA in Aarch32 */ 1330 regs->sp = user_sp; /* Used when running TA in Aarch64 */ 1331 /* Set frame pointer (user stack can't be unwound past this point) */ 1332 regs->x[29] = 0; 1333 #endif 1334 } 1335 1336 uint32_t thread_enter_user_mode(unsigned long a0, unsigned long a1, 1337 unsigned long a2, unsigned long a3, unsigned long user_sp, 1338 unsigned long entry_func, bool is_32bit, 1339 uint32_t *exit_status0, uint32_t *exit_status1) 1340 { 1341 uint32_t spsr = 0; 1342 uint32_t exceptions = 0; 1343 uint32_t rc = 0; 1344 struct thread_ctx_regs *regs = NULL; 1345 1346 tee_ta_update_session_utime_resume(); 1347 1348 /* Derive SPSR from current CPSR/PSTATE readout. */ 1349 if (!get_spsr(is_32bit, entry_func, &spsr)) { 1350 *exit_status0 = 1; /* panic */ 1351 *exit_status1 = 0xbadbadba; 1352 return 0; 1353 } 1354 1355 exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); 1356 /* 1357 * We're using the per thread location of saved context registers 1358 * for temporary storage. Now that exceptions are masked they will 1359 * not be used for any thing else until they are eventually 1360 * unmasked when user mode has been entered. 1361 */ 1362 regs = thread_get_ctx_regs(); 1363 set_ctx_regs(regs, a0, a1, a2, a3, user_sp, entry_func, spsr); 1364 rc = __thread_enter_user_mode(regs, exit_status0, exit_status1); 1365 thread_unmask_exceptions(exceptions); 1366 return rc; 1367 } 1368 1369 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 1370 void thread_get_user_kcode(struct mobj **mobj, size_t *offset, 1371 vaddr_t *va, size_t *sz) 1372 { 1373 core_mmu_get_user_va_range(va, NULL); 1374 *mobj = mobj_tee_ram; 1375 *offset = thread_user_kcode_va - VCORE_START_VA; 1376 *sz = thread_user_kcode_size; 1377 } 1378 #endif 1379 1380 #if defined(CFG_CORE_UNMAP_CORE_AT_EL0) && \ 1381 defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) 1382 void thread_get_user_kdata(struct mobj **mobj, size_t *offset, 1383 vaddr_t *va, size_t *sz) 1384 { 1385 vaddr_t v; 1386 1387 core_mmu_get_user_va_range(&v, NULL); 1388 *va = v + thread_user_kcode_size; 1389 *mobj = mobj_tee_ram; 1390 *offset = (vaddr_t)thread_user_kdata_page - VCORE_START_VA; 1391 *sz = sizeof(thread_user_kdata_page); 1392 } 1393 #endif 1394 1395 static void setup_unwind_user_mode(struct thread_svc_regs *regs) 1396 { 1397 #ifdef ARM32 1398 regs->lr = (uintptr_t)thread_unwind_user_mode; 1399 regs->spsr = read_cpsr(); 1400 #endif 1401 #ifdef ARM64 1402 regs->elr = (uintptr_t)thread_unwind_user_mode; 1403 regs->spsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, 0); 1404 regs->spsr |= read_daif(); 1405 /* 1406 * Regs is the value of stack pointer before calling the SVC 1407 * handler. By the addition matches for the reserved space at the 1408 * beginning of el0_sync_svc(). This prepares the stack when 1409 * returning to thread_unwind_user_mode instead of a normal 1410 * exception return. 1411 */ 1412 regs->sp_el0 = (uint64_t)(regs + 1); 1413 #endif 1414 } 1415 1416 /* 1417 * Note: this function is weak just to make it possible to exclude it from 1418 * the unpaged area. 1419 */ 1420 void __weak thread_svc_handler(struct thread_svc_regs *regs) 1421 { 1422 struct tee_ta_session *sess = NULL; 1423 uint32_t state = 0; 1424 1425 /* Enable native interrupts */ 1426 state = thread_get_exceptions(); 1427 thread_unmask_exceptions(state & ~THREAD_EXCP_NATIVE_INTR); 1428 1429 thread_user_save_vfp(); 1430 1431 /* TA has just entered kernel mode */ 1432 tee_ta_update_session_utime_suspend(); 1433 1434 /* Restore foreign interrupts which are disabled on exception entry */ 1435 thread_restore_foreign_intr(); 1436 1437 tee_ta_get_current_session(&sess); 1438 assert(sess && sess->ctx->ops && sess->ctx->ops->handle_svc); 1439 if (sess->ctx->ops->handle_svc(regs)) { 1440 /* We're about to switch back to user mode */ 1441 tee_ta_update_session_utime_resume(); 1442 } else { 1443 /* We're returning from __thread_enter_user_mode() */ 1444 setup_unwind_user_mode(regs); 1445 } 1446 } 1447