1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016, Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * Copyright (c) 2020-2021, Arm Limited 6 */ 7 8 #include <platform_config.h> 9 10 #include <arm.h> 11 #include <assert.h> 12 #include <config.h> 13 #include <io.h> 14 #include <keep.h> 15 #include <kernel/asan.h> 16 #include <kernel/boot.h> 17 #include <kernel/linker.h> 18 #include <kernel/lockdep.h> 19 #include <kernel/misc.h> 20 #include <kernel/panic.h> 21 #include <kernel/spinlock.h> 22 #include <kernel/spmc_sp_handler.h> 23 #include <kernel/tee_ta_manager.h> 24 #include <kernel/thread_defs.h> 25 #include <kernel/thread.h> 26 #include <kernel/user_mode_ctx_struct.h> 27 #include <kernel/virtualization.h> 28 #include <mm/core_memprot.h> 29 #include <mm/mobj.h> 30 #include <mm/tee_mm.h> 31 #include <mm/tee_pager.h> 32 #include <mm/vm.h> 33 #include <smccc.h> 34 #include <sm/sm.h> 35 #include <trace.h> 36 #include <util.h> 37 38 #include "thread_private.h" 39 40 struct thread_ctx threads[CFG_NUM_THREADS]; 41 42 struct thread_core_local thread_core_local[CFG_TEE_CORE_NB_CORE] __nex_bss; 43 44 /* 45 * Stacks 46 * 47 * [Lower addresses on the left] 48 * 49 * [ STACK_CANARY_SIZE/2 | STACK_CHECK_EXTRA | STACK_XXX_SIZE | STACK_CANARY_SIZE/2 ] 50 * ^ ^ ^ ^ 51 * stack_xxx[n] "hard" top "soft" top bottom 52 */ 53 54 #ifdef CFG_WITH_ARM_TRUSTED_FW 55 #define STACK_TMP_OFFS 0 56 #else 57 #define STACK_TMP_OFFS SM_STACK_TMP_RESERVE_SIZE 58 #endif 59 60 #ifdef ARM32 61 #ifdef CFG_CORE_SANITIZE_KADDRESS 62 #define STACK_TMP_SIZE (3072 + STACK_TMP_OFFS) 63 #else 64 #define STACK_TMP_SIZE (2048 + STACK_TMP_OFFS) 65 #endif 66 #define STACK_THREAD_SIZE 8192 67 68 #if defined(CFG_CORE_SANITIZE_KADDRESS) || defined(__clang__) 69 #define STACK_ABT_SIZE 3072 70 #else 71 #define STACK_ABT_SIZE 2048 72 #endif 73 74 #endif /*ARM32*/ 75 76 #ifdef ARM64 77 #if defined(__clang__) && !defined(__OPTIMIZE_SIZE__) 78 #define STACK_TMP_SIZE (4096 + STACK_TMP_OFFS) 79 #else 80 #define STACK_TMP_SIZE (2048 + STACK_TMP_OFFS) 81 #endif 82 #define STACK_THREAD_SIZE 8192 83 84 #if TRACE_LEVEL > 0 85 #define STACK_ABT_SIZE 3072 86 #else 87 #define STACK_ABT_SIZE 1024 88 #endif 89 #endif /*ARM64*/ 90 91 #ifdef CFG_WITH_STACK_CANARIES 92 #ifdef ARM32 93 #define STACK_CANARY_SIZE (4 * sizeof(uint32_t)) 94 #endif 95 #ifdef ARM64 96 #define STACK_CANARY_SIZE (8 * sizeof(uint32_t)) 97 #endif 98 #define START_CANARY_VALUE 0xdededede 99 #define END_CANARY_VALUE 0xabababab 100 #define GET_START_CANARY(name, stack_num) name[stack_num][0] 101 #define GET_END_CANARY(name, stack_num) \ 102 name[stack_num][sizeof(name[stack_num]) / sizeof(uint32_t) - 1] 103 #else 104 #define STACK_CANARY_SIZE 0 105 #endif 106 107 #ifdef CFG_CORE_DEBUG_CHECK_STACKS 108 /* 109 * Extra space added to each stack in order to reliably detect and dump stack 110 * overflows. Should cover the maximum expected overflow size caused by any C 111 * function (say, 512 bytes; no function should have that much local variables), 112 * plus the maximum stack space needed by __cyg_profile_func_exit(): about 1 KB, 113 * a large part of which is used to print the call stack. Total: 1.5 KB. 114 */ 115 #define STACK_CHECK_EXTRA 1536 116 #else 117 #define STACK_CHECK_EXTRA 0 118 #endif 119 120 #define DECLARE_STACK(name, num_stacks, stack_size, linkage) \ 121 linkage uint32_t name[num_stacks] \ 122 [ROUNDUP(stack_size + STACK_CANARY_SIZE + STACK_CHECK_EXTRA, \ 123 STACK_ALIGNMENT) / sizeof(uint32_t)] \ 124 __attribute__((section(".nozi_stack." # name), \ 125 aligned(STACK_ALIGNMENT))) 126 127 #define GET_STACK(stack) ((vaddr_t)(stack) + STACK_SIZE(stack)) 128 129 DECLARE_STACK(stack_tmp, CFG_TEE_CORE_NB_CORE, 130 STACK_TMP_SIZE + CFG_STACK_TMP_EXTRA, static); 131 DECLARE_STACK(stack_abt, CFG_TEE_CORE_NB_CORE, STACK_ABT_SIZE, static); 132 #ifndef CFG_WITH_PAGER 133 DECLARE_STACK(stack_thread, CFG_NUM_THREADS, 134 STACK_THREAD_SIZE + CFG_STACK_THREAD_EXTRA, static); 135 #endif 136 137 #define GET_STACK_TOP_HARD(stack, n) \ 138 ((vaddr_t)&(stack)[n] + STACK_CANARY_SIZE / 2) 139 #define GET_STACK_TOP_SOFT(stack, n) \ 140 (GET_STACK_TOP_HARD(stack, n) + STACK_CHECK_EXTRA) 141 #define GET_STACK_BOTTOM(stack, n) ((vaddr_t)&(stack)[n] + sizeof(stack[n]) - \ 142 STACK_CANARY_SIZE / 2) 143 144 const void *stack_tmp_export __section(".identity_map.stack_tmp_export") = 145 (void *)(GET_STACK_BOTTOM(stack_tmp, 0) - STACK_TMP_OFFS); 146 const uint32_t stack_tmp_stride __section(".identity_map.stack_tmp_stride") = 147 sizeof(stack_tmp[0]); 148 149 /* 150 * These stack setup info are required by secondary boot cores before they 151 * each locally enable the pager (the mmu). Hence kept in pager sections. 152 */ 153 DECLARE_KEEP_PAGER(stack_tmp_export); 154 DECLARE_KEEP_PAGER(stack_tmp_stride); 155 156 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 157 static vaddr_t thread_user_kcode_va __nex_bss; 158 long thread_user_kcode_offset __nex_bss; 159 static size_t thread_user_kcode_size __nex_bss; 160 #endif 161 162 #if defined(CFG_CORE_UNMAP_CORE_AT_EL0) && \ 163 defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) 164 long thread_user_kdata_sp_offset __nex_bss; 165 static uint8_t thread_user_kdata_page[ 166 ROUNDUP(sizeof(thread_core_local), SMALL_PAGE_SIZE)] 167 __aligned(SMALL_PAGE_SIZE) 168 #ifndef CFG_VIRTUALIZATION 169 __section(".nozi.kdata_page"); 170 #else 171 __section(".nex_nozi.kdata_page"); 172 #endif 173 #endif 174 175 static unsigned int thread_global_lock __nex_bss = SPINLOCK_UNLOCK; 176 177 static void init_canaries(void) 178 { 179 #ifdef CFG_WITH_STACK_CANARIES 180 size_t n; 181 #define INIT_CANARY(name) \ 182 for (n = 0; n < ARRAY_SIZE(name); n++) { \ 183 uint32_t *start_canary = &GET_START_CANARY(name, n); \ 184 uint32_t *end_canary = &GET_END_CANARY(name, n); \ 185 \ 186 *start_canary = START_CANARY_VALUE; \ 187 *end_canary = END_CANARY_VALUE; \ 188 } 189 190 INIT_CANARY(stack_tmp); 191 INIT_CANARY(stack_abt); 192 #if !defined(CFG_WITH_PAGER) && !defined(CFG_VIRTUALIZATION) 193 INIT_CANARY(stack_thread); 194 #endif 195 #endif/*CFG_WITH_STACK_CANARIES*/ 196 } 197 198 #define CANARY_DIED(stack, loc, n, addr) \ 199 do { \ 200 EMSG_RAW("Dead canary at %s of '%s[%zu]' (%p)", #loc, #stack, \ 201 n, (void *)addr); \ 202 panic(); \ 203 } while (0) 204 205 void thread_check_canaries(void) 206 { 207 #ifdef CFG_WITH_STACK_CANARIES 208 uint32_t *canary = NULL; 209 size_t n = 0; 210 211 for (n = 0; n < ARRAY_SIZE(stack_tmp); n++) { 212 canary = &GET_START_CANARY(stack_tmp, n); 213 if (*canary != START_CANARY_VALUE) 214 CANARY_DIED(stack_tmp, start, n, canary); 215 canary = &GET_END_CANARY(stack_tmp, n); 216 if (*canary != END_CANARY_VALUE) 217 CANARY_DIED(stack_tmp, end, n, canary); 218 } 219 220 for (n = 0; n < ARRAY_SIZE(stack_abt); n++) { 221 canary = &GET_START_CANARY(stack_abt, n); 222 if (*canary != START_CANARY_VALUE) 223 CANARY_DIED(stack_abt, start, n, canary); 224 canary = &GET_END_CANARY(stack_abt, n); 225 if (*canary != END_CANARY_VALUE) 226 CANARY_DIED(stack_abt, end, n, canary); 227 228 } 229 #if !defined(CFG_WITH_PAGER) && !defined(CFG_VIRTUALIZATION) 230 for (n = 0; n < ARRAY_SIZE(stack_thread); n++) { 231 canary = &GET_START_CANARY(stack_thread, n); 232 if (*canary != START_CANARY_VALUE) 233 CANARY_DIED(stack_thread, start, n, canary); 234 canary = &GET_END_CANARY(stack_thread, n); 235 if (*canary != END_CANARY_VALUE) 236 CANARY_DIED(stack_thread, end, n, canary); 237 } 238 #endif 239 #endif/*CFG_WITH_STACK_CANARIES*/ 240 } 241 242 void thread_lock_global(void) 243 { 244 cpu_spin_lock(&thread_global_lock); 245 } 246 247 void thread_unlock_global(void) 248 { 249 cpu_spin_unlock(&thread_global_lock); 250 } 251 252 #ifdef ARM32 253 uint32_t __nostackcheck thread_get_exceptions(void) 254 { 255 uint32_t cpsr = read_cpsr(); 256 257 return (cpsr >> CPSR_F_SHIFT) & THREAD_EXCP_ALL; 258 } 259 260 void __nostackcheck thread_set_exceptions(uint32_t exceptions) 261 { 262 uint32_t cpsr = read_cpsr(); 263 264 /* Foreign interrupts must not be unmasked while holding a spinlock */ 265 if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) 266 assert_have_no_spinlock(); 267 268 cpsr &= ~(THREAD_EXCP_ALL << CPSR_F_SHIFT); 269 cpsr |= ((exceptions & THREAD_EXCP_ALL) << CPSR_F_SHIFT); 270 271 barrier(); 272 write_cpsr(cpsr); 273 barrier(); 274 } 275 #endif /*ARM32*/ 276 277 #ifdef ARM64 278 uint32_t __nostackcheck thread_get_exceptions(void) 279 { 280 uint32_t daif = read_daif(); 281 282 return (daif >> DAIF_F_SHIFT) & THREAD_EXCP_ALL; 283 } 284 285 void __nostackcheck thread_set_exceptions(uint32_t exceptions) 286 { 287 uint32_t daif = read_daif(); 288 289 /* Foreign interrupts must not be unmasked while holding a spinlock */ 290 if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) 291 assert_have_no_spinlock(); 292 293 daif &= ~(THREAD_EXCP_ALL << DAIF_F_SHIFT); 294 daif |= ((exceptions & THREAD_EXCP_ALL) << DAIF_F_SHIFT); 295 296 barrier(); 297 write_daif(daif); 298 barrier(); 299 } 300 #endif /*ARM64*/ 301 302 uint32_t __nostackcheck thread_mask_exceptions(uint32_t exceptions) 303 { 304 uint32_t state = thread_get_exceptions(); 305 306 thread_set_exceptions(state | (exceptions & THREAD_EXCP_ALL)); 307 return state; 308 } 309 310 void __nostackcheck thread_unmask_exceptions(uint32_t state) 311 { 312 thread_set_exceptions(state & THREAD_EXCP_ALL); 313 } 314 315 316 static struct thread_core_local * __nostackcheck 317 get_core_local(unsigned int pos) 318 { 319 /* 320 * Foreign interrupts must be disabled before playing with core_local 321 * since we otherwise may be rescheduled to a different core in the 322 * middle of this function. 323 */ 324 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 325 326 assert(pos < CFG_TEE_CORE_NB_CORE); 327 return &thread_core_local[pos]; 328 } 329 330 struct thread_core_local * __nostackcheck thread_get_core_local(void) 331 { 332 unsigned int pos = get_core_pos(); 333 334 return get_core_local(pos); 335 } 336 337 #ifdef CFG_CORE_DEBUG_CHECK_STACKS 338 static void print_stack_limits(void) 339 { 340 size_t n = 0; 341 vaddr_t __maybe_unused start = 0; 342 vaddr_t __maybe_unused end = 0; 343 344 for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { 345 start = GET_STACK_TOP_SOFT(stack_tmp, n); 346 end = GET_STACK_BOTTOM(stack_tmp, n); 347 DMSG("tmp [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 348 } 349 for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { 350 start = GET_STACK_TOP_SOFT(stack_abt, n); 351 end = GET_STACK_BOTTOM(stack_abt, n); 352 DMSG("abt [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 353 } 354 for (n = 0; n < CFG_NUM_THREADS; n++) { 355 end = threads[n].stack_va_end; 356 start = end - STACK_THREAD_SIZE; 357 DMSG("thr [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 358 } 359 } 360 361 static void check_stack_limits(void) 362 { 363 vaddr_t stack_start = 0; 364 vaddr_t stack_end = 0; 365 /* Any value in the current stack frame will do */ 366 vaddr_t current_sp = (vaddr_t)&stack_start; 367 368 if (!get_stack_soft_limits(&stack_start, &stack_end)) 369 panic("Unknown stack limits"); 370 if (current_sp < stack_start || current_sp > stack_end) { 371 DMSG("Stack pointer out of range (0x%" PRIxVA ")", current_sp); 372 print_stack_limits(); 373 panic(); 374 } 375 } 376 377 static bool * __nostackcheck get_stackcheck_recursion_flag(void) 378 { 379 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 380 unsigned int pos = get_core_pos(); 381 struct thread_core_local *l = get_core_local(pos); 382 int ct = l->curr_thread; 383 bool *p = NULL; 384 385 if (l->flags & (THREAD_CLF_ABORT | THREAD_CLF_TMP)) 386 p = &l->stackcheck_recursion; 387 else if (!l->flags) 388 p = &threads[ct].tsd.stackcheck_recursion; 389 390 thread_unmask_exceptions(exceptions); 391 return p; 392 } 393 394 void __cyg_profile_func_enter(void *this_fn, void *call_site); 395 void __nostackcheck __cyg_profile_func_enter(void *this_fn __unused, 396 void *call_site __unused) 397 { 398 bool *p = get_stackcheck_recursion_flag(); 399 400 assert(p); 401 if (*p) 402 return; 403 *p = true; 404 check_stack_limits(); 405 *p = false; 406 } 407 408 void __cyg_profile_func_exit(void *this_fn, void *call_site); 409 void __nostackcheck __cyg_profile_func_exit(void *this_fn __unused, 410 void *call_site __unused) 411 { 412 } 413 #else 414 static void print_stack_limits(void) 415 { 416 } 417 #endif 418 419 static void thread_lazy_save_ns_vfp(void) 420 { 421 #ifdef CFG_WITH_VFP 422 struct thread_ctx *thr = threads + thread_get_id(); 423 424 thr->vfp_state.ns_saved = false; 425 vfp_lazy_save_state_init(&thr->vfp_state.ns); 426 #endif /*CFG_WITH_VFP*/ 427 } 428 429 static void thread_lazy_restore_ns_vfp(void) 430 { 431 #ifdef CFG_WITH_VFP 432 struct thread_ctx *thr = threads + thread_get_id(); 433 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 434 435 assert(!thr->vfp_state.sec_lazy_saved && !thr->vfp_state.sec_saved); 436 437 if (tuv && tuv->lazy_saved && !tuv->saved) { 438 vfp_lazy_save_state_final(&tuv->vfp, false /*!force_save*/); 439 tuv->saved = true; 440 } 441 442 vfp_lazy_restore_state(&thr->vfp_state.ns, thr->vfp_state.ns_saved); 443 thr->vfp_state.ns_saved = false; 444 #endif /*CFG_WITH_VFP*/ 445 } 446 447 #ifdef ARM32 448 static void init_regs(struct thread_ctx *thread, uint32_t a0, uint32_t a1, 449 uint32_t a2, uint32_t a3, uint32_t a4, uint32_t a5, 450 uint32_t a6, uint32_t a7, void *pc) 451 { 452 thread->regs.pc = (uint32_t)pc; 453 454 /* 455 * Stdcalls starts in SVC mode with masked foreign interrupts, masked 456 * Asynchronous abort and unmasked native interrupts. 457 */ 458 thread->regs.cpsr = read_cpsr() & ARM32_CPSR_E; 459 thread->regs.cpsr |= CPSR_MODE_SVC | CPSR_A | 460 (THREAD_EXCP_FOREIGN_INTR << ARM32_CPSR_F_SHIFT); 461 /* Enable thumb mode if it's a thumb instruction */ 462 if (thread->regs.pc & 1) 463 thread->regs.cpsr |= CPSR_T; 464 /* Reinitialize stack pointer */ 465 thread->regs.svc_sp = thread->stack_va_end; 466 467 /* 468 * Copy arguments into context. This will make the 469 * arguments appear in r0-r7 when thread is started. 470 */ 471 thread->regs.r0 = a0; 472 thread->regs.r1 = a1; 473 thread->regs.r2 = a2; 474 thread->regs.r3 = a3; 475 thread->regs.r4 = a4; 476 thread->regs.r5 = a5; 477 thread->regs.r6 = a6; 478 thread->regs.r7 = a7; 479 } 480 #endif /*ARM32*/ 481 482 #ifdef ARM64 483 static void init_regs(struct thread_ctx *thread, uint32_t a0, uint32_t a1, 484 uint32_t a2, uint32_t a3, uint32_t a4, uint32_t a5, 485 uint32_t a6, uint32_t a7, void *pc) 486 { 487 thread->regs.pc = (uint64_t)pc; 488 489 /* 490 * Stdcalls starts in SVC mode with masked foreign interrupts, masked 491 * Asynchronous abort and unmasked native interrupts. 492 */ 493 thread->regs.cpsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, 494 THREAD_EXCP_FOREIGN_INTR | DAIFBIT_ABT); 495 /* Reinitialize stack pointer */ 496 thread->regs.sp = thread->stack_va_end; 497 498 /* 499 * Copy arguments into context. This will make the 500 * arguments appear in x0-x7 when thread is started. 501 */ 502 thread->regs.x[0] = a0; 503 thread->regs.x[1] = a1; 504 thread->regs.x[2] = a2; 505 thread->regs.x[3] = a3; 506 thread->regs.x[4] = a4; 507 thread->regs.x[5] = a5; 508 thread->regs.x[6] = a6; 509 thread->regs.x[7] = a7; 510 511 /* Set up frame pointer as per the Aarch64 AAPCS */ 512 thread->regs.x[29] = 0; 513 } 514 #endif /*ARM64*/ 515 516 void thread_init_boot_thread(void) 517 { 518 struct thread_core_local *l = thread_get_core_local(); 519 520 thread_init_threads(); 521 522 l->curr_thread = 0; 523 threads[0].state = THREAD_STATE_ACTIVE; 524 } 525 526 void __nostackcheck thread_clr_boot_thread(void) 527 { 528 struct thread_core_local *l = thread_get_core_local(); 529 530 assert(l->curr_thread >= 0 && l->curr_thread < CFG_NUM_THREADS); 531 assert(threads[l->curr_thread].state == THREAD_STATE_ACTIVE); 532 threads[l->curr_thread].state = THREAD_STATE_FREE; 533 l->curr_thread = -1; 534 } 535 536 static void __thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, 537 uint32_t a3, uint32_t a4, uint32_t a5, 538 uint32_t a6, uint32_t a7, 539 void *pc) 540 { 541 size_t n; 542 struct thread_core_local *l = thread_get_core_local(); 543 bool found_thread = false; 544 545 assert(l->curr_thread == -1); 546 547 thread_lock_global(); 548 549 for (n = 0; n < CFG_NUM_THREADS; n++) { 550 if (threads[n].state == THREAD_STATE_FREE) { 551 threads[n].state = THREAD_STATE_ACTIVE; 552 found_thread = true; 553 break; 554 } 555 } 556 557 thread_unlock_global(); 558 559 if (!found_thread) 560 return; 561 562 l->curr_thread = n; 563 564 threads[n].flags = 0; 565 init_regs(threads + n, a0, a1, a2, a3, a4, a5, a6, a7, pc); 566 567 thread_lazy_save_ns_vfp(); 568 569 l->flags &= ~THREAD_CLF_TMP; 570 thread_resume(&threads[n].regs); 571 /*NOTREACHED*/ 572 panic(); 573 } 574 575 void thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3) 576 { 577 __thread_alloc_and_run(a0, a1, a2, a3, 0, 0, 0, 0, 578 thread_std_smc_entry); 579 } 580 581 #ifdef CFG_SECURE_PARTITION 582 void thread_sp_alloc_and_run(struct thread_smc_args *args __maybe_unused) 583 { 584 __thread_alloc_and_run(args->a0, args->a1, args->a2, args->a3, args->a4, 585 args->a5, args->a6, args->a7, 586 spmc_sp_thread_entry); 587 } 588 #endif 589 590 #ifdef ARM32 591 static void copy_a0_to_a3(struct thread_ctx_regs *regs, uint32_t a0, 592 uint32_t a1, uint32_t a2, uint32_t a3) 593 { 594 /* 595 * Update returned values from RPC, values will appear in 596 * r0-r3 when thread is resumed. 597 */ 598 regs->r0 = a0; 599 regs->r1 = a1; 600 regs->r2 = a2; 601 regs->r3 = a3; 602 } 603 #endif /*ARM32*/ 604 605 #ifdef ARM64 606 static void copy_a0_to_a3(struct thread_ctx_regs *regs, uint32_t a0, 607 uint32_t a1, uint32_t a2, uint32_t a3) 608 { 609 /* 610 * Update returned values from RPC, values will appear in 611 * x0-x3 when thread is resumed. 612 */ 613 regs->x[0] = a0; 614 regs->x[1] = a1; 615 regs->x[2] = a2; 616 regs->x[3] = a3; 617 } 618 #endif /*ARM64*/ 619 620 #ifdef ARM32 621 static bool is_from_user(uint32_t cpsr) 622 { 623 return (cpsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; 624 } 625 #endif 626 627 #ifdef ARM64 628 static bool is_from_user(uint32_t cpsr) 629 { 630 if (cpsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) 631 return true; 632 if (((cpsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == 633 SPSR_64_MODE_EL0) 634 return true; 635 return false; 636 } 637 #endif 638 639 #ifdef CFG_SYSCALL_FTRACE 640 static void __noprof ftrace_suspend(void) 641 { 642 struct ts_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); 643 644 if (s && s->fbuf) 645 s->fbuf->syscall_trace_suspended = true; 646 } 647 648 static void __noprof ftrace_resume(void) 649 { 650 struct ts_session *s = TAILQ_FIRST(&thread_get_tsd()->sess_stack); 651 652 if (s && s->fbuf) 653 s->fbuf->syscall_trace_suspended = false; 654 } 655 #else 656 static void __noprof ftrace_suspend(void) 657 { 658 } 659 660 static void __noprof ftrace_resume(void) 661 { 662 } 663 #endif 664 665 static bool is_user_mode(struct thread_ctx_regs *regs) 666 { 667 return is_from_user((uint32_t)regs->cpsr); 668 } 669 670 void thread_resume_from_rpc(uint32_t thread_id, uint32_t a0, uint32_t a1, 671 uint32_t a2, uint32_t a3) 672 { 673 size_t n = thread_id; 674 struct thread_core_local *l = thread_get_core_local(); 675 bool found_thread = false; 676 677 assert(l->curr_thread == -1); 678 679 thread_lock_global(); 680 681 if (n < CFG_NUM_THREADS && threads[n].state == THREAD_STATE_SUSPENDED) { 682 threads[n].state = THREAD_STATE_ACTIVE; 683 found_thread = true; 684 } 685 686 thread_unlock_global(); 687 688 if (!found_thread) 689 return; 690 691 l->curr_thread = n; 692 693 if (threads[n].have_user_map) { 694 core_mmu_set_user_map(&threads[n].user_map); 695 if (threads[n].flags & THREAD_FLAGS_EXIT_ON_FOREIGN_INTR) 696 tee_ta_ftrace_update_times_resume(); 697 } 698 699 if (is_user_mode(&threads[n].regs)) 700 tee_ta_update_session_utime_resume(); 701 702 /* 703 * Return from RPC to request service of a foreign interrupt must not 704 * get parameters from non-secure world. 705 */ 706 if (threads[n].flags & THREAD_FLAGS_COPY_ARGS_ON_RETURN) { 707 copy_a0_to_a3(&threads[n].regs, a0, a1, a2, a3); 708 threads[n].flags &= ~THREAD_FLAGS_COPY_ARGS_ON_RETURN; 709 } 710 711 thread_lazy_save_ns_vfp(); 712 713 if (threads[n].have_user_map) 714 ftrace_resume(); 715 716 l->flags &= ~THREAD_CLF_TMP; 717 thread_resume(&threads[n].regs); 718 /*NOTREACHED*/ 719 panic(); 720 } 721 722 void __nostackcheck *thread_get_tmp_sp(void) 723 { 724 struct thread_core_local *l = thread_get_core_local(); 725 726 /* 727 * Called from assembly when switching to the temporary stack, so flags 728 * need updating 729 */ 730 l->flags |= THREAD_CLF_TMP; 731 732 return (void *)l->tmp_stack_va_end; 733 } 734 735 #ifdef ARM64 736 vaddr_t thread_get_saved_thread_sp(void) 737 { 738 struct thread_core_local *l = thread_get_core_local(); 739 int ct = l->curr_thread; 740 741 assert(ct != -1); 742 return threads[ct].kern_sp; 743 } 744 #endif /*ARM64*/ 745 746 vaddr_t thread_stack_start(void) 747 { 748 struct thread_ctx *thr; 749 int ct = thread_get_id_may_fail(); 750 751 if (ct == -1) 752 return 0; 753 754 thr = threads + ct; 755 return thr->stack_va_end - STACK_THREAD_SIZE; 756 } 757 758 size_t thread_stack_size(void) 759 { 760 return STACK_THREAD_SIZE; 761 } 762 763 bool get_stack_limits(vaddr_t *start, vaddr_t *end, bool hard) 764 { 765 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 766 unsigned int pos = get_core_pos(); 767 struct thread_core_local *l = get_core_local(pos); 768 int ct = l->curr_thread; 769 bool ret = false; 770 771 if (l->flags & THREAD_CLF_TMP) { 772 if (hard) 773 *start = GET_STACK_TOP_HARD(stack_tmp, pos); 774 else 775 *start = GET_STACK_TOP_SOFT(stack_tmp, pos); 776 *end = GET_STACK_BOTTOM(stack_tmp, pos); 777 ret = true; 778 } else if (l->flags & THREAD_CLF_ABORT) { 779 if (hard) 780 *start = GET_STACK_TOP_HARD(stack_abt, pos); 781 else 782 *start = GET_STACK_TOP_SOFT(stack_abt, pos); 783 *end = GET_STACK_BOTTOM(stack_abt, pos); 784 ret = true; 785 } else if (!l->flags) { 786 if (ct < 0 || ct >= CFG_NUM_THREADS) 787 goto out; 788 789 *end = threads[ct].stack_va_end; 790 *start = *end - STACK_THREAD_SIZE; 791 if (!hard) 792 *start += STACK_CHECK_EXTRA; 793 ret = true; 794 } 795 out: 796 thread_unmask_exceptions(exceptions); 797 return ret; 798 } 799 800 bool thread_is_from_abort_mode(void) 801 { 802 struct thread_core_local *l = thread_get_core_local(); 803 804 return (l->flags >> THREAD_CLF_SAVED_SHIFT) & THREAD_CLF_ABORT; 805 } 806 807 #ifdef ARM32 808 bool thread_is_in_normal_mode(void) 809 { 810 return (read_cpsr() & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_SVC; 811 } 812 #endif 813 814 #ifdef ARM64 815 bool thread_is_in_normal_mode(void) 816 { 817 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 818 struct thread_core_local *l = thread_get_core_local(); 819 bool ret; 820 821 /* 822 * If any bit in l->flags is set aside from THREAD_CLF_TMP we're 823 * handling some exception. 824 */ 825 ret = (l->curr_thread != -1) && !(l->flags & ~THREAD_CLF_TMP); 826 thread_unmask_exceptions(exceptions); 827 828 return ret; 829 } 830 #endif 831 832 void thread_state_free(void) 833 { 834 struct thread_core_local *l = thread_get_core_local(); 835 int ct = l->curr_thread; 836 837 assert(ct != -1); 838 839 thread_lazy_restore_ns_vfp(); 840 tee_pager_release_phys( 841 (void *)(threads[ct].stack_va_end - STACK_THREAD_SIZE), 842 STACK_THREAD_SIZE); 843 844 thread_lock_global(); 845 846 assert(threads[ct].state == THREAD_STATE_ACTIVE); 847 threads[ct].state = THREAD_STATE_FREE; 848 threads[ct].flags = 0; 849 l->curr_thread = -1; 850 851 #ifdef CFG_VIRTUALIZATION 852 virt_unset_guest(); 853 #endif 854 thread_unlock_global(); 855 } 856 857 #ifdef CFG_WITH_PAGER 858 static void release_unused_kernel_stack(struct thread_ctx *thr, 859 uint32_t cpsr __maybe_unused) 860 { 861 #ifdef ARM64 862 /* 863 * If we're from user mode then thr->regs.sp is the saved user 864 * stack pointer and thr->kern_sp holds the last kernel stack 865 * pointer. But if we're from kernel mode then thr->kern_sp isn't 866 * up to date so we need to read from thr->regs.sp instead. 867 */ 868 vaddr_t sp = is_from_user(cpsr) ? thr->kern_sp : thr->regs.sp; 869 #else 870 vaddr_t sp = thr->regs.svc_sp; 871 #endif 872 vaddr_t base = thr->stack_va_end - STACK_THREAD_SIZE; 873 size_t len = sp - base; 874 875 tee_pager_release_phys((void *)base, len); 876 } 877 #else 878 static void release_unused_kernel_stack(struct thread_ctx *thr __unused, 879 uint32_t cpsr __unused) 880 { 881 } 882 #endif 883 884 int thread_state_suspend(uint32_t flags, uint32_t cpsr, vaddr_t pc) 885 { 886 struct thread_core_local *l = thread_get_core_local(); 887 int ct = l->curr_thread; 888 889 assert(ct != -1); 890 891 if (core_mmu_user_mapping_is_active()) 892 ftrace_suspend(); 893 894 thread_check_canaries(); 895 896 release_unused_kernel_stack(threads + ct, cpsr); 897 898 if (is_from_user(cpsr)) { 899 thread_user_save_vfp(); 900 tee_ta_update_session_utime_suspend(); 901 tee_ta_gprof_sample_pc(pc); 902 } 903 thread_lazy_restore_ns_vfp(); 904 905 thread_lock_global(); 906 907 assert(threads[ct].state == THREAD_STATE_ACTIVE); 908 threads[ct].flags |= flags; 909 threads[ct].regs.cpsr = cpsr; 910 threads[ct].regs.pc = pc; 911 threads[ct].state = THREAD_STATE_SUSPENDED; 912 913 threads[ct].have_user_map = core_mmu_user_mapping_is_active(); 914 if (threads[ct].have_user_map) { 915 if (threads[ct].flags & THREAD_FLAGS_EXIT_ON_FOREIGN_INTR) 916 tee_ta_ftrace_update_times_suspend(); 917 core_mmu_get_user_map(&threads[ct].user_map); 918 core_mmu_set_user_map(NULL); 919 } 920 921 l->curr_thread = -1; 922 923 #ifdef CFG_VIRTUALIZATION 924 virt_unset_guest(); 925 #endif 926 927 thread_unlock_global(); 928 929 return ct; 930 } 931 932 #ifdef ARM32 933 static void set_tmp_stack(struct thread_core_local *l, vaddr_t sp) 934 { 935 l->tmp_stack_va_end = sp; 936 thread_set_irq_sp(sp); 937 thread_set_fiq_sp(sp); 938 } 939 940 static void set_abt_stack(struct thread_core_local *l, vaddr_t sp) 941 { 942 l->abt_stack_va_end = sp; 943 thread_set_abt_sp((vaddr_t)l); 944 thread_set_und_sp((vaddr_t)l); 945 } 946 #endif /*ARM32*/ 947 948 #ifdef ARM64 949 static void set_tmp_stack(struct thread_core_local *l, vaddr_t sp) 950 { 951 /* 952 * We're already using the tmp stack when this function is called 953 * so there's no need to assign it to any stack pointer. However, 954 * we'll need to restore it at different times so store it here. 955 */ 956 l->tmp_stack_va_end = sp; 957 } 958 959 static void set_abt_stack(struct thread_core_local *l, vaddr_t sp) 960 { 961 l->abt_stack_va_end = sp; 962 } 963 #endif /*ARM64*/ 964 965 bool thread_init_stack(uint32_t thread_id, vaddr_t sp) 966 { 967 if (thread_id >= CFG_NUM_THREADS) 968 return false; 969 threads[thread_id].stack_va_end = sp; 970 return true; 971 } 972 973 short int thread_get_id_may_fail(void) 974 { 975 /* 976 * thread_get_core_local() requires foreign interrupts to be disabled 977 */ 978 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 979 struct thread_core_local *l = thread_get_core_local(); 980 short int ct = l->curr_thread; 981 982 thread_unmask_exceptions(exceptions); 983 return ct; 984 } 985 986 short int thread_get_id(void) 987 { 988 short int ct = thread_get_id_may_fail(); 989 990 /* Thread ID has to fit in a short int */ 991 COMPILE_TIME_ASSERT(CFG_NUM_THREADS <= SHRT_MAX); 992 assert(ct >= 0 && ct < CFG_NUM_THREADS); 993 return ct; 994 } 995 996 #ifdef CFG_WITH_PAGER 997 static void init_thread_stacks(void) 998 { 999 size_t n = 0; 1000 1001 /* 1002 * Allocate virtual memory for thread stacks. 1003 */ 1004 for (n = 0; n < CFG_NUM_THREADS; n++) { 1005 tee_mm_entry_t *mm = NULL; 1006 vaddr_t sp = 0; 1007 size_t num_pages = 0; 1008 struct fobj *fobj = NULL; 1009 1010 /* Find vmem for thread stack and its protection gap */ 1011 mm = tee_mm_alloc(&tee_mm_vcore, 1012 SMALL_PAGE_SIZE + STACK_THREAD_SIZE); 1013 assert(mm); 1014 1015 /* Claim eventual physical page */ 1016 tee_pager_add_pages(tee_mm_get_smem(mm), tee_mm_get_size(mm), 1017 true); 1018 1019 num_pages = tee_mm_get_bytes(mm) / SMALL_PAGE_SIZE - 1; 1020 fobj = fobj_locked_paged_alloc(num_pages); 1021 1022 /* Add the area to the pager */ 1023 tee_pager_add_core_area(tee_mm_get_smem(mm) + SMALL_PAGE_SIZE, 1024 PAGER_AREA_TYPE_LOCK, fobj); 1025 fobj_put(fobj); 1026 1027 /* init effective stack */ 1028 sp = tee_mm_get_smem(mm) + tee_mm_get_bytes(mm); 1029 asan_tag_access((void *)tee_mm_get_smem(mm), (void *)sp); 1030 if (!thread_init_stack(n, sp)) 1031 panic("init stack failed"); 1032 } 1033 } 1034 #else 1035 static void init_thread_stacks(void) 1036 { 1037 size_t n; 1038 1039 /* Assign the thread stacks */ 1040 for (n = 0; n < CFG_NUM_THREADS; n++) { 1041 if (!thread_init_stack(n, GET_STACK_BOTTOM(stack_thread, n))) 1042 panic("thread_init_stack failed"); 1043 } 1044 } 1045 #endif /*CFG_WITH_PAGER*/ 1046 1047 static void init_user_kcode(void) 1048 { 1049 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 1050 vaddr_t v = (vaddr_t)thread_excp_vect; 1051 vaddr_t ve = (vaddr_t)thread_excp_vect_end; 1052 1053 thread_user_kcode_va = ROUNDDOWN(v, CORE_MMU_USER_CODE_SIZE); 1054 ve = ROUNDUP(ve, CORE_MMU_USER_CODE_SIZE); 1055 thread_user_kcode_size = ve - thread_user_kcode_va; 1056 1057 core_mmu_get_user_va_range(&v, NULL); 1058 thread_user_kcode_offset = thread_user_kcode_va - v; 1059 1060 #if defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) 1061 /* 1062 * When transitioning to EL0 subtract SP with this much to point to 1063 * this special kdata page instead. SP is restored by add this much 1064 * while transitioning back to EL1. 1065 */ 1066 v += thread_user_kcode_size; 1067 thread_user_kdata_sp_offset = (vaddr_t)thread_core_local - v; 1068 #endif 1069 #endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ 1070 } 1071 1072 void thread_init_threads(void) 1073 { 1074 size_t n = 0; 1075 1076 init_thread_stacks(); 1077 print_stack_limits(); 1078 pgt_init(); 1079 1080 mutex_lockdep_init(); 1081 1082 for (n = 0; n < CFG_NUM_THREADS; n++) { 1083 TAILQ_INIT(&threads[n].tsd.sess_stack); 1084 SLIST_INIT(&threads[n].tsd.pgt_cache); 1085 } 1086 } 1087 1088 void __nostackcheck thread_init_thread_core_local(void) 1089 { 1090 size_t n = 0; 1091 struct thread_core_local *tcl = thread_core_local; 1092 1093 for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { 1094 tcl[n].curr_thread = -1; 1095 tcl[n].flags = THREAD_CLF_TMP; 1096 } 1097 1098 tcl[0].tmp_stack_va_end = GET_STACK_BOTTOM(stack_tmp, 0); 1099 } 1100 1101 void thread_init_primary(void) 1102 { 1103 /* Initialize canaries around the stacks */ 1104 init_canaries(); 1105 1106 init_user_kcode(); 1107 } 1108 1109 static void init_sec_mon_stack(size_t pos __maybe_unused) 1110 { 1111 #if !defined(CFG_WITH_ARM_TRUSTED_FW) 1112 /* Initialize secure monitor */ 1113 sm_init(GET_STACK_BOTTOM(stack_tmp, pos)); 1114 #endif 1115 } 1116 1117 static uint32_t __maybe_unused get_midr_implementer(uint32_t midr) 1118 { 1119 return (midr >> MIDR_IMPLEMENTER_SHIFT) & MIDR_IMPLEMENTER_MASK; 1120 } 1121 1122 static uint32_t __maybe_unused get_midr_primary_part(uint32_t midr) 1123 { 1124 return (midr >> MIDR_PRIMARY_PART_NUM_SHIFT) & 1125 MIDR_PRIMARY_PART_NUM_MASK; 1126 } 1127 1128 #ifdef ARM64 1129 static bool probe_workaround_available(void) 1130 { 1131 int32_t r; 1132 1133 r = thread_smc(SMCCC_VERSION, 0, 0, 0); 1134 if (r < 0) 1135 return false; 1136 if (r < 0x10001) /* compare with version 1.1 */ 1137 return false; 1138 1139 /* Version >= 1.1, so SMCCC_ARCH_FEATURES is available */ 1140 r = thread_smc(SMCCC_ARCH_FEATURES, SMCCC_ARCH_WORKAROUND_1, 0, 0); 1141 return r >= 0; 1142 } 1143 1144 static vaddr_t __maybe_unused select_vector(vaddr_t a) 1145 { 1146 if (probe_workaround_available()) { 1147 DMSG("SMCCC_ARCH_WORKAROUND_1 (%#08" PRIx32 ") available", 1148 SMCCC_ARCH_WORKAROUND_1); 1149 DMSG("SMC Workaround for CVE-2017-5715 used"); 1150 return a; 1151 } 1152 1153 DMSG("SMCCC_ARCH_WORKAROUND_1 (%#08" PRIx32 ") unavailable", 1154 SMCCC_ARCH_WORKAROUND_1); 1155 DMSG("SMC Workaround for CVE-2017-5715 not needed (if ARM-TF is up to date)"); 1156 return (vaddr_t)thread_excp_vect; 1157 } 1158 #else 1159 static vaddr_t __maybe_unused select_vector(vaddr_t a) 1160 { 1161 return a; 1162 } 1163 #endif 1164 1165 static vaddr_t get_excp_vect(void) 1166 { 1167 #ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC 1168 uint32_t midr = read_midr(); 1169 1170 if (get_midr_implementer(midr) != MIDR_IMPLEMENTER_ARM) 1171 return (vaddr_t)thread_excp_vect; 1172 1173 switch (get_midr_primary_part(midr)) { 1174 #ifdef ARM32 1175 case CORTEX_A8_PART_NUM: 1176 case CORTEX_A9_PART_NUM: 1177 case CORTEX_A17_PART_NUM: 1178 #endif 1179 case CORTEX_A57_PART_NUM: 1180 case CORTEX_A72_PART_NUM: 1181 case CORTEX_A73_PART_NUM: 1182 case CORTEX_A75_PART_NUM: 1183 return select_vector((vaddr_t)thread_excp_vect_workaround); 1184 #ifdef ARM32 1185 case CORTEX_A15_PART_NUM: 1186 return select_vector((vaddr_t)thread_excp_vect_workaround_a15); 1187 #endif 1188 default: 1189 return (vaddr_t)thread_excp_vect; 1190 } 1191 #endif /*CFG_CORE_WORKAROUND_SPECTRE_BP_SEC*/ 1192 1193 return (vaddr_t)thread_excp_vect; 1194 } 1195 1196 void thread_init_per_cpu(void) 1197 { 1198 size_t pos = get_core_pos(); 1199 struct thread_core_local *l = thread_get_core_local(); 1200 1201 init_sec_mon_stack(pos); 1202 1203 set_tmp_stack(l, GET_STACK_BOTTOM(stack_tmp, pos) - STACK_TMP_OFFS); 1204 set_abt_stack(l, GET_STACK_BOTTOM(stack_abt, pos)); 1205 1206 thread_init_vbar(get_excp_vect()); 1207 1208 #ifdef CFG_FTRACE_SUPPORT 1209 /* 1210 * Enable accesses to frequency register and physical counter 1211 * register in EL0/PL0 required for timestamping during 1212 * function tracing. 1213 */ 1214 write_cntkctl(read_cntkctl() | CNTKCTL_PL0PCTEN); 1215 #endif 1216 } 1217 1218 struct thread_specific_data *thread_get_tsd(void) 1219 { 1220 return &threads[thread_get_id()].tsd; 1221 } 1222 1223 struct thread_ctx_regs * __nostackcheck thread_get_ctx_regs(void) 1224 { 1225 struct thread_core_local *l = thread_get_core_local(); 1226 1227 assert(l->curr_thread != -1); 1228 return &threads[l->curr_thread].regs; 1229 } 1230 1231 void thread_set_foreign_intr(bool enable) 1232 { 1233 /* thread_get_core_local() requires foreign interrupts to be disabled */ 1234 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1235 struct thread_core_local *l; 1236 1237 l = thread_get_core_local(); 1238 1239 assert(l->curr_thread != -1); 1240 1241 if (enable) { 1242 threads[l->curr_thread].flags |= 1243 THREAD_FLAGS_FOREIGN_INTR_ENABLE; 1244 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 1245 } else { 1246 /* 1247 * No need to disable foreign interrupts here since they're 1248 * already disabled above. 1249 */ 1250 threads[l->curr_thread].flags &= 1251 ~THREAD_FLAGS_FOREIGN_INTR_ENABLE; 1252 } 1253 } 1254 1255 void thread_restore_foreign_intr(void) 1256 { 1257 /* thread_get_core_local() requires foreign interrupts to be disabled */ 1258 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1259 struct thread_core_local *l; 1260 1261 l = thread_get_core_local(); 1262 1263 assert(l->curr_thread != -1); 1264 1265 if (threads[l->curr_thread].flags & THREAD_FLAGS_FOREIGN_INTR_ENABLE) 1266 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 1267 } 1268 1269 #ifdef CFG_WITH_VFP 1270 uint32_t thread_kernel_enable_vfp(void) 1271 { 1272 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1273 struct thread_ctx *thr = threads + thread_get_id(); 1274 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1275 1276 assert(!vfp_is_enabled()); 1277 1278 if (!thr->vfp_state.ns_saved) { 1279 vfp_lazy_save_state_final(&thr->vfp_state.ns, 1280 true /*force_save*/); 1281 thr->vfp_state.ns_saved = true; 1282 } else if (thr->vfp_state.sec_lazy_saved && 1283 !thr->vfp_state.sec_saved) { 1284 /* 1285 * This happens when we're handling an abort while the 1286 * thread was using the VFP state. 1287 */ 1288 vfp_lazy_save_state_final(&thr->vfp_state.sec, 1289 false /*!force_save*/); 1290 thr->vfp_state.sec_saved = true; 1291 } else if (tuv && tuv->lazy_saved && !tuv->saved) { 1292 /* 1293 * This can happen either during syscall or abort 1294 * processing (while processing a syscall). 1295 */ 1296 vfp_lazy_save_state_final(&tuv->vfp, false /*!force_save*/); 1297 tuv->saved = true; 1298 } 1299 1300 vfp_enable(); 1301 return exceptions; 1302 } 1303 1304 void thread_kernel_disable_vfp(uint32_t state) 1305 { 1306 uint32_t exceptions; 1307 1308 assert(vfp_is_enabled()); 1309 1310 vfp_disable(); 1311 exceptions = thread_get_exceptions(); 1312 assert(exceptions & THREAD_EXCP_FOREIGN_INTR); 1313 exceptions &= ~THREAD_EXCP_FOREIGN_INTR; 1314 exceptions |= state & THREAD_EXCP_FOREIGN_INTR; 1315 thread_set_exceptions(exceptions); 1316 } 1317 1318 void thread_kernel_save_vfp(void) 1319 { 1320 struct thread_ctx *thr = threads + thread_get_id(); 1321 1322 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1323 if (vfp_is_enabled()) { 1324 vfp_lazy_save_state_init(&thr->vfp_state.sec); 1325 thr->vfp_state.sec_lazy_saved = true; 1326 } 1327 } 1328 1329 void thread_kernel_restore_vfp(void) 1330 { 1331 struct thread_ctx *thr = threads + thread_get_id(); 1332 1333 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1334 assert(!vfp_is_enabled()); 1335 if (thr->vfp_state.sec_lazy_saved) { 1336 vfp_lazy_restore_state(&thr->vfp_state.sec, 1337 thr->vfp_state.sec_saved); 1338 thr->vfp_state.sec_saved = false; 1339 thr->vfp_state.sec_lazy_saved = false; 1340 } 1341 } 1342 1343 void thread_user_enable_vfp(struct thread_user_vfp_state *uvfp) 1344 { 1345 struct thread_ctx *thr = threads + thread_get_id(); 1346 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1347 1348 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1349 assert(!vfp_is_enabled()); 1350 1351 if (!thr->vfp_state.ns_saved) { 1352 vfp_lazy_save_state_final(&thr->vfp_state.ns, 1353 true /*force_save*/); 1354 thr->vfp_state.ns_saved = true; 1355 } else if (tuv && uvfp != tuv) { 1356 if (tuv->lazy_saved && !tuv->saved) { 1357 vfp_lazy_save_state_final(&tuv->vfp, 1358 false /*!force_save*/); 1359 tuv->saved = true; 1360 } 1361 } 1362 1363 if (uvfp->lazy_saved) 1364 vfp_lazy_restore_state(&uvfp->vfp, uvfp->saved); 1365 uvfp->lazy_saved = false; 1366 uvfp->saved = false; 1367 1368 thr->vfp_state.uvfp = uvfp; 1369 vfp_enable(); 1370 } 1371 1372 void thread_user_save_vfp(void) 1373 { 1374 struct thread_ctx *thr = threads + thread_get_id(); 1375 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1376 1377 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1378 if (!vfp_is_enabled()) 1379 return; 1380 1381 assert(tuv && !tuv->lazy_saved && !tuv->saved); 1382 vfp_lazy_save_state_init(&tuv->vfp); 1383 tuv->lazy_saved = true; 1384 } 1385 1386 void thread_user_clear_vfp(struct user_mode_ctx *uctx) 1387 { 1388 struct thread_user_vfp_state *uvfp = &uctx->vfp; 1389 struct thread_ctx *thr = threads + thread_get_id(); 1390 1391 if (uvfp == thr->vfp_state.uvfp) 1392 thr->vfp_state.uvfp = NULL; 1393 uvfp->lazy_saved = false; 1394 uvfp->saved = false; 1395 } 1396 #endif /*CFG_WITH_VFP*/ 1397 1398 #ifdef ARM32 1399 static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) 1400 { 1401 uint32_t s; 1402 1403 if (!is_32bit) 1404 return false; 1405 1406 s = read_cpsr(); 1407 s &= ~(CPSR_MODE_MASK | CPSR_T | CPSR_IT_MASK1 | CPSR_IT_MASK2); 1408 s |= CPSR_MODE_USR; 1409 if (entry_func & 1) 1410 s |= CPSR_T; 1411 *spsr = s; 1412 return true; 1413 } 1414 #endif 1415 1416 #ifdef ARM64 1417 static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) 1418 { 1419 uint32_t s; 1420 1421 if (is_32bit) { 1422 s = read_daif() & (SPSR_32_AIF_MASK << SPSR_32_AIF_SHIFT); 1423 s |= SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT; 1424 s |= (entry_func & SPSR_32_T_MASK) << SPSR_32_T_SHIFT; 1425 } else { 1426 s = read_daif() & (SPSR_64_DAIF_MASK << SPSR_64_DAIF_SHIFT); 1427 } 1428 1429 *spsr = s; 1430 return true; 1431 } 1432 #endif 1433 1434 static void set_ctx_regs(struct thread_ctx_regs *regs, unsigned long a0, 1435 unsigned long a1, unsigned long a2, unsigned long a3, 1436 unsigned long user_sp, unsigned long entry_func, 1437 uint32_t spsr) 1438 { 1439 /* 1440 * First clear all registers to avoid leaking information from 1441 * other TAs or even the Core itself. 1442 */ 1443 *regs = (struct thread_ctx_regs){ }; 1444 #ifdef ARM32 1445 regs->r0 = a0; 1446 regs->r1 = a1; 1447 regs->r2 = a2; 1448 regs->r3 = a3; 1449 regs->usr_sp = user_sp; 1450 regs->pc = entry_func; 1451 regs->cpsr = spsr; 1452 #endif 1453 #ifdef ARM64 1454 regs->x[0] = a0; 1455 regs->x[1] = a1; 1456 regs->x[2] = a2; 1457 regs->x[3] = a3; 1458 regs->sp = user_sp; 1459 regs->pc = entry_func; 1460 regs->cpsr = spsr; 1461 regs->x[13] = user_sp; /* Used when running TA in Aarch32 */ 1462 regs->sp = user_sp; /* Used when running TA in Aarch64 */ 1463 /* Set frame pointer (user stack can't be unwound past this point) */ 1464 regs->x[29] = 0; 1465 #endif 1466 } 1467 1468 uint32_t thread_enter_user_mode(unsigned long a0, unsigned long a1, 1469 unsigned long a2, unsigned long a3, unsigned long user_sp, 1470 unsigned long entry_func, bool is_32bit, 1471 uint32_t *exit_status0, uint32_t *exit_status1) 1472 { 1473 uint32_t spsr = 0; 1474 uint32_t exceptions = 0; 1475 uint32_t rc = 0; 1476 struct thread_ctx_regs *regs = NULL; 1477 1478 tee_ta_update_session_utime_resume(); 1479 1480 /* Derive SPSR from current CPSR/PSTATE readout. */ 1481 if (!get_spsr(is_32bit, entry_func, &spsr)) { 1482 *exit_status0 = 1; /* panic */ 1483 *exit_status1 = 0xbadbadba; 1484 return 0; 1485 } 1486 1487 exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); 1488 /* 1489 * We're using the per thread location of saved context registers 1490 * for temporary storage. Now that exceptions are masked they will 1491 * not be used for any thing else until they are eventually 1492 * unmasked when user mode has been entered. 1493 */ 1494 regs = thread_get_ctx_regs(); 1495 set_ctx_regs(regs, a0, a1, a2, a3, user_sp, entry_func, spsr); 1496 rc = __thread_enter_user_mode(regs, exit_status0, exit_status1); 1497 thread_unmask_exceptions(exceptions); 1498 return rc; 1499 } 1500 1501 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 1502 void thread_get_user_kcode(struct mobj **mobj, size_t *offset, 1503 vaddr_t *va, size_t *sz) 1504 { 1505 core_mmu_get_user_va_range(va, NULL); 1506 *mobj = mobj_tee_ram; 1507 *offset = thread_user_kcode_va - VCORE_START_VA; 1508 *sz = thread_user_kcode_size; 1509 } 1510 #endif 1511 1512 #if defined(CFG_CORE_UNMAP_CORE_AT_EL0) && \ 1513 defined(CFG_CORE_WORKAROUND_SPECTRE_BP_SEC) && defined(ARM64) 1514 void thread_get_user_kdata(struct mobj **mobj, size_t *offset, 1515 vaddr_t *va, size_t *sz) 1516 { 1517 vaddr_t v; 1518 1519 core_mmu_get_user_va_range(&v, NULL); 1520 *va = v + thread_user_kcode_size; 1521 *mobj = mobj_tee_ram; 1522 *offset = (vaddr_t)thread_user_kdata_page - VCORE_START_VA; 1523 *sz = sizeof(thread_user_kdata_page); 1524 } 1525 #endif 1526 1527 static void setup_unwind_user_mode(struct thread_svc_regs *regs) 1528 { 1529 #ifdef ARM32 1530 regs->lr = (uintptr_t)thread_unwind_user_mode; 1531 regs->spsr = read_cpsr(); 1532 #endif 1533 #ifdef ARM64 1534 regs->elr = (uintptr_t)thread_unwind_user_mode; 1535 regs->spsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, 0); 1536 regs->spsr |= read_daif(); 1537 /* 1538 * Regs is the value of stack pointer before calling the SVC 1539 * handler. By the addition matches for the reserved space at the 1540 * beginning of el0_sync_svc(). This prepares the stack when 1541 * returning to thread_unwind_user_mode instead of a normal 1542 * exception return. 1543 */ 1544 regs->sp_el0 = (uint64_t)(regs + 1); 1545 #endif 1546 } 1547 1548 static void gprof_set_status(struct ts_session *s __maybe_unused, 1549 enum ts_gprof_status status __maybe_unused) 1550 { 1551 #ifdef CFG_TA_GPROF_SUPPORT 1552 if (s->ctx->ops->gprof_set_status) 1553 s->ctx->ops->gprof_set_status(status); 1554 #endif 1555 } 1556 1557 /* 1558 * Note: this function is weak just to make it possible to exclude it from 1559 * the unpaged area. 1560 */ 1561 void __weak thread_svc_handler(struct thread_svc_regs *regs) 1562 { 1563 struct ts_session *sess = NULL; 1564 uint32_t state = 0; 1565 1566 /* Enable native interrupts */ 1567 state = thread_get_exceptions(); 1568 thread_unmask_exceptions(state & ~THREAD_EXCP_NATIVE_INTR); 1569 1570 thread_user_save_vfp(); 1571 1572 sess = ts_get_current_session(); 1573 /* 1574 * User mode service has just entered kernel mode, suspend gprof 1575 * collection until we're about to switch back again. 1576 */ 1577 gprof_set_status(sess, TS_GPROF_SUSPEND); 1578 1579 /* Restore foreign interrupts which are disabled on exception entry */ 1580 thread_restore_foreign_intr(); 1581 1582 assert(sess && sess->handle_svc); 1583 if (sess->handle_svc(regs)) { 1584 /* We're about to switch back to user mode */ 1585 gprof_set_status(sess, TS_GPROF_RESUME); 1586 } else { 1587 /* We're returning from __thread_enter_user_mode() */ 1588 setup_unwind_user_mode(regs); 1589 } 1590 } 1591 1592 static struct mobj *alloc_shm(enum thread_shm_type shm_type, size_t size) 1593 { 1594 switch (shm_type) { 1595 case THREAD_SHM_TYPE_APPLICATION: 1596 return thread_rpc_alloc_payload(size); 1597 case THREAD_SHM_TYPE_KERNEL_PRIVATE: 1598 return thread_rpc_alloc_kernel_payload(size); 1599 case THREAD_SHM_TYPE_GLOBAL: 1600 return thread_rpc_alloc_global_payload(size); 1601 default: 1602 return NULL; 1603 } 1604 } 1605 1606 static void clear_shm_cache_entry(struct thread_shm_cache_entry *ce) 1607 { 1608 if (ce->mobj) { 1609 switch (ce->type) { 1610 case THREAD_SHM_TYPE_APPLICATION: 1611 thread_rpc_free_payload(ce->mobj); 1612 break; 1613 case THREAD_SHM_TYPE_KERNEL_PRIVATE: 1614 thread_rpc_free_kernel_payload(ce->mobj); 1615 break; 1616 case THREAD_SHM_TYPE_GLOBAL: 1617 thread_rpc_free_global_payload(ce->mobj); 1618 break; 1619 default: 1620 assert(0); /* "can't happen" */ 1621 break; 1622 } 1623 } 1624 ce->mobj = NULL; 1625 ce->size = 0; 1626 } 1627 1628 static struct thread_shm_cache_entry * 1629 get_shm_cache_entry(enum thread_shm_cache_user user) 1630 { 1631 struct thread_shm_cache *cache = &threads[thread_get_id()].shm_cache; 1632 struct thread_shm_cache_entry *ce = NULL; 1633 1634 SLIST_FOREACH(ce, cache, link) 1635 if (ce->user == user) 1636 return ce; 1637 1638 ce = calloc(1, sizeof(*ce)); 1639 if (ce) { 1640 ce->user = user; 1641 SLIST_INSERT_HEAD(cache, ce, link); 1642 } 1643 1644 return ce; 1645 } 1646 1647 void *thread_rpc_shm_cache_alloc(enum thread_shm_cache_user user, 1648 enum thread_shm_type shm_type, 1649 size_t size, struct mobj **mobj) 1650 { 1651 struct thread_shm_cache_entry *ce = NULL; 1652 size_t sz = size; 1653 paddr_t p = 0; 1654 void *va = NULL; 1655 1656 if (!size) 1657 return NULL; 1658 1659 ce = get_shm_cache_entry(user); 1660 if (!ce) 1661 return NULL; 1662 1663 /* 1664 * Always allocate in page chunks as normal world allocates payload 1665 * memory as complete pages. 1666 */ 1667 sz = ROUNDUP(size, SMALL_PAGE_SIZE); 1668 1669 if (ce->type != shm_type || sz > ce->size) { 1670 clear_shm_cache_entry(ce); 1671 1672 ce->mobj = alloc_shm(shm_type, sz); 1673 if (!ce->mobj) 1674 return NULL; 1675 1676 if (mobj_get_pa(ce->mobj, 0, 0, &p)) 1677 goto err; 1678 1679 if (!ALIGNMENT_IS_OK(p, uint64_t)) 1680 goto err; 1681 1682 va = mobj_get_va(ce->mobj, 0); 1683 if (!va) 1684 goto err; 1685 1686 ce->size = sz; 1687 ce->type = shm_type; 1688 } else { 1689 va = mobj_get_va(ce->mobj, 0); 1690 if (!va) 1691 goto err; 1692 } 1693 *mobj = ce->mobj; 1694 1695 return va; 1696 err: 1697 clear_shm_cache_entry(ce); 1698 return NULL; 1699 } 1700 1701 void thread_rpc_shm_cache_clear(struct thread_shm_cache *cache) 1702 { 1703 while (true) { 1704 struct thread_shm_cache_entry *ce = SLIST_FIRST(cache); 1705 1706 if (!ce) 1707 break; 1708 SLIST_REMOVE_HEAD(cache, link); 1709 clear_shm_cache_entry(ce); 1710 free(ce); 1711 } 1712 } 1713 1714 #ifdef CFG_WITH_ARM_TRUSTED_FW 1715 /* 1716 * These five functions are __weak to allow platforms to override them if 1717 * needed. 1718 */ 1719 unsigned long __weak thread_cpu_off_handler(unsigned long a0 __unused, 1720 unsigned long a1 __unused) 1721 { 1722 return 0; 1723 } 1724 DECLARE_KEEP_PAGER(thread_cpu_off_handler); 1725 1726 unsigned long __weak thread_cpu_suspend_handler(unsigned long a0 __unused, 1727 unsigned long a1 __unused) 1728 { 1729 return 0; 1730 } 1731 DECLARE_KEEP_PAGER(thread_cpu_suspend_handler); 1732 1733 unsigned long __weak thread_cpu_resume_handler(unsigned long a0 __unused, 1734 unsigned long a1 __unused) 1735 { 1736 return 0; 1737 } 1738 DECLARE_KEEP_PAGER(thread_cpu_resume_handler); 1739 1740 unsigned long __weak thread_system_off_handler(unsigned long a0 __unused, 1741 unsigned long a1 __unused) 1742 { 1743 return 0; 1744 } 1745 DECLARE_KEEP_PAGER(thread_system_off_handler); 1746 1747 unsigned long __weak thread_system_reset_handler(unsigned long a0 __unused, 1748 unsigned long a1 __unused) 1749 { 1750 return 0; 1751 } 1752 DECLARE_KEEP_PAGER(thread_system_reset_handler); 1753 #endif /*CFG_WITH_ARM_TRUSTED_FW*/ 1754