1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016-2022, Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * Copyright (c) 2020-2021, Arm Limited 6 */ 7 8 #include <config.h> 9 #include <crypto/crypto.h> 10 #include <kernel/asan.h> 11 #include <kernel/boot.h> 12 #include <kernel/lockdep.h> 13 #include <kernel/misc.h> 14 #include <kernel/panic.h> 15 #include <kernel/spinlock.h> 16 #include <kernel/thread.h> 17 #include <kernel/thread_private.h> 18 #include <mm/mobj.h> 19 #include <mm/page_alloc.h> 20 #include <stdalign.h> 21 22 static struct thread_ctx __threads[CFG_NUM_THREADS]; 23 struct thread_ctx *threads = __threads; 24 #if defined(CFG_DYN_STACK_CONFIG) 25 struct thread_core_local *thread_core_local __nex_bss; 26 size_t thread_core_count __nex_bss; 27 #else 28 static struct thread_core_local 29 __thread_core_local[CFG_TEE_CORE_NB_CORE] __nex_bss; 30 struct thread_core_local *thread_core_local __nex_data = __thread_core_local; 31 size_t thread_core_count __nex_data = CFG_TEE_CORE_NB_CORE; 32 #endif 33 size_t thread_count = CFG_NUM_THREADS; 34 unsigned long thread_core_local_pa __nex_bss; 35 struct thread_core_local *__thread_core_local_new __nex_bss; 36 size_t __thread_core_count_new __nex_bss; 37 38 /* 39 * Stacks 40 * 41 * [Lower addresses on the left] 42 * 43 * [ STACK_CANARY_SIZE/2 | STACK_CHECK_EXTRA | STACK_XXX_SIZE | STACK_CANARY_SIZE/2 ] 44 * ^ ^ ^ ^ 45 * stack_xxx[n] "hard" top "soft" top bottom 46 */ 47 48 static uint32_t start_canary_value = 0xdedede00; 49 static uint32_t end_canary_value = 0xababab00; 50 51 #define DECLARE_STACK(name, num_stacks, stack_size, linkage) \ 52 linkage uint32_t name[num_stacks] \ 53 [ROUNDUP(stack_size + STACK_CANARY_SIZE + STACK_CHECK_EXTRA, \ 54 STACK_ALIGNMENT) / sizeof(uint32_t)] \ 55 __attribute__((section(".nozi_stack." # name), \ 56 aligned(STACK_ALIGNMENT))) 57 58 #ifndef CFG_DYN_STACK_CONFIG 59 DECLARE_STACK(stack_tmp, CFG_TEE_CORE_NB_CORE, STACK_TMP_SIZE, 60 /* global linkage */); 61 DECLARE_STACK(stack_abt, CFG_TEE_CORE_NB_CORE, STACK_ABT_SIZE, static); 62 #define GET_STACK_BOTTOM(stack, n) ((vaddr_t)&(stack)[n] + sizeof(stack[n]) - \ 63 STACK_CANARY_SIZE / 2) 64 #else 65 /* Not used */ 66 #define GET_STACK_BOTTOM(stack, n) 0 67 #endif 68 #ifndef CFG_WITH_PAGER 69 DECLARE_STACK(stack_thread, CFG_NUM_THREADS, STACK_THREAD_SIZE, static); 70 #define GET_STACK_THREAD_BOTTOM(n) \ 71 ((vaddr_t)&stack_thread[n] + sizeof(stack_thread[n]) - \ 72 STACK_CANARY_SIZE / 2) 73 #endif 74 75 76 #ifndef CFG_DYN_STACK_CONFIG 77 const uint32_t stack_tmp_stride __section(".identity_map.stack_tmp_stride") = 78 sizeof(stack_tmp[0]); 79 80 /* 81 * This stack setup info is required by secondary boot cores before they 82 * each locally enable the pager (the mmu). Hence kept in pager sections. 83 */ 84 DECLARE_KEEP_PAGER(stack_tmp_stride); 85 #endif 86 87 static unsigned int thread_global_lock __nex_bss = SPINLOCK_UNLOCK; 88 89 static size_t stack_size_to_alloc_size(size_t stack_size) 90 { 91 return ROUNDUP(stack_size + STACK_CANARY_SIZE + STACK_CHECK_EXTRA, 92 STACK_ALIGNMENT); 93 } 94 95 static vaddr_t stack_end_va_to_top_hard(size_t stack_size, vaddr_t end_va) 96 { 97 size_t l = stack_size_to_alloc_size(stack_size); 98 99 return end_va - l + STACK_CANARY_SIZE; 100 } 101 102 static vaddr_t stack_end_va_to_top_soft(size_t stack_size, vaddr_t end_va) 103 { 104 return stack_end_va_to_top_hard(stack_size, end_va) + STACK_CHECK_EXTRA; 105 } 106 107 static vaddr_t stack_end_va_to_bottom(size_t stack_size __unused, 108 vaddr_t end_va) 109 { 110 return end_va; 111 } 112 113 static uint32_t *stack_end_va_to_start_canary(size_t stack_size, vaddr_t end_va) 114 { 115 return (uint32_t *)(stack_end_va_to_top_hard(stack_size, end_va) - 116 STACK_CANARY_SIZE / 2); 117 } 118 119 static uint32_t *stack_end_va_to_end_canary(size_t stack_size __unused, 120 vaddr_t end_va) 121 { 122 return (uint32_t *)(end_va + STACK_CANARY_SIZE / 2 - sizeof(uint32_t)); 123 } 124 125 static void init_canaries(size_t stack_size, vaddr_t va_end) 126 { 127 uint32_t *canary = NULL; 128 129 assert(va_end); 130 canary = stack_end_va_to_start_canary(stack_size, va_end); 131 *canary = start_canary_value; 132 canary = stack_end_va_to_end_canary(stack_size, va_end); 133 *canary = end_canary_value; 134 } 135 136 void thread_init_canaries(void) 137 { 138 vaddr_t va = 0; 139 size_t n = 0; 140 141 if (IS_ENABLED(CFG_WITH_STACK_CANARIES)) { 142 for (n = 0; n < thread_core_count; n++) { 143 if (thread_core_local[n].tmp_stack_va_end) { 144 va = thread_core_local[n].tmp_stack_va_end + 145 STACK_TMP_OFFS; 146 init_canaries(STACK_TMP_SIZE, va); 147 } 148 va = thread_core_local[n].abt_stack_va_end; 149 if (va) 150 init_canaries(STACK_ABT_SIZE, va); 151 } 152 153 } 154 155 if (IS_ENABLED(CFG_WITH_STACK_CANARIES) && 156 !IS_ENABLED(CFG_WITH_PAGER) && !IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 157 for (n = 0; n < CFG_NUM_THREADS; n++) { 158 va = threads[n].stack_va_end; 159 if (va) 160 init_canaries(STACK_THREAD_SIZE, va); 161 } 162 } 163 } 164 165 #if defined(CFG_WITH_STACK_CANARIES) 166 void thread_update_canaries(void) 167 { 168 uint32_t canary[2] = { }; 169 uint32_t exceptions = 0; 170 171 plat_get_random_stack_canaries(canary, ARRAY_SIZE(canary), 172 sizeof(canary[0])); 173 174 exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); 175 176 thread_check_canaries(); 177 178 start_canary_value = canary[0]; 179 end_canary_value = canary[1]; 180 thread_init_canaries(); 181 182 thread_unmask_exceptions(exceptions); 183 } 184 #endif 185 186 static void check_stack_canary(const char *stack_name __maybe_unused, 187 size_t n __maybe_unused, 188 size_t stack_size, vaddr_t end_va) 189 { 190 uint32_t *canary = NULL; 191 192 canary = stack_end_va_to_start_canary(stack_size, end_va); 193 if (*canary != start_canary_value) { 194 EMSG_RAW("Dead canary at start of '%s[%zu]' (%p)", 195 stack_name, n, (void *)canary); 196 panic(); 197 } 198 199 canary = stack_end_va_to_end_canary(stack_size, end_va); 200 if (*canary != end_canary_value) { 201 EMSG_RAW("Dead canary at end of '%s[%zu]' (%p)", 202 stack_name, n, (void *)canary); 203 panic(); 204 } 205 } 206 207 void thread_check_canaries(void) 208 { 209 vaddr_t va = 0; 210 size_t n = 0; 211 212 if (IS_ENABLED(CFG_WITH_STACK_CANARIES)) { 213 for (n = 0; n < thread_core_count; n++) { 214 if (thread_core_local[n].tmp_stack_va_end) { 215 va = thread_core_local[n].tmp_stack_va_end + 216 STACK_TMP_OFFS; 217 check_stack_canary("tmp_stack", n, 218 STACK_TMP_SIZE, va); 219 } 220 221 va = thread_core_local[n].abt_stack_va_end; 222 if (va) 223 check_stack_canary("abt_stack", n, 224 STACK_ABT_SIZE, va); 225 } 226 } 227 228 if (IS_ENABLED(CFG_WITH_STACK_CANARIES) && 229 !IS_ENABLED(CFG_WITH_PAGER) && !IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 230 for (n = 0; n < CFG_NUM_THREADS; n++) { 231 va = threads[n].stack_va_end; 232 if (va) 233 check_stack_canary("thread_stack", n, 234 STACK_THREAD_SIZE, va); 235 } 236 } 237 } 238 239 void thread_lock_global(void) 240 { 241 cpu_spin_lock(&thread_global_lock); 242 } 243 244 void thread_unlock_global(void) 245 { 246 cpu_spin_unlock(&thread_global_lock); 247 } 248 249 static struct thread_core_local * __nostackcheck 250 get_core_local(unsigned int pos) 251 { 252 /* 253 * Foreign interrupts must be disabled before playing with core_local 254 * since we otherwise may be rescheduled to a different core in the 255 * middle of this function. 256 */ 257 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 258 259 /* 260 * With CFG_BOOT_INIT_CURRENT_THREAD_CORE_LOCAL, we boot on a 261 * single core and have allocated only one struct thread_core_local 262 * so we return that regardless of pos. 263 */ 264 if (IS_ENABLED(CFG_DYN_STACK_CONFIG) && 265 thread_core_local != __thread_core_local_new) 266 return thread_core_local; 267 268 assert(pos < thread_core_count); 269 return &thread_core_local[pos]; 270 } 271 272 struct thread_core_local * __nostackcheck thread_get_core_local(void) 273 { 274 unsigned int pos = get_core_pos(); 275 276 return get_core_local(pos); 277 } 278 279 #ifdef CFG_CORE_DEBUG_CHECK_STACKS 280 static void print_stack_limits(void) 281 { 282 size_t n = 0; 283 vaddr_t __maybe_unused start = 0; 284 vaddr_t __maybe_unused end = 0; 285 vaddr_t va = 0; 286 287 for (n = 0; n < thread_core_count; n++) { 288 va = thread_core_local[n].tmp_stack_va_end + STACK_TMP_OFFS; 289 start = stack_end_va_to_top_soft(STACK_TMP_SIZE, va); 290 end = stack_end_va_to_bottom(STACK_TMP_SIZE, va); 291 DMSG("tmp [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 292 293 va = thread_core_local[n].abt_stack_va_end; 294 start = stack_end_va_to_top_soft(STACK_ABT_SIZE, va); 295 end = stack_end_va_to_bottom(STACK_ABT_SIZE, va); 296 DMSG("abt [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 297 } 298 299 for (n = 0; n < CFG_NUM_THREADS; n++) { 300 va = threads[n].stack_va_end; 301 start = stack_end_va_to_top_soft(STACK_THREAD_SIZE, va); 302 end = stack_end_va_to_bottom(STACK_THREAD_SIZE, va); 303 DMSG("thr [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 304 } 305 } 306 307 static void check_stack_limits(void) 308 { 309 vaddr_t stack_start = 0; 310 vaddr_t stack_end = 0; 311 /* Any value in the current stack frame will do */ 312 vaddr_t current_sp = (vaddr_t)&stack_start; 313 314 if (!get_stack_soft_limits(&stack_start, &stack_end)) 315 panic("Unknown stack limits"); 316 if (current_sp < stack_start || current_sp > stack_end) { 317 EMSG("Stack pointer out of range: 0x%" PRIxVA " not in [0x%" 318 PRIxVA " .. 0x%" PRIxVA "]", current_sp, stack_start, 319 stack_end); 320 print_stack_limits(); 321 panic(); 322 } 323 } 324 325 static bool * __nostackcheck get_stackcheck_recursion_flag(void) 326 { 327 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 328 unsigned int pos = get_core_pos(); 329 struct thread_core_local *l = get_core_local(pos); 330 int ct = l->curr_thread; 331 bool *p = NULL; 332 333 if (l->flags & (THREAD_CLF_ABORT | THREAD_CLF_TMP)) 334 p = &l->stackcheck_recursion; 335 else if (!l->flags) 336 p = &threads[ct].tsd.stackcheck_recursion; 337 338 thread_unmask_exceptions(exceptions); 339 return p; 340 } 341 342 void __cyg_profile_func_enter(void *this_fn, void *call_site); 343 void __nostackcheck __cyg_profile_func_enter(void *this_fn __unused, 344 void *call_site __unused) 345 { 346 bool *p = get_stackcheck_recursion_flag(); 347 348 assert(p); 349 if (*p) 350 return; 351 *p = true; 352 check_stack_limits(); 353 *p = false; 354 } 355 356 void __cyg_profile_func_exit(void *this_fn, void *call_site); 357 void __nostackcheck __cyg_profile_func_exit(void *this_fn __unused, 358 void *call_site __unused) 359 { 360 } 361 #else 362 static void print_stack_limits(void) 363 { 364 } 365 #endif 366 367 void thread_init_boot_thread(void) 368 { 369 struct thread_core_local *l = thread_get_core_local(); 370 371 l->curr_thread = 0; 372 threads[0].state = THREAD_STATE_ACTIVE; 373 } 374 375 void __nostackcheck thread_clr_boot_thread(void) 376 { 377 struct thread_core_local *l = thread_get_core_local(); 378 379 assert(l->curr_thread >= 0 && l->curr_thread < CFG_NUM_THREADS); 380 assert(threads[l->curr_thread].state == THREAD_STATE_ACTIVE); 381 threads[l->curr_thread].state = THREAD_STATE_FREE; 382 l->curr_thread = THREAD_ID_INVALID; 383 print_stack_limits(); 384 } 385 386 void __nostackcheck *thread_get_tmp_sp(void) 387 { 388 struct thread_core_local *l = thread_get_core_local(); 389 390 /* 391 * Called from assembly when switching to the temporary stack, so flags 392 * need updating 393 */ 394 l->flags |= THREAD_CLF_TMP; 395 396 return (void *)l->tmp_stack_va_end; 397 } 398 399 vaddr_t thread_stack_start(void) 400 { 401 struct thread_ctx *thr; 402 int ct = thread_get_id_may_fail(); 403 404 if (ct == THREAD_ID_INVALID) 405 return 0; 406 407 thr = threads + ct; 408 return stack_end_va_to_top_soft(STACK_THREAD_SIZE, thr->stack_va_end); 409 } 410 411 size_t thread_stack_size(void) 412 { 413 return STACK_THREAD_SIZE; 414 } 415 416 bool get_stack_limits(vaddr_t *start, vaddr_t *end, bool hard) 417 { 418 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 419 unsigned int pos = get_core_pos(); 420 struct thread_core_local *l = get_core_local(pos); 421 int ct = l->curr_thread; 422 size_t stack_size = 0; 423 bool ret = true; 424 vaddr_t va = 0; 425 426 if (l->flags & THREAD_CLF_TMP) { 427 va = l->tmp_stack_va_end + STACK_TMP_OFFS; 428 stack_size = STACK_TMP_SIZE; 429 } else if (l->flags & THREAD_CLF_ABORT) { 430 va = l->abt_stack_va_end; 431 stack_size = STACK_ABT_SIZE; 432 } else if (!l->flags && ct >= 0 && ct < CFG_NUM_THREADS) { 433 va = threads[ct].stack_va_end; 434 stack_size = STACK_THREAD_SIZE; 435 } else { 436 ret = false; 437 goto out; 438 } 439 440 *end = stack_end_va_to_bottom(stack_size, va); 441 if (hard) 442 *start = stack_end_va_to_top_hard(stack_size, va); 443 else 444 *start = stack_end_va_to_top_soft(stack_size, va); 445 out: 446 thread_unmask_exceptions(exceptions); 447 return ret; 448 } 449 450 bool thread_is_from_abort_mode(void) 451 { 452 struct thread_core_local *l = thread_get_core_local(); 453 454 return (l->flags >> THREAD_CLF_SAVED_SHIFT) & THREAD_CLF_ABORT; 455 } 456 457 /* 458 * This function should always be accurate, but it might be possible to 459 * implement a more efficient depending on cpu architecture. 460 */ 461 bool __weak __noprof thread_is_in_normal_mode(void) 462 { 463 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 464 struct thread_core_local *l = thread_get_core_local(); 465 bool ret; 466 467 /* 468 * If any bit in l->flags is set aside from THREAD_CLF_TMP we're 469 * handling some exception. 470 */ 471 ret = (l->curr_thread != THREAD_ID_INVALID) && 472 !(l->flags & ~THREAD_CLF_TMP); 473 thread_unmask_exceptions(exceptions); 474 475 return ret; 476 } 477 478 short int __noprof thread_get_id_may_fail(void) 479 { 480 /* 481 * thread_get_core_local() requires foreign interrupts to be disabled 482 */ 483 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 484 struct thread_core_local *l = thread_get_core_local(); 485 short int ct = l->curr_thread; 486 487 thread_unmask_exceptions(exceptions); 488 return ct; 489 } 490 491 short int __noprof thread_get_id(void) 492 { 493 short int ct = thread_get_id_may_fail(); 494 495 /* Thread ID has to fit in a short int */ 496 COMPILE_TIME_ASSERT(CFG_NUM_THREADS <= SHRT_MAX); 497 assert(ct >= 0 && ct < CFG_NUM_THREADS); 498 return ct; 499 } 500 501 #ifdef CFG_WITH_PAGER 502 static void init_thread_stacks(void) 503 { 504 size_t n = 0; 505 506 /* 507 * Allocate virtual memory for thread stacks. 508 */ 509 for (n = 0; n < CFG_NUM_THREADS; n++) { 510 tee_mm_entry_t *mm = NULL; 511 vaddr_t sp = 0; 512 size_t num_pages = 0; 513 struct fobj *fobj = NULL; 514 515 /* Find vmem for thread stack and its protection gap */ 516 mm = tee_mm_alloc(&core_virt_mem_pool, 517 SMALL_PAGE_SIZE + STACK_THREAD_SIZE); 518 assert(mm); 519 520 /* Claim eventual physical page */ 521 tee_pager_add_pages(tee_mm_get_smem(mm), tee_mm_get_size(mm), 522 true); 523 524 num_pages = tee_mm_get_bytes(mm) / SMALL_PAGE_SIZE - 1; 525 fobj = fobj_locked_paged_alloc(num_pages); 526 527 /* Add the region to the pager */ 528 tee_pager_add_core_region(tee_mm_get_smem(mm) + SMALL_PAGE_SIZE, 529 PAGED_REGION_TYPE_LOCK, fobj); 530 fobj_put(fobj); 531 532 /* init effective stack */ 533 sp = tee_mm_get_smem(mm) + tee_mm_get_bytes(mm); 534 asan_tag_access((void *)tee_mm_get_smem(mm), (void *)sp); 535 threads[n].stack_va_end = sp; 536 } 537 } 538 #else 539 static void init_thread_stacks(void) 540 { 541 vaddr_t va = 0; 542 size_t n = 0; 543 544 /* Assign the thread stacks */ 545 for (n = 0; n < CFG_NUM_THREADS; n++) { 546 va = GET_STACK_THREAD_BOTTOM(n); 547 threads[n].stack_va_end = va; 548 if (IS_ENABLED(CFG_WITH_STACK_CANARIES)) 549 init_canaries(STACK_THREAD_SIZE, va); 550 } 551 } 552 #endif /*CFG_WITH_PAGER*/ 553 554 void thread_init_threads(size_t count __maybe_unused) 555 { 556 size_t n = 0; 557 558 assert(count == CFG_NUM_THREADS); 559 init_thread_stacks(); 560 print_stack_limits(); 561 pgt_init(); 562 563 mutex_lockdep_init(); 564 565 for (n = 0; n < CFG_NUM_THREADS; n++) 566 TAILQ_INIT(&threads[n].tsd.sess_stack); 567 } 568 569 #ifndef CFG_DYN_STACK_CONFIG 570 vaddr_t __nostackcheck thread_get_abt_stack(void) 571 { 572 return GET_STACK_BOTTOM(stack_abt, get_core_pos()); 573 } 574 #endif 575 576 #ifdef CFG_BOOT_INIT_CURRENT_THREAD_CORE_LOCAL 577 static vaddr_t alloc_stack(size_t stack_size, bool nex) 578 { 579 size_t l = stack_size_to_alloc_size(stack_size); 580 size_t rl = ROUNDUP(l, SMALL_PAGE_SIZE); 581 uint32_t flags = MAF_GUARD_HEAD; 582 vaddr_t end_va = 0; 583 vaddr_t va = 0; 584 585 if (nex) 586 flags |= MAF_NEX; 587 va = virt_page_alloc(rl / SMALL_PAGE_SIZE, flags); 588 if (!va) 589 panic(); 590 591 end_va = va + l - STACK_CANARY_SIZE / 2; 592 if (IS_ENABLED(CFG_WITH_STACK_CANARIES)) 593 init_canaries(stack_size, end_va); 594 595 return end_va; 596 } 597 598 void thread_init_thread_core_local(size_t core_count) 599 { 600 struct thread_core_local *tcl = NULL; 601 const size_t core_pos = get_core_pos(); 602 vaddr_t va = 0; 603 size_t n = 0; 604 605 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) { 606 assert(core_count <= CFG_TEE_CORE_NB_CORE); 607 tcl = nex_calloc(core_count, sizeof(*tcl)); 608 if (!tcl) 609 panic(); 610 __thread_core_local_new = tcl; 611 __thread_core_count_new = core_count; 612 } else { 613 tcl = thread_core_local; 614 assert(core_count == CFG_TEE_CORE_NB_CORE); 615 616 for (n = 0; n < thread_core_count; n++) { 617 init_canaries(STACK_TMP_SIZE, 618 GET_STACK_BOTTOM(stack_tmp, n)); 619 init_canaries(STACK_ABT_SIZE, 620 GET_STACK_BOTTOM(stack_abt, n)); 621 } 622 } 623 624 for (n = 0; n < core_count; n++) { 625 if (n == core_pos) { 626 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) 627 tcl[n] = thread_core_local[0]; 628 else 629 continue; 630 } else { 631 tcl[n].curr_thread = THREAD_ID_INVALID; 632 tcl[n].flags = THREAD_CLF_TMP; 633 } 634 635 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) 636 va = alloc_stack(STACK_TMP_SIZE, true); 637 else 638 va = GET_STACK_BOTTOM(stack_tmp, n); 639 tcl[n].tmp_stack_va_end = va - STACK_TMP_OFFS; 640 #ifdef ARM32 641 tcl[n].tmp_stack_pa_end = 642 vaddr_to_phys(tcl[n].tmp_stack_va_end); 643 #endif 644 645 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) 646 va = alloc_stack(STACK_ABT_SIZE, true); 647 else 648 va = GET_STACK_BOTTOM(stack_abt, n); 649 tcl[n].abt_stack_va_end = va; 650 } 651 } 652 #else 653 void __nostackcheck 654 thread_init_thread_core_local(size_t core_count __maybe_unused) 655 { 656 size_t n = 0; 657 struct thread_core_local *tcl = thread_core_local; 658 659 assert(core_count == CFG_TEE_CORE_NB_CORE); 660 for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { 661 tcl[n].curr_thread = THREAD_ID_INVALID; 662 tcl[n].flags = THREAD_CLF_TMP; 663 } 664 tcl[0].tmp_stack_va_end = GET_STACK_BOTTOM(stack_tmp, 0); 665 } 666 667 void __nostackcheck thread_init_core_local_stacks(void) 668 { 669 size_t n = 0; 670 struct thread_core_local *tcl = thread_core_local; 671 672 for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) { 673 tcl[n].tmp_stack_va_end = GET_STACK_BOTTOM(stack_tmp, n) - 674 STACK_TMP_OFFS; 675 tcl[n].abt_stack_va_end = GET_STACK_BOTTOM(stack_abt, n); 676 } 677 } 678 #endif /*CFG_BOOT_INIT_CURRENT_THREAD_CORE_LOCAL*/ 679 680 #if defined(CFG_CORE_PAUTH) 681 void thread_init_thread_pauth_keys(void) 682 { 683 size_t n = 0; 684 685 for (n = 0; n < CFG_NUM_THREADS; n++) 686 if (crypto_rng_read(&threads[n].keys, sizeof(threads[n].keys))) 687 panic("Failed to init thread pauth keys"); 688 } 689 690 void thread_init_core_local_pauth_keys(void) 691 { 692 struct thread_core_local *tcl = thread_core_local; 693 size_t n = 0; 694 695 for (n = 0; n < thread_core_count; n++) 696 if (crypto_rng_read(&tcl[n].keys, sizeof(tcl[n].keys))) 697 panic("Failed to init core local pauth keys"); 698 } 699 #endif 700 701 struct thread_specific_data * __noprof thread_get_tsd(void) 702 { 703 return &threads[thread_get_id()].tsd; 704 } 705 706 struct thread_ctx_regs * __nostackcheck thread_get_ctx_regs(void) 707 { 708 struct thread_core_local *l = thread_get_core_local(); 709 710 assert(l->curr_thread != THREAD_ID_INVALID); 711 return &threads[l->curr_thread].regs; 712 } 713 714 void thread_set_foreign_intr(bool enable) 715 { 716 /* thread_get_core_local() requires foreign interrupts to be disabled */ 717 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 718 struct thread_core_local *l; 719 720 l = thread_get_core_local(); 721 722 assert(l->curr_thread != THREAD_ID_INVALID); 723 724 if (enable) { 725 threads[l->curr_thread].flags |= 726 THREAD_FLAGS_FOREIGN_INTR_ENABLE; 727 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 728 } else { 729 /* 730 * No need to disable foreign interrupts here since they're 731 * already disabled above. 732 */ 733 threads[l->curr_thread].flags &= 734 ~THREAD_FLAGS_FOREIGN_INTR_ENABLE; 735 } 736 } 737 738 void thread_restore_foreign_intr(void) 739 { 740 /* thread_get_core_local() requires foreign interrupts to be disabled */ 741 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 742 struct thread_core_local *l; 743 744 l = thread_get_core_local(); 745 746 assert(l->curr_thread != THREAD_ID_INVALID); 747 748 if (threads[l->curr_thread].flags & THREAD_FLAGS_FOREIGN_INTR_ENABLE) 749 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 750 } 751 752 static struct mobj *alloc_shm(enum thread_shm_type shm_type, size_t size) 753 { 754 switch (shm_type) { 755 case THREAD_SHM_TYPE_APPLICATION: 756 return thread_rpc_alloc_payload(size); 757 case THREAD_SHM_TYPE_KERNEL_PRIVATE: 758 return thread_rpc_alloc_kernel_payload(size); 759 case THREAD_SHM_TYPE_GLOBAL: 760 return thread_rpc_alloc_global_payload(size); 761 default: 762 return NULL; 763 } 764 } 765 766 static void clear_shm_cache_entry(struct thread_shm_cache_entry *ce) 767 { 768 if (ce->mobj) { 769 switch (ce->type) { 770 case THREAD_SHM_TYPE_APPLICATION: 771 thread_rpc_free_payload(ce->mobj); 772 break; 773 case THREAD_SHM_TYPE_KERNEL_PRIVATE: 774 thread_rpc_free_kernel_payload(ce->mobj); 775 break; 776 case THREAD_SHM_TYPE_GLOBAL: 777 thread_rpc_free_global_payload(ce->mobj); 778 break; 779 default: 780 assert(0); /* "can't happen" */ 781 break; 782 } 783 } 784 ce->mobj = NULL; 785 ce->size = 0; 786 } 787 788 static struct thread_shm_cache_entry * 789 get_shm_cache_entry(enum thread_shm_cache_user user) 790 { 791 struct thread_shm_cache *cache = &threads[thread_get_id()].shm_cache; 792 struct thread_shm_cache_entry *ce = NULL; 793 794 SLIST_FOREACH(ce, cache, link) 795 if (ce->user == user) 796 return ce; 797 798 ce = calloc(1, sizeof(*ce)); 799 if (ce) { 800 ce->user = user; 801 SLIST_INSERT_HEAD(cache, ce, link); 802 } 803 804 return ce; 805 } 806 807 void *thread_rpc_shm_cache_alloc(enum thread_shm_cache_user user, 808 enum thread_shm_type shm_type, 809 size_t size, struct mobj **mobj) 810 { 811 struct thread_shm_cache_entry *ce = NULL; 812 size_t sz = size; 813 paddr_t p = 0; 814 void *va = NULL; 815 816 if (!size) 817 return NULL; 818 819 ce = get_shm_cache_entry(user); 820 if (!ce) 821 return NULL; 822 823 /* 824 * Always allocate in page chunks as normal world allocates payload 825 * memory as complete pages. 826 */ 827 sz = ROUNDUP(size, SMALL_PAGE_SIZE); 828 829 if (ce->type != shm_type || sz > ce->size) { 830 clear_shm_cache_entry(ce); 831 832 ce->mobj = alloc_shm(shm_type, sz); 833 if (!ce->mobj) 834 return NULL; 835 836 if (mobj_get_pa(ce->mobj, 0, 0, &p)) 837 goto err; 838 839 if (!IS_ALIGNED_WITH_TYPE(p, uint64_t)) 840 goto err; 841 842 va = mobj_get_va(ce->mobj, 0, sz); 843 if (!va) 844 goto err; 845 846 ce->size = sz; 847 ce->type = shm_type; 848 } else { 849 va = mobj_get_va(ce->mobj, 0, sz); 850 if (!va) 851 goto err; 852 } 853 *mobj = ce->mobj; 854 855 return va; 856 err: 857 clear_shm_cache_entry(ce); 858 return NULL; 859 } 860 861 void thread_rpc_shm_cache_clear(struct thread_shm_cache *cache) 862 { 863 while (true) { 864 struct thread_shm_cache_entry *ce = SLIST_FIRST(cache); 865 866 if (!ce) 867 break; 868 SLIST_REMOVE_HEAD(cache, link); 869 clear_shm_cache_entry(ce); 870 free(ce); 871 } 872 } 873