1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016-2022, Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * Copyright (c) 2020-2021, Arm Limited 6 */ 7 8 #include <config.h> 9 #include <crypto/crypto.h> 10 #include <kernel/asan.h> 11 #include <kernel/boot.h> 12 #include <kernel/lockdep.h> 13 #include <kernel/misc.h> 14 #include <kernel/panic.h> 15 #include <kernel/spinlock.h> 16 #include <kernel/thread.h> 17 #include <kernel/thread_private.h> 18 #include <mm/mobj.h> 19 #include <mm/page_alloc.h> 20 #include <stdalign.h> 21 22 #if defined(CFG_DYN_STACK_CONFIG) 23 struct thread_core_local *thread_core_local __nex_bss; 24 size_t thread_core_count __nex_bss; 25 struct thread_ctx *threads; 26 size_t thread_count; 27 #else 28 static struct thread_core_local 29 __thread_core_local[CFG_TEE_CORE_NB_CORE] __nex_bss; 30 struct thread_core_local *thread_core_local __nex_data = __thread_core_local; 31 size_t thread_core_count __nex_data = CFG_TEE_CORE_NB_CORE; 32 static struct thread_ctx __threads[CFG_NUM_THREADS]; 33 struct thread_ctx *threads = __threads; 34 size_t thread_count = CFG_NUM_THREADS; 35 #endif 36 unsigned long thread_core_local_pa __nex_bss; 37 struct thread_core_local *__thread_core_local_new __nex_bss; 38 size_t __thread_core_count_new __nex_bss; 39 40 /* 41 * Stacks 42 * 43 * [Lower addresses on the left] 44 * 45 * [ STACK_CANARY_SIZE/2 | STACK_CHECK_EXTRA | STACK_XXX_SIZE | STACK_CANARY_SIZE/2 ] 46 * ^ ^ ^ ^ 47 * stack_xxx[n] "hard" top "soft" top bottom 48 */ 49 50 static uint32_t start_canary_value = 0xdedede00; 51 static uint32_t end_canary_value = 0xababab00; 52 53 #define DECLARE_STACK(name, num_stacks, stack_size, linkage) \ 54 linkage uint32_t name[num_stacks] \ 55 [ROUNDUP(stack_size + STACK_CANARY_SIZE + STACK_CHECK_EXTRA, \ 56 STACK_ALIGNMENT) / sizeof(uint32_t)] \ 57 __attribute__((section(".nozi_stack." # name), \ 58 aligned(STACK_ALIGNMENT))) 59 60 #ifndef CFG_DYN_STACK_CONFIG 61 DECLARE_STACK(stack_tmp, CFG_TEE_CORE_NB_CORE, STACK_TMP_SIZE, 62 /* global linkage */); 63 DECLARE_STACK(stack_abt, CFG_TEE_CORE_NB_CORE, STACK_ABT_SIZE, static); 64 #define GET_STACK_BOTTOM(stack, n) ((vaddr_t)&(stack)[n] + sizeof(stack[n]) - \ 65 STACK_CANARY_SIZE / 2) 66 #else 67 /* Not used */ 68 #define GET_STACK_BOTTOM(stack, n) 0 69 #endif 70 71 #if defined(CFG_DYN_STACK_CONFIG) || defined(CFG_WITH_PAGER) 72 /* Not used */ 73 #define GET_STACK_THREAD_BOTTOM(n) 0 74 #else 75 DECLARE_STACK(stack_thread, CFG_NUM_THREADS, STACK_THREAD_SIZE, static); 76 #define GET_STACK_THREAD_BOTTOM(n) \ 77 ((vaddr_t)&stack_thread[n] + sizeof(stack_thread[n]) - \ 78 STACK_CANARY_SIZE / 2) 79 #endif 80 81 #ifndef CFG_DYN_STACK_CONFIG 82 const uint32_t stack_tmp_stride __section(".identity_map.stack_tmp_stride") = 83 sizeof(stack_tmp[0]); 84 85 /* 86 * This stack setup info is required by secondary boot cores before they 87 * each locally enable the pager (the mmu). Hence kept in pager sections. 88 */ 89 DECLARE_KEEP_PAGER(stack_tmp_stride); 90 #endif 91 92 static unsigned int thread_global_lock __nex_bss = SPINLOCK_UNLOCK; 93 94 static size_t stack_size_to_alloc_size(size_t stack_size) 95 { 96 return ROUNDUP(stack_size + STACK_CANARY_SIZE + STACK_CHECK_EXTRA, 97 STACK_ALIGNMENT); 98 } 99 100 static vaddr_t stack_end_va_to_top_hard(size_t stack_size, vaddr_t end_va) 101 { 102 size_t l = stack_size_to_alloc_size(stack_size); 103 104 return end_va - l + STACK_CANARY_SIZE; 105 } 106 107 static vaddr_t stack_end_va_to_top_soft(size_t stack_size, vaddr_t end_va) 108 { 109 return stack_end_va_to_top_hard(stack_size, end_va) + STACK_CHECK_EXTRA; 110 } 111 112 static vaddr_t stack_end_va_to_bottom(size_t stack_size __unused, 113 vaddr_t end_va) 114 { 115 return end_va; 116 } 117 118 static uint32_t *stack_end_va_to_start_canary(size_t stack_size, vaddr_t end_va) 119 { 120 return (uint32_t *)(stack_end_va_to_top_hard(stack_size, end_va) - 121 STACK_CANARY_SIZE / 2); 122 } 123 124 static uint32_t *stack_end_va_to_end_canary(size_t stack_size __unused, 125 vaddr_t end_va) 126 { 127 return (uint32_t *)(end_va + STACK_CANARY_SIZE / 2 - sizeof(uint32_t)); 128 } 129 130 static void init_canaries(size_t stack_size, vaddr_t va_end) 131 { 132 uint32_t *canary = NULL; 133 134 assert(va_end); 135 canary = stack_end_va_to_start_canary(stack_size, va_end); 136 *canary = start_canary_value; 137 canary = stack_end_va_to_end_canary(stack_size, va_end); 138 *canary = end_canary_value; 139 } 140 141 void thread_init_canaries(void) 142 { 143 vaddr_t va = 0; 144 size_t n = 0; 145 146 if (IS_ENABLED(CFG_WITH_STACK_CANARIES)) { 147 for (n = 0; n < thread_core_count; n++) { 148 if (thread_core_local[n].tmp_stack_va_end) { 149 va = thread_core_local[n].tmp_stack_va_end + 150 STACK_TMP_OFFS; 151 init_canaries(STACK_TMP_SIZE, va); 152 } 153 va = thread_core_local[n].abt_stack_va_end; 154 if (va) 155 init_canaries(STACK_ABT_SIZE, va); 156 } 157 158 } 159 160 if (IS_ENABLED(CFG_WITH_STACK_CANARIES) && 161 !IS_ENABLED(CFG_WITH_PAGER) && 162 !IS_ENABLED(CFG_NS_VIRTUALIZATION) && threads) { 163 for (n = 0; n < thread_count; n++) { 164 va = threads[n].stack_va_end; 165 if (va) 166 init_canaries(STACK_THREAD_SIZE, va); 167 } 168 } 169 } 170 171 #if defined(CFG_WITH_STACK_CANARIES) 172 void thread_update_canaries(void) 173 { 174 uint32_t canary[2] = { }; 175 uint32_t exceptions = 0; 176 177 plat_get_random_stack_canaries(canary, ARRAY_SIZE(canary), 178 sizeof(canary[0])); 179 180 exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); 181 182 thread_check_canaries(); 183 184 start_canary_value = canary[0]; 185 end_canary_value = canary[1]; 186 thread_init_canaries(); 187 188 thread_unmask_exceptions(exceptions); 189 } 190 #endif 191 192 static void check_stack_canary(const char *stack_name __maybe_unused, 193 size_t n __maybe_unused, 194 size_t stack_size, vaddr_t end_va) 195 { 196 uint32_t *canary = NULL; 197 198 canary = stack_end_va_to_start_canary(stack_size, end_va); 199 if (*canary != start_canary_value) { 200 EMSG_RAW("Dead canary at start of '%s[%zu]' (%p)", 201 stack_name, n, (void *)canary); 202 panic(); 203 } 204 205 canary = stack_end_va_to_end_canary(stack_size, end_va); 206 if (*canary != end_canary_value) { 207 EMSG_RAW("Dead canary at end of '%s[%zu]' (%p)", 208 stack_name, n, (void *)canary); 209 panic(); 210 } 211 } 212 213 void thread_check_canaries(void) 214 { 215 vaddr_t va = 0; 216 size_t n = 0; 217 218 if (IS_ENABLED(CFG_WITH_STACK_CANARIES)) { 219 for (n = 0; n < thread_core_count; n++) { 220 if (thread_core_local[n].tmp_stack_va_end) { 221 va = thread_core_local[n].tmp_stack_va_end + 222 STACK_TMP_OFFS; 223 check_stack_canary("tmp_stack", n, 224 STACK_TMP_SIZE, va); 225 } 226 227 va = thread_core_local[n].abt_stack_va_end; 228 if (va) 229 check_stack_canary("abt_stack", n, 230 STACK_ABT_SIZE, va); 231 } 232 } 233 234 if (IS_ENABLED(CFG_WITH_STACK_CANARIES) && 235 !IS_ENABLED(CFG_WITH_PAGER) && !IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 236 for (n = 0; n < thread_count; n++) { 237 va = threads[n].stack_va_end; 238 if (va) 239 check_stack_canary("thread_stack", n, 240 STACK_THREAD_SIZE, va); 241 } 242 } 243 } 244 245 void thread_lock_global(void) 246 { 247 cpu_spin_lock(&thread_global_lock); 248 } 249 250 void thread_unlock_global(void) 251 { 252 cpu_spin_unlock(&thread_global_lock); 253 } 254 255 static struct thread_core_local * __nostackcheck 256 get_core_local(unsigned int pos) 257 { 258 /* 259 * Foreign interrupts must be disabled before playing with core_local 260 * since we otherwise may be rescheduled to a different core in the 261 * middle of this function. 262 */ 263 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 264 265 /* 266 * We boot on a single core and have allocated only one struct 267 * thread_core_local so we return that regardless of pos. 268 */ 269 if (IS_ENABLED(CFG_DYN_STACK_CONFIG) && 270 thread_core_local != __thread_core_local_new) 271 return thread_core_local; 272 273 assert(pos < thread_core_count); 274 return &thread_core_local[pos]; 275 } 276 277 struct thread_core_local * __nostackcheck thread_get_core_local(void) 278 { 279 unsigned int pos = get_core_pos(); 280 281 return get_core_local(pos); 282 } 283 284 #ifdef CFG_CORE_DEBUG_CHECK_STACKS 285 static void print_stack_limits(void) 286 { 287 size_t n = 0; 288 vaddr_t __maybe_unused start = 0; 289 vaddr_t __maybe_unused end = 0; 290 vaddr_t va = 0; 291 292 for (n = 0; n < thread_core_count; n++) { 293 va = thread_core_local[n].tmp_stack_va_end + STACK_TMP_OFFS; 294 start = stack_end_va_to_top_soft(STACK_TMP_SIZE, va); 295 end = stack_end_va_to_bottom(STACK_TMP_SIZE, va); 296 DMSG("tmp [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 297 298 va = thread_core_local[n].abt_stack_va_end; 299 start = stack_end_va_to_top_soft(STACK_ABT_SIZE, va); 300 end = stack_end_va_to_bottom(STACK_ABT_SIZE, va); 301 DMSG("abt [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 302 } 303 304 for (n = 0; n < thread_count; n++) { 305 va = threads[n].stack_va_end; 306 start = stack_end_va_to_top_soft(STACK_THREAD_SIZE, va); 307 end = stack_end_va_to_bottom(STACK_THREAD_SIZE, va); 308 DMSG("thr [%zu] 0x%" PRIxVA "..0x%" PRIxVA, n, start, end); 309 } 310 } 311 312 static void check_stack_limits(void) 313 { 314 vaddr_t stack_start = 0; 315 vaddr_t stack_end = 0; 316 /* Any value in the current stack frame will do */ 317 vaddr_t current_sp = (vaddr_t)&stack_start; 318 319 if (!get_stack_soft_limits(&stack_start, &stack_end)) 320 panic("Unknown stack limits"); 321 if (current_sp < stack_start || current_sp > stack_end) { 322 EMSG("Stack pointer out of range: 0x%" PRIxVA " not in [0x%" 323 PRIxVA " .. 0x%" PRIxVA "]", current_sp, stack_start, 324 stack_end); 325 print_stack_limits(); 326 panic(); 327 } 328 } 329 330 static bool * __nostackcheck get_stackcheck_recursion_flag(void) 331 { 332 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 333 unsigned int pos = get_core_pos(); 334 struct thread_core_local *l = get_core_local(pos); 335 int ct = l->curr_thread; 336 bool *p = NULL; 337 338 if (l->flags & (THREAD_CLF_ABORT | THREAD_CLF_TMP)) 339 p = &l->stackcheck_recursion; 340 else if (!l->flags) 341 p = &threads[ct].tsd.stackcheck_recursion; 342 343 thread_unmask_exceptions(exceptions); 344 return p; 345 } 346 347 void __cyg_profile_func_enter(void *this_fn, void *call_site); 348 void __nostackcheck __cyg_profile_func_enter(void *this_fn __unused, 349 void *call_site __unused) 350 { 351 bool *p = get_stackcheck_recursion_flag(); 352 353 assert(p); 354 if (*p) 355 return; 356 *p = true; 357 check_stack_limits(); 358 *p = false; 359 } 360 361 void __cyg_profile_func_exit(void *this_fn, void *call_site); 362 void __nostackcheck __cyg_profile_func_exit(void *this_fn __unused, 363 void *call_site __unused) 364 { 365 } 366 #else 367 static void print_stack_limits(void) 368 { 369 } 370 #endif 371 372 void thread_init_boot_thread(void) 373 { 374 struct thread_core_local *l = thread_get_core_local(); 375 376 l->curr_thread = 0; 377 threads[0].state = THREAD_STATE_ACTIVE; 378 } 379 380 void __nostackcheck thread_clr_boot_thread(void) 381 { 382 struct thread_core_local *l = thread_get_core_local(); 383 384 assert(l->curr_thread >= 0 && l->curr_thread < CFG_NUM_THREADS); 385 assert(threads[l->curr_thread].state == THREAD_STATE_ACTIVE); 386 threads[l->curr_thread].state = THREAD_STATE_FREE; 387 l->curr_thread = THREAD_ID_INVALID; 388 print_stack_limits(); 389 } 390 391 void __nostackcheck *thread_get_tmp_sp(void) 392 { 393 struct thread_core_local *l = thread_get_core_local(); 394 395 /* 396 * Called from assembly when switching to the temporary stack, so flags 397 * need updating 398 */ 399 l->flags |= THREAD_CLF_TMP; 400 401 return (void *)l->tmp_stack_va_end; 402 } 403 404 vaddr_t thread_stack_start(void) 405 { 406 struct thread_ctx *thr; 407 int ct = thread_get_id_may_fail(); 408 409 if (ct == THREAD_ID_INVALID) 410 return 0; 411 412 thr = threads + ct; 413 return stack_end_va_to_top_soft(STACK_THREAD_SIZE, thr->stack_va_end); 414 } 415 416 size_t thread_stack_size(void) 417 { 418 return STACK_THREAD_SIZE; 419 } 420 421 bool get_stack_limits(vaddr_t *start, vaddr_t *end, bool hard) 422 { 423 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 424 unsigned int pos = get_core_pos(); 425 struct thread_core_local *l = get_core_local(pos); 426 int ct = l->curr_thread; 427 size_t stack_size = 0; 428 bool ret = true; 429 vaddr_t va = 0; 430 431 if (l->flags & THREAD_CLF_TMP) { 432 va = l->tmp_stack_va_end + STACK_TMP_OFFS; 433 stack_size = STACK_TMP_SIZE; 434 } else if (l->flags & THREAD_CLF_ABORT) { 435 va = l->abt_stack_va_end; 436 stack_size = STACK_ABT_SIZE; 437 } else if (!l->flags && ct >= 0 && (size_t)ct < thread_count) { 438 va = threads[ct].stack_va_end; 439 stack_size = STACK_THREAD_SIZE; 440 } else { 441 ret = false; 442 goto out; 443 } 444 445 *end = stack_end_va_to_bottom(stack_size, va); 446 if (hard) 447 *start = stack_end_va_to_top_hard(stack_size, va); 448 else 449 *start = stack_end_va_to_top_soft(stack_size, va); 450 out: 451 thread_unmask_exceptions(exceptions); 452 return ret; 453 } 454 455 bool thread_is_from_abort_mode(void) 456 { 457 struct thread_core_local *l = thread_get_core_local(); 458 459 return (l->flags >> THREAD_CLF_SAVED_SHIFT) & THREAD_CLF_ABORT; 460 } 461 462 /* 463 * This function should always be accurate, but it might be possible to 464 * implement a more efficient depending on cpu architecture. 465 */ 466 bool __weak __noprof thread_is_in_normal_mode(void) 467 { 468 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 469 struct thread_core_local *l = thread_get_core_local(); 470 bool ret; 471 472 /* 473 * If any bit in l->flags is set aside from THREAD_CLF_TMP we're 474 * handling some exception. 475 */ 476 ret = (l->curr_thread != THREAD_ID_INVALID) && 477 !(l->flags & ~THREAD_CLF_TMP); 478 thread_unmask_exceptions(exceptions); 479 480 return ret; 481 } 482 483 short int __noprof thread_get_id_may_fail(void) 484 { 485 /* 486 * thread_get_core_local() requires foreign interrupts to be disabled 487 */ 488 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 489 struct thread_core_local *l = thread_get_core_local(); 490 short int ct = l->curr_thread; 491 492 thread_unmask_exceptions(exceptions); 493 return ct; 494 } 495 496 short int __noprof thread_get_id(void) 497 { 498 short int ct = thread_get_id_may_fail(); 499 500 /* Thread ID has to fit in a short int */ 501 COMPILE_TIME_ASSERT(CFG_NUM_THREADS <= SHRT_MAX); 502 assert(ct >= 0 && ct < CFG_NUM_THREADS); 503 return ct; 504 } 505 506 static vaddr_t alloc_stack(size_t stack_size, bool nex) 507 { 508 size_t l = stack_size_to_alloc_size(stack_size); 509 size_t rl = ROUNDUP(l, SMALL_PAGE_SIZE); 510 uint32_t flags = MAF_GUARD_HEAD; 511 vaddr_t end_va = 0; 512 vaddr_t va = 0; 513 514 if (nex) 515 flags |= MAF_NEX; 516 va = virt_page_alloc(rl / SMALL_PAGE_SIZE, flags); 517 if (!va) 518 panic(); 519 520 end_va = va + l - STACK_CANARY_SIZE / 2; 521 if (IS_ENABLED(CFG_WITH_STACK_CANARIES)) 522 init_canaries(stack_size, end_va); 523 524 return end_va; 525 } 526 527 #ifdef CFG_WITH_PAGER 528 static void init_thread_stacks(void) 529 { 530 size_t n = 0; 531 532 /* 533 * Allocate virtual memory for thread stacks. 534 */ 535 for (n = 0; n < thread_count; n++) { 536 tee_mm_entry_t *mm = NULL; 537 vaddr_t sp = 0; 538 size_t num_pages = 0; 539 struct fobj *fobj = NULL; 540 541 /* Find vmem for thread stack and its protection gap */ 542 mm = tee_mm_alloc(&core_virt_mem_pool, 543 SMALL_PAGE_SIZE + STACK_THREAD_SIZE); 544 assert(mm); 545 546 /* Claim eventual physical page */ 547 tee_pager_add_pages(tee_mm_get_smem(mm), tee_mm_get_size(mm), 548 true); 549 550 num_pages = tee_mm_get_bytes(mm) / SMALL_PAGE_SIZE - 1; 551 fobj = fobj_locked_paged_alloc(num_pages); 552 553 /* Add the region to the pager */ 554 tee_pager_add_core_region(tee_mm_get_smem(mm) + SMALL_PAGE_SIZE, 555 PAGED_REGION_TYPE_LOCK, fobj); 556 fobj_put(fobj); 557 558 /* init effective stack */ 559 sp = tee_mm_get_smem(mm) + tee_mm_get_bytes(mm); 560 asan_tag_access((void *)tee_mm_get_smem(mm), (void *)sp); 561 threads[n].stack_va_end = sp; 562 } 563 } 564 #else 565 static void init_thread_stacks(void) 566 { 567 vaddr_t va = 0; 568 size_t n = 0; 569 570 /* Assign the thread stacks */ 571 for (n = 0; n < thread_count; n++) { 572 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) 573 va = alloc_stack(STACK_THREAD_SIZE, false); 574 else 575 va = GET_STACK_THREAD_BOTTOM(n); 576 threads[n].stack_va_end = va; 577 if (IS_ENABLED(CFG_WITH_STACK_CANARIES)) 578 init_canaries(STACK_THREAD_SIZE, va); 579 } 580 } 581 #endif /*CFG_WITH_PAGER*/ 582 583 void thread_init_threads(size_t count) 584 { 585 size_t n = 0; 586 587 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) { 588 assert(count <= CFG_NUM_THREADS); 589 threads = calloc(count, sizeof(*threads)); 590 if (!threads) 591 panic(); 592 thread_count = count; 593 } else { 594 assert(count == CFG_NUM_THREADS); 595 } 596 597 init_thread_stacks(); 598 print_stack_limits(); 599 pgt_init(); 600 601 mutex_lockdep_init(); 602 603 for (n = 0; n < thread_count; n++) 604 TAILQ_INIT(&threads[n].tsd.sess_stack); 605 } 606 607 #ifndef CFG_DYN_STACK_CONFIG 608 vaddr_t __nostackcheck thread_get_abt_stack(void) 609 { 610 return GET_STACK_BOTTOM(stack_abt, get_core_pos()); 611 } 612 #endif 613 614 void thread_init_thread_core_local(size_t core_count) 615 { 616 struct thread_core_local *tcl = NULL; 617 const size_t core_pos = get_core_pos(); 618 vaddr_t va = 0; 619 size_t n = 0; 620 621 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) { 622 assert(core_count <= CFG_TEE_CORE_NB_CORE); 623 tcl = nex_calloc(core_count, sizeof(*tcl)); 624 if (!tcl) 625 panic(); 626 __thread_core_local_new = tcl; 627 __thread_core_count_new = core_count; 628 } else { 629 tcl = thread_core_local; 630 assert(core_count == CFG_TEE_CORE_NB_CORE); 631 632 for (n = 0; n < thread_core_count; n++) { 633 init_canaries(STACK_TMP_SIZE, 634 GET_STACK_BOTTOM(stack_tmp, n)); 635 init_canaries(STACK_ABT_SIZE, 636 GET_STACK_BOTTOM(stack_abt, n)); 637 } 638 } 639 640 for (n = 0; n < core_count; n++) { 641 if (n == core_pos) { 642 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) 643 tcl[n] = thread_core_local[0]; 644 else 645 continue; 646 } else { 647 tcl[n].curr_thread = THREAD_ID_INVALID; 648 tcl[n].flags = THREAD_CLF_TMP; 649 } 650 651 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) 652 va = alloc_stack(STACK_TMP_SIZE, true); 653 else 654 va = GET_STACK_BOTTOM(stack_tmp, n); 655 tcl[n].tmp_stack_va_end = va - STACK_TMP_OFFS; 656 #ifdef ARM32 657 tcl[n].tmp_stack_pa_end = 658 vaddr_to_phys(tcl[n].tmp_stack_va_end); 659 #endif 660 661 if (IS_ENABLED(CFG_DYN_STACK_CONFIG)) 662 va = alloc_stack(STACK_ABT_SIZE, true); 663 else 664 va = GET_STACK_BOTTOM(stack_abt, n); 665 tcl[n].abt_stack_va_end = va; 666 } 667 } 668 669 #if defined(CFG_CORE_PAUTH) 670 void thread_init_thread_pauth_keys(void) 671 { 672 size_t n = 0; 673 674 for (n = 0; n < thread_count; n++) 675 if (crypto_rng_read(&threads[n].keys, sizeof(threads[n].keys))) 676 panic("Failed to init thread pauth keys"); 677 } 678 679 void thread_init_core_local_pauth_keys(void) 680 { 681 struct thread_core_local *tcl = thread_core_local; 682 size_t n = 0; 683 684 for (n = 0; n < thread_core_count; n++) 685 if (crypto_rng_read(&tcl[n].keys, sizeof(tcl[n].keys))) 686 panic("Failed to init core local pauth keys"); 687 } 688 #endif 689 690 struct thread_specific_data * __noprof thread_get_tsd(void) 691 { 692 return &threads[thread_get_id()].tsd; 693 } 694 695 struct thread_ctx_regs * __nostackcheck thread_get_ctx_regs(void) 696 { 697 struct thread_core_local *l = thread_get_core_local(); 698 699 assert(l->curr_thread != THREAD_ID_INVALID); 700 return &threads[l->curr_thread].regs; 701 } 702 703 void thread_set_foreign_intr(bool enable) 704 { 705 /* thread_get_core_local() requires foreign interrupts to be disabled */ 706 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 707 struct thread_core_local *l; 708 709 l = thread_get_core_local(); 710 711 assert(l->curr_thread != THREAD_ID_INVALID); 712 713 if (enable) { 714 threads[l->curr_thread].flags |= 715 THREAD_FLAGS_FOREIGN_INTR_ENABLE; 716 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 717 } else { 718 /* 719 * No need to disable foreign interrupts here since they're 720 * already disabled above. 721 */ 722 threads[l->curr_thread].flags &= 723 ~THREAD_FLAGS_FOREIGN_INTR_ENABLE; 724 } 725 } 726 727 void thread_restore_foreign_intr(void) 728 { 729 /* thread_get_core_local() requires foreign interrupts to be disabled */ 730 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 731 struct thread_core_local *l; 732 733 l = thread_get_core_local(); 734 735 assert(l->curr_thread != THREAD_ID_INVALID); 736 737 if (threads[l->curr_thread].flags & THREAD_FLAGS_FOREIGN_INTR_ENABLE) 738 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 739 } 740 741 static struct mobj *alloc_shm(enum thread_shm_type shm_type, size_t size) 742 { 743 switch (shm_type) { 744 case THREAD_SHM_TYPE_APPLICATION: 745 return thread_rpc_alloc_payload(size); 746 case THREAD_SHM_TYPE_KERNEL_PRIVATE: 747 return thread_rpc_alloc_kernel_payload(size); 748 case THREAD_SHM_TYPE_GLOBAL: 749 return thread_rpc_alloc_global_payload(size); 750 default: 751 return NULL; 752 } 753 } 754 755 static void clear_shm_cache_entry(struct thread_shm_cache_entry *ce) 756 { 757 if (ce->mobj) { 758 switch (ce->type) { 759 case THREAD_SHM_TYPE_APPLICATION: 760 thread_rpc_free_payload(ce->mobj); 761 break; 762 case THREAD_SHM_TYPE_KERNEL_PRIVATE: 763 thread_rpc_free_kernel_payload(ce->mobj); 764 break; 765 case THREAD_SHM_TYPE_GLOBAL: 766 thread_rpc_free_global_payload(ce->mobj); 767 break; 768 default: 769 assert(0); /* "can't happen" */ 770 break; 771 } 772 } 773 ce->mobj = NULL; 774 ce->size = 0; 775 } 776 777 static struct thread_shm_cache_entry * 778 get_shm_cache_entry(enum thread_shm_cache_user user) 779 { 780 struct thread_shm_cache *cache = &threads[thread_get_id()].shm_cache; 781 struct thread_shm_cache_entry *ce = NULL; 782 783 SLIST_FOREACH(ce, cache, link) 784 if (ce->user == user) 785 return ce; 786 787 ce = calloc(1, sizeof(*ce)); 788 if (ce) { 789 ce->user = user; 790 SLIST_INSERT_HEAD(cache, ce, link); 791 } 792 793 return ce; 794 } 795 796 void *thread_rpc_shm_cache_alloc(enum thread_shm_cache_user user, 797 enum thread_shm_type shm_type, 798 size_t size, struct mobj **mobj) 799 { 800 struct thread_shm_cache_entry *ce = NULL; 801 size_t sz = size; 802 paddr_t p = 0; 803 void *va = NULL; 804 805 if (!size) 806 return NULL; 807 808 ce = get_shm_cache_entry(user); 809 if (!ce) 810 return NULL; 811 812 /* 813 * Always allocate in page chunks as normal world allocates payload 814 * memory as complete pages. 815 */ 816 sz = ROUNDUP(size, SMALL_PAGE_SIZE); 817 818 if (ce->type != shm_type || sz > ce->size) { 819 clear_shm_cache_entry(ce); 820 821 ce->mobj = alloc_shm(shm_type, sz); 822 if (!ce->mobj) 823 return NULL; 824 825 if (mobj_get_pa(ce->mobj, 0, 0, &p)) 826 goto err; 827 828 if (!IS_ALIGNED_WITH_TYPE(p, uint64_t)) 829 goto err; 830 831 va = mobj_get_va(ce->mobj, 0, sz); 832 if (!va) 833 goto err; 834 835 ce->size = sz; 836 ce->type = shm_type; 837 } else { 838 va = mobj_get_va(ce->mobj, 0, sz); 839 if (!va) 840 goto err; 841 } 842 *mobj = ce->mobj; 843 844 return va; 845 err: 846 clear_shm_cache_entry(ce); 847 return NULL; 848 } 849 850 void thread_rpc_shm_cache_clear(struct thread_shm_cache *cache) 851 { 852 while (true) { 853 struct thread_shm_cache_entry *ce = SLIST_FIRST(cache); 854 855 if (!ce) 856 break; 857 SLIST_REMOVE_HEAD(cache, link); 858 clear_shm_cache_entry(ce); 859 free(ce); 860 } 861 } 862