1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016, Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 21 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <platform_config.h> 31 32 #include <arm.h> 33 #include <assert.h> 34 #include <keep.h> 35 #include <kernel/asan.h> 36 #include <kernel/misc.h> 37 #include <kernel/msg_param.h> 38 #include <kernel/panic.h> 39 #include <kernel/spinlock.h> 40 #include <kernel/tee_ta_manager.h> 41 #include <kernel/thread_defs.h> 42 #include <kernel/thread.h> 43 #include <mm/core_memprot.h> 44 #include <mm/mobj.h> 45 #include <mm/tee_mm.h> 46 #include <mm/tee_mmu.h> 47 #include <mm/tee_pager.h> 48 #include <optee_msg.h> 49 #include <sm/optee_smc.h> 50 #include <sm/sm.h> 51 #include <tee/tee_cryp_utl.h> 52 #include <tee/tee_fs_rpc.h> 53 #include <trace.h> 54 #include <util.h> 55 56 #include "thread_private.h" 57 58 #ifdef CFG_WITH_ARM_TRUSTED_FW 59 #define STACK_TMP_OFFS 0 60 #else 61 #define STACK_TMP_OFFS SM_STACK_TMP_RESERVE_SIZE 62 #endif 63 64 65 #ifdef ARM32 66 #ifdef CFG_CORE_SANITIZE_KADDRESS 67 #define STACK_TMP_SIZE (3072 + STACK_TMP_OFFS) 68 #else 69 #define STACK_TMP_SIZE (1536 + STACK_TMP_OFFS) 70 #endif 71 #define STACK_THREAD_SIZE 8192 72 73 #ifdef CFG_CORE_SANITIZE_KADDRESS 74 #define STACK_ABT_SIZE 3072 75 #else 76 #define STACK_ABT_SIZE 2048 77 #endif 78 79 #endif /*ARM32*/ 80 81 #ifdef ARM64 82 #define STACK_TMP_SIZE (2048 + STACK_TMP_OFFS) 83 #define STACK_THREAD_SIZE 8192 84 85 #if TRACE_LEVEL > 0 86 #define STACK_ABT_SIZE 3072 87 #else 88 #define STACK_ABT_SIZE 1024 89 #endif 90 #endif /*ARM64*/ 91 92 struct thread_ctx threads[CFG_NUM_THREADS]; 93 94 struct thread_core_local thread_core_local[CFG_TEE_CORE_NB_CORE]; 95 96 #ifdef CFG_WITH_STACK_CANARIES 97 #ifdef ARM32 98 #define STACK_CANARY_SIZE (4 * sizeof(uint32_t)) 99 #endif 100 #ifdef ARM64 101 #define STACK_CANARY_SIZE (8 * sizeof(uint32_t)) 102 #endif 103 #define START_CANARY_VALUE 0xdededede 104 #define END_CANARY_VALUE 0xabababab 105 #define GET_START_CANARY(name, stack_num) name[stack_num][0] 106 #define GET_END_CANARY(name, stack_num) \ 107 name[stack_num][sizeof(name[stack_num]) / sizeof(uint32_t) - 1] 108 #else 109 #define STACK_CANARY_SIZE 0 110 #endif 111 112 #define DECLARE_STACK(name, num_stacks, stack_size, linkage) \ 113 linkage uint32_t name[num_stacks] \ 114 [ROUNDUP(stack_size + STACK_CANARY_SIZE, STACK_ALIGNMENT) / \ 115 sizeof(uint32_t)] \ 116 __attribute__((section(".nozi_stack"), \ 117 aligned(STACK_ALIGNMENT))) 118 119 #define STACK_SIZE(stack) (sizeof(stack) - STACK_CANARY_SIZE / 2) 120 121 #define GET_STACK(stack) \ 122 ((vaddr_t)(stack) + STACK_SIZE(stack)) 123 124 DECLARE_STACK(stack_tmp, CFG_TEE_CORE_NB_CORE, STACK_TMP_SIZE, static); 125 DECLARE_STACK(stack_abt, CFG_TEE_CORE_NB_CORE, STACK_ABT_SIZE, static); 126 #ifndef CFG_WITH_PAGER 127 DECLARE_STACK(stack_thread, CFG_NUM_THREADS, STACK_THREAD_SIZE, static); 128 #endif 129 130 const void *stack_tmp_export = (uint8_t *)stack_tmp + sizeof(stack_tmp[0]) - 131 (STACK_TMP_OFFS + STACK_CANARY_SIZE / 2); 132 const uint32_t stack_tmp_stride = sizeof(stack_tmp[0]); 133 134 /* 135 * These stack setup info are required by secondary boot cores before they 136 * each locally enable the pager (the mmu). Hence kept in pager sections. 137 */ 138 KEEP_PAGER(stack_tmp_export); 139 KEEP_PAGER(stack_tmp_stride); 140 141 thread_smc_handler_t thread_std_smc_handler_ptr; 142 static thread_smc_handler_t thread_fast_smc_handler_ptr; 143 thread_nintr_handler_t thread_nintr_handler_ptr; 144 thread_pm_handler_t thread_cpu_on_handler_ptr; 145 thread_pm_handler_t thread_cpu_off_handler_ptr; 146 thread_pm_handler_t thread_cpu_suspend_handler_ptr; 147 thread_pm_handler_t thread_cpu_resume_handler_ptr; 148 thread_pm_handler_t thread_system_off_handler_ptr; 149 thread_pm_handler_t thread_system_reset_handler_ptr; 150 151 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 152 static vaddr_t thread_user_kcode_va; 153 long thread_user_kcode_offset; 154 static size_t thread_user_kcode_size; 155 #endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ 156 157 static unsigned int thread_global_lock = SPINLOCK_UNLOCK; 158 static bool thread_prealloc_rpc_cache; 159 160 static void init_canaries(void) 161 { 162 #ifdef CFG_WITH_STACK_CANARIES 163 size_t n; 164 #define INIT_CANARY(name) \ 165 for (n = 0; n < ARRAY_SIZE(name); n++) { \ 166 uint32_t *start_canary = &GET_START_CANARY(name, n); \ 167 uint32_t *end_canary = &GET_END_CANARY(name, n); \ 168 \ 169 *start_canary = START_CANARY_VALUE; \ 170 *end_canary = END_CANARY_VALUE; \ 171 DMSG("#Stack canaries for %s[%zu] with top at %p\n", \ 172 #name, n, (void *)(end_canary - 1)); \ 173 DMSG("watch *%p\n", (void *)end_canary); \ 174 } 175 176 INIT_CANARY(stack_tmp); 177 INIT_CANARY(stack_abt); 178 #ifndef CFG_WITH_PAGER 179 INIT_CANARY(stack_thread); 180 #endif 181 #endif/*CFG_WITH_STACK_CANARIES*/ 182 } 183 184 #define CANARY_DIED(stack, loc, n) \ 185 do { \ 186 EMSG_RAW("Dead canary at %s of '%s[%zu]'", #loc, #stack, n); \ 187 panic(); \ 188 } while (0) 189 190 void thread_check_canaries(void) 191 { 192 #ifdef CFG_WITH_STACK_CANARIES 193 size_t n; 194 195 for (n = 0; n < ARRAY_SIZE(stack_tmp); n++) { 196 if (GET_START_CANARY(stack_tmp, n) != START_CANARY_VALUE) 197 CANARY_DIED(stack_tmp, start, n); 198 if (GET_END_CANARY(stack_tmp, n) != END_CANARY_VALUE) 199 CANARY_DIED(stack_tmp, end, n); 200 } 201 202 for (n = 0; n < ARRAY_SIZE(stack_abt); n++) { 203 if (GET_START_CANARY(stack_abt, n) != START_CANARY_VALUE) 204 CANARY_DIED(stack_abt, start, n); 205 if (GET_END_CANARY(stack_abt, n) != END_CANARY_VALUE) 206 CANARY_DIED(stack_abt, end, n); 207 208 } 209 #ifndef CFG_WITH_PAGER 210 for (n = 0; n < ARRAY_SIZE(stack_thread); n++) { 211 if (GET_START_CANARY(stack_thread, n) != START_CANARY_VALUE) 212 CANARY_DIED(stack_thread, start, n); 213 if (GET_END_CANARY(stack_thread, n) != END_CANARY_VALUE) 214 CANARY_DIED(stack_thread, end, n); 215 } 216 #endif 217 #endif/*CFG_WITH_STACK_CANARIES*/ 218 } 219 220 static void lock_global(void) 221 { 222 cpu_spin_lock(&thread_global_lock); 223 } 224 225 static void unlock_global(void) 226 { 227 cpu_spin_unlock(&thread_global_lock); 228 } 229 230 #ifdef ARM32 231 uint32_t thread_get_exceptions(void) 232 { 233 uint32_t cpsr = read_cpsr(); 234 235 return (cpsr >> CPSR_F_SHIFT) & THREAD_EXCP_ALL; 236 } 237 238 void thread_set_exceptions(uint32_t exceptions) 239 { 240 uint32_t cpsr = read_cpsr(); 241 242 /* Foreign interrupts must not be unmasked while holding a spinlock */ 243 if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) 244 assert_have_no_spinlock(); 245 246 cpsr &= ~(THREAD_EXCP_ALL << CPSR_F_SHIFT); 247 cpsr |= ((exceptions & THREAD_EXCP_ALL) << CPSR_F_SHIFT); 248 write_cpsr(cpsr); 249 } 250 #endif /*ARM32*/ 251 252 #ifdef ARM64 253 uint32_t thread_get_exceptions(void) 254 { 255 uint32_t daif = read_daif(); 256 257 return (daif >> DAIF_F_SHIFT) & THREAD_EXCP_ALL; 258 } 259 260 void thread_set_exceptions(uint32_t exceptions) 261 { 262 uint32_t daif = read_daif(); 263 264 /* Foreign interrupts must not be unmasked while holding a spinlock */ 265 if (!(exceptions & THREAD_EXCP_FOREIGN_INTR)) 266 assert_have_no_spinlock(); 267 268 daif &= ~(THREAD_EXCP_ALL << DAIF_F_SHIFT); 269 daif |= ((exceptions & THREAD_EXCP_ALL) << DAIF_F_SHIFT); 270 write_daif(daif); 271 } 272 #endif /*ARM64*/ 273 274 uint32_t thread_mask_exceptions(uint32_t exceptions) 275 { 276 uint32_t state = thread_get_exceptions(); 277 278 thread_set_exceptions(state | (exceptions & THREAD_EXCP_ALL)); 279 return state; 280 } 281 282 void thread_unmask_exceptions(uint32_t state) 283 { 284 thread_set_exceptions(state & THREAD_EXCP_ALL); 285 } 286 287 288 struct thread_core_local *thread_get_core_local(void) 289 { 290 uint32_t cpu_id = get_core_pos(); 291 292 /* 293 * Foreign interrupts must be disabled before playing with core_local 294 * since we otherwise may be rescheduled to a different core in the 295 * middle of this function. 296 */ 297 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 298 299 assert(cpu_id < CFG_TEE_CORE_NB_CORE); 300 return &thread_core_local[cpu_id]; 301 } 302 303 static void thread_lazy_save_ns_vfp(void) 304 { 305 #ifdef CFG_WITH_VFP 306 struct thread_ctx *thr = threads + thread_get_id(); 307 308 thr->vfp_state.ns_saved = false; 309 #if defined(ARM64) && defined(CFG_WITH_ARM_TRUSTED_FW) 310 /* 311 * ARM TF saves and restores CPACR_EL1, so we must assume NS world 312 * uses VFP and always preserve the register file when secure world 313 * is about to use it 314 */ 315 thr->vfp_state.ns.force_save = true; 316 #endif 317 vfp_lazy_save_state_init(&thr->vfp_state.ns); 318 #endif /*CFG_WITH_VFP*/ 319 } 320 321 static void thread_lazy_restore_ns_vfp(void) 322 { 323 #ifdef CFG_WITH_VFP 324 struct thread_ctx *thr = threads + thread_get_id(); 325 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 326 327 assert(!thr->vfp_state.sec_lazy_saved && !thr->vfp_state.sec_saved); 328 329 if (tuv && tuv->lazy_saved && !tuv->saved) { 330 vfp_lazy_save_state_final(&tuv->vfp); 331 tuv->saved = true; 332 } 333 334 vfp_lazy_restore_state(&thr->vfp_state.ns, thr->vfp_state.ns_saved); 335 thr->vfp_state.ns_saved = false; 336 #endif /*CFG_WITH_VFP*/ 337 } 338 339 #ifdef ARM32 340 static void init_regs(struct thread_ctx *thread, 341 struct thread_smc_args *args) 342 { 343 thread->regs.pc = (uint32_t)thread_std_smc_entry; 344 345 /* 346 * Stdcalls starts in SVC mode with masked foreign interrupts, masked 347 * Asynchronous abort and unmasked native interrupts. 348 */ 349 thread->regs.cpsr = read_cpsr() & ARM32_CPSR_E; 350 thread->regs.cpsr |= CPSR_MODE_SVC | CPSR_A | 351 (THREAD_EXCP_FOREIGN_INTR << ARM32_CPSR_F_SHIFT); 352 /* Enable thumb mode if it's a thumb instruction */ 353 if (thread->regs.pc & 1) 354 thread->regs.cpsr |= CPSR_T; 355 /* Reinitialize stack pointer */ 356 thread->regs.svc_sp = thread->stack_va_end; 357 358 /* 359 * Copy arguments into context. This will make the 360 * arguments appear in r0-r7 when thread is started. 361 */ 362 thread->regs.r0 = args->a0; 363 thread->regs.r1 = args->a1; 364 thread->regs.r2 = args->a2; 365 thread->regs.r3 = args->a3; 366 thread->regs.r4 = args->a4; 367 thread->regs.r5 = args->a5; 368 thread->regs.r6 = args->a6; 369 thread->regs.r7 = args->a7; 370 } 371 #endif /*ARM32*/ 372 373 #ifdef ARM64 374 static void init_regs(struct thread_ctx *thread, 375 struct thread_smc_args *args) 376 { 377 thread->regs.pc = (uint64_t)thread_std_smc_entry; 378 379 /* 380 * Stdcalls starts in SVC mode with masked foreign interrupts, masked 381 * Asynchronous abort and unmasked native interrupts. 382 */ 383 thread->regs.cpsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, 384 THREAD_EXCP_FOREIGN_INTR | DAIFBIT_ABT); 385 /* Reinitialize stack pointer */ 386 thread->regs.sp = thread->stack_va_end; 387 388 /* 389 * Copy arguments into context. This will make the 390 * arguments appear in x0-x7 when thread is started. 391 */ 392 thread->regs.x[0] = args->a0; 393 thread->regs.x[1] = args->a1; 394 thread->regs.x[2] = args->a2; 395 thread->regs.x[3] = args->a3; 396 thread->regs.x[4] = args->a4; 397 thread->regs.x[5] = args->a5; 398 thread->regs.x[6] = args->a6; 399 thread->regs.x[7] = args->a7; 400 401 /* Set up frame pointer as per the Aarch64 AAPCS */ 402 thread->regs.x[29] = 0; 403 } 404 #endif /*ARM64*/ 405 406 void thread_init_boot_thread(void) 407 { 408 struct thread_core_local *l = thread_get_core_local(); 409 size_t n; 410 411 for (n = 0; n < CFG_NUM_THREADS; n++) { 412 TAILQ_INIT(&threads[n].mutexes); 413 TAILQ_INIT(&threads[n].tsd.sess_stack); 414 SLIST_INIT(&threads[n].tsd.pgt_cache); 415 } 416 417 for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) 418 thread_core_local[n].curr_thread = -1; 419 420 l->curr_thread = 0; 421 threads[0].state = THREAD_STATE_ACTIVE; 422 } 423 424 void thread_clr_boot_thread(void) 425 { 426 struct thread_core_local *l = thread_get_core_local(); 427 428 assert(l->curr_thread >= 0 && l->curr_thread < CFG_NUM_THREADS); 429 assert(threads[l->curr_thread].state == THREAD_STATE_ACTIVE); 430 assert(TAILQ_EMPTY(&threads[l->curr_thread].mutexes)); 431 threads[l->curr_thread].state = THREAD_STATE_FREE; 432 l->curr_thread = -1; 433 } 434 435 static void thread_alloc_and_run(struct thread_smc_args *args) 436 { 437 size_t n; 438 struct thread_core_local *l = thread_get_core_local(); 439 bool found_thread = false; 440 441 assert(l->curr_thread == -1); 442 443 lock_global(); 444 445 for (n = 0; n < CFG_NUM_THREADS; n++) { 446 if (threads[n].state == THREAD_STATE_FREE) { 447 threads[n].state = THREAD_STATE_ACTIVE; 448 found_thread = true; 449 break; 450 } 451 } 452 453 unlock_global(); 454 455 if (!found_thread) { 456 args->a0 = OPTEE_SMC_RETURN_ETHREAD_LIMIT; 457 return; 458 } 459 460 l->curr_thread = n; 461 462 threads[n].flags = 0; 463 init_regs(threads + n, args); 464 465 /* Save Hypervisor Client ID */ 466 threads[n].hyp_clnt_id = args->a7; 467 468 thread_lazy_save_ns_vfp(); 469 thread_resume(&threads[n].regs); 470 } 471 472 #ifdef ARM32 473 static void copy_a0_to_a5(struct thread_ctx_regs *regs, 474 struct thread_smc_args *args) 475 { 476 /* 477 * Update returned values from RPC, values will appear in 478 * r0-r3 when thread is resumed. 479 */ 480 regs->r0 = args->a0; 481 regs->r1 = args->a1; 482 regs->r2 = args->a2; 483 regs->r3 = args->a3; 484 regs->r4 = args->a4; 485 regs->r5 = args->a5; 486 } 487 #endif /*ARM32*/ 488 489 #ifdef ARM64 490 static void copy_a0_to_a5(struct thread_ctx_regs *regs, 491 struct thread_smc_args *args) 492 { 493 /* 494 * Update returned values from RPC, values will appear in 495 * x0-x3 when thread is resumed. 496 */ 497 regs->x[0] = args->a0; 498 regs->x[1] = args->a1; 499 regs->x[2] = args->a2; 500 regs->x[3] = args->a3; 501 regs->x[4] = args->a4; 502 regs->x[5] = args->a5; 503 } 504 #endif /*ARM64*/ 505 506 #ifdef ARM32 507 static bool is_from_user(uint32_t cpsr) 508 { 509 return (cpsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; 510 } 511 #endif 512 513 #ifdef ARM64 514 static bool is_from_user(uint32_t cpsr) 515 { 516 if (cpsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) 517 return true; 518 if (((cpsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == 519 SPSR_64_MODE_EL0) 520 return true; 521 return false; 522 } 523 #endif 524 525 static bool is_user_mode(struct thread_ctx_regs *regs) 526 { 527 return is_from_user((uint32_t)regs->cpsr); 528 } 529 530 static void thread_resume_from_rpc(struct thread_smc_args *args) 531 { 532 size_t n = args->a3; /* thread id */ 533 struct thread_core_local *l = thread_get_core_local(); 534 uint32_t rv = 0; 535 536 assert(l->curr_thread == -1); 537 538 lock_global(); 539 540 if (n < CFG_NUM_THREADS && 541 threads[n].state == THREAD_STATE_SUSPENDED && 542 args->a7 == threads[n].hyp_clnt_id) 543 threads[n].state = THREAD_STATE_ACTIVE; 544 else 545 rv = OPTEE_SMC_RETURN_ERESUME; 546 547 unlock_global(); 548 549 if (rv) { 550 args->a0 = rv; 551 return; 552 } 553 554 l->curr_thread = n; 555 556 if (is_user_mode(&threads[n].regs)) 557 tee_ta_update_session_utime_resume(); 558 559 if (threads[n].have_user_map) 560 core_mmu_set_user_map(&threads[n].user_map); 561 562 /* 563 * Return from RPC to request service of a foreign interrupt must not 564 * get parameters from non-secure world. 565 */ 566 if (threads[n].flags & THREAD_FLAGS_COPY_ARGS_ON_RETURN) { 567 copy_a0_to_a5(&threads[n].regs, args); 568 threads[n].flags &= ~THREAD_FLAGS_COPY_ARGS_ON_RETURN; 569 } 570 571 thread_lazy_save_ns_vfp(); 572 thread_resume(&threads[n].regs); 573 } 574 575 void thread_handle_fast_smc(struct thread_smc_args *args) 576 { 577 thread_check_canaries(); 578 thread_fast_smc_handler_ptr(args); 579 /* Fast handlers must not unmask any exceptions */ 580 assert(thread_get_exceptions() == THREAD_EXCP_ALL); 581 } 582 583 void thread_handle_std_smc(struct thread_smc_args *args) 584 { 585 thread_check_canaries(); 586 587 if (args->a0 == OPTEE_SMC_CALL_RETURN_FROM_RPC) 588 thread_resume_from_rpc(args); 589 else 590 thread_alloc_and_run(args); 591 } 592 593 /* 594 * Helper routine for the assembly function thread_std_smc_entry() 595 * 596 * Note: this function is weak just to make it possible to exclude it from 597 * the unpaged area. 598 */ 599 void __weak __thread_std_smc_entry(struct thread_smc_args *args) 600 { 601 thread_std_smc_handler_ptr(args); 602 603 if (args->a0 == OPTEE_SMC_RETURN_OK) { 604 struct thread_ctx *thr = threads + thread_get_id(); 605 606 tee_fs_rpc_cache_clear(&thr->tsd); 607 if (!thread_prealloc_rpc_cache) { 608 thread_rpc_free_arg(thr->rpc_carg); 609 mobj_free(thr->rpc_mobj); 610 thr->rpc_carg = 0; 611 thr->rpc_arg = 0; 612 thr->rpc_mobj = NULL; 613 } 614 } 615 } 616 617 void *thread_get_tmp_sp(void) 618 { 619 struct thread_core_local *l = thread_get_core_local(); 620 621 return (void *)l->tmp_stack_va_end; 622 } 623 624 #ifdef ARM64 625 vaddr_t thread_get_saved_thread_sp(void) 626 { 627 struct thread_core_local *l = thread_get_core_local(); 628 int ct = l->curr_thread; 629 630 assert(ct != -1); 631 return threads[ct].kern_sp; 632 } 633 #endif /*ARM64*/ 634 635 vaddr_t thread_stack_start(void) 636 { 637 struct thread_ctx *thr; 638 int ct = thread_get_id_may_fail(); 639 640 if (ct == -1) 641 return 0; 642 643 thr = threads + ct; 644 return thr->stack_va_end - STACK_THREAD_SIZE; 645 } 646 647 size_t thread_stack_size(void) 648 { 649 return STACK_THREAD_SIZE; 650 } 651 652 bool thread_is_from_abort_mode(void) 653 { 654 struct thread_core_local *l = thread_get_core_local(); 655 656 return (l->flags >> THREAD_CLF_SAVED_SHIFT) & THREAD_CLF_ABORT; 657 } 658 659 #ifdef ARM32 660 bool thread_is_in_normal_mode(void) 661 { 662 return (read_cpsr() & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_SVC; 663 } 664 #endif 665 666 #ifdef ARM64 667 bool thread_is_in_normal_mode(void) 668 { 669 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 670 struct thread_core_local *l = thread_get_core_local(); 671 bool ret; 672 673 /* If any bit in l->flags is set we're handling some exception. */ 674 ret = !l->flags; 675 thread_unmask_exceptions(exceptions); 676 677 return ret; 678 } 679 #endif 680 681 void thread_state_free(void) 682 { 683 struct thread_core_local *l = thread_get_core_local(); 684 int ct = l->curr_thread; 685 686 assert(ct != -1); 687 assert(TAILQ_EMPTY(&threads[ct].mutexes)); 688 689 thread_lazy_restore_ns_vfp(); 690 tee_pager_release_phys( 691 (void *)(threads[ct].stack_va_end - STACK_THREAD_SIZE), 692 STACK_THREAD_SIZE); 693 694 lock_global(); 695 696 assert(threads[ct].state == THREAD_STATE_ACTIVE); 697 threads[ct].state = THREAD_STATE_FREE; 698 threads[ct].flags = 0; 699 l->curr_thread = -1; 700 701 unlock_global(); 702 } 703 704 #ifdef CFG_WITH_PAGER 705 static void release_unused_kernel_stack(struct thread_ctx *thr, 706 uint32_t cpsr __maybe_unused) 707 { 708 #ifdef ARM64 709 /* 710 * If we're from user mode then thr->regs.sp is the saved user 711 * stack pointer and thr->kern_sp holds the last kernel stack 712 * pointer. But if we're from kernel mode then thr->kern_sp isn't 713 * up to date so we need to read from thr->regs.sp instead. 714 */ 715 vaddr_t sp = is_from_user(cpsr) ? thr->kern_sp : thr->regs.sp; 716 #else 717 vaddr_t sp = thr->regs.svc_sp; 718 #endif 719 vaddr_t base = thr->stack_va_end - STACK_THREAD_SIZE; 720 size_t len = sp - base; 721 722 tee_pager_release_phys((void *)base, len); 723 } 724 #else 725 static void release_unused_kernel_stack(struct thread_ctx *thr __unused, 726 uint32_t cpsr __unused) 727 { 728 } 729 #endif 730 731 int thread_state_suspend(uint32_t flags, uint32_t cpsr, vaddr_t pc) 732 { 733 struct thread_core_local *l = thread_get_core_local(); 734 int ct = l->curr_thread; 735 736 assert(ct != -1); 737 738 thread_check_canaries(); 739 740 release_unused_kernel_stack(threads + ct, cpsr); 741 742 if (is_from_user(cpsr)) { 743 thread_user_save_vfp(); 744 tee_ta_update_session_utime_suspend(); 745 tee_ta_gprof_sample_pc(pc); 746 } 747 thread_lazy_restore_ns_vfp(); 748 749 lock_global(); 750 751 assert(threads[ct].state == THREAD_STATE_ACTIVE); 752 threads[ct].flags |= flags; 753 threads[ct].regs.cpsr = cpsr; 754 threads[ct].regs.pc = pc; 755 threads[ct].state = THREAD_STATE_SUSPENDED; 756 757 threads[ct].have_user_map = core_mmu_user_mapping_is_active(); 758 if (threads[ct].have_user_map) { 759 core_mmu_get_user_map(&threads[ct].user_map); 760 core_mmu_set_user_map(NULL); 761 } 762 763 l->curr_thread = -1; 764 765 unlock_global(); 766 767 return ct; 768 } 769 770 #ifdef ARM32 771 static void set_tmp_stack(struct thread_core_local *l, vaddr_t sp) 772 { 773 l->tmp_stack_va_end = sp; 774 thread_set_irq_sp(sp); 775 thread_set_fiq_sp(sp); 776 } 777 778 static void set_abt_stack(struct thread_core_local *l, vaddr_t sp) 779 { 780 l->abt_stack_va_end = sp; 781 thread_set_abt_sp((vaddr_t)l); 782 thread_set_und_sp((vaddr_t)l); 783 } 784 #endif /*ARM32*/ 785 786 #ifdef ARM64 787 static void set_tmp_stack(struct thread_core_local *l, vaddr_t sp) 788 { 789 /* 790 * We're already using the tmp stack when this function is called 791 * so there's no need to assign it to any stack pointer. However, 792 * we'll need to restore it at different times so store it here. 793 */ 794 l->tmp_stack_va_end = sp; 795 } 796 797 static void set_abt_stack(struct thread_core_local *l, vaddr_t sp) 798 { 799 l->abt_stack_va_end = sp; 800 } 801 #endif /*ARM64*/ 802 803 bool thread_init_stack(uint32_t thread_id, vaddr_t sp) 804 { 805 if (thread_id >= CFG_NUM_THREADS) 806 return false; 807 threads[thread_id].stack_va_end = sp; 808 return true; 809 } 810 811 int thread_get_id_may_fail(void) 812 { 813 /* 814 * thread_get_core_local() requires foreign interrupts to be disabled 815 */ 816 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 817 struct thread_core_local *l = thread_get_core_local(); 818 int ct = l->curr_thread; 819 820 thread_unmask_exceptions(exceptions); 821 return ct; 822 } 823 824 int thread_get_id(void) 825 { 826 int ct = thread_get_id_may_fail(); 827 828 assert(ct >= 0 && ct < CFG_NUM_THREADS); 829 return ct; 830 } 831 832 static void init_handlers(const struct thread_handlers *handlers) 833 { 834 thread_std_smc_handler_ptr = handlers->std_smc; 835 thread_fast_smc_handler_ptr = handlers->fast_smc; 836 thread_nintr_handler_ptr = handlers->nintr; 837 thread_cpu_on_handler_ptr = handlers->cpu_on; 838 thread_cpu_off_handler_ptr = handlers->cpu_off; 839 thread_cpu_suspend_handler_ptr = handlers->cpu_suspend; 840 thread_cpu_resume_handler_ptr = handlers->cpu_resume; 841 thread_system_off_handler_ptr = handlers->system_off; 842 thread_system_reset_handler_ptr = handlers->system_reset; 843 } 844 845 #ifdef CFG_WITH_PAGER 846 static void init_thread_stacks(void) 847 { 848 size_t n; 849 850 /* 851 * Allocate virtual memory for thread stacks. 852 */ 853 for (n = 0; n < CFG_NUM_THREADS; n++) { 854 tee_mm_entry_t *mm; 855 vaddr_t sp; 856 857 /* Find vmem for thread stack and its protection gap */ 858 mm = tee_mm_alloc(&tee_mm_vcore, 859 SMALL_PAGE_SIZE + STACK_THREAD_SIZE); 860 assert(mm); 861 862 /* Claim eventual physical page */ 863 tee_pager_add_pages(tee_mm_get_smem(mm), tee_mm_get_size(mm), 864 true); 865 866 /* Add the area to the pager */ 867 tee_pager_add_core_area(tee_mm_get_smem(mm) + SMALL_PAGE_SIZE, 868 tee_mm_get_bytes(mm) - SMALL_PAGE_SIZE, 869 TEE_MATTR_PRW | TEE_MATTR_LOCKED, 870 NULL, NULL); 871 872 /* init effective stack */ 873 sp = tee_mm_get_smem(mm) + tee_mm_get_bytes(mm); 874 asan_tag_access((void *)tee_mm_get_smem(mm), (void *)sp); 875 if (!thread_init_stack(n, sp)) 876 panic("init stack failed"); 877 } 878 } 879 #else 880 static void init_thread_stacks(void) 881 { 882 size_t n; 883 884 /* Assign the thread stacks */ 885 for (n = 0; n < CFG_NUM_THREADS; n++) { 886 if (!thread_init_stack(n, GET_STACK(stack_thread[n]))) 887 panic("thread_init_stack failed"); 888 } 889 } 890 #endif /*CFG_WITH_PAGER*/ 891 892 static void init_user_kcode(void) 893 { 894 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 895 vaddr_t v; 896 897 v = (vaddr_t)thread_excp_vect; 898 thread_user_kcode_va = ROUNDDOWN(v, CORE_MMU_USER_CODE_SIZE); 899 /* 900 * The maximum size of the exception vector and associated code is 901 * something slightly larger than 2 KiB. Worst case the exception 902 * vector can span two pages. 903 */ 904 thread_user_kcode_size = CORE_MMU_USER_CODE_SIZE * 2; 905 906 core_mmu_get_user_va_range(&v, NULL); 907 thread_user_kcode_offset = thread_user_kcode_va - v; 908 #endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ 909 } 910 911 void thread_init_primary(const struct thread_handlers *handlers) 912 { 913 init_handlers(handlers); 914 915 /* Initialize canaries around the stacks */ 916 init_canaries(); 917 918 init_thread_stacks(); 919 pgt_init(); 920 921 init_user_kcode(); 922 } 923 924 static void init_sec_mon(size_t pos __maybe_unused) 925 { 926 #if !defined(CFG_WITH_ARM_TRUSTED_FW) 927 /* Initialize secure monitor */ 928 sm_init(GET_STACK(stack_tmp[pos])); 929 #endif 930 } 931 932 static uint32_t __maybe_unused get_midr_implementer(uint32_t midr) 933 { 934 return (midr >> MIDR_IMPLEMENTER_SHIFT) & MIDR_IMPLEMENTER_MASK; 935 } 936 937 static uint32_t __maybe_unused get_midr_primary_part(uint32_t midr) 938 { 939 return (midr >> MIDR_PRIMARY_PART_NUM_SHIFT) & 940 MIDR_PRIMARY_PART_NUM_MASK; 941 } 942 943 static vaddr_t get_excp_vect(void) 944 { 945 #ifdef CFG_CORE_WORKAROUND_SPECTRE_BP_SEC 946 uint32_t midr = read_midr(); 947 948 if (get_midr_implementer(midr) != MIDR_IMPLEMENTER_ARM) 949 return (vaddr_t)thread_excp_vect; 950 951 switch (get_midr_primary_part(midr)) { 952 #ifdef ARM32 953 case CORTEX_A8_PART_NUM: 954 case CORTEX_A9_PART_NUM: 955 case CORTEX_A17_PART_NUM: 956 #endif 957 case CORTEX_A57_PART_NUM: 958 case CORTEX_A72_PART_NUM: 959 case CORTEX_A73_PART_NUM: 960 case CORTEX_A75_PART_NUM: 961 return (vaddr_t)thread_excp_vect_workaround; 962 #ifdef ARM32 963 case CORTEX_A15_PART_NUM: 964 return (vaddr_t)thread_excp_vect_workaround_a15; 965 #endif 966 default: 967 return (vaddr_t)thread_excp_vect; 968 } 969 #endif /*CFG_CORE_WORKAROUND_SPECTRE_BP_SEC*/ 970 971 return (vaddr_t)thread_excp_vect; 972 } 973 974 void thread_init_per_cpu(void) 975 { 976 size_t pos = get_core_pos(); 977 struct thread_core_local *l = thread_get_core_local(); 978 979 init_sec_mon(pos); 980 981 set_tmp_stack(l, GET_STACK(stack_tmp[pos]) - STACK_TMP_OFFS); 982 set_abt_stack(l, GET_STACK(stack_abt[pos])); 983 984 thread_init_vbar(get_excp_vect()); 985 } 986 987 struct thread_specific_data *thread_get_tsd(void) 988 { 989 return &threads[thread_get_id()].tsd; 990 } 991 992 struct thread_ctx_regs *thread_get_ctx_regs(void) 993 { 994 struct thread_core_local *l = thread_get_core_local(); 995 996 assert(l->curr_thread != -1); 997 return &threads[l->curr_thread].regs; 998 } 999 1000 void thread_set_foreign_intr(bool enable) 1001 { 1002 /* thread_get_core_local() requires foreign interrupts to be disabled */ 1003 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1004 struct thread_core_local *l; 1005 1006 l = thread_get_core_local(); 1007 1008 assert(l->curr_thread != -1); 1009 1010 if (enable) { 1011 threads[l->curr_thread].flags |= 1012 THREAD_FLAGS_FOREIGN_INTR_ENABLE; 1013 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 1014 } else { 1015 /* 1016 * No need to disable foreign interrupts here since they're 1017 * already disabled above. 1018 */ 1019 threads[l->curr_thread].flags &= 1020 ~THREAD_FLAGS_FOREIGN_INTR_ENABLE; 1021 } 1022 } 1023 1024 void thread_restore_foreign_intr(void) 1025 { 1026 /* thread_get_core_local() requires foreign interrupts to be disabled */ 1027 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1028 struct thread_core_local *l; 1029 1030 l = thread_get_core_local(); 1031 1032 assert(l->curr_thread != -1); 1033 1034 if (threads[l->curr_thread].flags & THREAD_FLAGS_FOREIGN_INTR_ENABLE) 1035 thread_set_exceptions(exceptions & ~THREAD_EXCP_FOREIGN_INTR); 1036 } 1037 1038 #ifdef CFG_WITH_VFP 1039 uint32_t thread_kernel_enable_vfp(void) 1040 { 1041 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1042 struct thread_ctx *thr = threads + thread_get_id(); 1043 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1044 1045 assert(!vfp_is_enabled()); 1046 1047 if (!thr->vfp_state.ns_saved) { 1048 vfp_lazy_save_state_final(&thr->vfp_state.ns); 1049 thr->vfp_state.ns_saved = true; 1050 } else if (thr->vfp_state.sec_lazy_saved && 1051 !thr->vfp_state.sec_saved) { 1052 /* 1053 * This happens when we're handling an abort while the 1054 * thread was using the VFP state. 1055 */ 1056 vfp_lazy_save_state_final(&thr->vfp_state.sec); 1057 thr->vfp_state.sec_saved = true; 1058 } else if (tuv && tuv->lazy_saved && !tuv->saved) { 1059 /* 1060 * This can happen either during syscall or abort 1061 * processing (while processing a syscall). 1062 */ 1063 vfp_lazy_save_state_final(&tuv->vfp); 1064 tuv->saved = true; 1065 } 1066 1067 vfp_enable(); 1068 return exceptions; 1069 } 1070 1071 void thread_kernel_disable_vfp(uint32_t state) 1072 { 1073 uint32_t exceptions; 1074 1075 assert(vfp_is_enabled()); 1076 1077 vfp_disable(); 1078 exceptions = thread_get_exceptions(); 1079 assert(exceptions & THREAD_EXCP_FOREIGN_INTR); 1080 exceptions &= ~THREAD_EXCP_FOREIGN_INTR; 1081 exceptions |= state & THREAD_EXCP_FOREIGN_INTR; 1082 thread_set_exceptions(exceptions); 1083 } 1084 1085 void thread_kernel_save_vfp(void) 1086 { 1087 struct thread_ctx *thr = threads + thread_get_id(); 1088 1089 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1090 if (vfp_is_enabled()) { 1091 vfp_lazy_save_state_init(&thr->vfp_state.sec); 1092 thr->vfp_state.sec_lazy_saved = true; 1093 } 1094 } 1095 1096 void thread_kernel_restore_vfp(void) 1097 { 1098 struct thread_ctx *thr = threads + thread_get_id(); 1099 1100 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1101 assert(!vfp_is_enabled()); 1102 if (thr->vfp_state.sec_lazy_saved) { 1103 vfp_lazy_restore_state(&thr->vfp_state.sec, 1104 thr->vfp_state.sec_saved); 1105 thr->vfp_state.sec_saved = false; 1106 thr->vfp_state.sec_lazy_saved = false; 1107 } 1108 } 1109 1110 void thread_user_enable_vfp(struct thread_user_vfp_state *uvfp) 1111 { 1112 struct thread_ctx *thr = threads + thread_get_id(); 1113 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1114 1115 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1116 assert(!vfp_is_enabled()); 1117 1118 if (!thr->vfp_state.ns_saved) { 1119 vfp_lazy_save_state_final(&thr->vfp_state.ns); 1120 thr->vfp_state.ns_saved = true; 1121 } else if (tuv && uvfp != tuv) { 1122 if (tuv->lazy_saved && !tuv->saved) { 1123 vfp_lazy_save_state_final(&tuv->vfp); 1124 tuv->saved = true; 1125 } 1126 } 1127 1128 if (uvfp->lazy_saved) 1129 vfp_lazy_restore_state(&uvfp->vfp, uvfp->saved); 1130 uvfp->lazy_saved = false; 1131 uvfp->saved = false; 1132 1133 thr->vfp_state.uvfp = uvfp; 1134 vfp_enable(); 1135 } 1136 1137 void thread_user_save_vfp(void) 1138 { 1139 struct thread_ctx *thr = threads + thread_get_id(); 1140 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1141 1142 assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); 1143 if (!vfp_is_enabled()) 1144 return; 1145 1146 assert(tuv && !tuv->lazy_saved && !tuv->saved); 1147 vfp_lazy_save_state_init(&tuv->vfp); 1148 tuv->lazy_saved = true; 1149 } 1150 1151 void thread_user_clear_vfp(struct thread_user_vfp_state *uvfp) 1152 { 1153 struct thread_ctx *thr = threads + thread_get_id(); 1154 1155 if (uvfp == thr->vfp_state.uvfp) 1156 thr->vfp_state.uvfp = NULL; 1157 uvfp->lazy_saved = false; 1158 uvfp->saved = false; 1159 } 1160 #endif /*CFG_WITH_VFP*/ 1161 1162 #ifdef ARM32 1163 static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) 1164 { 1165 uint32_t s; 1166 1167 if (!is_32bit) 1168 return false; 1169 1170 s = read_spsr(); 1171 s &= ~(CPSR_MODE_MASK | CPSR_T | CPSR_IT_MASK1 | CPSR_IT_MASK2); 1172 s |= CPSR_MODE_USR; 1173 if (entry_func & 1) 1174 s |= CPSR_T; 1175 *spsr = s; 1176 return true; 1177 } 1178 #endif 1179 1180 #ifdef ARM64 1181 static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) 1182 { 1183 uint32_t s; 1184 1185 if (is_32bit) { 1186 s = read_daif() & (SPSR_32_AIF_MASK << SPSR_32_AIF_SHIFT); 1187 s |= SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT; 1188 s |= (entry_func & SPSR_32_T_MASK) << SPSR_32_T_SHIFT; 1189 } else { 1190 s = read_daif() & (SPSR_64_DAIF_MASK << SPSR_64_DAIF_SHIFT); 1191 } 1192 1193 *spsr = s; 1194 return true; 1195 } 1196 #endif 1197 1198 uint32_t thread_enter_user_mode(unsigned long a0, unsigned long a1, 1199 unsigned long a2, unsigned long a3, unsigned long user_sp, 1200 unsigned long entry_func, bool is_32bit, 1201 uint32_t *exit_status0, uint32_t *exit_status1) 1202 { 1203 uint32_t spsr; 1204 1205 tee_ta_update_session_utime_resume(); 1206 1207 if (!get_spsr(is_32bit, entry_func, &spsr)) { 1208 *exit_status0 = 1; /* panic */ 1209 *exit_status1 = 0xbadbadba; 1210 return 0; 1211 } 1212 return __thread_enter_user_mode(a0, a1, a2, a3, user_sp, entry_func, 1213 spsr, exit_status0, exit_status1); 1214 } 1215 1216 #ifdef CFG_CORE_UNMAP_CORE_AT_EL0 1217 void thread_get_user_kcode(struct mobj **mobj, size_t *offset, 1218 vaddr_t *va, size_t *sz) 1219 { 1220 core_mmu_get_user_va_range(va, NULL); 1221 *mobj = mobj_tee_ram; 1222 *offset = thread_user_kcode_va - CFG_TEE_RAM_START; 1223 *sz = thread_user_kcode_size; 1224 } 1225 #endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ 1226 1227 void thread_add_mutex(struct mutex *m) 1228 { 1229 struct thread_core_local *l = thread_get_core_local(); 1230 int ct = l->curr_thread; 1231 1232 assert(ct != -1 && threads[ct].state == THREAD_STATE_ACTIVE); 1233 assert(m->owner_id == MUTEX_OWNER_ID_NONE); 1234 m->owner_id = ct; 1235 TAILQ_INSERT_TAIL(&threads[ct].mutexes, m, link); 1236 } 1237 1238 void thread_rem_mutex(struct mutex *m) 1239 { 1240 struct thread_core_local *l = thread_get_core_local(); 1241 int ct = l->curr_thread; 1242 1243 assert(ct != -1 && threads[ct].state == THREAD_STATE_ACTIVE); 1244 assert(m->owner_id == ct); 1245 m->owner_id = MUTEX_OWNER_ID_NONE; 1246 TAILQ_REMOVE(&threads[ct].mutexes, m, link); 1247 } 1248 1249 bool thread_disable_prealloc_rpc_cache(uint64_t *cookie) 1250 { 1251 bool rv; 1252 size_t n; 1253 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1254 1255 lock_global(); 1256 1257 for (n = 0; n < CFG_NUM_THREADS; n++) { 1258 if (threads[n].state != THREAD_STATE_FREE) { 1259 rv = false; 1260 goto out; 1261 } 1262 } 1263 1264 rv = true; 1265 for (n = 0; n < CFG_NUM_THREADS; n++) { 1266 if (threads[n].rpc_arg) { 1267 mobj_free(threads[n].rpc_mobj); 1268 *cookie = threads[n].rpc_carg; 1269 threads[n].rpc_carg = 0; 1270 threads[n].rpc_arg = NULL; 1271 goto out; 1272 } 1273 } 1274 1275 *cookie = 0; 1276 thread_prealloc_rpc_cache = false; 1277 out: 1278 unlock_global(); 1279 thread_unmask_exceptions(exceptions); 1280 return rv; 1281 } 1282 1283 bool thread_enable_prealloc_rpc_cache(void) 1284 { 1285 bool rv; 1286 size_t n; 1287 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 1288 1289 lock_global(); 1290 1291 for (n = 0; n < CFG_NUM_THREADS; n++) { 1292 if (threads[n].state != THREAD_STATE_FREE) { 1293 rv = false; 1294 goto out; 1295 } 1296 } 1297 1298 rv = true; 1299 thread_prealloc_rpc_cache = true; 1300 out: 1301 unlock_global(); 1302 thread_unmask_exceptions(exceptions); 1303 return rv; 1304 } 1305 1306 void thread_rpc_free_arg(uint64_t cookie) 1307 { 1308 if (cookie) { 1309 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { 1310 OPTEE_SMC_RETURN_RPC_FREE 1311 }; 1312 1313 reg_pair_from_64(cookie, rpc_args + 1, rpc_args + 2); 1314 thread_rpc(rpc_args); 1315 } 1316 } 1317 1318 struct mobj *thread_rpc_alloc_arg(size_t size, uint64_t *cookie) 1319 { 1320 paddr_t pa; 1321 uint64_t co; 1322 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { 1323 OPTEE_SMC_RETURN_RPC_ALLOC, size 1324 }; 1325 struct mobj *mobj = NULL; 1326 1327 thread_rpc(rpc_args); 1328 1329 pa = reg_pair_to_64(rpc_args[1], rpc_args[2]); 1330 co = reg_pair_to_64(rpc_args[4], rpc_args[5]); 1331 1332 if (!ALIGNMENT_IS_OK(pa, struct optee_msg_arg)) 1333 goto err; 1334 1335 /* Check if this region is in static shared space */ 1336 if (core_pbuf_is(CORE_MEM_NSEC_SHM, pa, size)) 1337 mobj = mobj_shm_alloc(pa, size); 1338 else if ((!(pa & SMALL_PAGE_MASK)) && size <= SMALL_PAGE_SIZE) 1339 mobj = mobj_mapped_shm_alloc(&pa, 1, 0, co); 1340 1341 if (!mobj) 1342 goto err; 1343 1344 *cookie = co; 1345 return mobj; 1346 err: 1347 thread_rpc_free_arg(co); 1348 mobj_free(mobj); 1349 *cookie = 0; 1350 return NULL; 1351 } 1352 1353 static bool get_rpc_arg(uint32_t cmd, size_t num_params, 1354 struct optee_msg_arg **arg_ret, uint64_t *carg_ret) 1355 { 1356 struct thread_ctx *thr = threads + thread_get_id(); 1357 struct optee_msg_arg *arg = thr->rpc_arg; 1358 struct mobj *mobj; 1359 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1360 uint64_t c; 1361 1362 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 1363 return false; 1364 1365 if (!arg) { 1366 mobj = thread_rpc_alloc_arg(sz, &c); 1367 if (!mobj) 1368 return false; 1369 1370 arg = mobj_get_va(mobj, 0); 1371 if (!arg) 1372 goto bad; 1373 1374 thr->rpc_arg = arg; 1375 thr->rpc_carg = c; 1376 thr->rpc_mobj = mobj; 1377 } 1378 1379 memset(arg, 0, OPTEE_MSG_GET_ARG_SIZE(num_params)); 1380 arg->cmd = cmd; 1381 arg->num_params = num_params; 1382 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 1383 1384 *arg_ret = arg; 1385 *carg_ret = thr->rpc_carg; 1386 return true; 1387 1388 bad: 1389 thread_rpc_free_arg(c); 1390 return false; 1391 } 1392 1393 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 1394 struct optee_msg_param *params) 1395 { 1396 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; 1397 struct optee_msg_arg *arg; 1398 uint64_t carg; 1399 size_t n; 1400 1401 /* 1402 * Break recursion in case plat_prng_add_jitter_entropy_norpc() 1403 * sleeps on a mutex or unlocks a mutex with a sleeper (contended 1404 * mutex). 1405 */ 1406 if (cmd != OPTEE_MSG_RPC_CMD_WAIT_QUEUE) 1407 plat_prng_add_jitter_entropy_norpc(); 1408 1409 if (!get_rpc_arg(cmd, num_params, &arg, &carg)) 1410 return TEE_ERROR_OUT_OF_MEMORY; 1411 1412 memcpy(arg->params, params, sizeof(*params) * num_params); 1413 1414 reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); 1415 thread_rpc(rpc_args); 1416 for (n = 0; n < num_params; n++) { 1417 switch (params[n].attr & OPTEE_MSG_ATTR_TYPE_MASK) { 1418 case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT: 1419 case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT: 1420 case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT: 1421 case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT: 1422 case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT: 1423 case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT: 1424 params[n] = arg->params[n]; 1425 break; 1426 default: 1427 break; 1428 } 1429 } 1430 return arg->ret; 1431 } 1432 1433 /** 1434 * Free physical memory previously allocated with thread_rpc_alloc() 1435 * 1436 * @cookie: cookie received when allocating the buffer 1437 * @bt: must be the same as supplied when allocating 1438 * @mobj: mobj that describes allocated buffer 1439 * 1440 * This function also frees corresponding mobj. 1441 */ 1442 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 1443 { 1444 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; 1445 struct optee_msg_arg *arg; 1446 uint64_t carg; 1447 1448 if (!get_rpc_arg(OPTEE_MSG_RPC_CMD_SHM_FREE, 1, &arg, &carg)) 1449 return; 1450 1451 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 1452 arg->params[0].u.value.a = bt; 1453 arg->params[0].u.value.b = cookie; 1454 arg->params[0].u.value.c = 0; 1455 1456 mobj_free(mobj); 1457 1458 reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); 1459 thread_rpc(rpc_args); 1460 } 1461 1462 /** 1463 * Allocates shared memory buffer via RPC 1464 * 1465 * @size: size in bytes of shared memory buffer 1466 * @align: required alignment of buffer 1467 * @bt: buffer type OPTEE_MSG_RPC_SHM_TYPE_* 1468 * @payload: returned physical pointer to buffer, 0 if allocation 1469 * failed. 1470 * @cookie: returned cookie used when freeing the buffer 1471 */ 1472 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt, 1473 uint64_t *cookie) 1474 { 1475 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; 1476 struct optee_msg_arg *arg; 1477 uint64_t carg; 1478 struct mobj *mobj = NULL; 1479 1480 if (!get_rpc_arg(OPTEE_MSG_RPC_CMD_SHM_ALLOC, 1, &arg, &carg)) 1481 goto fail; 1482 1483 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 1484 arg->params[0].u.value.a = bt; 1485 arg->params[0].u.value.b = size; 1486 arg->params[0].u.value.c = align; 1487 1488 reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); 1489 thread_rpc(rpc_args); 1490 1491 if (arg->ret != TEE_SUCCESS) 1492 goto fail; 1493 1494 if (arg->num_params != 1) 1495 goto fail; 1496 1497 if (arg->params[0].attr == OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT) { 1498 *cookie = arg->params[0].u.tmem.shm_ref; 1499 mobj = mobj_shm_alloc(arg->params[0].u.tmem.buf_ptr, 1500 arg->params[0].u.tmem.size); 1501 } else if (arg->params[0].attr == (OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT | 1502 OPTEE_MSG_ATTR_NONCONTIG)) { 1503 *cookie = arg->params[0].u.tmem.shm_ref; 1504 mobj = msg_param_mobj_from_noncontig( 1505 arg->params[0].u.tmem.buf_ptr, 1506 arg->params[0].u.tmem.size, 1507 *cookie, 1508 true); 1509 } else 1510 goto fail; 1511 1512 if (!mobj) 1513 goto free_first; 1514 1515 assert(mobj_is_nonsec(mobj)); 1516 return mobj; 1517 1518 free_first: 1519 thread_rpc_free(bt, *cookie, mobj); 1520 fail: 1521 *cookie = 0; 1522 return NULL; 1523 } 1524 1525 struct mobj *thread_rpc_alloc_payload(size_t size, uint64_t *cookie) 1526 { 1527 return thread_rpc_alloc(size, 8, OPTEE_MSG_RPC_SHM_TYPE_APPL, cookie); 1528 } 1529 1530 void thread_rpc_free_payload(uint64_t cookie, struct mobj *mobj) 1531 { 1532 thread_rpc_free(OPTEE_MSG_RPC_SHM_TYPE_APPL, cookie, mobj); 1533 } 1534