1 /* 2 * Copyright (c) 2016, Linaro Limited 3 * Copyright (c) 2014, STMicroelectronics International N.V. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, 10 * this list of conditions and the following disclaimer. 11 * 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <platform_config.h> 30 31 #include <arm.h> 32 #include <assert.h> 33 #include <keep.h> 34 #include <kernel/misc.h> 35 #include <kernel/panic.h> 36 #include <kernel/tee_ta_manager.h> 37 #include <kernel/thread_defs.h> 38 #include <kernel/thread.h> 39 #include <kernel/tz_proc_def.h> 40 #include <kernel/tz_proc.h> 41 #include <mm/core_memprot.h> 42 #include <mm/tee_mm.h> 43 #include <mm/tee_mmu_defs.h> 44 #include <mm/tee_mmu.h> 45 #include <mm/tee_pager.h> 46 #include <optee_msg.h> 47 #include <sm/optee_smc.h> 48 #include <sm/sm.h> 49 #include <tee/tee_fs_rpc.h> 50 #include <trace.h> 51 #include <util.h> 52 53 #include "thread_private.h" 54 55 #ifdef ARM32 56 #ifdef CFG_CORE_SANITIZE_KADDRESS 57 #define STACK_TMP_SIZE 3072 58 #else 59 #define STACK_TMP_SIZE 1024 60 #endif 61 #define STACK_THREAD_SIZE 8192 62 63 #if TRACE_LEVEL > 0 64 #ifdef CFG_CORE_SANITIZE_KADDRESS 65 #define STACK_ABT_SIZE 3072 66 #else 67 #define STACK_ABT_SIZE 2048 68 #endif 69 #else 70 #define STACK_ABT_SIZE 1024 71 #endif 72 73 #endif /*ARM32*/ 74 75 #ifdef ARM64 76 #define STACK_TMP_SIZE 2048 77 #define STACK_THREAD_SIZE 8192 78 79 #if TRACE_LEVEL > 0 80 #define STACK_ABT_SIZE 3072 81 #else 82 #define STACK_ABT_SIZE 1024 83 #endif 84 #endif /*ARM64*/ 85 86 struct thread_ctx threads[CFG_NUM_THREADS]; 87 88 static struct thread_core_local thread_core_local[CFG_TEE_CORE_NB_CORE]; 89 90 #ifdef CFG_WITH_STACK_CANARIES 91 #ifdef ARM32 92 #define STACK_CANARY_SIZE (4 * sizeof(uint32_t)) 93 #endif 94 #ifdef ARM64 95 #define STACK_CANARY_SIZE (8 * sizeof(uint32_t)) 96 #endif 97 #define START_CANARY_VALUE 0xdededede 98 #define END_CANARY_VALUE 0xabababab 99 #define GET_START_CANARY(name, stack_num) name[stack_num][0] 100 #define GET_END_CANARY(name, stack_num) \ 101 name[stack_num][sizeof(name[stack_num]) / sizeof(uint32_t) - 1] 102 #else 103 #define STACK_CANARY_SIZE 0 104 #endif 105 106 #define DECLARE_STACK(name, num_stacks, stack_size, linkage) \ 107 linkage uint32_t name[num_stacks] \ 108 [ROUNDUP(stack_size + STACK_CANARY_SIZE, STACK_ALIGNMENT) / \ 109 sizeof(uint32_t)] \ 110 __attribute__((section(".nozi_stack"), \ 111 aligned(STACK_ALIGNMENT))) 112 113 #define STACK_SIZE(stack) (sizeof(stack) - STACK_CANARY_SIZE / 2) 114 115 #define GET_STACK(stack) \ 116 ((vaddr_t)(stack) + STACK_SIZE(stack)) 117 118 DECLARE_STACK(stack_tmp, CFG_TEE_CORE_NB_CORE, STACK_TMP_SIZE, /* global */); 119 DECLARE_STACK(stack_abt, CFG_TEE_CORE_NB_CORE, STACK_ABT_SIZE, static); 120 #if !defined(CFG_WITH_ARM_TRUSTED_FW) 121 DECLARE_STACK(stack_sm, CFG_TEE_CORE_NB_CORE, SM_STACK_SIZE, static); 122 #endif 123 #ifndef CFG_WITH_PAGER 124 DECLARE_STACK(stack_thread, CFG_NUM_THREADS, STACK_THREAD_SIZE, static); 125 #endif 126 127 const uint32_t stack_tmp_stride = STACK_SIZE(stack_tmp[0]); 128 129 KEEP_PAGER(stack_tmp); 130 KEEP_PAGER(stack_tmp_stride); 131 132 thread_smc_handler_t thread_std_smc_handler_ptr; 133 static thread_smc_handler_t thread_fast_smc_handler_ptr; 134 thread_fiq_handler_t thread_fiq_handler_ptr; 135 thread_pm_handler_t thread_cpu_on_handler_ptr; 136 thread_pm_handler_t thread_cpu_off_handler_ptr; 137 thread_pm_handler_t thread_cpu_suspend_handler_ptr; 138 thread_pm_handler_t thread_cpu_resume_handler_ptr; 139 thread_pm_handler_t thread_system_off_handler_ptr; 140 thread_pm_handler_t thread_system_reset_handler_ptr; 141 142 143 static unsigned int thread_global_lock = SPINLOCK_UNLOCK; 144 static bool thread_prealloc_rpc_cache; 145 146 static void init_canaries(void) 147 { 148 #ifdef CFG_WITH_STACK_CANARIES 149 size_t n; 150 #define INIT_CANARY(name) \ 151 for (n = 0; n < ARRAY_SIZE(name); n++) { \ 152 uint32_t *start_canary = &GET_START_CANARY(name, n); \ 153 uint32_t *end_canary = &GET_END_CANARY(name, n); \ 154 \ 155 *start_canary = START_CANARY_VALUE; \ 156 *end_canary = END_CANARY_VALUE; \ 157 DMSG("#Stack canaries for %s[%zu] with top at %p\n", \ 158 #name, n, (void *)(end_canary - 1)); \ 159 DMSG("watch *%p\n", (void *)end_canary); \ 160 } 161 162 INIT_CANARY(stack_tmp); 163 INIT_CANARY(stack_abt); 164 #if !defined(CFG_WITH_ARM_TRUSTED_FW) 165 INIT_CANARY(stack_sm); 166 #endif 167 #ifndef CFG_WITH_PAGER 168 INIT_CANARY(stack_thread); 169 #endif 170 #endif/*CFG_WITH_STACK_CANARIES*/ 171 } 172 173 #define CANARY_DIED(stack, loc, n) \ 174 do { \ 175 EMSG_RAW("Dead canary at %s of '%s[%zu]'", #loc, #stack, n); \ 176 panic(); \ 177 } while (0) 178 179 void thread_check_canaries(void) 180 { 181 #ifdef CFG_WITH_STACK_CANARIES 182 size_t n; 183 184 for (n = 0; n < ARRAY_SIZE(stack_tmp); n++) { 185 if (GET_START_CANARY(stack_tmp, n) != START_CANARY_VALUE) 186 CANARY_DIED(stack_tmp, start, n); 187 if (GET_END_CANARY(stack_tmp, n) != END_CANARY_VALUE) 188 CANARY_DIED(stack_tmp, end, n); 189 } 190 191 for (n = 0; n < ARRAY_SIZE(stack_abt); n++) { 192 if (GET_START_CANARY(stack_abt, n) != START_CANARY_VALUE) 193 CANARY_DIED(stack_abt, start, n); 194 if (GET_END_CANARY(stack_abt, n) != END_CANARY_VALUE) 195 CANARY_DIED(stack_abt, end, n); 196 197 } 198 #if !defined(CFG_WITH_ARM_TRUSTED_FW) 199 for (n = 0; n < ARRAY_SIZE(stack_sm); n++) { 200 if (GET_START_CANARY(stack_sm, n) != START_CANARY_VALUE) 201 CANARY_DIED(stack_sm, start, n); 202 if (GET_END_CANARY(stack_sm, n) != END_CANARY_VALUE) 203 CANARY_DIED(stack_sm, end, n); 204 } 205 #endif 206 #ifndef CFG_WITH_PAGER 207 for (n = 0; n < ARRAY_SIZE(stack_thread); n++) { 208 if (GET_START_CANARY(stack_thread, n) != START_CANARY_VALUE) 209 CANARY_DIED(stack_thread, start, n); 210 if (GET_END_CANARY(stack_thread, n) != END_CANARY_VALUE) 211 CANARY_DIED(stack_thread, end, n); 212 } 213 #endif 214 #endif/*CFG_WITH_STACK_CANARIES*/ 215 } 216 217 static void lock_global(void) 218 { 219 cpu_spin_lock(&thread_global_lock); 220 } 221 222 static void unlock_global(void) 223 { 224 cpu_spin_unlock(&thread_global_lock); 225 } 226 227 #ifdef ARM32 228 uint32_t thread_get_exceptions(void) 229 { 230 uint32_t cpsr = read_cpsr(); 231 232 return (cpsr >> CPSR_F_SHIFT) & THREAD_EXCP_ALL; 233 } 234 235 void thread_set_exceptions(uint32_t exceptions) 236 { 237 uint32_t cpsr = read_cpsr(); 238 239 cpsr &= ~(THREAD_EXCP_ALL << CPSR_F_SHIFT); 240 cpsr |= ((exceptions & THREAD_EXCP_ALL) << CPSR_F_SHIFT); 241 write_cpsr(cpsr); 242 } 243 #endif /*ARM32*/ 244 245 #ifdef ARM64 246 uint32_t thread_get_exceptions(void) 247 { 248 uint32_t daif = read_daif(); 249 250 return (daif >> DAIF_F_SHIFT) & THREAD_EXCP_ALL; 251 } 252 253 void thread_set_exceptions(uint32_t exceptions) 254 { 255 uint32_t daif = read_daif(); 256 257 daif &= ~(THREAD_EXCP_ALL << DAIF_F_SHIFT); 258 daif |= ((exceptions & THREAD_EXCP_ALL) << DAIF_F_SHIFT); 259 write_daif(daif); 260 } 261 #endif /*ARM64*/ 262 263 uint32_t thread_mask_exceptions(uint32_t exceptions) 264 { 265 uint32_t state = thread_get_exceptions(); 266 267 thread_set_exceptions(state | (exceptions & THREAD_EXCP_ALL)); 268 return state; 269 } 270 271 void thread_unmask_exceptions(uint32_t state) 272 { 273 thread_set_exceptions(state & THREAD_EXCP_ALL); 274 } 275 276 277 struct thread_core_local *thread_get_core_local(void) 278 { 279 uint32_t cpu_id = get_core_pos(); 280 281 /* 282 * IRQs must be disabled before playing with core_local since 283 * we otherwise may be rescheduled to a different core in the 284 * middle of this function. 285 */ 286 assert(thread_get_exceptions() & THREAD_EXCP_IRQ); 287 288 assert(cpu_id < CFG_TEE_CORE_NB_CORE); 289 return &thread_core_local[cpu_id]; 290 } 291 292 static void thread_lazy_save_ns_vfp(void) 293 { 294 #ifdef CFG_WITH_VFP 295 struct thread_ctx *thr = threads + thread_get_id(); 296 297 thr->vfp_state.ns_saved = false; 298 #if defined(ARM64) && defined(CFG_WITH_ARM_TRUSTED_FW) 299 /* 300 * ARM TF saves and restores CPACR_EL1, so we must assume NS world 301 * uses VFP and always preserve the register file when secure world 302 * is about to use it 303 */ 304 thr->vfp_state.ns.force_save = true; 305 #endif 306 vfp_lazy_save_state_init(&thr->vfp_state.ns); 307 #endif /*CFG_WITH_VFP*/ 308 } 309 310 static void thread_lazy_restore_ns_vfp(void) 311 { 312 #ifdef CFG_WITH_VFP 313 struct thread_ctx *thr = threads + thread_get_id(); 314 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 315 316 assert(!thr->vfp_state.sec_lazy_saved && !thr->vfp_state.sec_saved); 317 318 if (tuv && tuv->lazy_saved && !tuv->saved) { 319 vfp_lazy_save_state_final(&tuv->vfp); 320 tuv->saved = true; 321 } 322 323 vfp_lazy_restore_state(&thr->vfp_state.ns, thr->vfp_state.ns_saved); 324 thr->vfp_state.ns_saved = false; 325 #endif /*CFG_WITH_VFP*/ 326 } 327 328 #ifdef ARM32 329 static void init_regs(struct thread_ctx *thread, 330 struct thread_smc_args *args) 331 { 332 thread->regs.pc = (uint32_t)thread_std_smc_entry; 333 334 /* 335 * Stdcalls starts in SVC mode with masked IRQ, masked Asynchronous 336 * abort and unmasked FIQ. 337 */ 338 thread->regs.cpsr = read_cpsr() & ARM32_CPSR_E; 339 thread->regs.cpsr |= CPSR_MODE_SVC | CPSR_I | CPSR_A; 340 /* Enable thumb mode if it's a thumb instruction */ 341 if (thread->regs.pc & 1) 342 thread->regs.cpsr |= CPSR_T; 343 /* Reinitialize stack pointer */ 344 thread->regs.svc_sp = thread->stack_va_end; 345 346 /* 347 * Copy arguments into context. This will make the 348 * arguments appear in r0-r7 when thread is started. 349 */ 350 thread->regs.r0 = args->a0; 351 thread->regs.r1 = args->a1; 352 thread->regs.r2 = args->a2; 353 thread->regs.r3 = args->a3; 354 thread->regs.r4 = args->a4; 355 thread->regs.r5 = args->a5; 356 thread->regs.r6 = args->a6; 357 thread->regs.r7 = args->a7; 358 } 359 #endif /*ARM32*/ 360 361 #ifdef ARM64 362 static void init_regs(struct thread_ctx *thread, 363 struct thread_smc_args *args) 364 { 365 thread->regs.pc = (uint64_t)thread_std_smc_entry; 366 367 /* 368 * Stdcalls starts in SVC mode with masked IRQ, masked Asynchronous 369 * abort and unmasked FIQ. 370 */ 371 thread->regs.cpsr = SPSR_64(SPSR_64_MODE_EL1, SPSR_64_MODE_SP_EL0, 372 DAIFBIT_IRQ | DAIFBIT_ABT); 373 /* Reinitialize stack pointer */ 374 thread->regs.sp = thread->stack_va_end; 375 376 /* 377 * Copy arguments into context. This will make the 378 * arguments appear in x0-x7 when thread is started. 379 */ 380 thread->regs.x[0] = args->a0; 381 thread->regs.x[1] = args->a1; 382 thread->regs.x[2] = args->a2; 383 thread->regs.x[3] = args->a3; 384 thread->regs.x[4] = args->a4; 385 thread->regs.x[5] = args->a5; 386 thread->regs.x[6] = args->a6; 387 thread->regs.x[7] = args->a7; 388 389 /* Set up frame pointer as per the Aarch64 AAPCS */ 390 thread->regs.x[29] = 0; 391 } 392 #endif /*ARM64*/ 393 394 void thread_init_boot_thread(void) 395 { 396 struct thread_core_local *l = thread_get_core_local(); 397 size_t n; 398 399 for (n = 0; n < CFG_NUM_THREADS; n++) { 400 TAILQ_INIT(&threads[n].mutexes); 401 TAILQ_INIT(&threads[n].tsd.sess_stack); 402 #ifdef CFG_SMALL_PAGE_USER_TA 403 SLIST_INIT(&threads[n].tsd.pgt_cache); 404 #endif 405 } 406 407 for (n = 0; n < CFG_TEE_CORE_NB_CORE; n++) 408 thread_core_local[n].curr_thread = -1; 409 410 l->curr_thread = 0; 411 threads[0].state = THREAD_STATE_ACTIVE; 412 } 413 414 void thread_clr_boot_thread(void) 415 { 416 struct thread_core_local *l = thread_get_core_local(); 417 418 assert(l->curr_thread >= 0 && l->curr_thread < CFG_NUM_THREADS); 419 assert(threads[l->curr_thread].state == THREAD_STATE_ACTIVE); 420 assert(TAILQ_EMPTY(&threads[l->curr_thread].mutexes)); 421 threads[l->curr_thread].state = THREAD_STATE_FREE; 422 l->curr_thread = -1; 423 } 424 425 static void thread_alloc_and_run(struct thread_smc_args *args) 426 { 427 size_t n; 428 struct thread_core_local *l = thread_get_core_local(); 429 bool found_thread = false; 430 431 assert(l->curr_thread == -1); 432 433 lock_global(); 434 435 for (n = 0; n < CFG_NUM_THREADS; n++) { 436 if (threads[n].state == THREAD_STATE_FREE) { 437 threads[n].state = THREAD_STATE_ACTIVE; 438 found_thread = true; 439 break; 440 } 441 } 442 443 unlock_global(); 444 445 if (!found_thread) { 446 args->a0 = OPTEE_SMC_RETURN_ETHREAD_LIMIT; 447 return; 448 } 449 450 l->curr_thread = n; 451 452 threads[n].flags = 0; 453 init_regs(threads + n, args); 454 455 /* Save Hypervisor Client ID */ 456 threads[n].hyp_clnt_id = args->a7; 457 458 thread_lazy_save_ns_vfp(); 459 thread_resume(&threads[n].regs); 460 } 461 462 #ifdef ARM32 463 static void copy_a0_to_a5(struct thread_ctx_regs *regs, 464 struct thread_smc_args *args) 465 { 466 /* 467 * Update returned values from RPC, values will appear in 468 * r0-r3 when thread is resumed. 469 */ 470 regs->r0 = args->a0; 471 regs->r1 = args->a1; 472 regs->r2 = args->a2; 473 regs->r3 = args->a3; 474 regs->r4 = args->a4; 475 regs->r5 = args->a5; 476 } 477 #endif /*ARM32*/ 478 479 #ifdef ARM64 480 static void copy_a0_to_a5(struct thread_ctx_regs *regs, 481 struct thread_smc_args *args) 482 { 483 /* 484 * Update returned values from RPC, values will appear in 485 * x0-x3 when thread is resumed. 486 */ 487 regs->x[0] = args->a0; 488 regs->x[1] = args->a1; 489 regs->x[2] = args->a2; 490 regs->x[3] = args->a3; 491 regs->x[4] = args->a4; 492 regs->x[5] = args->a5; 493 } 494 #endif /*ARM64*/ 495 496 static void thread_resume_from_rpc(struct thread_smc_args *args) 497 { 498 size_t n = args->a3; /* thread id */ 499 struct thread_core_local *l = thread_get_core_local(); 500 uint32_t rv = 0; 501 502 assert(l->curr_thread == -1); 503 504 lock_global(); 505 506 if (n < CFG_NUM_THREADS && 507 threads[n].state == THREAD_STATE_SUSPENDED && 508 args->a7 == threads[n].hyp_clnt_id) 509 threads[n].state = THREAD_STATE_ACTIVE; 510 else 511 rv = OPTEE_SMC_RETURN_ERESUME; 512 513 unlock_global(); 514 515 if (rv) { 516 args->a0 = rv; 517 return; 518 } 519 520 l->curr_thread = n; 521 522 if (threads[n].have_user_map) 523 core_mmu_set_user_map(&threads[n].user_map); 524 525 /* 526 * Return from RPC to request service of an IRQ must not 527 * get parameters from non-secure world. 528 */ 529 if (threads[n].flags & THREAD_FLAGS_COPY_ARGS_ON_RETURN) { 530 copy_a0_to_a5(&threads[n].regs, args); 531 threads[n].flags &= ~THREAD_FLAGS_COPY_ARGS_ON_RETURN; 532 } 533 534 thread_lazy_save_ns_vfp(); 535 thread_resume(&threads[n].regs); 536 } 537 538 void thread_handle_fast_smc(struct thread_smc_args *args) 539 { 540 thread_check_canaries(); 541 thread_fast_smc_handler_ptr(args); 542 /* Fast handlers must not unmask any exceptions */ 543 assert(thread_get_exceptions() == THREAD_EXCP_ALL); 544 } 545 546 void thread_handle_std_smc(struct thread_smc_args *args) 547 { 548 thread_check_canaries(); 549 550 if (args->a0 == OPTEE_SMC_CALL_RETURN_FROM_RPC) 551 thread_resume_from_rpc(args); 552 else 553 thread_alloc_and_run(args); 554 } 555 556 /* Helper routine for the assembly function thread_std_smc_entry() */ 557 void __thread_std_smc_entry(struct thread_smc_args *args) 558 { 559 struct thread_ctx *thr = threads + thread_get_id(); 560 561 if (!thr->rpc_arg) { 562 paddr_t parg; 563 uint64_t carg; 564 void *arg; 565 566 thread_rpc_alloc_arg( 567 OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS), 568 &parg, &carg); 569 if (!parg || !ALIGNMENT_IS_OK(parg, struct optee_msg_arg) || 570 !(arg = phys_to_virt(parg, CORE_MEM_NSEC_SHM))) { 571 thread_rpc_free_arg(carg); 572 args->a0 = OPTEE_SMC_RETURN_ENOMEM; 573 return; 574 } 575 576 thr->rpc_arg = arg; 577 thr->rpc_carg = carg; 578 } 579 580 thread_std_smc_handler_ptr(args); 581 582 tee_fs_rpc_cache_clear(&thr->tsd); 583 if (!thread_prealloc_rpc_cache) { 584 thread_rpc_free_arg(thr->rpc_carg); 585 thr->rpc_carg = 0; 586 thr->rpc_arg = 0; 587 } 588 } 589 590 void *thread_get_tmp_sp(void) 591 { 592 struct thread_core_local *l = thread_get_core_local(); 593 594 return (void *)l->tmp_stack_va_end; 595 } 596 597 #ifdef ARM64 598 vaddr_t thread_get_saved_thread_sp(void) 599 { 600 struct thread_core_local *l = thread_get_core_local(); 601 int ct = l->curr_thread; 602 603 assert(ct != -1); 604 return threads[ct].kern_sp; 605 } 606 #endif /*ARM64*/ 607 608 bool thread_addr_is_in_stack(vaddr_t va) 609 { 610 struct thread_ctx *thr; 611 int ct = thread_get_id_may_fail(); 612 613 if (ct == -1) 614 return false; 615 616 thr = threads + ct; 617 return va < thr->stack_va_end && 618 va >= (thr->stack_va_end - STACK_THREAD_SIZE); 619 } 620 621 void thread_state_free(void) 622 { 623 struct thread_core_local *l = thread_get_core_local(); 624 int ct = l->curr_thread; 625 626 assert(ct != -1); 627 assert(TAILQ_EMPTY(&threads[ct].mutexes)); 628 629 thread_lazy_restore_ns_vfp(); 630 tee_pager_release_phys( 631 (void *)(threads[ct].stack_va_end - STACK_THREAD_SIZE), 632 STACK_THREAD_SIZE); 633 634 lock_global(); 635 636 assert(threads[ct].state == THREAD_STATE_ACTIVE); 637 threads[ct].state = THREAD_STATE_FREE; 638 threads[ct].flags = 0; 639 l->curr_thread = -1; 640 641 unlock_global(); 642 } 643 644 #ifdef ARM32 645 static bool is_from_user(uint32_t cpsr) 646 { 647 return (cpsr & ARM32_CPSR_MODE_MASK) == ARM32_CPSR_MODE_USR; 648 } 649 #endif 650 651 #ifdef ARM64 652 static bool is_from_user(uint32_t cpsr) 653 { 654 if (cpsr & (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT)) 655 return true; 656 if (((cpsr >> SPSR_64_MODE_EL_SHIFT) & SPSR_64_MODE_EL_MASK) == 657 SPSR_64_MODE_EL0) 658 return true; 659 return false; 660 } 661 #endif 662 663 #ifdef CFG_WITH_PAGER 664 static void release_unused_kernel_stack(struct thread_ctx *thr) 665 { 666 vaddr_t sp = thr->regs.svc_sp; 667 vaddr_t base = thr->stack_va_end - STACK_THREAD_SIZE; 668 size_t len = sp - base; 669 670 tee_pager_release_phys((void *)base, len); 671 } 672 #else 673 static void release_unused_kernel_stack(struct thread_ctx *thr __unused) 674 { 675 } 676 #endif 677 678 int thread_state_suspend(uint32_t flags, uint32_t cpsr, vaddr_t pc) 679 { 680 struct thread_core_local *l = thread_get_core_local(); 681 int ct = l->curr_thread; 682 683 assert(ct != -1); 684 685 thread_check_canaries(); 686 687 release_unused_kernel_stack(threads + ct); 688 689 if (is_from_user(cpsr)) 690 thread_user_save_vfp(); 691 thread_lazy_restore_ns_vfp(); 692 693 lock_global(); 694 695 assert(threads[ct].state == THREAD_STATE_ACTIVE); 696 threads[ct].flags |= flags; 697 threads[ct].regs.cpsr = cpsr; 698 threads[ct].regs.pc = pc; 699 threads[ct].state = THREAD_STATE_SUSPENDED; 700 701 threads[ct].have_user_map = core_mmu_user_mapping_is_active(); 702 if (threads[ct].have_user_map) { 703 core_mmu_get_user_map(&threads[ct].user_map); 704 core_mmu_set_user_map(NULL); 705 } 706 707 l->curr_thread = -1; 708 709 unlock_global(); 710 711 return ct; 712 } 713 714 #ifdef ARM32 715 static void set_tmp_stack(struct thread_core_local *l, vaddr_t sp) 716 { 717 l->tmp_stack_va_end = sp; 718 thread_set_irq_sp(sp); 719 thread_set_fiq_sp(sp); 720 } 721 722 static void set_abt_stack(struct thread_core_local *l __unused, vaddr_t sp) 723 { 724 thread_set_abt_sp(sp); 725 } 726 #endif /*ARM32*/ 727 728 #ifdef ARM64 729 static void set_tmp_stack(struct thread_core_local *l, vaddr_t sp) 730 { 731 /* 732 * We're already using the tmp stack when this function is called 733 * so there's no need to assign it to any stack pointer. However, 734 * we'll need to restore it at different times so store it here. 735 */ 736 l->tmp_stack_va_end = sp; 737 } 738 739 static void set_abt_stack(struct thread_core_local *l, vaddr_t sp) 740 { 741 l->abt_stack_va_end = sp; 742 } 743 #endif /*ARM64*/ 744 745 bool thread_init_stack(uint32_t thread_id, vaddr_t sp) 746 { 747 if (thread_id >= CFG_NUM_THREADS) 748 return false; 749 threads[thread_id].stack_va_end = sp; 750 return true; 751 } 752 753 int thread_get_id_may_fail(void) 754 { 755 /* thread_get_core_local() requires IRQs to be disabled */ 756 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_IRQ); 757 struct thread_core_local *l = thread_get_core_local(); 758 int ct = l->curr_thread; 759 760 thread_unmask_exceptions(exceptions); 761 return ct; 762 } 763 764 int thread_get_id(void) 765 { 766 int ct = thread_get_id_may_fail(); 767 768 assert(ct >= 0 && ct < CFG_NUM_THREADS); 769 return ct; 770 } 771 772 static void init_handlers(const struct thread_handlers *handlers) 773 { 774 thread_std_smc_handler_ptr = handlers->std_smc; 775 thread_fast_smc_handler_ptr = handlers->fast_smc; 776 thread_fiq_handler_ptr = handlers->fiq; 777 thread_cpu_on_handler_ptr = handlers->cpu_on; 778 thread_cpu_off_handler_ptr = handlers->cpu_off; 779 thread_cpu_suspend_handler_ptr = handlers->cpu_suspend; 780 thread_cpu_resume_handler_ptr = handlers->cpu_resume; 781 thread_system_off_handler_ptr = handlers->system_off; 782 thread_system_reset_handler_ptr = handlers->system_reset; 783 } 784 785 #ifdef CFG_WITH_PAGER 786 static void init_thread_stacks(void) 787 { 788 size_t n; 789 790 /* 791 * Allocate virtual memory for thread stacks. 792 */ 793 for (n = 0; n < CFG_NUM_THREADS; n++) { 794 tee_mm_entry_t *mm; 795 vaddr_t sp; 796 797 /* Find vmem for thread stack and its protection gap */ 798 mm = tee_mm_alloc(&tee_mm_vcore, 799 SMALL_PAGE_SIZE + STACK_THREAD_SIZE); 800 assert(mm); 801 802 /* Claim eventual physical page */ 803 tee_pager_add_pages(tee_mm_get_smem(mm), tee_mm_get_size(mm), 804 true); 805 806 /* Add the area to the pager */ 807 tee_pager_add_core_area(tee_mm_get_smem(mm) + SMALL_PAGE_SIZE, 808 tee_mm_get_bytes(mm) - SMALL_PAGE_SIZE, 809 TEE_MATTR_PRW | TEE_MATTR_LOCKED, 810 NULL, NULL); 811 812 /* init effective stack */ 813 sp = tee_mm_get_smem(mm) + tee_mm_get_bytes(mm); 814 if (!thread_init_stack(n, sp)) 815 panic("init stack failed"); 816 } 817 } 818 #else 819 static void init_thread_stacks(void) 820 { 821 size_t n; 822 823 /* Assign the thread stacks */ 824 for (n = 0; n < CFG_NUM_THREADS; n++) { 825 if (!thread_init_stack(n, GET_STACK(stack_thread[n]))) 826 panic("thread_init_stack failed"); 827 } 828 } 829 #endif /*CFG_WITH_PAGER*/ 830 831 void thread_init_primary(const struct thread_handlers *handlers) 832 { 833 init_handlers(handlers); 834 835 /* Initialize canaries around the stacks */ 836 init_canaries(); 837 838 init_thread_stacks(); 839 pgt_init(); 840 } 841 842 static void init_sec_mon(size_t pos __maybe_unused) 843 { 844 #if !defined(CFG_WITH_ARM_TRUSTED_FW) 845 /* Initialize secure monitor */ 846 sm_init(GET_STACK(stack_sm[pos])); 847 #endif 848 } 849 850 void thread_init_per_cpu(void) 851 { 852 size_t pos = get_core_pos(); 853 struct thread_core_local *l = thread_get_core_local(); 854 855 init_sec_mon(pos); 856 857 set_tmp_stack(l, GET_STACK(stack_tmp[pos])); 858 set_abt_stack(l, GET_STACK(stack_abt[pos])); 859 860 thread_init_vbar(); 861 } 862 863 struct thread_specific_data *thread_get_tsd(void) 864 { 865 return &threads[thread_get_id()].tsd; 866 } 867 868 struct thread_ctx_regs *thread_get_ctx_regs(void) 869 { 870 struct thread_core_local *l = thread_get_core_local(); 871 872 assert(l->curr_thread != -1); 873 return &threads[l->curr_thread].regs; 874 } 875 876 void thread_set_irq(bool enable) 877 { 878 /* thread_get_core_local() requires IRQs to be disabled */ 879 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_IRQ); 880 struct thread_core_local *l; 881 882 l = thread_get_core_local(); 883 884 assert(l->curr_thread != -1); 885 886 if (enable) { 887 threads[l->curr_thread].flags |= THREAD_FLAGS_IRQ_ENABLE; 888 thread_set_exceptions(exceptions & ~THREAD_EXCP_IRQ); 889 } else { 890 /* 891 * No need to disable IRQ here since it's already disabled 892 * above. 893 */ 894 threads[l->curr_thread].flags &= ~THREAD_FLAGS_IRQ_ENABLE; 895 } 896 } 897 898 void thread_restore_irq(void) 899 { 900 /* thread_get_core_local() requires IRQs to be disabled */ 901 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_IRQ); 902 struct thread_core_local *l; 903 904 l = thread_get_core_local(); 905 906 assert(l->curr_thread != -1); 907 908 if (threads[l->curr_thread].flags & THREAD_FLAGS_IRQ_ENABLE) 909 thread_set_exceptions(exceptions & ~THREAD_EXCP_IRQ); 910 } 911 912 #ifdef CFG_WITH_VFP 913 uint32_t thread_kernel_enable_vfp(void) 914 { 915 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_IRQ); 916 struct thread_ctx *thr = threads + thread_get_id(); 917 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 918 919 assert(!vfp_is_enabled()); 920 921 if (!thr->vfp_state.ns_saved) { 922 vfp_lazy_save_state_final(&thr->vfp_state.ns); 923 thr->vfp_state.ns_saved = true; 924 } else if (thr->vfp_state.sec_lazy_saved && 925 !thr->vfp_state.sec_saved) { 926 /* 927 * This happens when we're handling an abort while the 928 * thread was using the VFP state. 929 */ 930 vfp_lazy_save_state_final(&thr->vfp_state.sec); 931 thr->vfp_state.sec_saved = true; 932 } else if (tuv && tuv->lazy_saved && !tuv->saved) { 933 /* 934 * This can happen either during syscall or abort 935 * processing (while processing a syscall). 936 */ 937 vfp_lazy_save_state_final(&tuv->vfp); 938 tuv->saved = true; 939 } 940 941 vfp_enable(); 942 return exceptions; 943 } 944 945 void thread_kernel_disable_vfp(uint32_t state) 946 { 947 uint32_t exceptions; 948 949 assert(vfp_is_enabled()); 950 951 vfp_disable(); 952 exceptions = thread_get_exceptions(); 953 assert(exceptions & THREAD_EXCP_IRQ); 954 exceptions &= ~THREAD_EXCP_IRQ; 955 exceptions |= state & THREAD_EXCP_IRQ; 956 thread_set_exceptions(exceptions); 957 } 958 959 void thread_kernel_save_vfp(void) 960 { 961 struct thread_ctx *thr = threads + thread_get_id(); 962 963 assert(thread_get_exceptions() & THREAD_EXCP_IRQ); 964 if (vfp_is_enabled()) { 965 vfp_lazy_save_state_init(&thr->vfp_state.sec); 966 thr->vfp_state.sec_lazy_saved = true; 967 } 968 } 969 970 void thread_kernel_restore_vfp(void) 971 { 972 struct thread_ctx *thr = threads + thread_get_id(); 973 974 assert(thread_get_exceptions() & THREAD_EXCP_IRQ); 975 assert(!vfp_is_enabled()); 976 if (thr->vfp_state.sec_lazy_saved) { 977 vfp_lazy_restore_state(&thr->vfp_state.sec, 978 thr->vfp_state.sec_saved); 979 thr->vfp_state.sec_saved = false; 980 thr->vfp_state.sec_lazy_saved = false; 981 } 982 } 983 984 void thread_user_enable_vfp(struct thread_user_vfp_state *uvfp) 985 { 986 struct thread_ctx *thr = threads + thread_get_id(); 987 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 988 989 assert(thread_get_exceptions() & THREAD_EXCP_IRQ); 990 assert(!vfp_is_enabled()); 991 992 if (!thr->vfp_state.ns_saved) { 993 vfp_lazy_save_state_final(&thr->vfp_state.ns); 994 thr->vfp_state.ns_saved = true; 995 } else if (tuv && uvfp != tuv) { 996 if (tuv->lazy_saved && !tuv->saved) { 997 vfp_lazy_save_state_final(&tuv->vfp); 998 tuv->saved = true; 999 } 1000 } 1001 1002 if (uvfp->lazy_saved) 1003 vfp_lazy_restore_state(&uvfp->vfp, uvfp->saved); 1004 uvfp->lazy_saved = false; 1005 uvfp->saved = false; 1006 1007 thr->vfp_state.uvfp = uvfp; 1008 vfp_enable(); 1009 } 1010 1011 void thread_user_save_vfp(void) 1012 { 1013 struct thread_ctx *thr = threads + thread_get_id(); 1014 struct thread_user_vfp_state *tuv = thr->vfp_state.uvfp; 1015 1016 assert(thread_get_exceptions() & THREAD_EXCP_IRQ); 1017 if (!vfp_is_enabled()) 1018 return; 1019 1020 assert(tuv && !tuv->lazy_saved && !tuv->saved); 1021 vfp_lazy_save_state_init(&tuv->vfp); 1022 tuv->lazy_saved = true; 1023 } 1024 1025 void thread_user_clear_vfp(struct thread_user_vfp_state *uvfp) 1026 { 1027 struct thread_ctx *thr = threads + thread_get_id(); 1028 1029 if (uvfp == thr->vfp_state.uvfp) 1030 thr->vfp_state.uvfp = NULL; 1031 uvfp->lazy_saved = false; 1032 uvfp->saved = false; 1033 } 1034 #endif /*CFG_WITH_VFP*/ 1035 1036 #ifdef ARM32 1037 static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) 1038 { 1039 uint32_t s; 1040 1041 if (!is_32bit) 1042 return false; 1043 1044 s = read_spsr(); 1045 s &= ~(CPSR_MODE_MASK | CPSR_T | CPSR_IT_MASK1 | CPSR_IT_MASK2); 1046 s |= CPSR_MODE_USR; 1047 if (entry_func & 1) 1048 s |= CPSR_T; 1049 *spsr = s; 1050 return true; 1051 } 1052 #endif 1053 1054 #ifdef ARM64 1055 static bool get_spsr(bool is_32bit, unsigned long entry_func, uint32_t *spsr) 1056 { 1057 uint32_t s; 1058 1059 if (is_32bit) { 1060 s = read_daif() & (SPSR_32_AIF_MASK << SPSR_32_AIF_SHIFT); 1061 s |= SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT; 1062 s |= (entry_func & SPSR_32_T_MASK) << SPSR_32_T_SHIFT; 1063 } else { 1064 s = read_daif() & (SPSR_64_DAIF_MASK << SPSR_64_DAIF_SHIFT); 1065 } 1066 1067 *spsr = s; 1068 return true; 1069 } 1070 #endif 1071 1072 uint32_t thread_enter_user_mode(unsigned long a0, unsigned long a1, 1073 unsigned long a2, unsigned long a3, unsigned long user_sp, 1074 unsigned long entry_func, bool is_32bit, 1075 uint32_t *exit_status0, uint32_t *exit_status1) 1076 { 1077 uint32_t spsr; 1078 1079 if (!get_spsr(is_32bit, entry_func, &spsr)) { 1080 *exit_status0 = 1; /* panic */ 1081 *exit_status1 = 0xbadbadba; 1082 return 0; 1083 } 1084 return __thread_enter_user_mode(a0, a1, a2, a3, user_sp, entry_func, 1085 spsr, exit_status0, exit_status1); 1086 } 1087 1088 void thread_add_mutex(struct mutex *m) 1089 { 1090 struct thread_core_local *l = thread_get_core_local(); 1091 int ct = l->curr_thread; 1092 1093 assert(ct != -1 && threads[ct].state == THREAD_STATE_ACTIVE); 1094 assert(m->owner_id == -1); 1095 m->owner_id = ct; 1096 TAILQ_INSERT_TAIL(&threads[ct].mutexes, m, link); 1097 } 1098 1099 void thread_rem_mutex(struct mutex *m) 1100 { 1101 struct thread_core_local *l = thread_get_core_local(); 1102 int ct = l->curr_thread; 1103 1104 assert(ct != -1 && threads[ct].state == THREAD_STATE_ACTIVE); 1105 assert(m->owner_id == ct); 1106 m->owner_id = -1; 1107 TAILQ_REMOVE(&threads[ct].mutexes, m, link); 1108 } 1109 1110 bool thread_disable_prealloc_rpc_cache(uint64_t *cookie) 1111 { 1112 bool rv; 1113 size_t n; 1114 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_IRQ); 1115 1116 lock_global(); 1117 1118 for (n = 0; n < CFG_NUM_THREADS; n++) { 1119 if (threads[n].state != THREAD_STATE_FREE) { 1120 rv = false; 1121 goto out; 1122 } 1123 } 1124 1125 rv = true; 1126 for (n = 0; n < CFG_NUM_THREADS; n++) { 1127 if (threads[n].rpc_arg) { 1128 *cookie = threads[n].rpc_carg; 1129 threads[n].rpc_carg = 0; 1130 threads[n].rpc_arg = NULL; 1131 goto out; 1132 } 1133 } 1134 1135 *cookie = 0; 1136 thread_prealloc_rpc_cache = false; 1137 out: 1138 unlock_global(); 1139 thread_unmask_exceptions(exceptions); 1140 return rv; 1141 } 1142 1143 bool thread_enable_prealloc_rpc_cache(void) 1144 { 1145 bool rv; 1146 size_t n; 1147 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_IRQ); 1148 1149 lock_global(); 1150 1151 for (n = 0; n < CFG_NUM_THREADS; n++) { 1152 if (threads[n].state != THREAD_STATE_FREE) { 1153 rv = false; 1154 goto out; 1155 } 1156 } 1157 1158 rv = true; 1159 thread_prealloc_rpc_cache = true; 1160 out: 1161 unlock_global(); 1162 thread_unmask_exceptions(exceptions); 1163 return rv; 1164 } 1165 1166 static uint32_t rpc_cmd_nolock(uint32_t cmd, size_t num_params, 1167 struct optee_msg_param *params) 1168 { 1169 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; 1170 struct thread_ctx *thr = threads + thread_get_id(); 1171 struct optee_msg_arg *arg = thr->rpc_arg; 1172 uint64_t carg = thr->rpc_carg; 1173 const size_t params_size = sizeof(struct optee_msg_param) * num_params; 1174 size_t n; 1175 1176 assert(arg && carg && num_params <= THREAD_RPC_MAX_NUM_PARAMS); 1177 1178 memset(arg, 0, OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS)); 1179 arg->cmd = cmd; 1180 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 1181 arg->num_params = num_params; 1182 memcpy(OPTEE_MSG_GET_PARAMS(arg), params, params_size); 1183 1184 reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); 1185 thread_rpc(rpc_args); 1186 for (n = 0; n < num_params; n++) { 1187 switch (params[n].attr & OPTEE_MSG_ATTR_TYPE_MASK) { 1188 case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT: 1189 case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT: 1190 case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT: 1191 case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT: 1192 case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT: 1193 case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT: 1194 memcpy(params + n, OPTEE_MSG_GET_PARAMS(arg) + n, 1195 sizeof(struct optee_msg_param)); 1196 break; 1197 default: 1198 break; 1199 } 1200 } 1201 return arg->ret; 1202 } 1203 1204 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 1205 struct optee_msg_param *params) 1206 { 1207 uint32_t ret; 1208 1209 ret = rpc_cmd_nolock(cmd, num_params, params); 1210 1211 return ret; 1212 } 1213 1214 static bool check_alloced_shm(paddr_t pa, size_t len, size_t align) 1215 { 1216 if (pa & (align - 1)) 1217 return false; 1218 return core_pbuf_is(CORE_MEM_NSEC_SHM, pa, len); 1219 } 1220 1221 void thread_rpc_free_arg(uint64_t cookie) 1222 { 1223 if (cookie) { 1224 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { 1225 OPTEE_SMC_RETURN_RPC_FREE 1226 }; 1227 1228 reg_pair_from_64(cookie, rpc_args + 1, rpc_args + 2); 1229 thread_rpc(rpc_args); 1230 } 1231 } 1232 1233 void thread_rpc_alloc_arg(size_t size, paddr_t *arg, uint64_t *cookie) 1234 { 1235 paddr_t pa; 1236 uint64_t co; 1237 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { 1238 OPTEE_SMC_RETURN_RPC_ALLOC, size 1239 }; 1240 1241 thread_rpc(rpc_args); 1242 1243 pa = reg_pair_to_64(rpc_args[1], rpc_args[2]); 1244 co = reg_pair_to_64(rpc_args[4], rpc_args[5]); 1245 if (!check_alloced_shm(pa, size, sizeof(uint64_t))) { 1246 thread_rpc_free_arg(co); 1247 pa = 0; 1248 co = 0; 1249 } 1250 1251 *arg = pa; 1252 *cookie = co; 1253 } 1254 1255 /** 1256 * Free physical memory previously allocated with thread_rpc_alloc() 1257 * 1258 * @cookie: cookie received when allocating the buffer 1259 * @bt: must be the same as supplied when allocating 1260 */ 1261 static void thread_rpc_free(unsigned int bt, uint64_t cookie) 1262 { 1263 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; 1264 struct thread_ctx *thr = threads + thread_get_id(); 1265 struct optee_msg_arg *arg = thr->rpc_arg; 1266 uint64_t carg = thr->rpc_carg; 1267 struct optee_msg_param *params = OPTEE_MSG_GET_PARAMS(arg); 1268 1269 memset(arg, 0, OPTEE_MSG_GET_ARG_SIZE(1)); 1270 arg->cmd = OPTEE_MSG_RPC_CMD_SHM_FREE; 1271 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 1272 arg->num_params = 1; 1273 1274 params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 1275 params[0].u.value.a = bt; 1276 params[0].u.value.b = cookie; 1277 params[0].u.value.c = 0; 1278 1279 reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); 1280 thread_rpc(rpc_args); 1281 } 1282 1283 /** 1284 * Allocates shared memory buffer via RPC 1285 * 1286 * @size: size in bytes of shared memory buffer 1287 * @align: required alignment of buffer 1288 * @bt: buffer type OPTEE_MSG_RPC_SHM_TYPE_* 1289 * @payload: returned physical pointer to buffer, 0 if allocation 1290 * failed. 1291 * @cookie: returned cookie used when freeing the buffer 1292 */ 1293 static void thread_rpc_alloc(size_t size, size_t align, unsigned int bt, 1294 paddr_t *payload, uint64_t *cookie) 1295 { 1296 uint32_t rpc_args[THREAD_RPC_NUM_ARGS] = { OPTEE_SMC_RETURN_RPC_CMD }; 1297 struct thread_ctx *thr = threads + thread_get_id(); 1298 struct optee_msg_arg *arg = thr->rpc_arg; 1299 uint64_t carg = thr->rpc_carg; 1300 struct optee_msg_param *params = OPTEE_MSG_GET_PARAMS(arg); 1301 1302 memset(arg, 0, OPTEE_MSG_GET_ARG_SIZE(1)); 1303 arg->cmd = OPTEE_MSG_RPC_CMD_SHM_ALLOC; 1304 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 1305 arg->num_params = 1; 1306 1307 params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 1308 params[0].u.value.a = bt; 1309 params[0].u.value.b = size; 1310 params[0].u.value.c = align; 1311 1312 reg_pair_from_64(carg, rpc_args + 1, rpc_args + 2); 1313 thread_rpc(rpc_args); 1314 if (arg->ret != TEE_SUCCESS) 1315 goto fail; 1316 1317 if (arg->num_params != 1) 1318 goto fail; 1319 1320 if (params[0].attr != OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT) 1321 goto fail; 1322 1323 if (!check_alloced_shm(params[0].u.tmem.buf_ptr, size, align)) { 1324 thread_rpc_free(bt, params[0].u.tmem.shm_ref); 1325 goto fail; 1326 } 1327 1328 *payload = params[0].u.tmem.buf_ptr; 1329 *cookie = params[0].u.tmem.shm_ref; 1330 return; 1331 fail: 1332 *payload = 0; 1333 *cookie = 0; 1334 } 1335 1336 void thread_rpc_alloc_payload(size_t size, paddr_t *payload, uint64_t *cookie) 1337 { 1338 thread_rpc_alloc(size, 8, OPTEE_MSG_RPC_SHM_TYPE_APPL, payload, cookie); 1339 } 1340 1341 void thread_rpc_free_payload(uint64_t cookie) 1342 { 1343 thread_rpc_free(OPTEE_MSG_RPC_SHM_TYPE_APPL, cookie); 1344 } 1345