1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2020-2023, Linaro Limited. 4 * Copyright (c) 2019-2024, Arm Limited. All rights reserved. 5 */ 6 7 #include <assert.h> 8 #include <ffa.h> 9 #include <initcall.h> 10 #include <io.h> 11 #include <kernel/interrupt.h> 12 #include <kernel/notif.h> 13 #include <kernel/panic.h> 14 #include <kernel/secure_partition.h> 15 #include <kernel/spinlock.h> 16 #include <kernel/spmc_sp_handler.h> 17 #include <kernel/tee_misc.h> 18 #include <kernel/thread.h> 19 #include <kernel/thread_private.h> 20 #include <kernel/thread_spmc.h> 21 #include <kernel/virtualization.h> 22 #include <mm/core_mmu.h> 23 #include <mm/mobj.h> 24 #include <optee_ffa.h> 25 #include <optee_msg.h> 26 #include <optee_rpc_cmd.h> 27 #include <sm/optee_smc.h> 28 #include <string.h> 29 #include <sys/queue.h> 30 #include <tee/entry_std.h> 31 #include <tee/uuid.h> 32 #include <util.h> 33 34 #if defined(CFG_CORE_SEL1_SPMC) 35 struct mem_share_state { 36 struct mobj_ffa *mf; 37 unsigned int page_count; 38 unsigned int region_count; 39 unsigned int current_page_idx; 40 }; 41 42 struct mem_frag_state { 43 struct mem_share_state share; 44 tee_mm_entry_t *mm; 45 unsigned int frag_offset; 46 SLIST_ENTRY(mem_frag_state) link; 47 }; 48 #endif 49 50 static unsigned int spmc_notif_lock = SPINLOCK_UNLOCK; 51 static int do_bottom_half_value = -1; 52 static uint16_t notif_vm_id; 53 static bool spmc_notif_is_ready; 54 55 /* Initialized in spmc_init() below */ 56 uint16_t optee_endpoint_id __nex_bss; 57 uint16_t spmc_id __nex_bss; 58 #ifdef CFG_CORE_SEL1_SPMC 59 uint16_t spmd_id __nex_bss; 60 static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | 61 FFA_PART_PROP_DIRECT_REQ_SEND | 62 #ifdef CFG_NS_VIRTUALIZATION 63 FFA_PART_PROP_NOTIF_CREATED | 64 FFA_PART_PROP_NOTIF_DESTROYED | 65 #endif 66 #ifdef ARM64 67 FFA_PART_PROP_AARCH64_STATE | 68 #endif 69 FFA_PART_PROP_IS_PE_ID; 70 71 static uint32_t my_uuid_words[] = { 72 /* 73 * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 74 * SP, or 75 * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a 76 * logical partition, residing in the same exception level as the 77 * SPMC 78 * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 79 */ 80 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, 81 }; 82 83 /* 84 * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. 85 * 86 * struct ffa_rxtx::spin_lock protects the variables below from concurrent 87 * access this includes the use of content of struct ffa_rxtx::rx and 88 * @frag_state_head. 89 * 90 * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct 91 * ffa_rxtx::tx and false when it is owned by normal world. 92 * 93 * Note that we can't prevent normal world from updating the content of 94 * these buffers so we must always be careful when reading. while we hold 95 * the lock. 96 */ 97 98 static struct ffa_rxtx my_rxtx __nex_bss; 99 100 static bool is_nw_buf(struct ffa_rxtx *rxtx) 101 { 102 return rxtx == &my_rxtx; 103 } 104 105 static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = 106 SLIST_HEAD_INITIALIZER(&frag_state_head); 107 108 static uint64_t notif_pending_bitmap; 109 static uint64_t notif_bound_bitmap; 110 static bool notif_vm_id_valid; 111 static int notif_intid = -1; 112 #else 113 static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 114 static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 115 static struct ffa_rxtx my_rxtx = { 116 .rx = __rx_buf, 117 .tx = __tx_buf, 118 .size = sizeof(__rx_buf), 119 }; 120 #endif 121 122 static uint32_t swap_src_dst(uint32_t src_dst) 123 { 124 return (src_dst >> 16) | (src_dst << 16); 125 } 126 127 static uint16_t get_sender_id(uint32_t src_dst) 128 { 129 return src_dst >> 16; 130 } 131 132 void spmc_set_args(struct thread_smc_args *args, uint32_t fid, uint32_t src_dst, 133 uint32_t w2, uint32_t w3, uint32_t w4, uint32_t w5) 134 { 135 *args = (struct thread_smc_args){ .a0 = fid, 136 .a1 = src_dst, 137 .a2 = w2, 138 .a3 = w3, 139 .a4 = w4, 140 .a5 = w5, }; 141 } 142 143 static void set_simple_ret_val(struct thread_smc_args *args, int ffa_ret) 144 { 145 if (ffa_ret) 146 spmc_set_args(args, FFA_ERROR, 0, ffa_ret, 0, 0, 0); 147 else 148 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 149 } 150 151 uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) 152 { 153 /* 154 * No locking, if the caller does concurrent calls to this it's 155 * only making a mess for itself. We must be able to renegotiate 156 * the FF-A version in order to support differing versions between 157 * the loader and the driver. 158 */ 159 if (vers < FFA_VERSION_1_1) 160 rxtx->ffa_vers = FFA_VERSION_1_0; 161 else 162 rxtx->ffa_vers = FFA_VERSION_1_1; 163 164 return rxtx->ffa_vers; 165 } 166 167 static bool is_ffa_success(uint32_t fid) 168 { 169 #ifdef ARM64 170 if (fid == FFA_SUCCESS_64) 171 return true; 172 #endif 173 return fid == FFA_SUCCESS_32; 174 } 175 176 static int32_t get_ffa_ret_code(const struct thread_smc_args *args) 177 { 178 if (is_ffa_success(args->a0)) 179 return FFA_OK; 180 if (args->a0 == FFA_ERROR && args->a2) 181 return args->a2; 182 return FFA_NOT_SUPPORTED; 183 } 184 185 static int ffa_simple_call(uint32_t fid, unsigned long a1, unsigned long a2, 186 unsigned long a3, unsigned long a4) 187 { 188 struct thread_smc_args args = { 189 .a0 = fid, 190 .a1 = a1, 191 .a2 = a2, 192 .a3 = a3, 193 .a4 = a4, 194 }; 195 196 thread_smccc(&args); 197 198 return get_ffa_ret_code(&args); 199 } 200 201 static int __maybe_unused ffa_features(uint32_t id) 202 { 203 return ffa_simple_call(FFA_FEATURES, id, 0, 0, 0); 204 } 205 206 static int __maybe_unused ffa_set_notification(uint16_t dst, uint16_t src, 207 uint32_t flags, uint64_t bitmap) 208 { 209 return ffa_simple_call(FFA_NOTIFICATION_SET, 210 SHIFT_U32(src, 16) | dst, flags, 211 low32_from_64(bitmap), high32_from_64(bitmap)); 212 } 213 214 #if defined(CFG_CORE_SEL1_SPMC) 215 static void handle_features(struct thread_smc_args *args) 216 { 217 uint32_t ret_fid = FFA_ERROR; 218 uint32_t ret_w2 = FFA_NOT_SUPPORTED; 219 220 switch (args->a1) { 221 case FFA_FEATURE_SCHEDULE_RECV_INTR: 222 if (spmc_notif_is_ready) { 223 ret_fid = FFA_SUCCESS_32; 224 ret_w2 = notif_intid; 225 } 226 break; 227 228 #ifdef ARM64 229 case FFA_RXTX_MAP_64: 230 #endif 231 case FFA_RXTX_MAP_32: 232 ret_fid = FFA_SUCCESS_32; 233 ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ 234 break; 235 #ifdef ARM64 236 case FFA_MEM_SHARE_64: 237 #endif 238 case FFA_MEM_SHARE_32: 239 ret_fid = FFA_SUCCESS_32; 240 /* 241 * Partition manager supports transmission of a memory 242 * transaction descriptor in a buffer dynamically allocated 243 * by the endpoint. 244 */ 245 ret_w2 = BIT(0); 246 break; 247 248 case FFA_ERROR: 249 case FFA_VERSION: 250 case FFA_SUCCESS_32: 251 #ifdef ARM64 252 case FFA_SUCCESS_64: 253 #endif 254 case FFA_FEATURES: 255 case FFA_SPM_ID_GET: 256 case FFA_MEM_FRAG_TX: 257 case FFA_MEM_RECLAIM: 258 case FFA_MSG_SEND_DIRECT_REQ_64: 259 case FFA_MSG_SEND_DIRECT_REQ_32: 260 case FFA_INTERRUPT: 261 case FFA_PARTITION_INFO_GET: 262 case FFA_RXTX_UNMAP: 263 case FFA_RX_RELEASE: 264 case FFA_FEATURE_MANAGED_EXIT_INTR: 265 case FFA_NOTIFICATION_BITMAP_CREATE: 266 case FFA_NOTIFICATION_BITMAP_DESTROY: 267 case FFA_NOTIFICATION_BIND: 268 case FFA_NOTIFICATION_UNBIND: 269 case FFA_NOTIFICATION_SET: 270 case FFA_NOTIFICATION_GET: 271 case FFA_NOTIFICATION_INFO_GET_32: 272 #ifdef ARM64 273 case FFA_NOTIFICATION_INFO_GET_64: 274 #endif 275 ret_fid = FFA_SUCCESS_32; 276 ret_w2 = FFA_PARAM_MBZ; 277 break; 278 default: 279 break; 280 } 281 282 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, 283 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 284 } 285 286 static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) 287 { 288 tee_mm_entry_t *mm = NULL; 289 290 if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) 291 return FFA_INVALID_PARAMETERS; 292 293 mm = tee_mm_alloc(&core_virt_shm_pool, sz); 294 if (!mm) 295 return FFA_NO_MEMORY; 296 297 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, 298 sz / SMALL_PAGE_SIZE, 299 MEM_AREA_NSEC_SHM)) { 300 tee_mm_free(mm); 301 return FFA_INVALID_PARAMETERS; 302 } 303 304 *va_ret = (void *)tee_mm_get_smem(mm); 305 return 0; 306 } 307 308 void spmc_handle_spm_id_get(struct thread_smc_args *args) 309 { 310 spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, spmc_id, 311 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 312 } 313 314 static void unmap_buf(void *va, size_t sz) 315 { 316 tee_mm_entry_t *mm = tee_mm_find(&core_virt_shm_pool, (vaddr_t)va); 317 318 assert(mm); 319 core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); 320 tee_mm_free(mm); 321 } 322 323 void spmc_handle_rxtx_map(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 324 { 325 int rc = 0; 326 unsigned int sz = 0; 327 paddr_t rx_pa = 0; 328 paddr_t tx_pa = 0; 329 void *rx = NULL; 330 void *tx = NULL; 331 332 cpu_spin_lock(&rxtx->spinlock); 333 334 if (args->a3 & GENMASK_64(63, 6)) { 335 rc = FFA_INVALID_PARAMETERS; 336 goto out; 337 } 338 339 sz = args->a3 * SMALL_PAGE_SIZE; 340 if (!sz) { 341 rc = FFA_INVALID_PARAMETERS; 342 goto out; 343 } 344 /* TX/RX are swapped compared to the caller */ 345 tx_pa = args->a2; 346 rx_pa = args->a1; 347 348 if (rxtx->size) { 349 rc = FFA_DENIED; 350 goto out; 351 } 352 353 /* 354 * If the buffer comes from a SP the address is virtual and already 355 * mapped. 356 */ 357 if (is_nw_buf(rxtx)) { 358 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 359 enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; 360 bool tx_alloced = false; 361 362 /* 363 * With virtualization we establish this mapping in 364 * the nexus mapping which then is replicated to 365 * each partition. 366 * 367 * This means that this mapping must be done before 368 * any partition is created and then must not be 369 * changed. 370 */ 371 372 /* 373 * core_mmu_add_mapping() may reuse previous 374 * mappings. First check if there's any mappings to 375 * reuse so we know how to clean up in case of 376 * failure. 377 */ 378 tx = phys_to_virt(tx_pa, mt, sz); 379 rx = phys_to_virt(rx_pa, mt, sz); 380 if (!tx) { 381 tx = core_mmu_add_mapping(mt, tx_pa, sz); 382 if (!tx) { 383 rc = FFA_NO_MEMORY; 384 goto out; 385 } 386 tx_alloced = true; 387 } 388 if (!rx) 389 rx = core_mmu_add_mapping(mt, rx_pa, sz); 390 391 if (!rx) { 392 if (tx_alloced && tx) 393 core_mmu_remove_mapping(mt, tx, sz); 394 rc = FFA_NO_MEMORY; 395 goto out; 396 } 397 } else { 398 rc = map_buf(tx_pa, sz, &tx); 399 if (rc) 400 goto out; 401 rc = map_buf(rx_pa, sz, &rx); 402 if (rc) { 403 unmap_buf(tx, sz); 404 goto out; 405 } 406 } 407 rxtx->tx = tx; 408 rxtx->rx = rx; 409 } else { 410 if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { 411 rc = FFA_INVALID_PARAMETERS; 412 goto out; 413 } 414 415 if (!virt_to_phys((void *)tx_pa) || 416 !virt_to_phys((void *)rx_pa)) { 417 rc = FFA_INVALID_PARAMETERS; 418 goto out; 419 } 420 421 rxtx->tx = (void *)tx_pa; 422 rxtx->rx = (void *)rx_pa; 423 } 424 425 rxtx->size = sz; 426 rxtx->tx_is_mine = true; 427 DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); 428 DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); 429 out: 430 cpu_spin_unlock(&rxtx->spinlock); 431 set_simple_ret_val(args, rc); 432 } 433 434 void spmc_handle_rxtx_unmap(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 435 { 436 int rc = FFA_INVALID_PARAMETERS; 437 438 cpu_spin_lock(&rxtx->spinlock); 439 440 if (!rxtx->size) 441 goto out; 442 443 /* We don't unmap the SP memory as the SP might still use it */ 444 if (is_nw_buf(rxtx)) { 445 unmap_buf(rxtx->rx, rxtx->size); 446 unmap_buf(rxtx->tx, rxtx->size); 447 } 448 rxtx->size = 0; 449 rxtx->rx = NULL; 450 rxtx->tx = NULL; 451 rc = 0; 452 out: 453 cpu_spin_unlock(&rxtx->spinlock); 454 set_simple_ret_val(args, rc); 455 } 456 457 void spmc_handle_rx_release(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 458 { 459 int rc = 0; 460 461 cpu_spin_lock(&rxtx->spinlock); 462 /* The senders RX is our TX */ 463 if (!rxtx->size || rxtx->tx_is_mine) { 464 rc = FFA_DENIED; 465 } else { 466 rc = 0; 467 rxtx->tx_is_mine = true; 468 } 469 cpu_spin_unlock(&rxtx->spinlock); 470 471 set_simple_ret_val(args, rc); 472 } 473 474 static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 475 { 476 return !w0 && !w1 && !w2 && !w3; 477 } 478 479 static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 480 { 481 /* 482 * This depends on which UUID we have been assigned. 483 * TODO add a generic mechanism to obtain our UUID. 484 * 485 * The test below is for the hard coded UUID 486 * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 487 */ 488 return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && 489 w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; 490 } 491 492 TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, 493 size_t idx, uint16_t endpoint_id, 494 uint16_t execution_context, 495 uint32_t part_props, 496 const uint32_t uuid_words[4]) 497 { 498 struct ffa_partition_info_x *fpi = NULL; 499 size_t fpi_size = sizeof(*fpi); 500 501 if (ffa_vers >= FFA_VERSION_1_1) 502 fpi_size += FFA_UUID_SIZE; 503 504 if ((idx + 1) * fpi_size > blen) 505 return TEE_ERROR_OUT_OF_MEMORY; 506 507 fpi = (void *)((vaddr_t)buf + idx * fpi_size); 508 fpi->id = endpoint_id; 509 /* Number of execution contexts implemented by this partition */ 510 fpi->execution_context = execution_context; 511 512 fpi->partition_properties = part_props; 513 514 if (ffa_vers >= FFA_VERSION_1_1) { 515 if (uuid_words) 516 memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); 517 else 518 memset(fpi->uuid, 0, FFA_UUID_SIZE); 519 } 520 521 return TEE_SUCCESS; 522 } 523 524 static int handle_partition_info_get_all(size_t *elem_count, 525 struct ffa_rxtx *rxtx, bool count_only) 526 { 527 if (!count_only) { 528 /* Add OP-TEE SP */ 529 if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, 530 rxtx->size, 0, optee_endpoint_id, 531 CFG_TEE_CORE_NB_CORE, 532 my_part_props, my_uuid_words)) 533 return FFA_NO_MEMORY; 534 } 535 *elem_count = 1; 536 537 if (IS_ENABLED(CFG_SECURE_PARTITION)) { 538 if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, 539 NULL, elem_count, count_only)) 540 return FFA_NO_MEMORY; 541 } 542 543 return FFA_OK; 544 } 545 546 void spmc_handle_partition_info_get(struct thread_smc_args *args, 547 struct ffa_rxtx *rxtx) 548 { 549 TEE_Result res = TEE_SUCCESS; 550 uint32_t ret_fid = FFA_ERROR; 551 uint32_t fpi_size = 0; 552 uint32_t rc = 0; 553 bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; 554 555 if (!count_only) { 556 cpu_spin_lock(&rxtx->spinlock); 557 558 if (!rxtx->size || !rxtx->tx_is_mine) { 559 rc = FFA_BUSY; 560 goto out; 561 } 562 } 563 564 if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { 565 size_t elem_count = 0; 566 567 ret_fid = handle_partition_info_get_all(&elem_count, rxtx, 568 count_only); 569 570 if (ret_fid) { 571 rc = ret_fid; 572 ret_fid = FFA_ERROR; 573 } else { 574 ret_fid = FFA_SUCCESS_32; 575 rc = elem_count; 576 } 577 578 goto out; 579 } 580 581 if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { 582 if (!count_only) { 583 res = spmc_fill_partition_entry(rxtx->ffa_vers, 584 rxtx->tx, rxtx->size, 0, 585 optee_endpoint_id, 586 CFG_TEE_CORE_NB_CORE, 587 my_part_props, 588 my_uuid_words); 589 if (res) { 590 ret_fid = FFA_ERROR; 591 rc = FFA_INVALID_PARAMETERS; 592 goto out; 593 } 594 } 595 rc = 1; 596 } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { 597 uint32_t uuid_array[4] = { 0 }; 598 TEE_UUID uuid = { }; 599 size_t count = 0; 600 601 uuid_array[0] = args->a1; 602 uuid_array[1] = args->a2; 603 uuid_array[2] = args->a3; 604 uuid_array[3] = args->a4; 605 tee_uuid_from_octets(&uuid, (uint8_t *)uuid_array); 606 607 res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, 608 rxtx->size, &uuid, &count, 609 count_only); 610 if (res != TEE_SUCCESS) { 611 ret_fid = FFA_ERROR; 612 rc = FFA_INVALID_PARAMETERS; 613 goto out; 614 } 615 rc = count; 616 } else { 617 ret_fid = FFA_ERROR; 618 rc = FFA_INVALID_PARAMETERS; 619 goto out; 620 } 621 622 ret_fid = FFA_SUCCESS_32; 623 624 out: 625 if (ret_fid == FFA_SUCCESS_32 && !count_only && 626 rxtx->ffa_vers >= FFA_VERSION_1_1) 627 fpi_size = sizeof(struct ffa_partition_info_x) + FFA_UUID_SIZE; 628 629 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, fpi_size, 630 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 631 if (!count_only) { 632 rxtx->tx_is_mine = false; 633 cpu_spin_unlock(&rxtx->spinlock); 634 } 635 } 636 637 static void spmc_handle_run(struct thread_smc_args *args) 638 { 639 uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); 640 uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); 641 uint32_t rc = FFA_OK; 642 643 if (endpoint != optee_endpoint_id) { 644 /* 645 * The endpoint should be an SP, try to resume the SP from 646 * preempted into busy state. 647 */ 648 rc = spmc_sp_resume_from_preempted(endpoint); 649 if (rc) 650 goto out; 651 } 652 653 thread_resume_from_rpc(thread_id, 0, 0, 0, 0); 654 655 /* thread_resume_from_rpc return only of the thread_id is invalid */ 656 rc = FFA_INVALID_PARAMETERS; 657 658 out: 659 set_simple_ret_val(args, rc); 660 } 661 #endif /*CFG_CORE_SEL1_SPMC*/ 662 663 static uint32_t spmc_enable_async_notif(uint32_t bottom_half_value, 664 uint16_t vm_id) 665 { 666 uint32_t old_itr_status = 0; 667 668 if (!spmc_notif_is_ready) { 669 /* 670 * This should never happen, not if normal world respects the 671 * exchanged capabilities. 672 */ 673 EMSG("Asynchronous notifications are not ready"); 674 return TEE_ERROR_NOT_IMPLEMENTED; 675 } 676 677 if (bottom_half_value >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE) { 678 EMSG("Invalid bottom half value %"PRIu32, bottom_half_value); 679 return TEE_ERROR_BAD_PARAMETERS; 680 } 681 682 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 683 do_bottom_half_value = bottom_half_value; 684 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) 685 notif_vm_id = vm_id; 686 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 687 688 notif_deliver_atomic_event(NOTIF_EVENT_STARTED); 689 return TEE_SUCCESS; 690 } 691 692 static void handle_yielding_call(struct thread_smc_args *args, 693 uint32_t direct_resp_fid) 694 { 695 TEE_Result res = 0; 696 697 thread_check_canaries(); 698 699 #ifdef ARM64 700 /* Saving this for an eventual RPC */ 701 thread_get_core_local()->direct_resp_fid = direct_resp_fid; 702 #endif 703 704 if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { 705 /* Note connection to struct thread_rpc_arg::ret */ 706 thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, 707 0); 708 res = TEE_ERROR_BAD_PARAMETERS; 709 } else { 710 thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, 711 args->a6, args->a7); 712 res = TEE_ERROR_BUSY; 713 } 714 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 715 0, res, 0, 0); 716 } 717 718 static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) 719 { 720 uint64_t cookie = reg_pair_to_64(a5, a4); 721 uint32_t res = 0; 722 723 res = mobj_ffa_unregister_by_cookie(cookie); 724 switch (res) { 725 case TEE_SUCCESS: 726 case TEE_ERROR_ITEM_NOT_FOUND: 727 return 0; 728 case TEE_ERROR_BUSY: 729 EMSG("res %#"PRIx32, res); 730 return FFA_BUSY; 731 default: 732 EMSG("res %#"PRIx32, res); 733 return FFA_INVALID_PARAMETERS; 734 } 735 } 736 737 static void handle_blocking_call(struct thread_smc_args *args, 738 uint32_t direct_resp_fid) 739 { 740 uint32_t sec_caps = 0; 741 742 switch (args->a3) { 743 case OPTEE_FFA_GET_API_VERSION: 744 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 745 OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, 746 0); 747 break; 748 case OPTEE_FFA_GET_OS_VERSION: 749 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 750 CFG_OPTEE_REVISION_MAJOR, 751 CFG_OPTEE_REVISION_MINOR, 752 TEE_IMPL_GIT_SHA1 >> 32); 753 break; 754 case OPTEE_FFA_EXCHANGE_CAPABILITIES: 755 sec_caps = OPTEE_FFA_SEC_CAP_ARG_OFFSET; 756 if (spmc_notif_is_ready) 757 sec_caps |= OPTEE_FFA_SEC_CAP_ASYNC_NOTIF; 758 spmc_set_args(args, direct_resp_fid, 759 swap_src_dst(args->a1), 0, 0, 760 THREAD_RPC_MAX_NUM_PARAMS, sec_caps); 761 break; 762 case OPTEE_FFA_UNREGISTER_SHM: 763 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 764 handle_unregister_shm(args->a4, args->a5), 0, 0); 765 break; 766 case OPTEE_FFA_ENABLE_ASYNC_NOTIF: 767 spmc_set_args(args, direct_resp_fid, 768 swap_src_dst(args->a1), 0, 769 spmc_enable_async_notif(args->a4, 770 FFA_SRC(args->a1)), 771 0, 0); 772 break; 773 default: 774 EMSG("Unhandled blocking service ID %#"PRIx32, 775 (uint32_t)args->a3); 776 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 777 TEE_ERROR_BAD_PARAMETERS, 0, 0); 778 } 779 } 780 781 static void handle_framework_direct_request(struct thread_smc_args *args, 782 struct ffa_rxtx *rxtx, 783 uint32_t direct_resp_fid) 784 { 785 uint32_t w0 = FFA_ERROR; 786 uint32_t w1 = FFA_PARAM_MBZ; 787 uint32_t w2 = FFA_NOT_SUPPORTED; 788 uint32_t w3 = FFA_PARAM_MBZ; 789 790 switch (args->a2 & FFA_MSG_TYPE_MASK) { 791 case FFA_MSG_SEND_VM_CREATED: 792 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 793 uint16_t guest_id = args->a5; 794 TEE_Result res = virt_guest_created(guest_id); 795 796 w0 = direct_resp_fid; 797 w1 = swap_src_dst(args->a1); 798 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; 799 if (res == TEE_SUCCESS) 800 w3 = FFA_OK; 801 else if (res == TEE_ERROR_OUT_OF_MEMORY) 802 w3 = FFA_DENIED; 803 else 804 w3 = FFA_INVALID_PARAMETERS; 805 } 806 break; 807 case FFA_MSG_SEND_VM_DESTROYED: 808 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 809 uint16_t guest_id = args->a5; 810 TEE_Result res = virt_guest_destroyed(guest_id); 811 812 w0 = direct_resp_fid; 813 w1 = swap_src_dst(args->a1); 814 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_DESTROYED; 815 if (res == TEE_SUCCESS) 816 w3 = FFA_OK; 817 else 818 w3 = FFA_INVALID_PARAMETERS; 819 } 820 break; 821 case FFA_MSG_VERSION_REQ: 822 w0 = direct_resp_fid; 823 w1 = swap_src_dst(args->a1); 824 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; 825 w3 = spmc_exchange_version(args->a3, rxtx); 826 break; 827 default: 828 break; 829 } 830 spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 831 } 832 833 static void handle_direct_request(struct thread_smc_args *args, 834 struct ffa_rxtx *rxtx) 835 { 836 uint32_t direct_resp_fid = 0; 837 838 if (IS_ENABLED(CFG_SECURE_PARTITION) && 839 FFA_DST(args->a1) != spmc_id && 840 FFA_DST(args->a1) != optee_endpoint_id) { 841 spmc_sp_start_thread(args); 842 return; 843 } 844 845 if (OPTEE_SMC_IS_64(args->a0)) 846 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_64; 847 else 848 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_32; 849 850 if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { 851 handle_framework_direct_request(args, rxtx, direct_resp_fid); 852 return; 853 } 854 855 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 856 virt_set_guest(get_sender_id(args->a1))) { 857 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 858 TEE_ERROR_ITEM_NOT_FOUND, 0, 0); 859 return; 860 } 861 862 if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) 863 handle_yielding_call(args, direct_resp_fid); 864 else 865 handle_blocking_call(args, direct_resp_fid); 866 867 /* 868 * Note that handle_yielding_call() typically only returns if a 869 * thread cannot be allocated or found. virt_unset_guest() is also 870 * called from thread_state_suspend() and thread_state_free(). 871 */ 872 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 873 virt_unset_guest(); 874 } 875 876 int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, 877 struct ffa_mem_transaction_x *trans) 878 { 879 uint16_t mem_reg_attr = 0; 880 uint32_t flags = 0; 881 uint32_t count = 0; 882 uint32_t offs = 0; 883 uint32_t size = 0; 884 size_t n = 0; 885 886 if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) 887 return FFA_INVALID_PARAMETERS; 888 889 if (ffa_vers >= FFA_VERSION_1_1) { 890 struct ffa_mem_transaction_1_1 *descr = NULL; 891 892 if (blen < sizeof(*descr)) 893 return FFA_INVALID_PARAMETERS; 894 895 descr = buf; 896 trans->sender_id = READ_ONCE(descr->sender_id); 897 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 898 flags = READ_ONCE(descr->flags); 899 trans->global_handle = READ_ONCE(descr->global_handle); 900 trans->tag = READ_ONCE(descr->tag); 901 902 count = READ_ONCE(descr->mem_access_count); 903 size = READ_ONCE(descr->mem_access_size); 904 offs = READ_ONCE(descr->mem_access_offs); 905 } else { 906 struct ffa_mem_transaction_1_0 *descr = NULL; 907 908 if (blen < sizeof(*descr)) 909 return FFA_INVALID_PARAMETERS; 910 911 descr = buf; 912 trans->sender_id = READ_ONCE(descr->sender_id); 913 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 914 flags = READ_ONCE(descr->flags); 915 trans->global_handle = READ_ONCE(descr->global_handle); 916 trans->tag = READ_ONCE(descr->tag); 917 918 count = READ_ONCE(descr->mem_access_count); 919 size = sizeof(struct ffa_mem_access); 920 offs = offsetof(struct ffa_mem_transaction_1_0, 921 mem_access_array); 922 } 923 924 if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || 925 size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) 926 return FFA_INVALID_PARAMETERS; 927 928 /* Check that the endpoint memory access descriptor array fits */ 929 if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || 930 n > blen) 931 return FFA_INVALID_PARAMETERS; 932 933 trans->mem_reg_attr = mem_reg_attr; 934 trans->flags = flags; 935 trans->mem_access_size = size; 936 trans->mem_access_count = count; 937 trans->mem_access_offs = offs; 938 return 0; 939 } 940 941 #if defined(CFG_CORE_SEL1_SPMC) 942 static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, 943 unsigned int mem_access_count, uint8_t *acc_perms, 944 unsigned int *region_offs) 945 { 946 struct ffa_mem_access_perm *descr = NULL; 947 struct ffa_mem_access *mem_acc = NULL; 948 unsigned int n = 0; 949 950 for (n = 0; n < mem_access_count; n++) { 951 mem_acc = (void *)(mem_acc_base + mem_access_size * n); 952 descr = &mem_acc->access_perm; 953 if (READ_ONCE(descr->endpoint_id) == optee_endpoint_id) { 954 *acc_perms = READ_ONCE(descr->perm); 955 *region_offs = READ_ONCE(mem_acc[n].region_offs); 956 return 0; 957 } 958 } 959 960 return FFA_INVALID_PARAMETERS; 961 } 962 963 static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, 964 size_t blen, unsigned int *page_count, 965 unsigned int *region_count, size_t *addr_range_offs) 966 { 967 const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 968 const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; 969 struct ffa_mem_region *region_descr = NULL; 970 unsigned int region_descr_offs = 0; 971 uint8_t mem_acc_perm = 0; 972 size_t n = 0; 973 974 if (mem_trans->mem_reg_attr != exp_mem_reg_attr) 975 return FFA_INVALID_PARAMETERS; 976 977 /* Check that the access permissions matches what's expected */ 978 if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, 979 mem_trans->mem_access_size, 980 mem_trans->mem_access_count, 981 &mem_acc_perm, ®ion_descr_offs) || 982 mem_acc_perm != exp_mem_acc_perm) 983 return FFA_INVALID_PARAMETERS; 984 985 /* Check that the Composite memory region descriptor fits */ 986 if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || 987 n > blen) 988 return FFA_INVALID_PARAMETERS; 989 990 if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, 991 struct ffa_mem_region)) 992 return FFA_INVALID_PARAMETERS; 993 994 region_descr = (struct ffa_mem_region *)((vaddr_t)buf + 995 region_descr_offs); 996 *page_count = READ_ONCE(region_descr->total_page_count); 997 *region_count = READ_ONCE(region_descr->address_range_count); 998 *addr_range_offs = n; 999 return 0; 1000 } 1001 1002 static int add_mem_share_helper(struct mem_share_state *s, void *buf, 1003 size_t flen) 1004 { 1005 unsigned int region_count = flen / sizeof(struct ffa_address_range); 1006 struct ffa_address_range *arange = NULL; 1007 unsigned int n = 0; 1008 1009 if (region_count > s->region_count) 1010 region_count = s->region_count; 1011 1012 if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) 1013 return FFA_INVALID_PARAMETERS; 1014 arange = buf; 1015 1016 for (n = 0; n < region_count; n++) { 1017 unsigned int page_count = READ_ONCE(arange[n].page_count); 1018 uint64_t addr = READ_ONCE(arange[n].address); 1019 1020 if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, 1021 addr, page_count)) 1022 return FFA_INVALID_PARAMETERS; 1023 } 1024 1025 s->region_count -= region_count; 1026 if (s->region_count) 1027 return region_count * sizeof(*arange); 1028 1029 if (s->current_page_idx != s->page_count) 1030 return FFA_INVALID_PARAMETERS; 1031 1032 return 0; 1033 } 1034 1035 static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) 1036 { 1037 int rc = 0; 1038 1039 rc = add_mem_share_helper(&s->share, buf, flen); 1040 if (rc >= 0) { 1041 if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { 1042 /* We're not at the end of the descriptor yet */ 1043 if (s->share.region_count) 1044 return s->frag_offset; 1045 1046 /* We're done */ 1047 rc = 0; 1048 } else { 1049 rc = FFA_INVALID_PARAMETERS; 1050 } 1051 } 1052 1053 SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); 1054 if (rc < 0) 1055 mobj_ffa_sel1_spmc_delete(s->share.mf); 1056 else 1057 mobj_ffa_push_to_inactive(s->share.mf); 1058 free(s); 1059 1060 return rc; 1061 } 1062 1063 static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, 1064 void *buf) 1065 { 1066 struct ffa_mem_access_perm *perm = NULL; 1067 struct ffa_mem_access *mem_acc = NULL; 1068 1069 if (!IS_ENABLED(CFG_SECURE_PARTITION)) 1070 return false; 1071 1072 if (mem_trans->mem_access_count < 1) 1073 return false; 1074 1075 mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); 1076 perm = &mem_acc->access_perm; 1077 1078 /* 1079 * perm->endpoint_id is read here only to check if the endpoint is 1080 * OP-TEE. We do read it later on again, but there are some additional 1081 * checks there to make sure that the data is correct. 1082 */ 1083 return READ_ONCE(perm->endpoint_id) != optee_endpoint_id; 1084 } 1085 1086 static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, 1087 tee_mm_entry_t *mm, void *buf, size_t blen, 1088 size_t flen, uint64_t *global_handle) 1089 { 1090 int rc = 0; 1091 struct mem_share_state share = { }; 1092 size_t addr_range_offs = 0; 1093 uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1094 size_t n = 0; 1095 1096 rc = mem_share_init(mem_trans, buf, flen, &share.page_count, 1097 &share.region_count, &addr_range_offs); 1098 if (rc) 1099 return rc; 1100 1101 if (MUL_OVERFLOW(share.region_count, 1102 sizeof(struct ffa_address_range), &n) || 1103 ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) 1104 return FFA_INVALID_PARAMETERS; 1105 1106 if (mem_trans->global_handle) 1107 cookie = mem_trans->global_handle; 1108 share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); 1109 if (!share.mf) 1110 return FFA_NO_MEMORY; 1111 1112 if (flen != blen) { 1113 struct mem_frag_state *s = calloc(sizeof(*s), 1); 1114 1115 if (!s) { 1116 rc = FFA_NO_MEMORY; 1117 goto err; 1118 } 1119 s->share = share; 1120 s->mm = mm; 1121 s->frag_offset = addr_range_offs; 1122 1123 SLIST_INSERT_HEAD(&frag_state_head, s, link); 1124 rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, 1125 flen - addr_range_offs); 1126 1127 if (rc >= 0) 1128 *global_handle = mobj_ffa_get_cookie(share.mf); 1129 1130 return rc; 1131 } 1132 1133 rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, 1134 flen - addr_range_offs); 1135 if (rc) { 1136 /* 1137 * Number of consumed bytes may be returned instead of 0 for 1138 * done. 1139 */ 1140 rc = FFA_INVALID_PARAMETERS; 1141 goto err; 1142 } 1143 1144 *global_handle = mobj_ffa_push_to_inactive(share.mf); 1145 1146 return 0; 1147 err: 1148 mobj_ffa_sel1_spmc_delete(share.mf); 1149 return rc; 1150 } 1151 1152 static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, 1153 unsigned int page_count, 1154 uint64_t *global_handle, struct ffa_rxtx *rxtx) 1155 { 1156 struct ffa_mem_transaction_x mem_trans = { }; 1157 int rc = 0; 1158 size_t len = 0; 1159 void *buf = NULL; 1160 tee_mm_entry_t *mm = NULL; 1161 vaddr_t offs = pbuf & SMALL_PAGE_MASK; 1162 1163 if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) 1164 return FFA_INVALID_PARAMETERS; 1165 if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) 1166 return FFA_INVALID_PARAMETERS; 1167 1168 /* 1169 * Check that the length reported in flen is covered by len even 1170 * if the offset is taken into account. 1171 */ 1172 if (len < flen || len - offs < flen) 1173 return FFA_INVALID_PARAMETERS; 1174 1175 mm = tee_mm_alloc(&core_virt_shm_pool, len); 1176 if (!mm) 1177 return FFA_NO_MEMORY; 1178 1179 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, 1180 page_count, MEM_AREA_NSEC_SHM)) { 1181 rc = FFA_INVALID_PARAMETERS; 1182 goto out; 1183 } 1184 buf = (void *)(tee_mm_get_smem(mm) + offs); 1185 1186 cpu_spin_lock(&rxtx->spinlock); 1187 rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); 1188 if (rc) 1189 goto unlock; 1190 1191 if (is_sp_share(&mem_trans, buf)) { 1192 rc = spmc_sp_add_share(&mem_trans, buf, blen, flen, 1193 global_handle, NULL); 1194 goto unlock; 1195 } 1196 1197 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1198 virt_set_guest(mem_trans.sender_id)) { 1199 rc = FFA_DENIED; 1200 goto unlock; 1201 } 1202 1203 rc = add_mem_share(&mem_trans, mm, buf, blen, flen, global_handle); 1204 1205 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1206 virt_unset_guest(); 1207 1208 unlock: 1209 cpu_spin_unlock(&rxtx->spinlock); 1210 if (rc > 0) 1211 return rc; 1212 1213 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1214 out: 1215 tee_mm_free(mm); 1216 return rc; 1217 } 1218 1219 static int handle_mem_share_rxbuf(size_t blen, size_t flen, 1220 uint64_t *global_handle, 1221 struct ffa_rxtx *rxtx) 1222 { 1223 struct ffa_mem_transaction_x mem_trans = { }; 1224 int rc = FFA_DENIED; 1225 1226 cpu_spin_lock(&rxtx->spinlock); 1227 1228 if (!rxtx->rx || flen > rxtx->size) 1229 goto out; 1230 1231 rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, 1232 &mem_trans); 1233 if (rc) 1234 goto out; 1235 if (is_sp_share(&mem_trans, rxtx->rx)) { 1236 rc = spmc_sp_add_share(&mem_trans, rxtx, blen, flen, 1237 global_handle, NULL); 1238 goto out; 1239 } 1240 1241 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1242 virt_set_guest(mem_trans.sender_id)) 1243 goto out; 1244 1245 rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, 1246 global_handle); 1247 1248 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1249 virt_unset_guest(); 1250 1251 out: 1252 cpu_spin_unlock(&rxtx->spinlock); 1253 1254 return rc; 1255 } 1256 1257 static void handle_mem_share(struct thread_smc_args *args, 1258 struct ffa_rxtx *rxtx) 1259 { 1260 uint32_t tot_len = args->a1; 1261 uint32_t frag_len = args->a2; 1262 uint64_t addr = args->a3; 1263 uint32_t page_count = args->a4; 1264 uint32_t ret_w1 = 0; 1265 uint32_t ret_w2 = FFA_INVALID_PARAMETERS; 1266 uint32_t ret_w3 = 0; 1267 uint32_t ret_fid = FFA_ERROR; 1268 uint64_t global_handle = 0; 1269 int rc = 0; 1270 1271 /* Check that the MBZs are indeed 0 */ 1272 if (args->a5 || args->a6 || args->a7) 1273 goto out; 1274 1275 /* Check that fragment length doesn't exceed total length */ 1276 if (frag_len > tot_len) 1277 goto out; 1278 1279 /* Check for 32-bit calling convention */ 1280 if (args->a0 == FFA_MEM_SHARE_32) 1281 addr &= UINT32_MAX; 1282 1283 if (!addr) { 1284 /* 1285 * The memory transaction descriptor is passed via our rx 1286 * buffer. 1287 */ 1288 if (page_count) 1289 goto out; 1290 rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, 1291 rxtx); 1292 } else { 1293 rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, 1294 &global_handle, rxtx); 1295 } 1296 if (rc < 0) { 1297 ret_w2 = rc; 1298 } else if (rc > 0) { 1299 ret_fid = FFA_MEM_FRAG_RX; 1300 ret_w3 = rc; 1301 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1302 } else { 1303 ret_fid = FFA_SUCCESS_32; 1304 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1305 } 1306 out: 1307 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1308 } 1309 1310 static struct mem_frag_state *get_frag_state(uint64_t global_handle) 1311 { 1312 struct mem_frag_state *s = NULL; 1313 1314 SLIST_FOREACH(s, &frag_state_head, link) 1315 if (mobj_ffa_get_cookie(s->share.mf) == global_handle) 1316 return s; 1317 1318 return NULL; 1319 } 1320 1321 static void handle_mem_frag_tx(struct thread_smc_args *args, 1322 struct ffa_rxtx *rxtx) 1323 { 1324 uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); 1325 size_t flen = args->a3; 1326 uint32_t endpoint_id = args->a4; 1327 struct mem_frag_state *s = NULL; 1328 tee_mm_entry_t *mm = NULL; 1329 unsigned int page_count = 0; 1330 void *buf = NULL; 1331 uint32_t ret_w1 = 0; 1332 uint32_t ret_w2 = 0; 1333 uint32_t ret_w3 = 0; 1334 uint32_t ret_fid = 0; 1335 int rc = 0; 1336 1337 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1338 uint16_t guest_id = endpoint_id >> 16; 1339 1340 if (!guest_id || virt_set_guest(guest_id)) { 1341 rc = FFA_INVALID_PARAMETERS; 1342 goto out_set_rc; 1343 } 1344 } 1345 1346 /* 1347 * Currently we're only doing this for fragmented FFA_MEM_SHARE_* 1348 * requests. 1349 */ 1350 1351 cpu_spin_lock(&rxtx->spinlock); 1352 1353 s = get_frag_state(global_handle); 1354 if (!s) { 1355 rc = FFA_INVALID_PARAMETERS; 1356 goto out; 1357 } 1358 1359 mm = s->mm; 1360 if (mm) { 1361 if (flen > tee_mm_get_bytes(mm)) { 1362 rc = FFA_INVALID_PARAMETERS; 1363 goto out; 1364 } 1365 page_count = s->share.page_count; 1366 buf = (void *)tee_mm_get_smem(mm); 1367 } else { 1368 if (flen > rxtx->size) { 1369 rc = FFA_INVALID_PARAMETERS; 1370 goto out; 1371 } 1372 buf = rxtx->rx; 1373 } 1374 1375 rc = add_mem_share_frag(s, buf, flen); 1376 out: 1377 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1378 virt_unset_guest(); 1379 1380 cpu_spin_unlock(&rxtx->spinlock); 1381 1382 if (rc <= 0 && mm) { 1383 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1384 tee_mm_free(mm); 1385 } 1386 1387 out_set_rc: 1388 if (rc < 0) { 1389 ret_fid = FFA_ERROR; 1390 ret_w2 = rc; 1391 } else if (rc > 0) { 1392 ret_fid = FFA_MEM_FRAG_RX; 1393 ret_w3 = rc; 1394 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1395 } else { 1396 ret_fid = FFA_SUCCESS_32; 1397 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1398 } 1399 1400 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1401 } 1402 1403 static void handle_mem_reclaim(struct thread_smc_args *args) 1404 { 1405 int rc = FFA_INVALID_PARAMETERS; 1406 uint64_t cookie = 0; 1407 1408 if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) 1409 goto out; 1410 1411 cookie = reg_pair_to_64(args->a2, args->a1); 1412 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1413 uint16_t guest_id = 0; 1414 1415 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 1416 guest_id = virt_find_guest_by_cookie(cookie); 1417 } else { 1418 guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & 1419 FFA_MEMORY_HANDLE_PRTN_MASK; 1420 } 1421 if (!guest_id) 1422 goto out; 1423 if (virt_set_guest(guest_id)) { 1424 if (!virt_reclaim_cookie_from_destroyed_guest(guest_id, 1425 cookie)) 1426 rc = FFA_OK; 1427 goto out; 1428 } 1429 } 1430 1431 switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { 1432 case TEE_SUCCESS: 1433 rc = FFA_OK; 1434 break; 1435 case TEE_ERROR_ITEM_NOT_FOUND: 1436 DMSG("cookie %#"PRIx64" not found", cookie); 1437 rc = FFA_INVALID_PARAMETERS; 1438 break; 1439 default: 1440 DMSG("cookie %#"PRIx64" busy", cookie); 1441 rc = FFA_DENIED; 1442 break; 1443 } 1444 1445 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1446 virt_unset_guest(); 1447 1448 out: 1449 set_simple_ret_val(args, rc); 1450 } 1451 1452 static void handle_notification_bitmap_create(struct thread_smc_args *args) 1453 { 1454 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1455 uint32_t ret_fid = FFA_ERROR; 1456 uint32_t old_itr_status = 0; 1457 1458 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1459 !args->a5 && !args->a6 && !args->a7) { 1460 uint16_t vm_id = args->a1; 1461 1462 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1463 1464 if (notif_vm_id_valid) { 1465 if (vm_id == notif_vm_id) 1466 ret_val = FFA_DENIED; 1467 else 1468 ret_val = FFA_NO_MEMORY; 1469 } else { 1470 notif_vm_id = vm_id; 1471 notif_vm_id_valid = true; 1472 ret_val = FFA_OK; 1473 ret_fid = FFA_SUCCESS_32; 1474 } 1475 1476 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1477 } 1478 1479 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1480 } 1481 1482 static void handle_notification_bitmap_destroy(struct thread_smc_args *args) 1483 { 1484 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1485 uint32_t ret_fid = FFA_ERROR; 1486 uint32_t old_itr_status = 0; 1487 1488 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1489 !args->a5 && !args->a6 && !args->a7) { 1490 uint16_t vm_id = args->a1; 1491 1492 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1493 1494 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1495 if (notif_pending_bitmap || notif_bound_bitmap) { 1496 ret_val = FFA_DENIED; 1497 } else { 1498 notif_vm_id_valid = false; 1499 ret_val = FFA_OK; 1500 ret_fid = FFA_SUCCESS_32; 1501 } 1502 } 1503 1504 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1505 } 1506 1507 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1508 } 1509 1510 static void handle_notification_bind(struct thread_smc_args *args) 1511 { 1512 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1513 uint32_t ret_fid = FFA_ERROR; 1514 uint32_t old_itr_status = 0; 1515 uint64_t bitmap = 0; 1516 uint16_t vm_id = 0; 1517 1518 if (args->a5 || args->a6 || args->a7) 1519 goto out; 1520 if (args->a2) { 1521 /* We only deal with global notifications for now */ 1522 ret_val = FFA_NOT_SUPPORTED; 1523 goto out; 1524 } 1525 1526 /* The destination of the eventual notification */ 1527 vm_id = FFA_DST(args->a1); 1528 bitmap = reg_pair_to_64(args->a4, args->a3); 1529 1530 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1531 1532 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1533 if (bitmap & notif_bound_bitmap) { 1534 ret_val = FFA_DENIED; 1535 } else { 1536 notif_bound_bitmap |= bitmap; 1537 ret_val = FFA_OK; 1538 ret_fid = FFA_SUCCESS_32; 1539 } 1540 } 1541 1542 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1543 out: 1544 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1545 } 1546 1547 static void handle_notification_unbind(struct thread_smc_args *args) 1548 { 1549 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1550 uint32_t ret_fid = FFA_ERROR; 1551 uint32_t old_itr_status = 0; 1552 uint64_t bitmap = 0; 1553 uint16_t vm_id = 0; 1554 1555 if (args->a2 || args->a5 || args->a6 || args->a7) 1556 goto out; 1557 1558 /* The destination of the eventual notification */ 1559 vm_id = FFA_DST(args->a1); 1560 bitmap = reg_pair_to_64(args->a4, args->a3); 1561 1562 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1563 1564 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1565 /* 1566 * Spec says: 1567 * At least one notification is bound to another Sender or 1568 * is currently pending. 1569 * 1570 * Not sure what the intention is. 1571 */ 1572 if (bitmap & notif_pending_bitmap) { 1573 ret_val = FFA_DENIED; 1574 } else { 1575 notif_bound_bitmap &= ~bitmap; 1576 ret_val = FFA_OK; 1577 ret_fid = FFA_SUCCESS_32; 1578 } 1579 } 1580 1581 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1582 out: 1583 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1584 } 1585 1586 static void handle_notification_get(struct thread_smc_args *args) 1587 { 1588 uint32_t w2 = FFA_INVALID_PARAMETERS; 1589 uint32_t ret_fid = FFA_ERROR; 1590 uint32_t old_itr_status = 0; 1591 uint16_t vm_id = 0; 1592 uint32_t w3 = 0; 1593 1594 if (args->a5 || args->a6 || args->a7) 1595 goto out; 1596 if (!(args->a2 & 0x1)) { 1597 ret_fid = FFA_SUCCESS_32; 1598 w2 = 0; 1599 goto out; 1600 } 1601 vm_id = FFA_DST(args->a1); 1602 1603 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1604 1605 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1606 reg_pair_from_64(notif_pending_bitmap, &w3, &w2); 1607 notif_pending_bitmap = 0; 1608 ret_fid = FFA_SUCCESS_32; 1609 } 1610 1611 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1612 out: 1613 spmc_set_args(args, ret_fid, 0, w2, w3, 0, 0); 1614 } 1615 1616 static void handle_notification_info_get(struct thread_smc_args *args) 1617 { 1618 uint32_t w2 = FFA_INVALID_PARAMETERS; 1619 uint32_t ret_fid = FFA_ERROR; 1620 1621 if (args->a1 || args->a2 || args->a3 || args->a4 || args->a5 || 1622 args->a6 || args->a7) 1623 goto out; 1624 1625 if (OPTEE_SMC_IS_64(args->a0)) 1626 ret_fid = FFA_SUCCESS_64; 1627 else 1628 ret_fid = FFA_SUCCESS_32; 1629 1630 /* 1631 * Note, we're only supporting physical OS kernel in normal world 1632 * with Global Notifications. 1633 * So one list of ID list registers (BIT[11:7]) 1634 * and one count of IDs (BIT[13:12] + 1) 1635 * and the VM is always 0. 1636 */ 1637 w2 = SHIFT_U32(1, 7); 1638 out: 1639 spmc_set_args(args, ret_fid, 0, w2, 0, 0, 0); 1640 } 1641 1642 void thread_spmc_set_async_notif_intid(int intid) 1643 { 1644 assert(interrupt_can_raise_sgi(interrupt_get_main_chip())); 1645 notif_intid = intid; 1646 spmc_notif_is_ready = true; 1647 DMSG("Asynchronous notifications are ready"); 1648 } 1649 1650 void notif_send_async(uint32_t value) 1651 { 1652 uint32_t old_itr_status = 0; 1653 1654 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1655 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && spmc_notif_is_ready && 1656 do_bottom_half_value >= 0 && notif_intid >= 0); 1657 notif_pending_bitmap |= BIT64(do_bottom_half_value); 1658 interrupt_raise_sgi(interrupt_get_main_chip(), notif_intid, 1659 ITR_CPU_MASK_TO_THIS_CPU); 1660 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1661 } 1662 #else 1663 void notif_send_async(uint32_t value) 1664 { 1665 /* global notification, delay notification interrupt */ 1666 uint32_t flags = BIT32(1); 1667 int res = 0; 1668 1669 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && spmc_notif_is_ready && 1670 do_bottom_half_value >= 0); 1671 res = ffa_set_notification(notif_vm_id, optee_endpoint_id, flags, 1672 BIT64(do_bottom_half_value)); 1673 if (res) { 1674 EMSG("notification set failed with error %d", res); 1675 panic(); 1676 } 1677 } 1678 #endif 1679 1680 /* Only called from assembly */ 1681 void thread_spmc_msg_recv(struct thread_smc_args *args); 1682 void thread_spmc_msg_recv(struct thread_smc_args *args) 1683 { 1684 assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); 1685 switch (args->a0) { 1686 #if defined(CFG_CORE_SEL1_SPMC) 1687 case FFA_FEATURES: 1688 handle_features(args); 1689 break; 1690 case FFA_SPM_ID_GET: 1691 spmc_handle_spm_id_get(args); 1692 break; 1693 #ifdef ARM64 1694 case FFA_RXTX_MAP_64: 1695 #endif 1696 case FFA_RXTX_MAP_32: 1697 spmc_handle_rxtx_map(args, &my_rxtx); 1698 break; 1699 case FFA_RXTX_UNMAP: 1700 spmc_handle_rxtx_unmap(args, &my_rxtx); 1701 break; 1702 case FFA_RX_RELEASE: 1703 spmc_handle_rx_release(args, &my_rxtx); 1704 break; 1705 case FFA_PARTITION_INFO_GET: 1706 spmc_handle_partition_info_get(args, &my_rxtx); 1707 break; 1708 case FFA_RUN: 1709 spmc_handle_run(args); 1710 break; 1711 #endif /*CFG_CORE_SEL1_SPMC*/ 1712 case FFA_INTERRUPT: 1713 if (IS_ENABLED(CFG_CORE_SEL1_SPMC)) 1714 spmc_set_args(args, FFA_NORMAL_WORLD_RESUME, 0, 0, 0, 1715 0, 0); 1716 else 1717 spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); 1718 break; 1719 #ifdef ARM64 1720 case FFA_MSG_SEND_DIRECT_REQ_64: 1721 #endif 1722 case FFA_MSG_SEND_DIRECT_REQ_32: 1723 handle_direct_request(args, &my_rxtx); 1724 break; 1725 #if defined(CFG_CORE_SEL1_SPMC) 1726 #ifdef ARM64 1727 case FFA_MEM_SHARE_64: 1728 #endif 1729 case FFA_MEM_SHARE_32: 1730 handle_mem_share(args, &my_rxtx); 1731 break; 1732 case FFA_MEM_RECLAIM: 1733 if (!IS_ENABLED(CFG_SECURE_PARTITION) || 1734 !ffa_mem_reclaim(args, NULL)) 1735 handle_mem_reclaim(args); 1736 break; 1737 case FFA_MEM_FRAG_TX: 1738 handle_mem_frag_tx(args, &my_rxtx); 1739 break; 1740 case FFA_NOTIFICATION_BITMAP_CREATE: 1741 handle_notification_bitmap_create(args); 1742 break; 1743 case FFA_NOTIFICATION_BITMAP_DESTROY: 1744 handle_notification_bitmap_destroy(args); 1745 break; 1746 case FFA_NOTIFICATION_BIND: 1747 handle_notification_bind(args); 1748 break; 1749 case FFA_NOTIFICATION_UNBIND: 1750 handle_notification_unbind(args); 1751 break; 1752 case FFA_NOTIFICATION_GET: 1753 handle_notification_get(args); 1754 break; 1755 #ifdef ARM64 1756 case FFA_NOTIFICATION_INFO_GET_64: 1757 #endif 1758 case FFA_NOTIFICATION_INFO_GET_32: 1759 handle_notification_info_get(args); 1760 break; 1761 #endif /*CFG_CORE_SEL1_SPMC*/ 1762 case FFA_ERROR: 1763 EMSG("Cannot handle FFA_ERROR(%d)", (int)args->a2); 1764 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { 1765 /* 1766 * The SPMC will return an FFA_ERROR back so better 1767 * panic() now than flooding the log. 1768 */ 1769 panic("FFA_ERROR from SPMC is fatal"); 1770 } 1771 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, 1772 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 1773 break; 1774 default: 1775 EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); 1776 set_simple_ret_val(args, FFA_NOT_SUPPORTED); 1777 } 1778 } 1779 1780 static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) 1781 { 1782 size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1783 struct thread_ctx *thr = threads + thread_get_id(); 1784 TEE_Result res = TEE_ERROR_BAD_PARAMETERS; 1785 struct optee_msg_arg *arg = NULL; 1786 struct mobj *mobj = NULL; 1787 uint32_t num_params = 0; 1788 size_t sz = 0; 1789 1790 mobj = mobj_ffa_get_by_cookie(cookie, 0); 1791 if (!mobj) { 1792 EMSG("Can't find cookie %#"PRIx64, cookie); 1793 return TEE_ERROR_BAD_PARAMETERS; 1794 } 1795 1796 res = mobj_inc_map(mobj); 1797 if (res) 1798 goto out_put_mobj; 1799 1800 res = TEE_ERROR_BAD_PARAMETERS; 1801 arg = mobj_get_va(mobj, offset, sizeof(*arg)); 1802 if (!arg) 1803 goto out_dec_map; 1804 1805 num_params = READ_ONCE(arg->num_params); 1806 if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) 1807 goto out_dec_map; 1808 1809 sz = OPTEE_MSG_GET_ARG_SIZE(num_params); 1810 1811 thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); 1812 if (!thr->rpc_arg) 1813 goto out_dec_map; 1814 1815 virt_on_stdcall(); 1816 res = tee_entry_std(arg, num_params); 1817 1818 thread_rpc_shm_cache_clear(&thr->shm_cache); 1819 thr->rpc_arg = NULL; 1820 1821 out_dec_map: 1822 mobj_dec_map(mobj); 1823 out_put_mobj: 1824 mobj_put(mobj); 1825 return res; 1826 } 1827 1828 /* 1829 * Helper routine for the assembly function thread_std_smc_entry() 1830 * 1831 * Note: this function is weak just to make link_dummies_paged.c happy. 1832 */ 1833 uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, 1834 uint32_t a2, uint32_t a3, 1835 uint32_t a4, uint32_t a5 __unused) 1836 { 1837 /* 1838 * Arguments are supplied from handle_yielding_call() as: 1839 * a0 <- w1 1840 * a1 <- w3 1841 * a2 <- w4 1842 * a3 <- w5 1843 * a4 <- w6 1844 * a5 <- w7 1845 */ 1846 thread_get_tsd()->rpc_target_info = swap_src_dst(a0); 1847 if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) 1848 return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); 1849 return FFA_DENIED; 1850 } 1851 1852 static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) 1853 { 1854 uint64_t offs = tpm->u.memref.offs; 1855 1856 param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + 1857 OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; 1858 1859 param->u.fmem.offs_low = offs; 1860 param->u.fmem.offs_high = offs >> 32; 1861 if (param->u.fmem.offs_high != offs >> 32) 1862 return false; 1863 1864 param->u.fmem.size = tpm->u.memref.size; 1865 if (tpm->u.memref.mobj) { 1866 uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); 1867 1868 /* If a mobj is passed it better be one with a valid cookie. */ 1869 if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) 1870 return false; 1871 param->u.fmem.global_id = cookie; 1872 } else { 1873 param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1874 } 1875 1876 return true; 1877 } 1878 1879 static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, 1880 struct thread_param *params, 1881 struct optee_msg_arg **arg_ret) 1882 { 1883 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1884 struct thread_ctx *thr = threads + thread_get_id(); 1885 struct optee_msg_arg *arg = thr->rpc_arg; 1886 1887 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 1888 return TEE_ERROR_BAD_PARAMETERS; 1889 1890 if (!arg) { 1891 EMSG("rpc_arg not set"); 1892 return TEE_ERROR_GENERIC; 1893 } 1894 1895 memset(arg, 0, sz); 1896 arg->cmd = cmd; 1897 arg->num_params = num_params; 1898 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 1899 1900 for (size_t n = 0; n < num_params; n++) { 1901 switch (params[n].attr) { 1902 case THREAD_PARAM_ATTR_NONE: 1903 arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; 1904 break; 1905 case THREAD_PARAM_ATTR_VALUE_IN: 1906 case THREAD_PARAM_ATTR_VALUE_OUT: 1907 case THREAD_PARAM_ATTR_VALUE_INOUT: 1908 arg->params[n].attr = params[n].attr - 1909 THREAD_PARAM_ATTR_VALUE_IN + 1910 OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 1911 arg->params[n].u.value.a = params[n].u.value.a; 1912 arg->params[n].u.value.b = params[n].u.value.b; 1913 arg->params[n].u.value.c = params[n].u.value.c; 1914 break; 1915 case THREAD_PARAM_ATTR_MEMREF_IN: 1916 case THREAD_PARAM_ATTR_MEMREF_OUT: 1917 case THREAD_PARAM_ATTR_MEMREF_INOUT: 1918 if (!set_fmem(arg->params + n, params + n)) 1919 return TEE_ERROR_BAD_PARAMETERS; 1920 break; 1921 default: 1922 return TEE_ERROR_BAD_PARAMETERS; 1923 } 1924 } 1925 1926 if (arg_ret) 1927 *arg_ret = arg; 1928 1929 return TEE_SUCCESS; 1930 } 1931 1932 static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, 1933 struct thread_param *params) 1934 { 1935 for (size_t n = 0; n < num_params; n++) { 1936 switch (params[n].attr) { 1937 case THREAD_PARAM_ATTR_VALUE_OUT: 1938 case THREAD_PARAM_ATTR_VALUE_INOUT: 1939 params[n].u.value.a = arg->params[n].u.value.a; 1940 params[n].u.value.b = arg->params[n].u.value.b; 1941 params[n].u.value.c = arg->params[n].u.value.c; 1942 break; 1943 case THREAD_PARAM_ATTR_MEMREF_OUT: 1944 case THREAD_PARAM_ATTR_MEMREF_INOUT: 1945 params[n].u.memref.size = arg->params[n].u.fmem.size; 1946 break; 1947 default: 1948 break; 1949 } 1950 } 1951 1952 return arg->ret; 1953 } 1954 1955 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 1956 struct thread_param *params) 1957 { 1958 struct thread_rpc_arg rpc_arg = { .call = { 1959 .w1 = thread_get_tsd()->rpc_target_info, 1960 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1961 }, 1962 }; 1963 struct optee_msg_arg *arg = NULL; 1964 uint32_t ret = 0; 1965 1966 ret = get_rpc_arg(cmd, num_params, params, &arg); 1967 if (ret) 1968 return ret; 1969 1970 thread_rpc(&rpc_arg); 1971 1972 return get_rpc_arg_res(arg, num_params, params); 1973 } 1974 1975 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 1976 { 1977 struct thread_rpc_arg rpc_arg = { .call = { 1978 .w1 = thread_get_tsd()->rpc_target_info, 1979 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1980 }, 1981 }; 1982 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); 1983 uint32_t res2 = 0; 1984 uint32_t res = 0; 1985 1986 DMSG("freeing cookie %#"PRIx64, cookie); 1987 1988 res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); 1989 1990 mobj_put(mobj); 1991 res2 = mobj_ffa_unregister_by_cookie(cookie); 1992 if (res2) 1993 DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, 1994 cookie, res2); 1995 if (!res) 1996 thread_rpc(&rpc_arg); 1997 } 1998 1999 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) 2000 { 2001 struct thread_rpc_arg rpc_arg = { .call = { 2002 .w1 = thread_get_tsd()->rpc_target_info, 2003 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2004 }, 2005 }; 2006 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); 2007 struct optee_msg_arg *arg = NULL; 2008 unsigned int internal_offset = 0; 2009 struct mobj *mobj = NULL; 2010 uint64_t cookie = 0; 2011 2012 if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) 2013 return NULL; 2014 2015 thread_rpc(&rpc_arg); 2016 2017 if (arg->num_params != 1 || 2018 arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) 2019 return NULL; 2020 2021 internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); 2022 cookie = READ_ONCE(arg->params->u.fmem.global_id); 2023 mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); 2024 if (!mobj) { 2025 DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", 2026 cookie, internal_offset); 2027 return NULL; 2028 } 2029 2030 assert(mobj_is_nonsec(mobj)); 2031 2032 if (mobj->size < size) { 2033 DMSG("Mobj %#"PRIx64": wrong size", cookie); 2034 mobj_put(mobj); 2035 return NULL; 2036 } 2037 2038 if (mobj_inc_map(mobj)) { 2039 DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); 2040 mobj_put(mobj); 2041 return NULL; 2042 } 2043 2044 return mobj; 2045 } 2046 2047 struct mobj *thread_rpc_alloc_payload(size_t size) 2048 { 2049 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); 2050 } 2051 2052 struct mobj *thread_rpc_alloc_kernel_payload(size_t size) 2053 { 2054 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); 2055 } 2056 2057 void thread_rpc_free_kernel_payload(struct mobj *mobj) 2058 { 2059 if (mobj) 2060 thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, 2061 mobj_get_cookie(mobj), mobj); 2062 } 2063 2064 void thread_rpc_free_payload(struct mobj *mobj) 2065 { 2066 if (mobj) 2067 thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), 2068 mobj); 2069 } 2070 2071 struct mobj *thread_rpc_alloc_global_payload(size_t size) 2072 { 2073 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); 2074 } 2075 2076 void thread_rpc_free_global_payload(struct mobj *mobj) 2077 { 2078 if (mobj) 2079 thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, 2080 mobj_get_cookie(mobj), mobj); 2081 } 2082 2083 void thread_spmc_register_secondary_ep(vaddr_t ep) 2084 { 2085 unsigned long ret = 0; 2086 2087 /* Let the SPM know the entry point for secondary CPUs */ 2088 ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); 2089 2090 if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) 2091 EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); 2092 } 2093 2094 static uint16_t ffa_id_get(void) 2095 { 2096 /* 2097 * Ask the SPM component running at a higher EL to return our FF-A ID. 2098 * This can either be the SPMC ID (if the SPMC is enabled in OP-TEE) or 2099 * the partition ID (if not). 2100 */ 2101 struct thread_smc_args args = { 2102 .a0 = FFA_ID_GET, 2103 }; 2104 2105 thread_smccc(&args); 2106 if (!is_ffa_success(args.a0)) { 2107 if (args.a0 == FFA_ERROR) 2108 EMSG("Get id failed with error %ld", args.a2); 2109 else 2110 EMSG("Get id failed"); 2111 panic(); 2112 } 2113 2114 return args.a2; 2115 } 2116 2117 static uint16_t ffa_spm_id_get(void) 2118 { 2119 /* 2120 * Ask the SPM component running at a higher EL to return its ID. 2121 * If OP-TEE implements the S-EL1 SPMC, this will get the SPMD ID. 2122 * If not, the ID of the SPMC will be returned. 2123 */ 2124 struct thread_smc_args args = { 2125 .a0 = FFA_SPM_ID_GET, 2126 }; 2127 2128 thread_smccc(&args); 2129 if (!is_ffa_success(args.a0)) { 2130 if (args.a0 == FFA_ERROR) 2131 EMSG("Get spm id failed with error %ld", args.a2); 2132 else 2133 EMSG("Get spm id failed"); 2134 panic(); 2135 } 2136 2137 return args.a2; 2138 } 2139 2140 #if defined(CFG_CORE_SEL1_SPMC) 2141 static TEE_Result spmc_init(void) 2142 { 2143 spmd_id = ffa_spm_id_get(); 2144 DMSG("SPMD ID %#"PRIx16, spmd_id); 2145 2146 spmc_id = ffa_id_get(); 2147 DMSG("SPMC ID %#"PRIx16, spmc_id); 2148 2149 optee_endpoint_id = FFA_SWD_ID_MIN; 2150 while (optee_endpoint_id == spmd_id || optee_endpoint_id == spmc_id) 2151 optee_endpoint_id++; 2152 2153 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2154 2155 /* 2156 * If SPMD think we are version 1.0 it will report version 1.0 to 2157 * normal world regardless of what version we query the SPM with. 2158 * However, if SPMD think we are version 1.1 it will forward 2159 * queries from normal world to let us negotiate version. So by 2160 * setting version 1.0 here we should be compatible. 2161 * 2162 * Note that disagreement on negotiated version means that we'll 2163 * have communication problems with normal world. 2164 */ 2165 my_rxtx.ffa_vers = FFA_VERSION_1_0; 2166 2167 return TEE_SUCCESS; 2168 } 2169 #else /* !defined(CFG_CORE_SEL1_SPMC) */ 2170 static void spmc_rxtx_map(struct ffa_rxtx *rxtx) 2171 { 2172 struct thread_smc_args args = { 2173 #ifdef ARM64 2174 .a0 = FFA_RXTX_MAP_64, 2175 #else 2176 .a0 = FFA_RXTX_MAP_32, 2177 #endif 2178 .a1 = virt_to_phys(rxtx->tx), 2179 .a2 = virt_to_phys(rxtx->rx), 2180 .a3 = 1, 2181 }; 2182 2183 thread_smccc(&args); 2184 if (!is_ffa_success(args.a0)) { 2185 if (args.a0 == FFA_ERROR) 2186 EMSG("rxtx map failed with error %ld", args.a2); 2187 else 2188 EMSG("rxtx map failed"); 2189 panic(); 2190 } 2191 } 2192 2193 static uint32_t get_ffa_version(uint32_t my_version) 2194 { 2195 struct thread_smc_args args = { 2196 .a0 = FFA_VERSION, 2197 .a1 = my_version, 2198 }; 2199 2200 thread_smccc(&args); 2201 if (args.a0 & BIT(31)) { 2202 EMSG("FF-A version failed with error %ld", args.a0); 2203 panic(); 2204 } 2205 2206 return args.a0; 2207 } 2208 2209 static void *spmc_retrieve_req(uint64_t cookie, 2210 struct ffa_mem_transaction_x *trans) 2211 { 2212 struct ffa_mem_access *acc_descr_array = NULL; 2213 struct ffa_mem_access_perm *perm_descr = NULL; 2214 struct thread_smc_args args = { 2215 .a0 = FFA_MEM_RETRIEVE_REQ_32, 2216 .a3 = 0, /* Address, Using TX -> MBZ */ 2217 .a4 = 0, /* Using TX -> MBZ */ 2218 }; 2219 size_t size = 0; 2220 int rc = 0; 2221 2222 if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { 2223 struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; 2224 2225 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2226 memset(trans_descr, 0, size); 2227 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2228 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2229 trans_descr->global_handle = cookie; 2230 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2231 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2232 trans_descr->mem_access_count = 1; 2233 acc_descr_array = trans_descr->mem_access_array; 2234 } else { 2235 struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; 2236 2237 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2238 memset(trans_descr, 0, size); 2239 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2240 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2241 trans_descr->global_handle = cookie; 2242 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2243 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2244 trans_descr->mem_access_count = 1; 2245 trans_descr->mem_access_offs = sizeof(*trans_descr); 2246 trans_descr->mem_access_size = sizeof(struct ffa_mem_access); 2247 acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + 2248 sizeof(*trans_descr)); 2249 } 2250 acc_descr_array->region_offs = 0; 2251 acc_descr_array->reserved = 0; 2252 perm_descr = &acc_descr_array->access_perm; 2253 perm_descr->endpoint_id = optee_endpoint_id; 2254 perm_descr->perm = FFA_MEM_ACC_RW; 2255 perm_descr->flags = 0; 2256 2257 args.a1 = size; /* Total Length */ 2258 args.a2 = size; /* Frag Length == Total length */ 2259 thread_smccc(&args); 2260 if (args.a0 != FFA_MEM_RETRIEVE_RESP) { 2261 if (args.a0 == FFA_ERROR) 2262 EMSG("Failed to fetch cookie %#"PRIx64" error code %d", 2263 cookie, (int)args.a2); 2264 else 2265 EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, 2266 cookie, args.a0); 2267 return NULL; 2268 } 2269 rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.rx, 2270 my_rxtx.size, trans); 2271 if (rc) { 2272 EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", 2273 cookie, rc); 2274 return NULL; 2275 } 2276 2277 return my_rxtx.rx; 2278 } 2279 2280 void thread_spmc_relinquish(uint64_t cookie) 2281 { 2282 struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; 2283 struct thread_smc_args args = { 2284 .a0 = FFA_MEM_RELINQUISH, 2285 }; 2286 2287 memset(relinquish_desc, 0, sizeof(*relinquish_desc)); 2288 relinquish_desc->handle = cookie; 2289 relinquish_desc->flags = 0; 2290 relinquish_desc->endpoint_count = 1; 2291 relinquish_desc->endpoint_id_array[0] = optee_endpoint_id; 2292 thread_smccc(&args); 2293 if (!is_ffa_success(args.a0)) 2294 EMSG("Failed to relinquish cookie %#"PRIx64, cookie); 2295 } 2296 2297 static int set_pages(struct ffa_address_range *regions, 2298 unsigned int num_regions, unsigned int num_pages, 2299 struct mobj_ffa *mf) 2300 { 2301 unsigned int n = 0; 2302 unsigned int idx = 0; 2303 2304 for (n = 0; n < num_regions; n++) { 2305 unsigned int page_count = READ_ONCE(regions[n].page_count); 2306 uint64_t addr = READ_ONCE(regions[n].address); 2307 2308 if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) 2309 return FFA_INVALID_PARAMETERS; 2310 } 2311 2312 if (idx != num_pages) 2313 return FFA_INVALID_PARAMETERS; 2314 2315 return 0; 2316 } 2317 2318 struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) 2319 { 2320 struct mobj_ffa *ret = NULL; 2321 struct ffa_mem_transaction_x retrieve_desc = { }; 2322 struct ffa_mem_access *descr_array = NULL; 2323 struct ffa_mem_region *descr = NULL; 2324 struct mobj_ffa *mf = NULL; 2325 unsigned int num_pages = 0; 2326 unsigned int offs = 0; 2327 void *buf = NULL; 2328 struct thread_smc_args ffa_rx_release_args = { 2329 .a0 = FFA_RX_RELEASE 2330 }; 2331 2332 /* 2333 * OP-TEE is only supporting a single mem_region while the 2334 * specification allows for more than one. 2335 */ 2336 buf = spmc_retrieve_req(cookie, &retrieve_desc); 2337 if (!buf) { 2338 EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, 2339 cookie); 2340 return NULL; 2341 } 2342 2343 descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); 2344 offs = READ_ONCE(descr_array->region_offs); 2345 descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); 2346 2347 num_pages = READ_ONCE(descr->total_page_count); 2348 mf = mobj_ffa_spmc_new(cookie, num_pages); 2349 if (!mf) 2350 goto out; 2351 2352 if (set_pages(descr->address_range_array, 2353 READ_ONCE(descr->address_range_count), num_pages, mf)) { 2354 mobj_ffa_spmc_delete(mf); 2355 goto out; 2356 } 2357 2358 ret = mf; 2359 2360 out: 2361 /* Release RX buffer after the mem retrieve request. */ 2362 thread_smccc(&ffa_rx_release_args); 2363 2364 return ret; 2365 } 2366 2367 static TEE_Result spmc_init(void) 2368 { 2369 unsigned int major = 0; 2370 unsigned int minor __maybe_unused = 0; 2371 uint32_t my_vers = 0; 2372 uint32_t vers = 0; 2373 2374 my_vers = MAKE_FFA_VERSION(FFA_VERSION_MAJOR, FFA_VERSION_MINOR); 2375 vers = get_ffa_version(my_vers); 2376 major = (vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK; 2377 minor = (vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK; 2378 DMSG("SPMC reported version %u.%u", major, minor); 2379 if (major != FFA_VERSION_MAJOR) { 2380 EMSG("Incompatible major version %u, expected %u", 2381 major, FFA_VERSION_MAJOR); 2382 panic(); 2383 } 2384 if (vers < my_vers) 2385 my_vers = vers; 2386 DMSG("Using version %u.%u", 2387 (my_vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK, 2388 (my_vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK); 2389 my_rxtx.ffa_vers = my_vers; 2390 2391 spmc_rxtx_map(&my_rxtx); 2392 2393 spmc_id = ffa_spm_id_get(); 2394 DMSG("SPMC ID %#"PRIx16, spmc_id); 2395 2396 optee_endpoint_id = ffa_id_get(); 2397 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2398 2399 if (!ffa_features(FFA_NOTIFICATION_SET)) { 2400 spmc_notif_is_ready = true; 2401 DMSG("Asynchronous notifications are ready"); 2402 } 2403 2404 return TEE_SUCCESS; 2405 } 2406 #endif /* !defined(CFG_CORE_SEL1_SPMC) */ 2407 2408 /* 2409 * boot_final() is always done before exiting at end of boot 2410 * initialization. In case of virtualization the init-calls are done only 2411 * once a OP-TEE partition has been created. So with virtualization we have 2412 * to initialize via boot_final() to make sure we have a value assigned 2413 * before it's used the first time. 2414 */ 2415 #ifdef CFG_NS_VIRTUALIZATION 2416 boot_final(spmc_init); 2417 #else 2418 service_init(spmc_init); 2419 #endif 2420