1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2020-2023, Linaro Limited. 4 * Copyright (c) 2019-2024, Arm Limited. All rights reserved. 5 */ 6 7 #include <assert.h> 8 #include <ffa.h> 9 #include <initcall.h> 10 #include <io.h> 11 #include <kernel/interrupt.h> 12 #include <kernel/notif.h> 13 #include <kernel/panic.h> 14 #include <kernel/secure_partition.h> 15 #include <kernel/spinlock.h> 16 #include <kernel/spmc_sp_handler.h> 17 #include <kernel/tee_misc.h> 18 #include <kernel/thread.h> 19 #include <kernel/thread_private.h> 20 #include <kernel/thread_spmc.h> 21 #include <kernel/virtualization.h> 22 #include <mm/core_mmu.h> 23 #include <mm/mobj.h> 24 #include <optee_ffa.h> 25 #include <optee_msg.h> 26 #include <optee_rpc_cmd.h> 27 #include <sm/optee_smc.h> 28 #include <string.h> 29 #include <sys/queue.h> 30 #include <tee/entry_std.h> 31 #include <tee/uuid.h> 32 #include <util.h> 33 34 #if defined(CFG_CORE_SEL1_SPMC) 35 struct mem_share_state { 36 struct mobj_ffa *mf; 37 unsigned int page_count; 38 unsigned int region_count; 39 unsigned int current_page_idx; 40 }; 41 42 struct mem_frag_state { 43 struct mem_share_state share; 44 tee_mm_entry_t *mm; 45 unsigned int frag_offset; 46 SLIST_ENTRY(mem_frag_state) link; 47 }; 48 #endif 49 50 struct notif_vm_bitmap { 51 bool initialized; 52 int do_bottom_half_value; 53 uint64_t pending; 54 uint64_t bound; 55 }; 56 57 static unsigned int spmc_notif_lock __nex_data = SPINLOCK_UNLOCK; 58 static bool spmc_notif_is_ready __nex_bss; 59 static int notif_intid __nex_data __maybe_unused = -1; 60 61 /* Id used to look up the guest specific struct notif_vm_bitmap */ 62 static unsigned int notif_vm_bitmap_id __nex_bss; 63 /* Notification state when ns-virtualization isn't enabled */ 64 static struct notif_vm_bitmap default_notif_vm_bitmap; 65 66 /* Initialized in spmc_init() below */ 67 uint16_t optee_endpoint_id __nex_bss; 68 uint16_t spmc_id __nex_bss; 69 #ifdef CFG_CORE_SEL1_SPMC 70 uint16_t spmd_id __nex_bss; 71 static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | 72 FFA_PART_PROP_DIRECT_REQ_SEND | 73 #ifdef CFG_NS_VIRTUALIZATION 74 FFA_PART_PROP_NOTIF_CREATED | 75 FFA_PART_PROP_NOTIF_DESTROYED | 76 #endif 77 #ifdef ARM64 78 FFA_PART_PROP_AARCH64_STATE | 79 #endif 80 FFA_PART_PROP_IS_PE_ID; 81 82 static uint32_t my_uuid_words[] = { 83 /* 84 * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 85 * SP, or 86 * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a 87 * logical partition, residing in the same exception level as the 88 * SPMC 89 * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 90 */ 91 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, 92 }; 93 94 /* 95 * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. 96 * 97 * struct ffa_rxtx::spin_lock protects the variables below from concurrent 98 * access this includes the use of content of struct ffa_rxtx::rx and 99 * @frag_state_head. 100 * 101 * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct 102 * ffa_rxtx::tx and false when it is owned by normal world. 103 * 104 * Note that we can't prevent normal world from updating the content of 105 * these buffers so we must always be careful when reading. while we hold 106 * the lock. 107 */ 108 109 static struct ffa_rxtx my_rxtx __nex_bss; 110 111 static bool is_nw_buf(struct ffa_rxtx *rxtx) 112 { 113 return rxtx == &my_rxtx; 114 } 115 116 static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = 117 SLIST_HEAD_INITIALIZER(&frag_state_head); 118 119 #else 120 static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 121 static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 122 static struct ffa_rxtx my_rxtx = { 123 .rx = __rx_buf, 124 .tx = __tx_buf, 125 .size = sizeof(__rx_buf), 126 }; 127 #endif 128 129 static uint32_t swap_src_dst(uint32_t src_dst) 130 { 131 return (src_dst >> 16) | (src_dst << 16); 132 } 133 134 static uint16_t get_sender_id(uint32_t src_dst) 135 { 136 return src_dst >> 16; 137 } 138 139 void spmc_set_args(struct thread_smc_args *args, uint32_t fid, uint32_t src_dst, 140 uint32_t w2, uint32_t w3, uint32_t w4, uint32_t w5) 141 { 142 *args = (struct thread_smc_args){ .a0 = fid, 143 .a1 = src_dst, 144 .a2 = w2, 145 .a3 = w3, 146 .a4 = w4, 147 .a5 = w5, }; 148 } 149 150 static void set_simple_ret_val(struct thread_smc_args *args, int ffa_ret) 151 { 152 if (ffa_ret) 153 spmc_set_args(args, FFA_ERROR, 0, ffa_ret, 0, 0, 0); 154 else 155 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 156 } 157 158 uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) 159 { 160 /* 161 * No locking, if the caller does concurrent calls to this it's 162 * only making a mess for itself. We must be able to renegotiate 163 * the FF-A version in order to support differing versions between 164 * the loader and the driver. 165 */ 166 if (vers < FFA_VERSION_1_1) 167 rxtx->ffa_vers = FFA_VERSION_1_0; 168 else 169 rxtx->ffa_vers = FFA_VERSION_1_1; 170 171 return rxtx->ffa_vers; 172 } 173 174 static bool is_ffa_success(uint32_t fid) 175 { 176 #ifdef ARM64 177 if (fid == FFA_SUCCESS_64) 178 return true; 179 #endif 180 return fid == FFA_SUCCESS_32; 181 } 182 183 static int32_t get_ffa_ret_code(const struct thread_smc_args *args) 184 { 185 if (is_ffa_success(args->a0)) 186 return FFA_OK; 187 if (args->a0 == FFA_ERROR && args->a2) 188 return args->a2; 189 return FFA_NOT_SUPPORTED; 190 } 191 192 static int ffa_simple_call(uint32_t fid, unsigned long a1, unsigned long a2, 193 unsigned long a3, unsigned long a4) 194 { 195 struct thread_smc_args args = { 196 .a0 = fid, 197 .a1 = a1, 198 .a2 = a2, 199 .a3 = a3, 200 .a4 = a4, 201 }; 202 203 thread_smccc(&args); 204 205 return get_ffa_ret_code(&args); 206 } 207 208 static int __maybe_unused ffa_features(uint32_t id) 209 { 210 return ffa_simple_call(FFA_FEATURES, id, 0, 0, 0); 211 } 212 213 static int __maybe_unused ffa_set_notification(uint16_t dst, uint16_t src, 214 uint32_t flags, uint64_t bitmap) 215 { 216 return ffa_simple_call(FFA_NOTIFICATION_SET, 217 SHIFT_U32(src, 16) | dst, flags, 218 low32_from_64(bitmap), high32_from_64(bitmap)); 219 } 220 221 #if defined(CFG_CORE_SEL1_SPMC) 222 static void handle_features(struct thread_smc_args *args) 223 { 224 uint32_t ret_fid = FFA_ERROR; 225 uint32_t ret_w2 = FFA_NOT_SUPPORTED; 226 227 switch (args->a1) { 228 case FFA_FEATURE_SCHEDULE_RECV_INTR: 229 if (spmc_notif_is_ready) { 230 ret_fid = FFA_SUCCESS_32; 231 ret_w2 = notif_intid; 232 } 233 break; 234 235 #ifdef ARM64 236 case FFA_RXTX_MAP_64: 237 #endif 238 case FFA_RXTX_MAP_32: 239 ret_fid = FFA_SUCCESS_32; 240 ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ 241 break; 242 #ifdef ARM64 243 case FFA_MEM_SHARE_64: 244 #endif 245 case FFA_MEM_SHARE_32: 246 ret_fid = FFA_SUCCESS_32; 247 /* 248 * Partition manager supports transmission of a memory 249 * transaction descriptor in a buffer dynamically allocated 250 * by the endpoint. 251 */ 252 ret_w2 = BIT(0); 253 break; 254 255 case FFA_ERROR: 256 case FFA_VERSION: 257 case FFA_SUCCESS_32: 258 #ifdef ARM64 259 case FFA_SUCCESS_64: 260 #endif 261 case FFA_FEATURES: 262 case FFA_SPM_ID_GET: 263 case FFA_MEM_FRAG_TX: 264 case FFA_MEM_RECLAIM: 265 case FFA_MSG_SEND_DIRECT_REQ_64: 266 case FFA_MSG_SEND_DIRECT_REQ_32: 267 case FFA_INTERRUPT: 268 case FFA_PARTITION_INFO_GET: 269 case FFA_RXTX_UNMAP: 270 case FFA_RX_RELEASE: 271 case FFA_FEATURE_MANAGED_EXIT_INTR: 272 case FFA_NOTIFICATION_BITMAP_CREATE: 273 case FFA_NOTIFICATION_BITMAP_DESTROY: 274 case FFA_NOTIFICATION_BIND: 275 case FFA_NOTIFICATION_UNBIND: 276 case FFA_NOTIFICATION_SET: 277 case FFA_NOTIFICATION_GET: 278 case FFA_NOTIFICATION_INFO_GET_32: 279 #ifdef ARM64 280 case FFA_NOTIFICATION_INFO_GET_64: 281 #endif 282 ret_fid = FFA_SUCCESS_32; 283 ret_w2 = FFA_PARAM_MBZ; 284 break; 285 default: 286 break; 287 } 288 289 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, 290 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 291 } 292 293 static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) 294 { 295 tee_mm_entry_t *mm = NULL; 296 297 if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) 298 return FFA_INVALID_PARAMETERS; 299 300 mm = tee_mm_alloc(&core_virt_shm_pool, sz); 301 if (!mm) 302 return FFA_NO_MEMORY; 303 304 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, 305 sz / SMALL_PAGE_SIZE, 306 MEM_AREA_NSEC_SHM)) { 307 tee_mm_free(mm); 308 return FFA_INVALID_PARAMETERS; 309 } 310 311 *va_ret = (void *)tee_mm_get_smem(mm); 312 return 0; 313 } 314 315 void spmc_handle_spm_id_get(struct thread_smc_args *args) 316 { 317 spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, spmc_id, 318 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 319 } 320 321 static void unmap_buf(void *va, size_t sz) 322 { 323 tee_mm_entry_t *mm = tee_mm_find(&core_virt_shm_pool, (vaddr_t)va); 324 325 assert(mm); 326 core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); 327 tee_mm_free(mm); 328 } 329 330 void spmc_handle_rxtx_map(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 331 { 332 int rc = 0; 333 unsigned int sz = 0; 334 paddr_t rx_pa = 0; 335 paddr_t tx_pa = 0; 336 void *rx = NULL; 337 void *tx = NULL; 338 339 cpu_spin_lock(&rxtx->spinlock); 340 341 if (args->a3 & GENMASK_64(63, 6)) { 342 rc = FFA_INVALID_PARAMETERS; 343 goto out; 344 } 345 346 sz = args->a3 * SMALL_PAGE_SIZE; 347 if (!sz) { 348 rc = FFA_INVALID_PARAMETERS; 349 goto out; 350 } 351 /* TX/RX are swapped compared to the caller */ 352 tx_pa = args->a2; 353 rx_pa = args->a1; 354 355 if (rxtx->size) { 356 rc = FFA_DENIED; 357 goto out; 358 } 359 360 /* 361 * If the buffer comes from a SP the address is virtual and already 362 * mapped. 363 */ 364 if (is_nw_buf(rxtx)) { 365 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 366 enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; 367 bool tx_alloced = false; 368 369 /* 370 * With virtualization we establish this mapping in 371 * the nexus mapping which then is replicated to 372 * each partition. 373 * 374 * This means that this mapping must be done before 375 * any partition is created and then must not be 376 * changed. 377 */ 378 379 /* 380 * core_mmu_add_mapping() may reuse previous 381 * mappings. First check if there's any mappings to 382 * reuse so we know how to clean up in case of 383 * failure. 384 */ 385 tx = phys_to_virt(tx_pa, mt, sz); 386 rx = phys_to_virt(rx_pa, mt, sz); 387 if (!tx) { 388 tx = core_mmu_add_mapping(mt, tx_pa, sz); 389 if (!tx) { 390 rc = FFA_NO_MEMORY; 391 goto out; 392 } 393 tx_alloced = true; 394 } 395 if (!rx) 396 rx = core_mmu_add_mapping(mt, rx_pa, sz); 397 398 if (!rx) { 399 if (tx_alloced && tx) 400 core_mmu_remove_mapping(mt, tx, sz); 401 rc = FFA_NO_MEMORY; 402 goto out; 403 } 404 } else { 405 rc = map_buf(tx_pa, sz, &tx); 406 if (rc) 407 goto out; 408 rc = map_buf(rx_pa, sz, &rx); 409 if (rc) { 410 unmap_buf(tx, sz); 411 goto out; 412 } 413 } 414 rxtx->tx = tx; 415 rxtx->rx = rx; 416 } else { 417 if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { 418 rc = FFA_INVALID_PARAMETERS; 419 goto out; 420 } 421 422 if (!virt_to_phys((void *)tx_pa) || 423 !virt_to_phys((void *)rx_pa)) { 424 rc = FFA_INVALID_PARAMETERS; 425 goto out; 426 } 427 428 rxtx->tx = (void *)tx_pa; 429 rxtx->rx = (void *)rx_pa; 430 } 431 432 rxtx->size = sz; 433 rxtx->tx_is_mine = true; 434 DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); 435 DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); 436 out: 437 cpu_spin_unlock(&rxtx->spinlock); 438 set_simple_ret_val(args, rc); 439 } 440 441 void spmc_handle_rxtx_unmap(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 442 { 443 int rc = FFA_INVALID_PARAMETERS; 444 445 cpu_spin_lock(&rxtx->spinlock); 446 447 if (!rxtx->size) 448 goto out; 449 450 /* 451 * We don't unmap the SP memory as the SP might still use it. 452 * We avoid to make changes to nexus mappings at this stage since 453 * there currently isn't a way to replicate those changes to all 454 * partitions. 455 */ 456 if (is_nw_buf(rxtx) && !IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 457 unmap_buf(rxtx->rx, rxtx->size); 458 unmap_buf(rxtx->tx, rxtx->size); 459 } 460 rxtx->size = 0; 461 rxtx->rx = NULL; 462 rxtx->tx = NULL; 463 rc = 0; 464 out: 465 cpu_spin_unlock(&rxtx->spinlock); 466 set_simple_ret_val(args, rc); 467 } 468 469 void spmc_handle_rx_release(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 470 { 471 int rc = 0; 472 473 cpu_spin_lock(&rxtx->spinlock); 474 /* The senders RX is our TX */ 475 if (!rxtx->size || rxtx->tx_is_mine) { 476 rc = FFA_DENIED; 477 } else { 478 rc = 0; 479 rxtx->tx_is_mine = true; 480 } 481 cpu_spin_unlock(&rxtx->spinlock); 482 483 set_simple_ret_val(args, rc); 484 } 485 486 static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 487 { 488 return !w0 && !w1 && !w2 && !w3; 489 } 490 491 static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 492 { 493 /* 494 * This depends on which UUID we have been assigned. 495 * TODO add a generic mechanism to obtain our UUID. 496 * 497 * The test below is for the hard coded UUID 498 * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 499 */ 500 return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && 501 w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; 502 } 503 504 TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, 505 size_t idx, uint16_t endpoint_id, 506 uint16_t execution_context, 507 uint32_t part_props, 508 const uint32_t uuid_words[4]) 509 { 510 struct ffa_partition_info_x *fpi = NULL; 511 size_t fpi_size = sizeof(*fpi); 512 513 if (ffa_vers >= FFA_VERSION_1_1) 514 fpi_size += FFA_UUID_SIZE; 515 516 if ((idx + 1) * fpi_size > blen) 517 return TEE_ERROR_OUT_OF_MEMORY; 518 519 fpi = (void *)((vaddr_t)buf + idx * fpi_size); 520 fpi->id = endpoint_id; 521 /* Number of execution contexts implemented by this partition */ 522 fpi->execution_context = execution_context; 523 524 fpi->partition_properties = part_props; 525 526 /* In FF-A 1.0 only bits [2:0] are defined, let's mask others */ 527 if (ffa_vers < FFA_VERSION_1_1) 528 fpi->partition_properties &= FFA_PART_PROP_DIRECT_REQ_RECV | 529 FFA_PART_PROP_DIRECT_REQ_SEND | 530 FFA_PART_PROP_INDIRECT_MSGS; 531 532 if (ffa_vers >= FFA_VERSION_1_1) { 533 if (uuid_words) 534 memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); 535 else 536 memset(fpi->uuid, 0, FFA_UUID_SIZE); 537 } 538 539 return TEE_SUCCESS; 540 } 541 542 static int handle_partition_info_get_all(size_t *elem_count, 543 struct ffa_rxtx *rxtx, bool count_only) 544 { 545 if (!count_only) { 546 /* Add OP-TEE SP */ 547 if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, 548 rxtx->size, 0, optee_endpoint_id, 549 CFG_TEE_CORE_NB_CORE, 550 my_part_props, my_uuid_words)) 551 return FFA_NO_MEMORY; 552 } 553 *elem_count = 1; 554 555 if (IS_ENABLED(CFG_SECURE_PARTITION)) { 556 if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, 557 NULL, elem_count, count_only)) 558 return FFA_NO_MEMORY; 559 } 560 561 return FFA_OK; 562 } 563 564 void spmc_handle_partition_info_get(struct thread_smc_args *args, 565 struct ffa_rxtx *rxtx) 566 { 567 TEE_Result res = TEE_SUCCESS; 568 uint32_t ret_fid = FFA_ERROR; 569 uint32_t fpi_size = 0; 570 uint32_t rc = 0; 571 bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; 572 573 if (!count_only) { 574 cpu_spin_lock(&rxtx->spinlock); 575 576 if (!rxtx->size || !rxtx->tx_is_mine) { 577 rc = FFA_BUSY; 578 goto out; 579 } 580 } 581 582 if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { 583 size_t elem_count = 0; 584 585 ret_fid = handle_partition_info_get_all(&elem_count, rxtx, 586 count_only); 587 588 if (ret_fid) { 589 rc = ret_fid; 590 ret_fid = FFA_ERROR; 591 } else { 592 ret_fid = FFA_SUCCESS_32; 593 rc = elem_count; 594 } 595 596 goto out; 597 } 598 599 if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { 600 if (!count_only) { 601 res = spmc_fill_partition_entry(rxtx->ffa_vers, 602 rxtx->tx, rxtx->size, 0, 603 optee_endpoint_id, 604 CFG_TEE_CORE_NB_CORE, 605 my_part_props, 606 my_uuid_words); 607 if (res) { 608 ret_fid = FFA_ERROR; 609 rc = FFA_INVALID_PARAMETERS; 610 goto out; 611 } 612 } 613 rc = 1; 614 } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { 615 uint32_t uuid_array[4] = { 0 }; 616 TEE_UUID uuid = { }; 617 size_t count = 0; 618 619 uuid_array[0] = args->a1; 620 uuid_array[1] = args->a2; 621 uuid_array[2] = args->a3; 622 uuid_array[3] = args->a4; 623 tee_uuid_from_octets(&uuid, (uint8_t *)uuid_array); 624 625 res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, 626 rxtx->size, &uuid, &count, 627 count_only); 628 if (res != TEE_SUCCESS) { 629 ret_fid = FFA_ERROR; 630 rc = FFA_INVALID_PARAMETERS; 631 goto out; 632 } 633 rc = count; 634 } else { 635 ret_fid = FFA_ERROR; 636 rc = FFA_INVALID_PARAMETERS; 637 goto out; 638 } 639 640 ret_fid = FFA_SUCCESS_32; 641 642 out: 643 if (ret_fid == FFA_SUCCESS_32 && !count_only && 644 rxtx->ffa_vers >= FFA_VERSION_1_1) 645 fpi_size = sizeof(struct ffa_partition_info_x) + FFA_UUID_SIZE; 646 647 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, fpi_size, 648 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 649 if (!count_only) { 650 rxtx->tx_is_mine = false; 651 cpu_spin_unlock(&rxtx->spinlock); 652 } 653 } 654 655 static void spmc_handle_run(struct thread_smc_args *args) 656 { 657 uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); 658 uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); 659 uint32_t rc = FFA_OK; 660 661 if (endpoint != optee_endpoint_id) { 662 /* 663 * The endpoint should be an SP, try to resume the SP from 664 * preempted into busy state. 665 */ 666 rc = spmc_sp_resume_from_preempted(endpoint); 667 if (rc) 668 goto out; 669 } 670 671 thread_resume_from_rpc(thread_id, 0, 0, 0, 0); 672 673 /* thread_resume_from_rpc return only of the thread_id is invalid */ 674 rc = FFA_INVALID_PARAMETERS; 675 676 out: 677 set_simple_ret_val(args, rc); 678 } 679 #endif /*CFG_CORE_SEL1_SPMC*/ 680 681 static struct notif_vm_bitmap *get_notif_vm_bitmap(struct guest_partition *prtn, 682 uint16_t vm_id) 683 { 684 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 685 if (!prtn) 686 return NULL; 687 assert(vm_id == virt_get_guest_id(prtn)); 688 return virt_get_guest_spec_data(prtn, notif_vm_bitmap_id); 689 } 690 if (vm_id) 691 return NULL; 692 return &default_notif_vm_bitmap; 693 } 694 695 static uint32_t spmc_enable_async_notif(uint32_t bottom_half_value, 696 uint16_t vm_id) 697 { 698 struct guest_partition *prtn = NULL; 699 struct notif_vm_bitmap *nvb = NULL; 700 uint32_t old_itr_status = 0; 701 uint32_t res = 0; 702 703 if (!spmc_notif_is_ready) { 704 /* 705 * This should never happen, not if normal world respects the 706 * exchanged capabilities. 707 */ 708 EMSG("Asynchronous notifications are not ready"); 709 return TEE_ERROR_NOT_IMPLEMENTED; 710 } 711 712 if (bottom_half_value >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE) { 713 EMSG("Invalid bottom half value %"PRIu32, bottom_half_value); 714 return TEE_ERROR_BAD_PARAMETERS; 715 } 716 717 prtn = virt_get_guest(vm_id); 718 nvb = get_notif_vm_bitmap(prtn, vm_id); 719 if (!nvb) { 720 res = TEE_ERROR_BAD_PARAMETERS; 721 goto out; 722 } 723 724 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 725 nvb->do_bottom_half_value = bottom_half_value; 726 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 727 728 notif_deliver_atomic_event(NOTIF_EVENT_STARTED, vm_id); 729 res = TEE_SUCCESS; 730 out: 731 virt_put_guest(prtn); 732 return res; 733 } 734 735 static void handle_yielding_call(struct thread_smc_args *args, 736 uint32_t direct_resp_fid) 737 { 738 TEE_Result res = 0; 739 740 thread_check_canaries(); 741 742 #ifdef ARM64 743 /* Saving this for an eventual RPC */ 744 thread_get_core_local()->direct_resp_fid = direct_resp_fid; 745 #endif 746 747 if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { 748 /* Note connection to struct thread_rpc_arg::ret */ 749 thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, 750 0); 751 res = TEE_ERROR_BAD_PARAMETERS; 752 } else { 753 thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, 754 args->a6, args->a7); 755 res = TEE_ERROR_BUSY; 756 } 757 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 758 0, res, 0, 0); 759 } 760 761 static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) 762 { 763 uint64_t cookie = reg_pair_to_64(a5, a4); 764 uint32_t res = 0; 765 766 res = mobj_ffa_unregister_by_cookie(cookie); 767 switch (res) { 768 case TEE_SUCCESS: 769 case TEE_ERROR_ITEM_NOT_FOUND: 770 return 0; 771 case TEE_ERROR_BUSY: 772 EMSG("res %#"PRIx32, res); 773 return FFA_BUSY; 774 default: 775 EMSG("res %#"PRIx32, res); 776 return FFA_INVALID_PARAMETERS; 777 } 778 } 779 780 static void handle_blocking_call(struct thread_smc_args *args, 781 uint32_t direct_resp_fid) 782 { 783 uint32_t sec_caps = 0; 784 785 switch (args->a3) { 786 case OPTEE_FFA_GET_API_VERSION: 787 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 788 OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, 789 0); 790 break; 791 case OPTEE_FFA_GET_OS_VERSION: 792 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 793 CFG_OPTEE_REVISION_MAJOR, 794 CFG_OPTEE_REVISION_MINOR, 795 TEE_IMPL_GIT_SHA1 >> 32); 796 break; 797 case OPTEE_FFA_EXCHANGE_CAPABILITIES: 798 sec_caps = OPTEE_FFA_SEC_CAP_ARG_OFFSET; 799 if (spmc_notif_is_ready) 800 sec_caps |= OPTEE_FFA_SEC_CAP_ASYNC_NOTIF; 801 spmc_set_args(args, direct_resp_fid, 802 swap_src_dst(args->a1), 0, 0, 803 THREAD_RPC_MAX_NUM_PARAMS, sec_caps); 804 break; 805 case OPTEE_FFA_UNREGISTER_SHM: 806 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 807 handle_unregister_shm(args->a4, args->a5), 0, 0); 808 break; 809 case OPTEE_FFA_ENABLE_ASYNC_NOTIF: 810 spmc_set_args(args, direct_resp_fid, 811 swap_src_dst(args->a1), 0, 812 spmc_enable_async_notif(args->a4, 813 FFA_SRC(args->a1)), 814 0, 0); 815 break; 816 default: 817 EMSG("Unhandled blocking service ID %#"PRIx32, 818 (uint32_t)args->a3); 819 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 820 TEE_ERROR_BAD_PARAMETERS, 0, 0); 821 } 822 } 823 824 static void handle_framework_direct_request(struct thread_smc_args *args, 825 struct ffa_rxtx *rxtx, 826 uint32_t direct_resp_fid) 827 { 828 uint32_t w0 = FFA_ERROR; 829 uint32_t w1 = FFA_PARAM_MBZ; 830 uint32_t w2 = FFA_NOT_SUPPORTED; 831 uint32_t w3 = FFA_PARAM_MBZ; 832 833 switch (args->a2 & FFA_MSG_TYPE_MASK) { 834 case FFA_MSG_SEND_VM_CREATED: 835 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 836 uint16_t guest_id = args->a5; 837 TEE_Result res = virt_guest_created(guest_id); 838 839 w0 = direct_resp_fid; 840 w1 = swap_src_dst(args->a1); 841 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; 842 if (res == TEE_SUCCESS) 843 w3 = FFA_OK; 844 else if (res == TEE_ERROR_OUT_OF_MEMORY) 845 w3 = FFA_DENIED; 846 else 847 w3 = FFA_INVALID_PARAMETERS; 848 } 849 break; 850 case FFA_MSG_SEND_VM_DESTROYED: 851 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 852 uint16_t guest_id = args->a5; 853 TEE_Result res = virt_guest_destroyed(guest_id); 854 855 w0 = direct_resp_fid; 856 w1 = swap_src_dst(args->a1); 857 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_DESTROYED; 858 if (res == TEE_SUCCESS) 859 w3 = FFA_OK; 860 else 861 w3 = FFA_INVALID_PARAMETERS; 862 } 863 break; 864 case FFA_MSG_VERSION_REQ: 865 w0 = direct_resp_fid; 866 w1 = swap_src_dst(args->a1); 867 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; 868 w3 = spmc_exchange_version(args->a3, rxtx); 869 break; 870 default: 871 break; 872 } 873 spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 874 } 875 876 static void handle_direct_request(struct thread_smc_args *args, 877 struct ffa_rxtx *rxtx) 878 { 879 uint32_t direct_resp_fid = 0; 880 881 if (IS_ENABLED(CFG_SECURE_PARTITION) && 882 FFA_DST(args->a1) != spmc_id && 883 FFA_DST(args->a1) != optee_endpoint_id) { 884 spmc_sp_start_thread(args); 885 return; 886 } 887 888 if (OPTEE_SMC_IS_64(args->a0)) 889 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_64; 890 else 891 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_32; 892 893 if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { 894 handle_framework_direct_request(args, rxtx, direct_resp_fid); 895 return; 896 } 897 898 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 899 virt_set_guest(get_sender_id(args->a1))) { 900 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 901 TEE_ERROR_ITEM_NOT_FOUND, 0, 0); 902 return; 903 } 904 905 if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) 906 handle_yielding_call(args, direct_resp_fid); 907 else 908 handle_blocking_call(args, direct_resp_fid); 909 910 /* 911 * Note that handle_yielding_call() typically only returns if a 912 * thread cannot be allocated or found. virt_unset_guest() is also 913 * called from thread_state_suspend() and thread_state_free(). 914 */ 915 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 916 virt_unset_guest(); 917 } 918 919 int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, 920 struct ffa_mem_transaction_x *trans) 921 { 922 uint16_t mem_reg_attr = 0; 923 uint32_t flags = 0; 924 uint32_t count = 0; 925 uint32_t offs = 0; 926 uint32_t size = 0; 927 size_t n = 0; 928 929 if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) 930 return FFA_INVALID_PARAMETERS; 931 932 if (ffa_vers >= FFA_VERSION_1_1) { 933 struct ffa_mem_transaction_1_1 *descr = NULL; 934 935 if (blen < sizeof(*descr)) 936 return FFA_INVALID_PARAMETERS; 937 938 descr = buf; 939 trans->sender_id = READ_ONCE(descr->sender_id); 940 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 941 flags = READ_ONCE(descr->flags); 942 trans->global_handle = READ_ONCE(descr->global_handle); 943 trans->tag = READ_ONCE(descr->tag); 944 945 count = READ_ONCE(descr->mem_access_count); 946 size = READ_ONCE(descr->mem_access_size); 947 offs = READ_ONCE(descr->mem_access_offs); 948 } else { 949 struct ffa_mem_transaction_1_0 *descr = NULL; 950 951 if (blen < sizeof(*descr)) 952 return FFA_INVALID_PARAMETERS; 953 954 descr = buf; 955 trans->sender_id = READ_ONCE(descr->sender_id); 956 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 957 flags = READ_ONCE(descr->flags); 958 trans->global_handle = READ_ONCE(descr->global_handle); 959 trans->tag = READ_ONCE(descr->tag); 960 961 count = READ_ONCE(descr->mem_access_count); 962 size = sizeof(struct ffa_mem_access); 963 offs = offsetof(struct ffa_mem_transaction_1_0, 964 mem_access_array); 965 } 966 967 if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || 968 size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) 969 return FFA_INVALID_PARAMETERS; 970 971 /* Check that the endpoint memory access descriptor array fits */ 972 if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || 973 n > blen) 974 return FFA_INVALID_PARAMETERS; 975 976 trans->mem_reg_attr = mem_reg_attr; 977 trans->flags = flags; 978 trans->mem_access_size = size; 979 trans->mem_access_count = count; 980 trans->mem_access_offs = offs; 981 return 0; 982 } 983 984 #if defined(CFG_CORE_SEL1_SPMC) 985 static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, 986 unsigned int mem_access_count, uint8_t *acc_perms, 987 unsigned int *region_offs) 988 { 989 struct ffa_mem_access_perm *descr = NULL; 990 struct ffa_mem_access *mem_acc = NULL; 991 unsigned int n = 0; 992 993 for (n = 0; n < mem_access_count; n++) { 994 mem_acc = (void *)(mem_acc_base + mem_access_size * n); 995 descr = &mem_acc->access_perm; 996 if (READ_ONCE(descr->endpoint_id) == optee_endpoint_id) { 997 *acc_perms = READ_ONCE(descr->perm); 998 *region_offs = READ_ONCE(mem_acc[n].region_offs); 999 return 0; 1000 } 1001 } 1002 1003 return FFA_INVALID_PARAMETERS; 1004 } 1005 1006 static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, 1007 size_t blen, unsigned int *page_count, 1008 unsigned int *region_count, size_t *addr_range_offs) 1009 { 1010 const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 1011 const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; 1012 struct ffa_mem_region *region_descr = NULL; 1013 unsigned int region_descr_offs = 0; 1014 uint8_t mem_acc_perm = 0; 1015 size_t n = 0; 1016 1017 if (mem_trans->mem_reg_attr != exp_mem_reg_attr) 1018 return FFA_INVALID_PARAMETERS; 1019 1020 /* Check that the access permissions matches what's expected */ 1021 if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, 1022 mem_trans->mem_access_size, 1023 mem_trans->mem_access_count, 1024 &mem_acc_perm, ®ion_descr_offs) || 1025 mem_acc_perm != exp_mem_acc_perm) 1026 return FFA_INVALID_PARAMETERS; 1027 1028 /* Check that the Composite memory region descriptor fits */ 1029 if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || 1030 n > blen) 1031 return FFA_INVALID_PARAMETERS; 1032 1033 if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, 1034 struct ffa_mem_region)) 1035 return FFA_INVALID_PARAMETERS; 1036 1037 region_descr = (struct ffa_mem_region *)((vaddr_t)buf + 1038 region_descr_offs); 1039 *page_count = READ_ONCE(region_descr->total_page_count); 1040 *region_count = READ_ONCE(region_descr->address_range_count); 1041 *addr_range_offs = n; 1042 return 0; 1043 } 1044 1045 static int add_mem_share_helper(struct mem_share_state *s, void *buf, 1046 size_t flen) 1047 { 1048 unsigned int region_count = flen / sizeof(struct ffa_address_range); 1049 struct ffa_address_range *arange = NULL; 1050 unsigned int n = 0; 1051 1052 if (region_count > s->region_count) 1053 region_count = s->region_count; 1054 1055 if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) 1056 return FFA_INVALID_PARAMETERS; 1057 arange = buf; 1058 1059 for (n = 0; n < region_count; n++) { 1060 unsigned int page_count = READ_ONCE(arange[n].page_count); 1061 uint64_t addr = READ_ONCE(arange[n].address); 1062 1063 if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, 1064 addr, page_count)) 1065 return FFA_INVALID_PARAMETERS; 1066 } 1067 1068 s->region_count -= region_count; 1069 if (s->region_count) 1070 return region_count * sizeof(*arange); 1071 1072 if (s->current_page_idx != s->page_count) 1073 return FFA_INVALID_PARAMETERS; 1074 1075 return 0; 1076 } 1077 1078 static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) 1079 { 1080 int rc = 0; 1081 1082 rc = add_mem_share_helper(&s->share, buf, flen); 1083 if (rc >= 0) { 1084 if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { 1085 /* We're not at the end of the descriptor yet */ 1086 if (s->share.region_count) 1087 return s->frag_offset; 1088 1089 /* We're done */ 1090 rc = 0; 1091 } else { 1092 rc = FFA_INVALID_PARAMETERS; 1093 } 1094 } 1095 1096 SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); 1097 if (rc < 0) 1098 mobj_ffa_sel1_spmc_delete(s->share.mf); 1099 else 1100 mobj_ffa_push_to_inactive(s->share.mf); 1101 free(s); 1102 1103 return rc; 1104 } 1105 1106 static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, 1107 void *buf) 1108 { 1109 struct ffa_mem_access_perm *perm = NULL; 1110 struct ffa_mem_access *mem_acc = NULL; 1111 1112 if (!IS_ENABLED(CFG_SECURE_PARTITION)) 1113 return false; 1114 1115 if (mem_trans->mem_access_count < 1) 1116 return false; 1117 1118 mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); 1119 perm = &mem_acc->access_perm; 1120 1121 /* 1122 * perm->endpoint_id is read here only to check if the endpoint is 1123 * OP-TEE. We do read it later on again, but there are some additional 1124 * checks there to make sure that the data is correct. 1125 */ 1126 return READ_ONCE(perm->endpoint_id) != optee_endpoint_id; 1127 } 1128 1129 static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, 1130 tee_mm_entry_t *mm, void *buf, size_t blen, 1131 size_t flen, uint64_t *global_handle) 1132 { 1133 int rc = 0; 1134 struct mem_share_state share = { }; 1135 size_t addr_range_offs = 0; 1136 uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1137 size_t n = 0; 1138 1139 rc = mem_share_init(mem_trans, buf, flen, &share.page_count, 1140 &share.region_count, &addr_range_offs); 1141 if (rc) 1142 return rc; 1143 1144 if (!share.page_count || !share.region_count) 1145 return FFA_INVALID_PARAMETERS; 1146 1147 if (MUL_OVERFLOW(share.region_count, 1148 sizeof(struct ffa_address_range), &n) || 1149 ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) 1150 return FFA_INVALID_PARAMETERS; 1151 1152 if (mem_trans->global_handle) 1153 cookie = mem_trans->global_handle; 1154 share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); 1155 if (!share.mf) 1156 return FFA_NO_MEMORY; 1157 1158 if (flen != blen) { 1159 struct mem_frag_state *s = calloc(1, sizeof(*s)); 1160 1161 if (!s) { 1162 rc = FFA_NO_MEMORY; 1163 goto err; 1164 } 1165 s->share = share; 1166 s->mm = mm; 1167 s->frag_offset = addr_range_offs; 1168 1169 SLIST_INSERT_HEAD(&frag_state_head, s, link); 1170 rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, 1171 flen - addr_range_offs); 1172 1173 if (rc >= 0) 1174 *global_handle = mobj_ffa_get_cookie(share.mf); 1175 1176 return rc; 1177 } 1178 1179 rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, 1180 flen - addr_range_offs); 1181 if (rc) { 1182 /* 1183 * Number of consumed bytes may be returned instead of 0 for 1184 * done. 1185 */ 1186 rc = FFA_INVALID_PARAMETERS; 1187 goto err; 1188 } 1189 1190 *global_handle = mobj_ffa_push_to_inactive(share.mf); 1191 1192 return 0; 1193 err: 1194 mobj_ffa_sel1_spmc_delete(share.mf); 1195 return rc; 1196 } 1197 1198 static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, 1199 unsigned int page_count, 1200 uint64_t *global_handle, struct ffa_rxtx *rxtx) 1201 { 1202 struct ffa_mem_transaction_x mem_trans = { }; 1203 int rc = 0; 1204 size_t len = 0; 1205 void *buf = NULL; 1206 tee_mm_entry_t *mm = NULL; 1207 vaddr_t offs = pbuf & SMALL_PAGE_MASK; 1208 1209 if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) 1210 return FFA_INVALID_PARAMETERS; 1211 if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) 1212 return FFA_INVALID_PARAMETERS; 1213 1214 /* 1215 * Check that the length reported in flen is covered by len even 1216 * if the offset is taken into account. 1217 */ 1218 if (len < flen || len - offs < flen) 1219 return FFA_INVALID_PARAMETERS; 1220 1221 mm = tee_mm_alloc(&core_virt_shm_pool, len); 1222 if (!mm) 1223 return FFA_NO_MEMORY; 1224 1225 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, 1226 page_count, MEM_AREA_NSEC_SHM)) { 1227 rc = FFA_INVALID_PARAMETERS; 1228 goto out; 1229 } 1230 buf = (void *)(tee_mm_get_smem(mm) + offs); 1231 1232 cpu_spin_lock(&rxtx->spinlock); 1233 rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); 1234 if (rc) 1235 goto unlock; 1236 1237 if (is_sp_share(&mem_trans, buf)) { 1238 rc = spmc_sp_add_share(&mem_trans, buf, blen, flen, 1239 global_handle, NULL); 1240 goto unlock; 1241 } 1242 1243 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1244 virt_set_guest(mem_trans.sender_id)) { 1245 rc = FFA_DENIED; 1246 goto unlock; 1247 } 1248 1249 rc = add_mem_share(&mem_trans, mm, buf, blen, flen, global_handle); 1250 1251 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1252 virt_unset_guest(); 1253 1254 unlock: 1255 cpu_spin_unlock(&rxtx->spinlock); 1256 if (rc > 0) 1257 return rc; 1258 1259 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1260 out: 1261 tee_mm_free(mm); 1262 return rc; 1263 } 1264 1265 static int handle_mem_share_rxbuf(size_t blen, size_t flen, 1266 uint64_t *global_handle, 1267 struct ffa_rxtx *rxtx) 1268 { 1269 struct ffa_mem_transaction_x mem_trans = { }; 1270 int rc = FFA_DENIED; 1271 1272 cpu_spin_lock(&rxtx->spinlock); 1273 1274 if (!rxtx->rx || flen > rxtx->size) 1275 goto out; 1276 1277 rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, 1278 &mem_trans); 1279 if (rc) 1280 goto out; 1281 if (is_sp_share(&mem_trans, rxtx->rx)) { 1282 rc = spmc_sp_add_share(&mem_trans, rxtx, blen, flen, 1283 global_handle, NULL); 1284 goto out; 1285 } 1286 1287 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1288 virt_set_guest(mem_trans.sender_id)) 1289 goto out; 1290 1291 rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, 1292 global_handle); 1293 1294 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1295 virt_unset_guest(); 1296 1297 out: 1298 cpu_spin_unlock(&rxtx->spinlock); 1299 1300 return rc; 1301 } 1302 1303 static void handle_mem_share(struct thread_smc_args *args, 1304 struct ffa_rxtx *rxtx) 1305 { 1306 uint32_t tot_len = args->a1; 1307 uint32_t frag_len = args->a2; 1308 uint64_t addr = args->a3; 1309 uint32_t page_count = args->a4; 1310 uint32_t ret_w1 = 0; 1311 uint32_t ret_w2 = FFA_INVALID_PARAMETERS; 1312 uint32_t ret_w3 = 0; 1313 uint32_t ret_fid = FFA_ERROR; 1314 uint64_t global_handle = 0; 1315 int rc = 0; 1316 1317 /* Check that the MBZs are indeed 0 */ 1318 if (args->a5 || args->a6 || args->a7) 1319 goto out; 1320 1321 /* Check that fragment length doesn't exceed total length */ 1322 if (frag_len > tot_len) 1323 goto out; 1324 1325 /* Check for 32-bit calling convention */ 1326 if (args->a0 == FFA_MEM_SHARE_32) 1327 addr &= UINT32_MAX; 1328 1329 if (!addr) { 1330 /* 1331 * The memory transaction descriptor is passed via our rx 1332 * buffer. 1333 */ 1334 if (page_count) 1335 goto out; 1336 rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, 1337 rxtx); 1338 } else { 1339 rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, 1340 &global_handle, rxtx); 1341 } 1342 if (rc < 0) { 1343 ret_w2 = rc; 1344 } else if (rc > 0) { 1345 ret_fid = FFA_MEM_FRAG_RX; 1346 ret_w3 = rc; 1347 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1348 } else { 1349 ret_fid = FFA_SUCCESS_32; 1350 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1351 } 1352 out: 1353 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1354 } 1355 1356 static struct mem_frag_state *get_frag_state(uint64_t global_handle) 1357 { 1358 struct mem_frag_state *s = NULL; 1359 1360 SLIST_FOREACH(s, &frag_state_head, link) 1361 if (mobj_ffa_get_cookie(s->share.mf) == global_handle) 1362 return s; 1363 1364 return NULL; 1365 } 1366 1367 static void handle_mem_frag_tx(struct thread_smc_args *args, 1368 struct ffa_rxtx *rxtx) 1369 { 1370 uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); 1371 size_t flen = args->a3; 1372 uint32_t endpoint_id = args->a4; 1373 struct mem_frag_state *s = NULL; 1374 tee_mm_entry_t *mm = NULL; 1375 unsigned int page_count = 0; 1376 void *buf = NULL; 1377 uint32_t ret_w1 = 0; 1378 uint32_t ret_w2 = 0; 1379 uint32_t ret_w3 = 0; 1380 uint32_t ret_fid = 0; 1381 int rc = 0; 1382 1383 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1384 uint16_t guest_id = endpoint_id >> 16; 1385 1386 if (!guest_id || virt_set_guest(guest_id)) { 1387 rc = FFA_INVALID_PARAMETERS; 1388 goto out_set_rc; 1389 } 1390 } 1391 1392 /* 1393 * Currently we're only doing this for fragmented FFA_MEM_SHARE_* 1394 * requests. 1395 */ 1396 1397 cpu_spin_lock(&rxtx->spinlock); 1398 1399 s = get_frag_state(global_handle); 1400 if (!s) { 1401 rc = FFA_INVALID_PARAMETERS; 1402 goto out; 1403 } 1404 1405 mm = s->mm; 1406 if (mm) { 1407 if (flen > tee_mm_get_bytes(mm)) { 1408 rc = FFA_INVALID_PARAMETERS; 1409 goto out; 1410 } 1411 page_count = s->share.page_count; 1412 buf = (void *)tee_mm_get_smem(mm); 1413 } else { 1414 if (flen > rxtx->size) { 1415 rc = FFA_INVALID_PARAMETERS; 1416 goto out; 1417 } 1418 buf = rxtx->rx; 1419 } 1420 1421 rc = add_mem_share_frag(s, buf, flen); 1422 out: 1423 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1424 virt_unset_guest(); 1425 1426 cpu_spin_unlock(&rxtx->spinlock); 1427 1428 if (rc <= 0 && mm) { 1429 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1430 tee_mm_free(mm); 1431 } 1432 1433 out_set_rc: 1434 if (rc < 0) { 1435 ret_fid = FFA_ERROR; 1436 ret_w2 = rc; 1437 } else if (rc > 0) { 1438 ret_fid = FFA_MEM_FRAG_RX; 1439 ret_w3 = rc; 1440 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1441 } else { 1442 ret_fid = FFA_SUCCESS_32; 1443 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1444 } 1445 1446 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1447 } 1448 1449 static void handle_mem_reclaim(struct thread_smc_args *args) 1450 { 1451 int rc = FFA_INVALID_PARAMETERS; 1452 uint64_t cookie = 0; 1453 1454 if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) 1455 goto out; 1456 1457 cookie = reg_pair_to_64(args->a2, args->a1); 1458 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1459 uint16_t guest_id = 0; 1460 1461 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 1462 guest_id = virt_find_guest_by_cookie(cookie); 1463 } else { 1464 guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & 1465 FFA_MEMORY_HANDLE_PRTN_MASK; 1466 } 1467 if (!guest_id) 1468 goto out; 1469 if (virt_set_guest(guest_id)) { 1470 if (!virt_reclaim_cookie_from_destroyed_guest(guest_id, 1471 cookie)) 1472 rc = FFA_OK; 1473 goto out; 1474 } 1475 } 1476 1477 switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { 1478 case TEE_SUCCESS: 1479 rc = FFA_OK; 1480 break; 1481 case TEE_ERROR_ITEM_NOT_FOUND: 1482 DMSG("cookie %#"PRIx64" not found", cookie); 1483 rc = FFA_INVALID_PARAMETERS; 1484 break; 1485 default: 1486 DMSG("cookie %#"PRIx64" busy", cookie); 1487 rc = FFA_DENIED; 1488 break; 1489 } 1490 1491 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1492 virt_unset_guest(); 1493 1494 out: 1495 set_simple_ret_val(args, rc); 1496 } 1497 1498 static void handle_notification_bitmap_create(struct thread_smc_args *args) 1499 { 1500 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1501 uint32_t ret_fid = FFA_ERROR; 1502 uint32_t old_itr_status = 0; 1503 1504 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1505 !args->a5 && !args->a6 && !args->a7) { 1506 struct guest_partition *prtn = NULL; 1507 struct notif_vm_bitmap *nvb = NULL; 1508 uint16_t vm_id = args->a1; 1509 1510 prtn = virt_get_guest(vm_id); 1511 nvb = get_notif_vm_bitmap(prtn, vm_id); 1512 if (!nvb) { 1513 ret_val = FFA_INVALID_PARAMETERS; 1514 goto out_virt_put; 1515 } 1516 1517 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1518 1519 if (nvb->initialized) { 1520 ret_val = FFA_DENIED; 1521 goto out_unlock; 1522 } 1523 1524 nvb->initialized = true; 1525 nvb->do_bottom_half_value = -1; 1526 ret_val = FFA_OK; 1527 ret_fid = FFA_SUCCESS_32; 1528 out_unlock: 1529 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1530 out_virt_put: 1531 virt_put_guest(prtn); 1532 } 1533 1534 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1535 } 1536 1537 static void handle_notification_bitmap_destroy(struct thread_smc_args *args) 1538 { 1539 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1540 uint32_t ret_fid = FFA_ERROR; 1541 uint32_t old_itr_status = 0; 1542 1543 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1544 !args->a5 && !args->a6 && !args->a7) { 1545 struct guest_partition *prtn = NULL; 1546 struct notif_vm_bitmap *nvb = NULL; 1547 uint16_t vm_id = args->a1; 1548 1549 prtn = virt_get_guest(vm_id); 1550 nvb = get_notif_vm_bitmap(prtn, vm_id); 1551 if (!nvb) { 1552 ret_val = FFA_INVALID_PARAMETERS; 1553 goto out_virt_put; 1554 } 1555 1556 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1557 1558 if (nvb->pending || nvb->bound) { 1559 ret_val = FFA_DENIED; 1560 goto out_unlock; 1561 } 1562 1563 memset(nvb, 0, sizeof(*nvb)); 1564 ret_val = FFA_OK; 1565 ret_fid = FFA_SUCCESS_32; 1566 out_unlock: 1567 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1568 out_virt_put: 1569 virt_put_guest(prtn); 1570 } 1571 1572 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1573 } 1574 1575 static void handle_notification_bind(struct thread_smc_args *args) 1576 { 1577 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1578 struct guest_partition *prtn = NULL; 1579 struct notif_vm_bitmap *nvb = NULL; 1580 uint32_t ret_fid = FFA_ERROR; 1581 uint32_t old_itr_status = 0; 1582 uint64_t bitmap = 0; 1583 uint16_t vm_id = 0; 1584 1585 if (args->a5 || args->a6 || args->a7) 1586 goto out; 1587 if (args->a2) { 1588 /* We only deal with global notifications */ 1589 ret_val = FFA_DENIED; 1590 goto out; 1591 } 1592 1593 /* The destination of the eventual notification */ 1594 vm_id = FFA_DST(args->a1); 1595 bitmap = reg_pair_to_64(args->a4, args->a3); 1596 1597 prtn = virt_get_guest(vm_id); 1598 nvb = get_notif_vm_bitmap(prtn, vm_id); 1599 if (!nvb) { 1600 ret_val = FFA_INVALID_PARAMETERS; 1601 goto out_virt_put; 1602 } 1603 1604 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1605 1606 if ((bitmap & nvb->bound)) { 1607 ret_val = FFA_DENIED; 1608 } else { 1609 nvb->bound |= bitmap; 1610 ret_val = FFA_OK; 1611 ret_fid = FFA_SUCCESS_32; 1612 } 1613 1614 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1615 out_virt_put: 1616 virt_put_guest(prtn); 1617 out: 1618 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1619 } 1620 1621 static void handle_notification_unbind(struct thread_smc_args *args) 1622 { 1623 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1624 struct guest_partition *prtn = NULL; 1625 struct notif_vm_bitmap *nvb = NULL; 1626 uint32_t ret_fid = FFA_ERROR; 1627 uint32_t old_itr_status = 0; 1628 uint64_t bitmap = 0; 1629 uint16_t vm_id = 0; 1630 1631 if (args->a2 || args->a5 || args->a6 || args->a7) 1632 goto out; 1633 1634 /* The destination of the eventual notification */ 1635 vm_id = FFA_DST(args->a1); 1636 bitmap = reg_pair_to_64(args->a4, args->a3); 1637 1638 prtn = virt_get_guest(vm_id); 1639 nvb = get_notif_vm_bitmap(prtn, vm_id); 1640 if (!nvb) { 1641 ret_val = FFA_INVALID_PARAMETERS; 1642 goto out_virt_put; 1643 } 1644 1645 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1646 1647 if (bitmap & nvb->pending) { 1648 ret_val = FFA_DENIED; 1649 } else { 1650 nvb->bound &= ~bitmap; 1651 ret_val = FFA_OK; 1652 ret_fid = FFA_SUCCESS_32; 1653 } 1654 1655 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1656 out_virt_put: 1657 virt_put_guest(prtn); 1658 out: 1659 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1660 } 1661 1662 static void handle_notification_get(struct thread_smc_args *args) 1663 { 1664 uint32_t w2 = FFA_INVALID_PARAMETERS; 1665 struct guest_partition *prtn = NULL; 1666 struct notif_vm_bitmap *nvb = NULL; 1667 uint32_t ret_fid = FFA_ERROR; 1668 uint32_t old_itr_status = 0; 1669 uint16_t vm_id = 0; 1670 uint32_t w3 = 0; 1671 1672 if (args->a5 || args->a6 || args->a7) 1673 goto out; 1674 if (!(args->a2 & 0x1)) { 1675 ret_fid = FFA_SUCCESS_32; 1676 w2 = 0; 1677 goto out; 1678 } 1679 vm_id = FFA_DST(args->a1); 1680 1681 prtn = virt_get_guest(vm_id); 1682 nvb = get_notif_vm_bitmap(prtn, vm_id); 1683 if (!nvb) 1684 goto out_virt_put; 1685 1686 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1687 1688 reg_pair_from_64(nvb->pending, &w3, &w2); 1689 nvb->pending = 0; 1690 ret_fid = FFA_SUCCESS_32; 1691 1692 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1693 out_virt_put: 1694 virt_put_guest(prtn); 1695 out: 1696 spmc_set_args(args, ret_fid, 0, w2, w3, 0, 0); 1697 } 1698 1699 struct notif_info_get_state { 1700 struct thread_smc_args *args; 1701 unsigned int ids_per_reg; 1702 unsigned int ids_count; 1703 unsigned int id_pos; 1704 unsigned int count; 1705 unsigned int max_list_count; 1706 unsigned int list_count; 1707 }; 1708 1709 static unsigned long get_smc_arg(struct thread_smc_args *args, unsigned int idx) 1710 { 1711 switch (idx) { 1712 case 0: 1713 return args->a0; 1714 case 1: 1715 return args->a1; 1716 case 2: 1717 return args->a2; 1718 case 3: 1719 return args->a3; 1720 case 4: 1721 return args->a4; 1722 case 5: 1723 return args->a5; 1724 case 6: 1725 return args->a6; 1726 case 7: 1727 return args->a7; 1728 default: 1729 assert(0); 1730 return 0; 1731 } 1732 } 1733 1734 static void set_smc_arg(struct thread_smc_args *args, unsigned int idx, 1735 unsigned long val) 1736 { 1737 switch (idx) { 1738 case 0: 1739 args->a0 = val; 1740 break; 1741 case 1: 1742 args->a1 = val; 1743 break; 1744 case 2: 1745 args->a2 = val; 1746 break; 1747 case 3: 1748 args->a3 = val; 1749 break; 1750 case 4: 1751 args->a4 = val; 1752 break; 1753 case 5: 1754 args->a5 = val; 1755 break; 1756 case 6: 1757 args->a6 = val; 1758 break; 1759 case 7: 1760 args->a7 = val; 1761 break; 1762 default: 1763 assert(0); 1764 } 1765 } 1766 1767 static bool add_id_in_regs(struct notif_info_get_state *state, 1768 uint16_t id) 1769 { 1770 unsigned int reg_idx = state->id_pos / state->ids_per_reg + 3; 1771 unsigned int reg_shift = (state->id_pos % state->ids_per_reg) * 16; 1772 unsigned long v; 1773 1774 if (reg_idx > 7) 1775 return false; 1776 1777 v = get_smc_arg(state->args, reg_idx); 1778 v &= ~(0xffffUL << reg_shift); 1779 v |= (unsigned long)id << reg_shift; 1780 set_smc_arg(state->args, reg_idx, v); 1781 1782 state->id_pos++; 1783 state->count++; 1784 return true; 1785 } 1786 1787 static bool add_id_count(struct notif_info_get_state *state) 1788 { 1789 assert(state->list_count < state->max_list_count && 1790 state->count >= 1 && state->count <= 4); 1791 1792 state->ids_count |= (state->count - 1) << (state->list_count * 2 + 12); 1793 state->list_count++; 1794 state->count = 0; 1795 1796 return state->list_count < state->max_list_count; 1797 } 1798 1799 static bool add_nvb_to_state(struct notif_info_get_state *state, 1800 uint16_t guest_id, struct notif_vm_bitmap *nvb) 1801 { 1802 if (!nvb->pending) 1803 return true; 1804 /* 1805 * Add only the guest_id, meaning a global notification for this 1806 * guest. 1807 * 1808 * If notifications for one or more specific vCPUs we'd add those 1809 * before calling add_id_count(), but that's not supported. 1810 */ 1811 return add_id_in_regs(state, guest_id) && add_id_count(state); 1812 } 1813 1814 static void handle_notification_info_get(struct thread_smc_args *args) 1815 { 1816 struct notif_info_get_state state = { .args = args }; 1817 uint32_t ffa_res = FFA_INVALID_PARAMETERS; 1818 struct guest_partition *prtn = NULL; 1819 struct notif_vm_bitmap *nvb = NULL; 1820 uint32_t more_pending_flag = 0; 1821 uint32_t itr_state = 0; 1822 uint16_t guest_id = 0; 1823 1824 if (args->a1 || args->a2 || args->a3 || args->a4 || args->a5 || 1825 args->a6 || args->a7) 1826 goto err; 1827 1828 if (OPTEE_SMC_IS_64(args->a0)) { 1829 spmc_set_args(args, FFA_SUCCESS_64, 0, 0, 0, 0, 0); 1830 state.ids_per_reg = 4; 1831 state.max_list_count = 31; 1832 } else { 1833 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 1834 state.ids_per_reg = 2; 1835 state.max_list_count = 15; 1836 } 1837 1838 while (true) { 1839 /* 1840 * With NS-Virtualization we need to go through all 1841 * partitions to collect the notification bitmaps, without 1842 * we just check the only notification bitmap we have. 1843 */ 1844 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1845 prtn = virt_next_guest(prtn); 1846 if (!prtn) 1847 break; 1848 guest_id = virt_get_guest_id(prtn); 1849 } 1850 nvb = get_notif_vm_bitmap(prtn, guest_id); 1851 1852 itr_state = cpu_spin_lock_xsave(&spmc_notif_lock); 1853 if (!add_nvb_to_state(&state, guest_id, nvb)) 1854 more_pending_flag = BIT(0); 1855 cpu_spin_unlock_xrestore(&spmc_notif_lock, itr_state); 1856 1857 if (!IS_ENABLED(CFG_NS_VIRTUALIZATION) || more_pending_flag) 1858 break; 1859 } 1860 virt_put_guest(prtn); 1861 1862 if (!state.id_pos) { 1863 ffa_res = FFA_NO_DATA; 1864 goto err; 1865 } 1866 args->a2 = (state.list_count << FFA_NOTIF_INFO_GET_ID_COUNT_SHIFT) | 1867 (state.ids_count << FFA_NOTIF_INFO_GET_ID_LIST_SHIFT) | 1868 more_pending_flag; 1869 return; 1870 err: 1871 spmc_set_args(args, FFA_ERROR, 0, ffa_res, 0, 0, 0); 1872 } 1873 1874 void thread_spmc_set_async_notif_intid(int intid) 1875 { 1876 assert(interrupt_can_raise_sgi(interrupt_get_main_chip())); 1877 notif_intid = intid; 1878 spmc_notif_is_ready = true; 1879 DMSG("Asynchronous notifications are ready"); 1880 } 1881 1882 void notif_send_async(uint32_t value, uint16_t guest_id) 1883 { 1884 struct guest_partition *prtn = NULL; 1885 struct notif_vm_bitmap *nvb = NULL; 1886 uint32_t old_itr_status = 0; 1887 1888 prtn = virt_get_guest(guest_id); 1889 nvb = get_notif_vm_bitmap(prtn, guest_id); 1890 1891 if (nvb) { 1892 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1893 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && 1894 spmc_notif_is_ready && nvb->do_bottom_half_value >= 0 && 1895 notif_intid >= 0); 1896 nvb->pending |= BIT64(nvb->do_bottom_half_value); 1897 interrupt_raise_sgi(interrupt_get_main_chip(), notif_intid, 1898 ITR_CPU_MASK_TO_THIS_CPU); 1899 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1900 } 1901 1902 virt_put_guest(prtn); 1903 } 1904 #else 1905 void notif_send_async(uint32_t value, uint16_t guest_id) 1906 { 1907 struct guest_partition *prtn = NULL; 1908 struct notif_vm_bitmap *nvb = NULL; 1909 /* global notification, delay notification interrupt */ 1910 uint32_t flags = BIT32(1); 1911 int res = 0; 1912 1913 prtn = virt_get_guest(guest_id); 1914 nvb = get_notif_vm_bitmap(prtn, guest_id); 1915 1916 if (nvb) { 1917 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && 1918 spmc_notif_is_ready && nvb->do_bottom_half_value >= 0); 1919 res = ffa_set_notification(guest_id, optee_endpoint_id, flags, 1920 BIT64(nvb->do_bottom_half_value)); 1921 if (res) { 1922 EMSG("notification set failed with error %d", res); 1923 panic(); 1924 } 1925 } 1926 1927 virt_put_guest(prtn); 1928 } 1929 #endif 1930 1931 /* Only called from assembly */ 1932 void thread_spmc_msg_recv(struct thread_smc_args *args); 1933 void thread_spmc_msg_recv(struct thread_smc_args *args) 1934 { 1935 assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); 1936 switch (args->a0) { 1937 #if defined(CFG_CORE_SEL1_SPMC) 1938 case FFA_FEATURES: 1939 handle_features(args); 1940 break; 1941 case FFA_SPM_ID_GET: 1942 spmc_handle_spm_id_get(args); 1943 break; 1944 #ifdef ARM64 1945 case FFA_RXTX_MAP_64: 1946 #endif 1947 case FFA_RXTX_MAP_32: 1948 spmc_handle_rxtx_map(args, &my_rxtx); 1949 break; 1950 case FFA_RXTX_UNMAP: 1951 spmc_handle_rxtx_unmap(args, &my_rxtx); 1952 break; 1953 case FFA_RX_RELEASE: 1954 spmc_handle_rx_release(args, &my_rxtx); 1955 break; 1956 case FFA_PARTITION_INFO_GET: 1957 spmc_handle_partition_info_get(args, &my_rxtx); 1958 break; 1959 case FFA_RUN: 1960 spmc_handle_run(args); 1961 break; 1962 #endif /*CFG_CORE_SEL1_SPMC*/ 1963 case FFA_INTERRUPT: 1964 if (IS_ENABLED(CFG_CORE_SEL1_SPMC)) 1965 spmc_set_args(args, FFA_NORMAL_WORLD_RESUME, 0, 0, 0, 1966 0, 0); 1967 else 1968 spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); 1969 break; 1970 #ifdef ARM64 1971 case FFA_MSG_SEND_DIRECT_REQ_64: 1972 #endif 1973 case FFA_MSG_SEND_DIRECT_REQ_32: 1974 handle_direct_request(args, &my_rxtx); 1975 break; 1976 #if defined(CFG_CORE_SEL1_SPMC) 1977 #ifdef ARM64 1978 case FFA_MEM_SHARE_64: 1979 #endif 1980 case FFA_MEM_SHARE_32: 1981 handle_mem_share(args, &my_rxtx); 1982 break; 1983 case FFA_MEM_RECLAIM: 1984 if (!IS_ENABLED(CFG_SECURE_PARTITION) || 1985 !ffa_mem_reclaim(args, NULL)) 1986 handle_mem_reclaim(args); 1987 break; 1988 case FFA_MEM_FRAG_TX: 1989 handle_mem_frag_tx(args, &my_rxtx); 1990 break; 1991 case FFA_NOTIFICATION_BITMAP_CREATE: 1992 handle_notification_bitmap_create(args); 1993 break; 1994 case FFA_NOTIFICATION_BITMAP_DESTROY: 1995 handle_notification_bitmap_destroy(args); 1996 break; 1997 case FFA_NOTIFICATION_BIND: 1998 handle_notification_bind(args); 1999 break; 2000 case FFA_NOTIFICATION_UNBIND: 2001 handle_notification_unbind(args); 2002 break; 2003 case FFA_NOTIFICATION_GET: 2004 handle_notification_get(args); 2005 break; 2006 #ifdef ARM64 2007 case FFA_NOTIFICATION_INFO_GET_64: 2008 #endif 2009 case FFA_NOTIFICATION_INFO_GET_32: 2010 handle_notification_info_get(args); 2011 break; 2012 #endif /*CFG_CORE_SEL1_SPMC*/ 2013 case FFA_ERROR: 2014 EMSG("Cannot handle FFA_ERROR(%d)", (int)args->a2); 2015 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { 2016 /* 2017 * The SPMC will return an FFA_ERROR back so better 2018 * panic() now than flooding the log. 2019 */ 2020 panic("FFA_ERROR from SPMC is fatal"); 2021 } 2022 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, 2023 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 2024 break; 2025 default: 2026 EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); 2027 set_simple_ret_val(args, FFA_NOT_SUPPORTED); 2028 } 2029 } 2030 2031 static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) 2032 { 2033 size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 2034 struct thread_ctx *thr = threads + thread_get_id(); 2035 TEE_Result res = TEE_ERROR_BAD_PARAMETERS; 2036 struct optee_msg_arg *arg = NULL; 2037 struct mobj *mobj = NULL; 2038 uint32_t num_params = 0; 2039 size_t sz = 0; 2040 2041 mobj = mobj_ffa_get_by_cookie(cookie, 0); 2042 if (!mobj) { 2043 EMSG("Can't find cookie %#"PRIx64, cookie); 2044 return TEE_ERROR_BAD_PARAMETERS; 2045 } 2046 2047 res = mobj_inc_map(mobj); 2048 if (res) 2049 goto out_put_mobj; 2050 2051 res = TEE_ERROR_BAD_PARAMETERS; 2052 arg = mobj_get_va(mobj, offset, sizeof(*arg)); 2053 if (!arg) 2054 goto out_dec_map; 2055 2056 num_params = READ_ONCE(arg->num_params); 2057 if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) 2058 goto out_dec_map; 2059 2060 sz = OPTEE_MSG_GET_ARG_SIZE(num_params); 2061 2062 thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); 2063 if (!thr->rpc_arg) 2064 goto out_dec_map; 2065 2066 virt_on_stdcall(); 2067 res = tee_entry_std(arg, num_params); 2068 2069 thread_rpc_shm_cache_clear(&thr->shm_cache); 2070 thr->rpc_arg = NULL; 2071 2072 out_dec_map: 2073 mobj_dec_map(mobj); 2074 out_put_mobj: 2075 mobj_put(mobj); 2076 return res; 2077 } 2078 2079 /* 2080 * Helper routine for the assembly function thread_std_smc_entry() 2081 * 2082 * Note: this function is weak just to make link_dummies_paged.c happy. 2083 */ 2084 uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, 2085 uint32_t a2, uint32_t a3, 2086 uint32_t a4, uint32_t a5 __unused) 2087 { 2088 /* 2089 * Arguments are supplied from handle_yielding_call() as: 2090 * a0 <- w1 2091 * a1 <- w3 2092 * a2 <- w4 2093 * a3 <- w5 2094 * a4 <- w6 2095 * a5 <- w7 2096 */ 2097 thread_get_tsd()->rpc_target_info = swap_src_dst(a0); 2098 if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) 2099 return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); 2100 return FFA_DENIED; 2101 } 2102 2103 static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) 2104 { 2105 uint64_t offs = tpm->u.memref.offs; 2106 2107 param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + 2108 OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; 2109 2110 param->u.fmem.offs_low = offs; 2111 param->u.fmem.offs_high = offs >> 32; 2112 if (param->u.fmem.offs_high != offs >> 32) 2113 return false; 2114 2115 param->u.fmem.size = tpm->u.memref.size; 2116 if (tpm->u.memref.mobj) { 2117 uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); 2118 2119 /* If a mobj is passed it better be one with a valid cookie. */ 2120 if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) 2121 return false; 2122 param->u.fmem.global_id = cookie; 2123 } else { 2124 param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 2125 } 2126 2127 return true; 2128 } 2129 2130 static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, 2131 struct thread_param *params, 2132 struct optee_msg_arg **arg_ret) 2133 { 2134 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 2135 struct thread_ctx *thr = threads + thread_get_id(); 2136 struct optee_msg_arg *arg = thr->rpc_arg; 2137 2138 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 2139 return TEE_ERROR_BAD_PARAMETERS; 2140 2141 if (!arg) { 2142 EMSG("rpc_arg not set"); 2143 return TEE_ERROR_GENERIC; 2144 } 2145 2146 memset(arg, 0, sz); 2147 arg->cmd = cmd; 2148 arg->num_params = num_params; 2149 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 2150 2151 for (size_t n = 0; n < num_params; n++) { 2152 switch (params[n].attr) { 2153 case THREAD_PARAM_ATTR_NONE: 2154 arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; 2155 break; 2156 case THREAD_PARAM_ATTR_VALUE_IN: 2157 case THREAD_PARAM_ATTR_VALUE_OUT: 2158 case THREAD_PARAM_ATTR_VALUE_INOUT: 2159 arg->params[n].attr = params[n].attr - 2160 THREAD_PARAM_ATTR_VALUE_IN + 2161 OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 2162 arg->params[n].u.value.a = params[n].u.value.a; 2163 arg->params[n].u.value.b = params[n].u.value.b; 2164 arg->params[n].u.value.c = params[n].u.value.c; 2165 break; 2166 case THREAD_PARAM_ATTR_MEMREF_IN: 2167 case THREAD_PARAM_ATTR_MEMREF_OUT: 2168 case THREAD_PARAM_ATTR_MEMREF_INOUT: 2169 if (!set_fmem(arg->params + n, params + n)) 2170 return TEE_ERROR_BAD_PARAMETERS; 2171 break; 2172 default: 2173 return TEE_ERROR_BAD_PARAMETERS; 2174 } 2175 } 2176 2177 if (arg_ret) 2178 *arg_ret = arg; 2179 2180 return TEE_SUCCESS; 2181 } 2182 2183 static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, 2184 struct thread_param *params) 2185 { 2186 for (size_t n = 0; n < num_params; n++) { 2187 switch (params[n].attr) { 2188 case THREAD_PARAM_ATTR_VALUE_OUT: 2189 case THREAD_PARAM_ATTR_VALUE_INOUT: 2190 params[n].u.value.a = arg->params[n].u.value.a; 2191 params[n].u.value.b = arg->params[n].u.value.b; 2192 params[n].u.value.c = arg->params[n].u.value.c; 2193 break; 2194 case THREAD_PARAM_ATTR_MEMREF_OUT: 2195 case THREAD_PARAM_ATTR_MEMREF_INOUT: 2196 params[n].u.memref.size = arg->params[n].u.fmem.size; 2197 break; 2198 default: 2199 break; 2200 } 2201 } 2202 2203 return arg->ret; 2204 } 2205 2206 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 2207 struct thread_param *params) 2208 { 2209 struct thread_rpc_arg rpc_arg = { .call = { 2210 .w1 = thread_get_tsd()->rpc_target_info, 2211 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2212 }, 2213 }; 2214 struct optee_msg_arg *arg = NULL; 2215 uint32_t ret = 0; 2216 2217 ret = get_rpc_arg(cmd, num_params, params, &arg); 2218 if (ret) 2219 return ret; 2220 2221 thread_rpc(&rpc_arg); 2222 2223 return get_rpc_arg_res(arg, num_params, params); 2224 } 2225 2226 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 2227 { 2228 struct thread_rpc_arg rpc_arg = { .call = { 2229 .w1 = thread_get_tsd()->rpc_target_info, 2230 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2231 }, 2232 }; 2233 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); 2234 uint32_t res2 = 0; 2235 uint32_t res = 0; 2236 2237 DMSG("freeing cookie %#"PRIx64, cookie); 2238 2239 res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); 2240 2241 mobj_put(mobj); 2242 res2 = mobj_ffa_unregister_by_cookie(cookie); 2243 if (res2) 2244 DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, 2245 cookie, res2); 2246 if (!res) 2247 thread_rpc(&rpc_arg); 2248 } 2249 2250 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) 2251 { 2252 struct thread_rpc_arg rpc_arg = { .call = { 2253 .w1 = thread_get_tsd()->rpc_target_info, 2254 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2255 }, 2256 }; 2257 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); 2258 struct optee_msg_arg *arg = NULL; 2259 unsigned int internal_offset = 0; 2260 struct mobj *mobj = NULL; 2261 uint64_t cookie = 0; 2262 2263 if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) 2264 return NULL; 2265 2266 thread_rpc(&rpc_arg); 2267 2268 if (arg->num_params != 1 || 2269 arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) 2270 return NULL; 2271 2272 internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); 2273 cookie = READ_ONCE(arg->params->u.fmem.global_id); 2274 mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); 2275 if (!mobj) { 2276 DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", 2277 cookie, internal_offset); 2278 return NULL; 2279 } 2280 2281 assert(mobj_is_nonsec(mobj)); 2282 2283 if (mobj->size < size) { 2284 DMSG("Mobj %#"PRIx64": wrong size", cookie); 2285 mobj_put(mobj); 2286 return NULL; 2287 } 2288 2289 if (mobj_inc_map(mobj)) { 2290 DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); 2291 mobj_put(mobj); 2292 return NULL; 2293 } 2294 2295 return mobj; 2296 } 2297 2298 struct mobj *thread_rpc_alloc_payload(size_t size) 2299 { 2300 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); 2301 } 2302 2303 struct mobj *thread_rpc_alloc_kernel_payload(size_t size) 2304 { 2305 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); 2306 } 2307 2308 void thread_rpc_free_kernel_payload(struct mobj *mobj) 2309 { 2310 if (mobj) 2311 thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, 2312 mobj_get_cookie(mobj), mobj); 2313 } 2314 2315 void thread_rpc_free_payload(struct mobj *mobj) 2316 { 2317 if (mobj) 2318 thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), 2319 mobj); 2320 } 2321 2322 struct mobj *thread_rpc_alloc_global_payload(size_t size) 2323 { 2324 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); 2325 } 2326 2327 void thread_rpc_free_global_payload(struct mobj *mobj) 2328 { 2329 if (mobj) 2330 thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, 2331 mobj_get_cookie(mobj), mobj); 2332 } 2333 2334 void thread_spmc_register_secondary_ep(vaddr_t ep) 2335 { 2336 unsigned long ret = 0; 2337 2338 /* Let the SPM know the entry point for secondary CPUs */ 2339 ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); 2340 2341 if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) 2342 EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); 2343 } 2344 2345 static uint16_t ffa_id_get(void) 2346 { 2347 /* 2348 * Ask the SPM component running at a higher EL to return our FF-A ID. 2349 * This can either be the SPMC ID (if the SPMC is enabled in OP-TEE) or 2350 * the partition ID (if not). 2351 */ 2352 struct thread_smc_args args = { 2353 .a0 = FFA_ID_GET, 2354 }; 2355 2356 thread_smccc(&args); 2357 if (!is_ffa_success(args.a0)) { 2358 if (args.a0 == FFA_ERROR) 2359 EMSG("Get id failed with error %ld", args.a2); 2360 else 2361 EMSG("Get id failed"); 2362 panic(); 2363 } 2364 2365 return args.a2; 2366 } 2367 2368 static uint16_t ffa_spm_id_get(void) 2369 { 2370 /* 2371 * Ask the SPM component running at a higher EL to return its ID. 2372 * If OP-TEE implements the S-EL1 SPMC, this will get the SPMD ID. 2373 * If not, the ID of the SPMC will be returned. 2374 */ 2375 struct thread_smc_args args = { 2376 .a0 = FFA_SPM_ID_GET, 2377 }; 2378 2379 thread_smccc(&args); 2380 if (!is_ffa_success(args.a0)) { 2381 if (args.a0 == FFA_ERROR) 2382 EMSG("Get spm id failed with error %ld", args.a2); 2383 else 2384 EMSG("Get spm id failed"); 2385 panic(); 2386 } 2387 2388 return args.a2; 2389 } 2390 2391 #if defined(CFG_CORE_SEL1_SPMC) 2392 static TEE_Result spmc_init(void) 2393 { 2394 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 2395 virt_add_guest_spec_data(¬if_vm_bitmap_id, 2396 sizeof(struct notif_vm_bitmap), NULL)) 2397 panic("virt_add_guest_spec_data"); 2398 spmd_id = ffa_spm_id_get(); 2399 DMSG("SPMD ID %#"PRIx16, spmd_id); 2400 2401 spmc_id = ffa_id_get(); 2402 DMSG("SPMC ID %#"PRIx16, spmc_id); 2403 2404 optee_endpoint_id = FFA_SWD_ID_MIN; 2405 while (optee_endpoint_id == spmd_id || optee_endpoint_id == spmc_id) 2406 optee_endpoint_id++; 2407 2408 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2409 2410 /* 2411 * If SPMD think we are version 1.0 it will report version 1.0 to 2412 * normal world regardless of what version we query the SPM with. 2413 * However, if SPMD think we are version 1.1 it will forward 2414 * queries from normal world to let us negotiate version. So by 2415 * setting version 1.0 here we should be compatible. 2416 * 2417 * Note that disagreement on negotiated version means that we'll 2418 * have communication problems with normal world. 2419 */ 2420 my_rxtx.ffa_vers = FFA_VERSION_1_0; 2421 2422 return TEE_SUCCESS; 2423 } 2424 #else /* !defined(CFG_CORE_SEL1_SPMC) */ 2425 static void spmc_rxtx_map(struct ffa_rxtx *rxtx) 2426 { 2427 struct thread_smc_args args = { 2428 #ifdef ARM64 2429 .a0 = FFA_RXTX_MAP_64, 2430 #else 2431 .a0 = FFA_RXTX_MAP_32, 2432 #endif 2433 .a1 = virt_to_phys(rxtx->tx), 2434 .a2 = virt_to_phys(rxtx->rx), 2435 .a3 = 1, 2436 }; 2437 2438 thread_smccc(&args); 2439 if (!is_ffa_success(args.a0)) { 2440 if (args.a0 == FFA_ERROR) 2441 EMSG("rxtx map failed with error %ld", args.a2); 2442 else 2443 EMSG("rxtx map failed"); 2444 panic(); 2445 } 2446 } 2447 2448 static uint32_t get_ffa_version(uint32_t my_version) 2449 { 2450 struct thread_smc_args args = { 2451 .a0 = FFA_VERSION, 2452 .a1 = my_version, 2453 }; 2454 2455 thread_smccc(&args); 2456 if (args.a0 & BIT(31)) { 2457 EMSG("FF-A version failed with error %ld", args.a0); 2458 panic(); 2459 } 2460 2461 return args.a0; 2462 } 2463 2464 static void *spmc_retrieve_req(uint64_t cookie, 2465 struct ffa_mem_transaction_x *trans) 2466 { 2467 struct ffa_mem_access *acc_descr_array = NULL; 2468 struct ffa_mem_access_perm *perm_descr = NULL; 2469 struct thread_smc_args args = { 2470 .a0 = FFA_MEM_RETRIEVE_REQ_32, 2471 .a3 = 0, /* Address, Using TX -> MBZ */ 2472 .a4 = 0, /* Using TX -> MBZ */ 2473 }; 2474 size_t size = 0; 2475 int rc = 0; 2476 2477 if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { 2478 struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; 2479 2480 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2481 memset(trans_descr, 0, size); 2482 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2483 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2484 trans_descr->global_handle = cookie; 2485 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2486 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2487 trans_descr->mem_access_count = 1; 2488 acc_descr_array = trans_descr->mem_access_array; 2489 } else { 2490 struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; 2491 2492 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2493 memset(trans_descr, 0, size); 2494 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2495 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2496 trans_descr->global_handle = cookie; 2497 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2498 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2499 trans_descr->mem_access_count = 1; 2500 trans_descr->mem_access_offs = sizeof(*trans_descr); 2501 trans_descr->mem_access_size = sizeof(struct ffa_mem_access); 2502 acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + 2503 sizeof(*trans_descr)); 2504 } 2505 acc_descr_array->region_offs = 0; 2506 acc_descr_array->reserved = 0; 2507 perm_descr = &acc_descr_array->access_perm; 2508 perm_descr->endpoint_id = optee_endpoint_id; 2509 perm_descr->perm = FFA_MEM_ACC_RW; 2510 perm_descr->flags = 0; 2511 2512 args.a1 = size; /* Total Length */ 2513 args.a2 = size; /* Frag Length == Total length */ 2514 thread_smccc(&args); 2515 if (args.a0 != FFA_MEM_RETRIEVE_RESP) { 2516 if (args.a0 == FFA_ERROR) 2517 EMSG("Failed to fetch cookie %#"PRIx64" error code %d", 2518 cookie, (int)args.a2); 2519 else 2520 EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, 2521 cookie, args.a0); 2522 return NULL; 2523 } 2524 rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.rx, 2525 my_rxtx.size, trans); 2526 if (rc) { 2527 EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", 2528 cookie, rc); 2529 return NULL; 2530 } 2531 2532 return my_rxtx.rx; 2533 } 2534 2535 void thread_spmc_relinquish(uint64_t cookie) 2536 { 2537 struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; 2538 struct thread_smc_args args = { 2539 .a0 = FFA_MEM_RELINQUISH, 2540 }; 2541 2542 memset(relinquish_desc, 0, sizeof(*relinquish_desc)); 2543 relinquish_desc->handle = cookie; 2544 relinquish_desc->flags = 0; 2545 relinquish_desc->endpoint_count = 1; 2546 relinquish_desc->endpoint_id_array[0] = optee_endpoint_id; 2547 thread_smccc(&args); 2548 if (!is_ffa_success(args.a0)) 2549 EMSG("Failed to relinquish cookie %#"PRIx64, cookie); 2550 } 2551 2552 static int set_pages(struct ffa_address_range *regions, 2553 unsigned int num_regions, unsigned int num_pages, 2554 struct mobj_ffa *mf) 2555 { 2556 unsigned int n = 0; 2557 unsigned int idx = 0; 2558 2559 for (n = 0; n < num_regions; n++) { 2560 unsigned int page_count = READ_ONCE(regions[n].page_count); 2561 uint64_t addr = READ_ONCE(regions[n].address); 2562 2563 if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) 2564 return FFA_INVALID_PARAMETERS; 2565 } 2566 2567 if (idx != num_pages) 2568 return FFA_INVALID_PARAMETERS; 2569 2570 return 0; 2571 } 2572 2573 struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) 2574 { 2575 struct mobj_ffa *ret = NULL; 2576 struct ffa_mem_transaction_x retrieve_desc = { }; 2577 struct ffa_mem_access *descr_array = NULL; 2578 struct ffa_mem_region *descr = NULL; 2579 struct mobj_ffa *mf = NULL; 2580 unsigned int num_pages = 0; 2581 unsigned int offs = 0; 2582 void *buf = NULL; 2583 struct thread_smc_args ffa_rx_release_args = { 2584 .a0 = FFA_RX_RELEASE 2585 }; 2586 2587 /* 2588 * OP-TEE is only supporting a single mem_region while the 2589 * specification allows for more than one. 2590 */ 2591 buf = spmc_retrieve_req(cookie, &retrieve_desc); 2592 if (!buf) { 2593 EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, 2594 cookie); 2595 return NULL; 2596 } 2597 2598 descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); 2599 offs = READ_ONCE(descr_array->region_offs); 2600 descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); 2601 2602 num_pages = READ_ONCE(descr->total_page_count); 2603 mf = mobj_ffa_spmc_new(cookie, num_pages); 2604 if (!mf) 2605 goto out; 2606 2607 if (set_pages(descr->address_range_array, 2608 READ_ONCE(descr->address_range_count), num_pages, mf)) { 2609 mobj_ffa_spmc_delete(mf); 2610 goto out; 2611 } 2612 2613 ret = mf; 2614 2615 out: 2616 /* Release RX buffer after the mem retrieve request. */ 2617 thread_smccc(&ffa_rx_release_args); 2618 2619 return ret; 2620 } 2621 2622 static TEE_Result spmc_init(void) 2623 { 2624 unsigned int major = 0; 2625 unsigned int minor __maybe_unused = 0; 2626 uint32_t my_vers = 0; 2627 uint32_t vers = 0; 2628 2629 my_vers = MAKE_FFA_VERSION(FFA_VERSION_MAJOR, FFA_VERSION_MINOR); 2630 vers = get_ffa_version(my_vers); 2631 major = (vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK; 2632 minor = (vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK; 2633 DMSG("SPMC reported version %u.%u", major, minor); 2634 if (major != FFA_VERSION_MAJOR) { 2635 EMSG("Incompatible major version %u, expected %u", 2636 major, FFA_VERSION_MAJOR); 2637 panic(); 2638 } 2639 if (vers < my_vers) 2640 my_vers = vers; 2641 DMSG("Using version %u.%u", 2642 (my_vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK, 2643 (my_vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK); 2644 my_rxtx.ffa_vers = my_vers; 2645 2646 spmc_rxtx_map(&my_rxtx); 2647 2648 spmc_id = ffa_spm_id_get(); 2649 DMSG("SPMC ID %#"PRIx16, spmc_id); 2650 2651 optee_endpoint_id = ffa_id_get(); 2652 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2653 2654 if (!ffa_features(FFA_NOTIFICATION_SET)) { 2655 spmc_notif_is_ready = true; 2656 DMSG("Asynchronous notifications are ready"); 2657 } 2658 2659 return TEE_SUCCESS; 2660 } 2661 #endif /* !defined(CFG_CORE_SEL1_SPMC) */ 2662 2663 nex_service_init(spmc_init); 2664