1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2020-2025, Linaro Limited. 4 * Copyright (c) 2019-2024, Arm Limited. All rights reserved. 5 */ 6 7 #include <assert.h> 8 #include <ffa.h> 9 #include <initcall.h> 10 #include <io.h> 11 #include <kernel/dt.h> 12 #include <kernel/interrupt.h> 13 #include <kernel/notif.h> 14 #include <kernel/panic.h> 15 #include <kernel/secure_partition.h> 16 #include <kernel/spinlock.h> 17 #include <kernel/spmc_sp_handler.h> 18 #include <kernel/tee_misc.h> 19 #include <kernel/thread.h> 20 #include <kernel/thread_private.h> 21 #include <kernel/thread_spmc.h> 22 #include <kernel/virtualization.h> 23 #include <libfdt.h> 24 #include <mm/core_mmu.h> 25 #include <mm/mobj.h> 26 #include <optee_ffa.h> 27 #include <optee_msg.h> 28 #include <optee_rpc_cmd.h> 29 #include <sm/optee_smc.h> 30 #include <string.h> 31 #include <sys/queue.h> 32 #include <tee/entry_std.h> 33 #include <tee/uuid.h> 34 #include <util.h> 35 36 #if defined(CFG_CORE_SEL1_SPMC) 37 struct mem_share_state { 38 struct mobj_ffa *mf; 39 unsigned int page_count; 40 unsigned int region_count; 41 unsigned int current_page_idx; 42 }; 43 44 struct mem_frag_state { 45 struct mem_share_state share; 46 tee_mm_entry_t *mm; 47 unsigned int frag_offset; 48 SLIST_ENTRY(mem_frag_state) link; 49 }; 50 #endif 51 52 struct notif_vm_bitmap { 53 bool initialized; 54 int do_bottom_half_value; 55 uint64_t pending; 56 uint64_t bound; 57 }; 58 59 static unsigned int spmc_notif_lock __nex_data = SPINLOCK_UNLOCK; 60 static bool spmc_notif_is_ready __nex_bss; 61 static int notif_intid __nex_data __maybe_unused = -1; 62 63 /* Id used to look up the guest specific struct notif_vm_bitmap */ 64 static unsigned int notif_vm_bitmap_id __nex_bss; 65 /* Notification state when ns-virtualization isn't enabled */ 66 static struct notif_vm_bitmap default_notif_vm_bitmap; 67 68 /* Initialized in spmc_init() below */ 69 uint16_t optee_endpoint_id __nex_bss; 70 static uint16_t spmc_id __nex_bss; 71 #ifdef CFG_CORE_SEL1_SPMC 72 static uint16_t spmd_id __nex_bss; 73 static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | 74 FFA_PART_PROP_DIRECT_REQ_SEND | 75 #ifdef CFG_NS_VIRTUALIZATION 76 FFA_PART_PROP_NOTIF_CREATED | 77 FFA_PART_PROP_NOTIF_DESTROYED | 78 #endif 79 #ifdef ARM64 80 FFA_PART_PROP_AARCH64_STATE | 81 #endif 82 FFA_PART_PROP_IS_PE_ID; 83 84 static uint32_t my_uuid_words[] = { 85 /* 86 * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 87 * SP, or 88 * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a 89 * logical partition, residing in the same exception level as the 90 * SPMC 91 * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 92 */ 93 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, 94 }; 95 96 /* 97 * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. 98 * 99 * struct ffa_rxtx::spin_lock protects the variables below from concurrent 100 * access this includes the use of content of struct ffa_rxtx::rx and 101 * @frag_state_head. 102 * 103 * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct 104 * ffa_rxtx::tx and false when it is owned by normal world. 105 * 106 * Note that we can't prevent normal world from updating the content of 107 * these buffers so we must always be careful when reading. while we hold 108 * the lock. 109 */ 110 111 static struct ffa_rxtx my_rxtx __nex_bss; 112 113 static bool is_nw_buf(struct ffa_rxtx *rxtx) 114 { 115 return rxtx == &my_rxtx; 116 } 117 118 static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = 119 SLIST_HEAD_INITIALIZER(&frag_state_head); 120 121 #else 122 static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE) __nex_bss; 123 static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE) __nex_bss; 124 static struct ffa_rxtx my_rxtx __nex_data = { 125 .rx = __rx_buf, 126 .tx = __tx_buf, 127 .size = sizeof(__rx_buf), 128 }; 129 #endif 130 131 bool spmc_is_reserved_id(uint16_t id) 132 { 133 #ifdef CFG_CORE_SEL1_SPMC 134 if (id == spmd_id) 135 return true; 136 #endif 137 return id == spmc_id; 138 } 139 140 static uint32_t swap_src_dst(uint32_t src_dst) 141 { 142 return (src_dst >> 16) | (src_dst << 16); 143 } 144 145 static uint16_t get_sender_id(uint32_t src_dst) 146 { 147 return src_dst >> 16; 148 } 149 150 void spmc_set_args(struct thread_smc_1_2_regs *args, uint32_t fid, 151 uint32_t src_dst, uint32_t w2, uint32_t w3, uint32_t w4, 152 uint32_t w5) 153 { 154 *args = (struct thread_smc_1_2_regs){ 155 .a0 = fid, 156 .a1 = src_dst, 157 .a2 = w2, 158 .a3 = w3, 159 .a4 = w4, 160 .a5 = w5, 161 }; 162 } 163 164 static void set_simple_ret_val(struct thread_smc_1_2_regs *args, int ffa_ret) 165 { 166 if (ffa_ret) 167 spmc_set_args(args, FFA_ERROR, 0, ffa_ret, 0, 0, 0); 168 else 169 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 170 } 171 172 uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) 173 { 174 uint32_t major_vers = FFA_GET_MAJOR_VERSION(vers); 175 uint32_t minor_vers = FFA_GET_MINOR_VERSION(vers); 176 uint32_t my_vers = FFA_VERSION_1_2; 177 uint32_t my_major_vers = 0; 178 uint32_t my_minor_vers = 0; 179 180 my_major_vers = FFA_GET_MAJOR_VERSION(my_vers); 181 my_minor_vers = FFA_GET_MINOR_VERSION(my_vers); 182 183 /* 184 * No locking, if the caller does concurrent calls to this it's 185 * only making a mess for itself. We must be able to renegotiate 186 * the FF-A version in order to support differing versions between 187 * the loader and the driver. 188 * 189 * Callers should use the version requested if we return a matching 190 * major version and a matching or larger minor version. The caller 191 * should downgrade to our minor version if our minor version is 192 * smaller. Regardless, always return our version as recommended by 193 * the specification. 194 */ 195 if (major_vers == my_major_vers) { 196 if (minor_vers > my_minor_vers) 197 rxtx->ffa_vers = my_vers; 198 else 199 rxtx->ffa_vers = vers; 200 } 201 202 return my_vers; 203 } 204 205 static bool is_ffa_success(uint32_t fid) 206 { 207 #ifdef ARM64 208 if (fid == FFA_SUCCESS_64) 209 return true; 210 #endif 211 return fid == FFA_SUCCESS_32; 212 } 213 214 static int32_t get_ffa_ret_code(const struct thread_smc_args *args) 215 { 216 if (is_ffa_success(args->a0)) 217 return FFA_OK; 218 if (args->a0 == FFA_ERROR && args->a2) 219 return args->a2; 220 return FFA_NOT_SUPPORTED; 221 } 222 223 static int ffa_simple_call(uint32_t fid, unsigned long a1, unsigned long a2, 224 unsigned long a3, unsigned long a4) 225 { 226 struct thread_smc_args args = { 227 .a0 = fid, 228 .a1 = a1, 229 .a2 = a2, 230 .a3 = a3, 231 .a4 = a4, 232 }; 233 234 thread_smccc(&args); 235 236 return get_ffa_ret_code(&args); 237 } 238 239 static int __maybe_unused ffa_features(uint32_t id) 240 { 241 return ffa_simple_call(FFA_FEATURES, id, 0, 0, 0); 242 } 243 244 static int __maybe_unused ffa_set_notification(uint16_t dst, uint16_t src, 245 uint32_t flags, uint64_t bitmap) 246 { 247 return ffa_simple_call(FFA_NOTIFICATION_SET, 248 SHIFT_U32(src, 16) | dst, flags, 249 low32_from_64(bitmap), high32_from_64(bitmap)); 250 } 251 252 #if defined(CFG_CORE_SEL1_SPMC) 253 static void handle_features(struct thread_smc_1_2_regs *args) 254 { 255 uint32_t ret_fid = FFA_ERROR; 256 uint32_t ret_w2 = FFA_NOT_SUPPORTED; 257 258 switch (args->a1) { 259 case FFA_FEATURE_SCHEDULE_RECV_INTR: 260 if (spmc_notif_is_ready) { 261 ret_fid = FFA_SUCCESS_32; 262 ret_w2 = notif_intid; 263 } 264 break; 265 266 #ifdef ARM64 267 case FFA_RXTX_MAP_64: 268 #endif 269 case FFA_RXTX_MAP_32: 270 ret_fid = FFA_SUCCESS_32; 271 ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ 272 break; 273 #ifdef ARM64 274 case FFA_MEM_SHARE_64: 275 #endif 276 case FFA_MEM_SHARE_32: 277 ret_fid = FFA_SUCCESS_32; 278 /* 279 * Partition manager supports transmission of a memory 280 * transaction descriptor in a buffer dynamically allocated 281 * by the endpoint. 282 */ 283 ret_w2 = BIT(0); 284 break; 285 286 case FFA_ERROR: 287 case FFA_VERSION: 288 case FFA_SUCCESS_32: 289 #ifdef ARM64 290 case FFA_SUCCESS_64: 291 #endif 292 case FFA_FEATURES: 293 case FFA_SPM_ID_GET: 294 case FFA_MEM_FRAG_TX: 295 case FFA_MEM_RECLAIM: 296 case FFA_MSG_SEND_DIRECT_REQ_64: 297 case FFA_MSG_SEND_DIRECT_REQ_32: 298 case FFA_INTERRUPT: 299 case FFA_PARTITION_INFO_GET: 300 case FFA_RXTX_UNMAP: 301 case FFA_RX_RELEASE: 302 case FFA_FEATURE_MANAGED_EXIT_INTR: 303 case FFA_NOTIFICATION_BITMAP_CREATE: 304 case FFA_NOTIFICATION_BITMAP_DESTROY: 305 case FFA_NOTIFICATION_BIND: 306 case FFA_NOTIFICATION_UNBIND: 307 case FFA_NOTIFICATION_SET: 308 case FFA_NOTIFICATION_GET: 309 case FFA_NOTIFICATION_INFO_GET_32: 310 #ifdef ARM64 311 case FFA_NOTIFICATION_INFO_GET_64: 312 #endif 313 ret_fid = FFA_SUCCESS_32; 314 ret_w2 = FFA_PARAM_MBZ; 315 break; 316 default: 317 break; 318 } 319 320 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, 321 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 322 } 323 324 static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) 325 { 326 tee_mm_entry_t *mm = NULL; 327 328 if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) 329 return FFA_INVALID_PARAMETERS; 330 331 mm = tee_mm_alloc(&core_virt_shm_pool, sz); 332 if (!mm) 333 return FFA_NO_MEMORY; 334 335 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, 336 sz / SMALL_PAGE_SIZE, 337 MEM_AREA_NSEC_SHM)) { 338 tee_mm_free(mm); 339 return FFA_INVALID_PARAMETERS; 340 } 341 342 *va_ret = (void *)tee_mm_get_smem(mm); 343 return 0; 344 } 345 346 void spmc_handle_spm_id_get(struct thread_smc_1_2_regs *args) 347 { 348 spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, spmc_id, 349 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 350 } 351 352 static void unmap_buf(void *va, size_t sz) 353 { 354 tee_mm_entry_t *mm = tee_mm_find(&core_virt_shm_pool, (vaddr_t)va); 355 356 assert(mm); 357 core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); 358 tee_mm_free(mm); 359 } 360 361 void spmc_handle_rxtx_map(struct thread_smc_1_2_regs *args, 362 struct ffa_rxtx *rxtx) 363 { 364 int rc = 0; 365 unsigned int sz = 0; 366 paddr_t rx_pa = 0; 367 paddr_t tx_pa = 0; 368 void *rx = NULL; 369 void *tx = NULL; 370 371 cpu_spin_lock(&rxtx->spinlock); 372 373 if (args->a3 & GENMASK_64(63, 6)) { 374 rc = FFA_INVALID_PARAMETERS; 375 goto out; 376 } 377 378 sz = args->a3 * SMALL_PAGE_SIZE; 379 if (!sz) { 380 rc = FFA_INVALID_PARAMETERS; 381 goto out; 382 } 383 /* TX/RX are swapped compared to the caller */ 384 tx_pa = args->a2; 385 rx_pa = args->a1; 386 387 if (rxtx->size) { 388 rc = FFA_DENIED; 389 goto out; 390 } 391 392 /* 393 * If the buffer comes from a SP the address is virtual and already 394 * mapped. 395 */ 396 if (is_nw_buf(rxtx)) { 397 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 398 enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; 399 bool tx_alloced = false; 400 401 /* 402 * With virtualization we establish this mapping in 403 * the nexus mapping which then is replicated to 404 * each partition. 405 * 406 * This means that this mapping must be done before 407 * any partition is created and then must not be 408 * changed. 409 */ 410 411 /* 412 * core_mmu_add_mapping() may reuse previous 413 * mappings. First check if there's any mappings to 414 * reuse so we know how to clean up in case of 415 * failure. 416 */ 417 tx = phys_to_virt(tx_pa, mt, sz); 418 rx = phys_to_virt(rx_pa, mt, sz); 419 if (!tx) { 420 tx = core_mmu_add_mapping(mt, tx_pa, sz); 421 if (!tx) { 422 rc = FFA_NO_MEMORY; 423 goto out; 424 } 425 tx_alloced = true; 426 } 427 if (!rx) 428 rx = core_mmu_add_mapping(mt, rx_pa, sz); 429 430 if (!rx) { 431 if (tx_alloced && tx) 432 core_mmu_remove_mapping(mt, tx, sz); 433 rc = FFA_NO_MEMORY; 434 goto out; 435 } 436 } else { 437 rc = map_buf(tx_pa, sz, &tx); 438 if (rc) 439 goto out; 440 rc = map_buf(rx_pa, sz, &rx); 441 if (rc) { 442 unmap_buf(tx, sz); 443 goto out; 444 } 445 } 446 rxtx->tx = tx; 447 rxtx->rx = rx; 448 } else { 449 if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { 450 rc = FFA_INVALID_PARAMETERS; 451 goto out; 452 } 453 454 if (!virt_to_phys((void *)tx_pa) || 455 !virt_to_phys((void *)rx_pa)) { 456 rc = FFA_INVALID_PARAMETERS; 457 goto out; 458 } 459 460 rxtx->tx = (void *)tx_pa; 461 rxtx->rx = (void *)rx_pa; 462 } 463 464 rxtx->size = sz; 465 rxtx->tx_is_mine = true; 466 DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); 467 DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); 468 out: 469 cpu_spin_unlock(&rxtx->spinlock); 470 set_simple_ret_val(args, rc); 471 } 472 473 void spmc_handle_rxtx_unmap(struct thread_smc_1_2_regs *args, 474 struct ffa_rxtx *rxtx) 475 { 476 int rc = FFA_INVALID_PARAMETERS; 477 478 cpu_spin_lock(&rxtx->spinlock); 479 480 if (!rxtx->size) 481 goto out; 482 483 /* 484 * We don't unmap the SP memory as the SP might still use it. 485 * We avoid to make changes to nexus mappings at this stage since 486 * there currently isn't a way to replicate those changes to all 487 * partitions. 488 */ 489 if (is_nw_buf(rxtx) && !IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 490 unmap_buf(rxtx->rx, rxtx->size); 491 unmap_buf(rxtx->tx, rxtx->size); 492 } 493 rxtx->size = 0; 494 rxtx->rx = NULL; 495 rxtx->tx = NULL; 496 rc = 0; 497 out: 498 cpu_spin_unlock(&rxtx->spinlock); 499 set_simple_ret_val(args, rc); 500 } 501 502 void spmc_handle_rx_release(struct thread_smc_1_2_regs *args, 503 struct ffa_rxtx *rxtx) 504 { 505 int rc = 0; 506 507 cpu_spin_lock(&rxtx->spinlock); 508 /* The senders RX is our TX */ 509 if (!rxtx->size || rxtx->tx_is_mine) { 510 rc = FFA_DENIED; 511 } else { 512 rc = 0; 513 rxtx->tx_is_mine = true; 514 } 515 cpu_spin_unlock(&rxtx->spinlock); 516 517 set_simple_ret_val(args, rc); 518 } 519 520 static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 521 { 522 return !w0 && !w1 && !w2 && !w3; 523 } 524 525 static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 526 { 527 /* 528 * This depends on which UUID we have been assigned. 529 * TODO add a generic mechanism to obtain our UUID. 530 * 531 * The test below is for the hard coded UUID 532 * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 533 */ 534 return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && 535 w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; 536 } 537 538 TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, 539 size_t idx, uint16_t endpoint_id, 540 uint16_t execution_context, 541 uint32_t part_props, 542 const uint32_t uuid_words[4]) 543 { 544 struct ffa_partition_info_x *fpi = NULL; 545 size_t fpi_size = sizeof(*fpi); 546 547 if (ffa_vers >= FFA_VERSION_1_1) 548 fpi_size += FFA_UUID_SIZE; 549 550 if ((idx + 1) * fpi_size > blen) 551 return TEE_ERROR_OUT_OF_MEMORY; 552 553 fpi = (void *)((vaddr_t)buf + idx * fpi_size); 554 fpi->id = endpoint_id; 555 /* Number of execution contexts implemented by this partition */ 556 fpi->execution_context = execution_context; 557 558 fpi->partition_properties = part_props; 559 560 /* In FF-A 1.0 only bits [2:0] are defined, let's mask others */ 561 if (ffa_vers < FFA_VERSION_1_1) 562 fpi->partition_properties &= FFA_PART_PROP_DIRECT_REQ_RECV | 563 FFA_PART_PROP_DIRECT_REQ_SEND | 564 FFA_PART_PROP_INDIRECT_MSGS; 565 566 if (ffa_vers >= FFA_VERSION_1_1) { 567 if (uuid_words) 568 memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); 569 else 570 memset(fpi->uuid, 0, FFA_UUID_SIZE); 571 } 572 573 return TEE_SUCCESS; 574 } 575 576 static int handle_partition_info_get_all(size_t *elem_count, 577 struct ffa_rxtx *rxtx, bool count_only) 578 { 579 if (!count_only) { 580 /* Add OP-TEE SP */ 581 if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, 582 rxtx->size, 0, optee_endpoint_id, 583 CFG_TEE_CORE_NB_CORE, 584 my_part_props, my_uuid_words)) 585 return FFA_NO_MEMORY; 586 } 587 *elem_count = 1; 588 589 if (IS_ENABLED(CFG_SECURE_PARTITION)) { 590 if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, 591 NULL, elem_count, count_only)) 592 return FFA_NO_MEMORY; 593 } 594 595 return FFA_OK; 596 } 597 598 void spmc_handle_partition_info_get(struct thread_smc_1_2_regs *args, 599 struct ffa_rxtx *rxtx) 600 { 601 TEE_Result res = TEE_SUCCESS; 602 uint32_t ret_fid = FFA_ERROR; 603 uint32_t fpi_size = 0; 604 uint32_t rc = 0; 605 bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; 606 607 if (!count_only) { 608 cpu_spin_lock(&rxtx->spinlock); 609 610 if (!rxtx->size || !rxtx->tx_is_mine) { 611 rc = FFA_BUSY; 612 goto out; 613 } 614 } 615 616 if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { 617 size_t elem_count = 0; 618 619 ret_fid = handle_partition_info_get_all(&elem_count, rxtx, 620 count_only); 621 622 if (ret_fid) { 623 rc = ret_fid; 624 ret_fid = FFA_ERROR; 625 } else { 626 ret_fid = FFA_SUCCESS_32; 627 rc = elem_count; 628 } 629 630 goto out; 631 } 632 633 if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { 634 if (!count_only) { 635 res = spmc_fill_partition_entry(rxtx->ffa_vers, 636 rxtx->tx, rxtx->size, 0, 637 optee_endpoint_id, 638 CFG_TEE_CORE_NB_CORE, 639 my_part_props, 640 my_uuid_words); 641 if (res) { 642 ret_fid = FFA_ERROR; 643 rc = FFA_INVALID_PARAMETERS; 644 goto out; 645 } 646 } 647 rc = 1; 648 } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { 649 uint32_t uuid_array[4] = { 0 }; 650 size_t count = 0; 651 652 uuid_array[0] = args->a1; 653 uuid_array[1] = args->a2; 654 uuid_array[2] = args->a3; 655 uuid_array[3] = args->a4; 656 657 res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, 658 rxtx->size, uuid_array, &count, 659 count_only); 660 if (res != TEE_SUCCESS) { 661 ret_fid = FFA_ERROR; 662 rc = FFA_INVALID_PARAMETERS; 663 goto out; 664 } 665 rc = count; 666 } else { 667 ret_fid = FFA_ERROR; 668 rc = FFA_INVALID_PARAMETERS; 669 goto out; 670 } 671 672 ret_fid = FFA_SUCCESS_32; 673 674 out: 675 if (ret_fid == FFA_SUCCESS_32 && !count_only && 676 rxtx->ffa_vers >= FFA_VERSION_1_1) 677 fpi_size = sizeof(struct ffa_partition_info_x) + FFA_UUID_SIZE; 678 679 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, fpi_size, 680 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 681 if (!count_only) { 682 rxtx->tx_is_mine = false; 683 cpu_spin_unlock(&rxtx->spinlock); 684 } 685 } 686 687 static void spmc_handle_run(struct thread_smc_1_2_regs *args) 688 { 689 uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); 690 uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); 691 uint32_t rc = FFA_OK; 692 693 if (endpoint != optee_endpoint_id) { 694 /* 695 * The endpoint should be an SP, try to resume the SP from 696 * preempted into busy state. 697 */ 698 rc = spmc_sp_resume_from_preempted(endpoint); 699 if (rc) 700 goto out; 701 } 702 703 thread_resume_from_rpc(thread_id, 0, 0, 0, 0); 704 705 /* thread_resume_from_rpc return only of the thread_id is invalid */ 706 rc = FFA_INVALID_PARAMETERS; 707 708 out: 709 set_simple_ret_val(args, rc); 710 } 711 #endif /*CFG_CORE_SEL1_SPMC*/ 712 713 static struct notif_vm_bitmap *get_notif_vm_bitmap(struct guest_partition *prtn, 714 uint16_t vm_id) 715 { 716 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 717 if (!prtn) 718 return NULL; 719 assert(vm_id == virt_get_guest_id(prtn)); 720 return virt_get_guest_spec_data(prtn, notif_vm_bitmap_id); 721 } 722 if (vm_id) 723 return NULL; 724 return &default_notif_vm_bitmap; 725 } 726 727 static uint32_t spmc_enable_async_notif(uint32_t bottom_half_value, 728 uint16_t vm_id) 729 { 730 struct guest_partition *prtn = NULL; 731 struct notif_vm_bitmap *nvb = NULL; 732 uint32_t old_itr_status = 0; 733 uint32_t res = 0; 734 735 if (!spmc_notif_is_ready) { 736 /* 737 * This should never happen, not if normal world respects the 738 * exchanged capabilities. 739 */ 740 EMSG("Asynchronous notifications are not ready"); 741 return TEE_ERROR_NOT_IMPLEMENTED; 742 } 743 744 if (bottom_half_value >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE) { 745 EMSG("Invalid bottom half value %"PRIu32, bottom_half_value); 746 return TEE_ERROR_BAD_PARAMETERS; 747 } 748 749 prtn = virt_get_guest(vm_id); 750 nvb = get_notif_vm_bitmap(prtn, vm_id); 751 if (!nvb) { 752 res = TEE_ERROR_BAD_PARAMETERS; 753 goto out; 754 } 755 756 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 757 nvb->do_bottom_half_value = bottom_half_value; 758 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 759 760 notif_deliver_atomic_event(NOTIF_EVENT_STARTED, vm_id); 761 res = TEE_SUCCESS; 762 out: 763 virt_put_guest(prtn); 764 return res; 765 } 766 767 static void handle_yielding_call(struct thread_smc_1_2_regs *args, 768 uint32_t direct_resp_fid) 769 { 770 TEE_Result res = 0; 771 772 thread_check_canaries(); 773 774 #ifdef ARM64 775 /* Saving this for an eventual RPC */ 776 thread_get_core_local()->direct_resp_fid = direct_resp_fid; 777 #endif 778 779 if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { 780 /* Note connection to struct thread_rpc_arg::ret */ 781 thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, 782 0); 783 res = TEE_ERROR_BAD_PARAMETERS; 784 } else { 785 thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, 786 args->a6, args->a7); 787 res = TEE_ERROR_BUSY; 788 } 789 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 790 0, res, 0, 0); 791 } 792 793 static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) 794 { 795 uint64_t cookie = reg_pair_to_64(a5, a4); 796 uint32_t res = 0; 797 798 res = mobj_ffa_unregister_by_cookie(cookie); 799 switch (res) { 800 case TEE_SUCCESS: 801 case TEE_ERROR_ITEM_NOT_FOUND: 802 return 0; 803 case TEE_ERROR_BUSY: 804 EMSG("res %#"PRIx32, res); 805 return FFA_BUSY; 806 default: 807 EMSG("res %#"PRIx32, res); 808 return FFA_INVALID_PARAMETERS; 809 } 810 } 811 812 static void handle_blocking_call(struct thread_smc_1_2_regs *args, 813 uint32_t direct_resp_fid) 814 { 815 uint32_t sec_caps = 0; 816 817 switch (args->a3) { 818 case OPTEE_FFA_GET_API_VERSION: 819 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 820 OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, 821 0); 822 break; 823 case OPTEE_FFA_GET_OS_VERSION: 824 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 825 CFG_OPTEE_REVISION_MAJOR, 826 CFG_OPTEE_REVISION_MINOR, 827 TEE_IMPL_GIT_SHA1 >> 32); 828 break; 829 case OPTEE_FFA_EXCHANGE_CAPABILITIES: 830 sec_caps = OPTEE_FFA_SEC_CAP_ARG_OFFSET; 831 if (spmc_notif_is_ready) 832 sec_caps |= OPTEE_FFA_SEC_CAP_ASYNC_NOTIF; 833 if (IS_ENABLED(CFG_RPMB_ANNOUNCE_PROBE_CAP)) 834 sec_caps |= OPTEE_FFA_SEC_CAP_RPMB_PROBE; 835 spmc_set_args(args, direct_resp_fid, 836 swap_src_dst(args->a1), 0, 0, 837 THREAD_RPC_MAX_NUM_PARAMS, sec_caps); 838 break; 839 case OPTEE_FFA_UNREGISTER_SHM: 840 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 841 handle_unregister_shm(args->a4, args->a5), 0, 0); 842 break; 843 case OPTEE_FFA_ENABLE_ASYNC_NOTIF: 844 spmc_set_args(args, direct_resp_fid, 845 swap_src_dst(args->a1), 0, 846 spmc_enable_async_notif(args->a4, 847 FFA_SRC(args->a1)), 848 0, 0); 849 break; 850 default: 851 EMSG("Unhandled blocking service ID %#"PRIx32, 852 (uint32_t)args->a3); 853 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 854 TEE_ERROR_BAD_PARAMETERS, 0, 0); 855 } 856 } 857 858 static void handle_framework_direct_request(struct thread_smc_1_2_regs *args, 859 struct ffa_rxtx *rxtx, 860 uint32_t direct_resp_fid) 861 { 862 uint32_t w0 = FFA_ERROR; 863 uint32_t w1 = FFA_PARAM_MBZ; 864 uint32_t w2 = FFA_NOT_SUPPORTED; 865 uint32_t w3 = FFA_PARAM_MBZ; 866 867 switch (args->a2 & FFA_MSG_TYPE_MASK) { 868 case FFA_MSG_SEND_VM_CREATED: 869 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 870 uint16_t guest_id = args->a5; 871 TEE_Result res = virt_guest_created(guest_id); 872 873 w0 = direct_resp_fid; 874 w1 = swap_src_dst(args->a1); 875 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; 876 if (res == TEE_SUCCESS) 877 w3 = FFA_OK; 878 else if (res == TEE_ERROR_OUT_OF_MEMORY) 879 w3 = FFA_DENIED; 880 else 881 w3 = FFA_INVALID_PARAMETERS; 882 } 883 break; 884 case FFA_MSG_SEND_VM_DESTROYED: 885 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 886 uint16_t guest_id = args->a5; 887 TEE_Result res = virt_guest_destroyed(guest_id); 888 889 w0 = direct_resp_fid; 890 w1 = swap_src_dst(args->a1); 891 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_DESTROYED; 892 if (res == TEE_SUCCESS) 893 w3 = FFA_OK; 894 else 895 w3 = FFA_INVALID_PARAMETERS; 896 } 897 break; 898 case FFA_MSG_VERSION_REQ: 899 w0 = direct_resp_fid; 900 w1 = swap_src_dst(args->a1); 901 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; 902 w3 = spmc_exchange_version(args->a3, rxtx); 903 break; 904 default: 905 break; 906 } 907 spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 908 } 909 910 static void handle_direct_request(struct thread_smc_1_2_regs *args, 911 struct ffa_rxtx *rxtx) 912 { 913 uint32_t direct_resp_fid = 0; 914 915 if (IS_ENABLED(CFG_SECURE_PARTITION) && 916 FFA_DST(args->a1) != spmc_id && 917 FFA_DST(args->a1) != optee_endpoint_id) { 918 spmc_sp_start_thread(args); 919 return; 920 } 921 922 if (OPTEE_SMC_IS_64(args->a0)) 923 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_64; 924 else 925 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_32; 926 927 if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { 928 handle_framework_direct_request(args, rxtx, direct_resp_fid); 929 return; 930 } 931 932 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 933 virt_set_guest(get_sender_id(args->a1))) { 934 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 935 TEE_ERROR_ITEM_NOT_FOUND, 0, 0); 936 return; 937 } 938 939 if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) 940 handle_yielding_call(args, direct_resp_fid); 941 else 942 handle_blocking_call(args, direct_resp_fid); 943 944 /* 945 * Note that handle_yielding_call() typically only returns if a 946 * thread cannot be allocated or found. virt_unset_guest() is also 947 * called from thread_state_suspend() and thread_state_free(). 948 */ 949 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 950 virt_unset_guest(); 951 } 952 953 int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, 954 struct ffa_mem_transaction_x *trans) 955 { 956 uint16_t mem_reg_attr = 0; 957 uint32_t flags = 0; 958 uint32_t count = 0; 959 uint32_t offs = 0; 960 uint32_t size = 0; 961 size_t n = 0; 962 963 if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) 964 return FFA_INVALID_PARAMETERS; 965 966 if (ffa_vers >= FFA_VERSION_1_1) { 967 struct ffa_mem_transaction_1_1 *descr = NULL; 968 969 if (blen < sizeof(*descr)) 970 return FFA_INVALID_PARAMETERS; 971 972 descr = buf; 973 trans->sender_id = READ_ONCE(descr->sender_id); 974 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 975 flags = READ_ONCE(descr->flags); 976 trans->global_handle = READ_ONCE(descr->global_handle); 977 trans->tag = READ_ONCE(descr->tag); 978 979 count = READ_ONCE(descr->mem_access_count); 980 size = READ_ONCE(descr->mem_access_size); 981 offs = READ_ONCE(descr->mem_access_offs); 982 } else { 983 struct ffa_mem_transaction_1_0 *descr = NULL; 984 985 if (blen < sizeof(*descr)) 986 return FFA_INVALID_PARAMETERS; 987 988 descr = buf; 989 trans->sender_id = READ_ONCE(descr->sender_id); 990 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 991 flags = READ_ONCE(descr->flags); 992 trans->global_handle = READ_ONCE(descr->global_handle); 993 trans->tag = READ_ONCE(descr->tag); 994 995 count = READ_ONCE(descr->mem_access_count); 996 size = sizeof(struct ffa_mem_access); 997 offs = offsetof(struct ffa_mem_transaction_1_0, 998 mem_access_array); 999 } 1000 1001 if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || 1002 size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) 1003 return FFA_INVALID_PARAMETERS; 1004 1005 /* Check that the endpoint memory access descriptor array fits */ 1006 if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || 1007 n > blen) 1008 return FFA_INVALID_PARAMETERS; 1009 1010 trans->mem_reg_attr = mem_reg_attr; 1011 trans->flags = flags; 1012 trans->mem_access_size = size; 1013 trans->mem_access_count = count; 1014 trans->mem_access_offs = offs; 1015 return 0; 1016 } 1017 1018 #if defined(CFG_CORE_SEL1_SPMC) 1019 static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, 1020 unsigned int mem_access_count, uint8_t *acc_perms, 1021 unsigned int *region_offs) 1022 { 1023 struct ffa_mem_access_perm *descr = NULL; 1024 struct ffa_mem_access *mem_acc = NULL; 1025 unsigned int n = 0; 1026 1027 for (n = 0; n < mem_access_count; n++) { 1028 mem_acc = (void *)(mem_acc_base + mem_access_size * n); 1029 descr = &mem_acc->access_perm; 1030 if (READ_ONCE(descr->endpoint_id) == optee_endpoint_id) { 1031 *acc_perms = READ_ONCE(descr->perm); 1032 *region_offs = READ_ONCE(mem_acc[n].region_offs); 1033 return 0; 1034 } 1035 } 1036 1037 return FFA_INVALID_PARAMETERS; 1038 } 1039 1040 static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, 1041 size_t blen, unsigned int *page_count, 1042 unsigned int *region_count, size_t *addr_range_offs) 1043 { 1044 const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 1045 const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; 1046 struct ffa_mem_region *region_descr = NULL; 1047 unsigned int region_descr_offs = 0; 1048 uint8_t mem_acc_perm = 0; 1049 size_t n = 0; 1050 1051 if (mem_trans->mem_reg_attr != exp_mem_reg_attr) 1052 return FFA_INVALID_PARAMETERS; 1053 1054 /* Check that the access permissions matches what's expected */ 1055 if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, 1056 mem_trans->mem_access_size, 1057 mem_trans->mem_access_count, 1058 &mem_acc_perm, ®ion_descr_offs) || 1059 mem_acc_perm != exp_mem_acc_perm) 1060 return FFA_INVALID_PARAMETERS; 1061 1062 /* Check that the Composite memory region descriptor fits */ 1063 if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || 1064 n > blen) 1065 return FFA_INVALID_PARAMETERS; 1066 1067 if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, 1068 struct ffa_mem_region)) 1069 return FFA_INVALID_PARAMETERS; 1070 1071 region_descr = (struct ffa_mem_region *)((vaddr_t)buf + 1072 region_descr_offs); 1073 *page_count = READ_ONCE(region_descr->total_page_count); 1074 *region_count = READ_ONCE(region_descr->address_range_count); 1075 *addr_range_offs = n; 1076 return 0; 1077 } 1078 1079 static int add_mem_share_helper(struct mem_share_state *s, void *buf, 1080 size_t flen) 1081 { 1082 unsigned int region_count = flen / sizeof(struct ffa_address_range); 1083 struct ffa_address_range *arange = NULL; 1084 unsigned int n = 0; 1085 1086 if (region_count > s->region_count) 1087 region_count = s->region_count; 1088 1089 if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) 1090 return FFA_INVALID_PARAMETERS; 1091 arange = buf; 1092 1093 for (n = 0; n < region_count; n++) { 1094 unsigned int page_count = READ_ONCE(arange[n].page_count); 1095 uint64_t addr = READ_ONCE(arange[n].address); 1096 1097 if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, 1098 addr, page_count)) 1099 return FFA_INVALID_PARAMETERS; 1100 } 1101 1102 s->region_count -= region_count; 1103 if (s->region_count) 1104 return region_count * sizeof(*arange); 1105 1106 if (s->current_page_idx != s->page_count) 1107 return FFA_INVALID_PARAMETERS; 1108 1109 return 0; 1110 } 1111 1112 static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) 1113 { 1114 int rc = 0; 1115 1116 rc = add_mem_share_helper(&s->share, buf, flen); 1117 if (rc >= 0) { 1118 if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { 1119 /* We're not at the end of the descriptor yet */ 1120 if (s->share.region_count) 1121 return s->frag_offset; 1122 1123 /* We're done */ 1124 rc = 0; 1125 } else { 1126 rc = FFA_INVALID_PARAMETERS; 1127 } 1128 } 1129 1130 SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); 1131 if (rc < 0) 1132 mobj_ffa_sel1_spmc_delete(s->share.mf); 1133 else 1134 mobj_ffa_push_to_inactive(s->share.mf); 1135 free(s); 1136 1137 return rc; 1138 } 1139 1140 static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, 1141 void *buf) 1142 { 1143 struct ffa_mem_access_perm *perm = NULL; 1144 struct ffa_mem_access *mem_acc = NULL; 1145 1146 if (!IS_ENABLED(CFG_SECURE_PARTITION)) 1147 return false; 1148 1149 if (mem_trans->mem_access_count < 1) 1150 return false; 1151 1152 mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); 1153 perm = &mem_acc->access_perm; 1154 1155 /* 1156 * perm->endpoint_id is read here only to check if the endpoint is 1157 * OP-TEE. We do read it later on again, but there are some additional 1158 * checks there to make sure that the data is correct. 1159 */ 1160 return READ_ONCE(perm->endpoint_id) != optee_endpoint_id; 1161 } 1162 1163 static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, 1164 tee_mm_entry_t *mm, void *buf, size_t blen, 1165 size_t flen, uint64_t *global_handle) 1166 { 1167 int rc = 0; 1168 struct mem_share_state share = { }; 1169 size_t addr_range_offs = 0; 1170 uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1171 size_t n = 0; 1172 1173 rc = mem_share_init(mem_trans, buf, flen, &share.page_count, 1174 &share.region_count, &addr_range_offs); 1175 if (rc) 1176 return rc; 1177 1178 if (!share.page_count || !share.region_count) 1179 return FFA_INVALID_PARAMETERS; 1180 1181 if (MUL_OVERFLOW(share.region_count, 1182 sizeof(struct ffa_address_range), &n) || 1183 ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) 1184 return FFA_INVALID_PARAMETERS; 1185 1186 if (mem_trans->global_handle) 1187 cookie = mem_trans->global_handle; 1188 share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); 1189 if (!share.mf) 1190 return FFA_NO_MEMORY; 1191 1192 if (flen != blen) { 1193 struct mem_frag_state *s = calloc(1, sizeof(*s)); 1194 1195 if (!s) { 1196 rc = FFA_NO_MEMORY; 1197 goto err; 1198 } 1199 s->share = share; 1200 s->mm = mm; 1201 s->frag_offset = addr_range_offs; 1202 1203 SLIST_INSERT_HEAD(&frag_state_head, s, link); 1204 rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, 1205 flen - addr_range_offs); 1206 1207 if (rc >= 0) 1208 *global_handle = mobj_ffa_get_cookie(share.mf); 1209 1210 return rc; 1211 } 1212 1213 rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, 1214 flen - addr_range_offs); 1215 if (rc) { 1216 /* 1217 * Number of consumed bytes may be returned instead of 0 for 1218 * done. 1219 */ 1220 rc = FFA_INVALID_PARAMETERS; 1221 goto err; 1222 } 1223 1224 *global_handle = mobj_ffa_push_to_inactive(share.mf); 1225 1226 return 0; 1227 err: 1228 mobj_ffa_sel1_spmc_delete(share.mf); 1229 return rc; 1230 } 1231 1232 static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, 1233 unsigned int page_count, 1234 uint64_t *global_handle, struct ffa_rxtx *rxtx) 1235 { 1236 struct ffa_mem_transaction_x mem_trans = { }; 1237 int rc = 0; 1238 size_t len = 0; 1239 void *buf = NULL; 1240 tee_mm_entry_t *mm = NULL; 1241 vaddr_t offs = pbuf & SMALL_PAGE_MASK; 1242 1243 if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) 1244 return FFA_INVALID_PARAMETERS; 1245 if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) 1246 return FFA_INVALID_PARAMETERS; 1247 1248 /* 1249 * Check that the length reported in flen is covered by len even 1250 * if the offset is taken into account. 1251 */ 1252 if (len < flen || len - offs < flen) 1253 return FFA_INVALID_PARAMETERS; 1254 1255 mm = tee_mm_alloc(&core_virt_shm_pool, len); 1256 if (!mm) 1257 return FFA_NO_MEMORY; 1258 1259 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, 1260 page_count, MEM_AREA_NSEC_SHM)) { 1261 rc = FFA_INVALID_PARAMETERS; 1262 goto out; 1263 } 1264 buf = (void *)(tee_mm_get_smem(mm) + offs); 1265 1266 cpu_spin_lock(&rxtx->spinlock); 1267 rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); 1268 if (rc) 1269 goto unlock; 1270 1271 if (is_sp_share(&mem_trans, buf)) { 1272 rc = spmc_sp_add_share(&mem_trans, buf, blen, flen, 1273 global_handle, NULL); 1274 goto unlock; 1275 } 1276 1277 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1278 virt_set_guest(mem_trans.sender_id)) { 1279 rc = FFA_DENIED; 1280 goto unlock; 1281 } 1282 1283 rc = add_mem_share(&mem_trans, mm, buf, blen, flen, global_handle); 1284 1285 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1286 virt_unset_guest(); 1287 1288 unlock: 1289 cpu_spin_unlock(&rxtx->spinlock); 1290 if (rc > 0) 1291 return rc; 1292 1293 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1294 out: 1295 tee_mm_free(mm); 1296 return rc; 1297 } 1298 1299 static int handle_mem_share_rxbuf(size_t blen, size_t flen, 1300 uint64_t *global_handle, 1301 struct ffa_rxtx *rxtx) 1302 { 1303 struct ffa_mem_transaction_x mem_trans = { }; 1304 int rc = FFA_DENIED; 1305 1306 cpu_spin_lock(&rxtx->spinlock); 1307 1308 if (!rxtx->rx || flen > rxtx->size) 1309 goto out; 1310 1311 rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, 1312 &mem_trans); 1313 if (rc) 1314 goto out; 1315 if (is_sp_share(&mem_trans, rxtx->rx)) { 1316 rc = spmc_sp_add_share(&mem_trans, rxtx, blen, flen, 1317 global_handle, NULL); 1318 goto out; 1319 } 1320 1321 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1322 virt_set_guest(mem_trans.sender_id)) 1323 goto out; 1324 1325 rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, 1326 global_handle); 1327 1328 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1329 virt_unset_guest(); 1330 1331 out: 1332 cpu_spin_unlock(&rxtx->spinlock); 1333 1334 return rc; 1335 } 1336 1337 static void handle_mem_share(struct thread_smc_1_2_regs *args, 1338 struct ffa_rxtx *rxtx) 1339 { 1340 uint32_t tot_len = args->a1; 1341 uint32_t frag_len = args->a2; 1342 uint64_t addr = args->a3; 1343 uint32_t page_count = args->a4; 1344 uint32_t ret_w1 = 0; 1345 uint32_t ret_w2 = FFA_INVALID_PARAMETERS; 1346 uint32_t ret_w3 = 0; 1347 uint32_t ret_fid = FFA_ERROR; 1348 uint64_t global_handle = 0; 1349 int rc = 0; 1350 1351 /* Check that the MBZs are indeed 0 */ 1352 if (args->a5 || args->a6 || args->a7) 1353 goto out; 1354 1355 /* Check that fragment length doesn't exceed total length */ 1356 if (frag_len > tot_len) 1357 goto out; 1358 1359 /* Check for 32-bit calling convention */ 1360 if (args->a0 == FFA_MEM_SHARE_32) 1361 addr &= UINT32_MAX; 1362 1363 if (!addr) { 1364 /* 1365 * The memory transaction descriptor is passed via our rx 1366 * buffer. 1367 */ 1368 if (page_count) 1369 goto out; 1370 rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, 1371 rxtx); 1372 } else { 1373 rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, 1374 &global_handle, rxtx); 1375 } 1376 if (rc < 0) { 1377 ret_w2 = rc; 1378 } else if (rc > 0) { 1379 ret_fid = FFA_MEM_FRAG_RX; 1380 ret_w3 = rc; 1381 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1382 } else { 1383 ret_fid = FFA_SUCCESS_32; 1384 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1385 } 1386 out: 1387 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1388 } 1389 1390 static struct mem_frag_state *get_frag_state(uint64_t global_handle) 1391 { 1392 struct mem_frag_state *s = NULL; 1393 1394 SLIST_FOREACH(s, &frag_state_head, link) 1395 if (mobj_ffa_get_cookie(s->share.mf) == global_handle) 1396 return s; 1397 1398 return NULL; 1399 } 1400 1401 static void handle_mem_frag_tx(struct thread_smc_1_2_regs *args, 1402 struct ffa_rxtx *rxtx) 1403 { 1404 uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); 1405 size_t flen = args->a3; 1406 uint32_t endpoint_id = args->a4; 1407 struct mem_frag_state *s = NULL; 1408 tee_mm_entry_t *mm = NULL; 1409 unsigned int page_count = 0; 1410 void *buf = NULL; 1411 uint32_t ret_w1 = 0; 1412 uint32_t ret_w2 = 0; 1413 uint32_t ret_w3 = 0; 1414 uint32_t ret_fid = 0; 1415 int rc = 0; 1416 1417 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1418 uint16_t guest_id = endpoint_id >> 16; 1419 1420 if (!guest_id || virt_set_guest(guest_id)) { 1421 rc = FFA_INVALID_PARAMETERS; 1422 goto out_set_rc; 1423 } 1424 } 1425 1426 /* 1427 * Currently we're only doing this for fragmented FFA_MEM_SHARE_* 1428 * requests. 1429 */ 1430 1431 cpu_spin_lock(&rxtx->spinlock); 1432 1433 s = get_frag_state(global_handle); 1434 if (!s) { 1435 rc = FFA_INVALID_PARAMETERS; 1436 goto out; 1437 } 1438 1439 mm = s->mm; 1440 if (mm) { 1441 if (flen > tee_mm_get_bytes(mm)) { 1442 rc = FFA_INVALID_PARAMETERS; 1443 goto out; 1444 } 1445 page_count = s->share.page_count; 1446 buf = (void *)tee_mm_get_smem(mm); 1447 } else { 1448 if (flen > rxtx->size) { 1449 rc = FFA_INVALID_PARAMETERS; 1450 goto out; 1451 } 1452 buf = rxtx->rx; 1453 } 1454 1455 rc = add_mem_share_frag(s, buf, flen); 1456 out: 1457 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1458 virt_unset_guest(); 1459 1460 cpu_spin_unlock(&rxtx->spinlock); 1461 1462 if (rc <= 0 && mm) { 1463 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1464 tee_mm_free(mm); 1465 } 1466 1467 out_set_rc: 1468 if (rc < 0) { 1469 ret_fid = FFA_ERROR; 1470 ret_w2 = rc; 1471 } else if (rc > 0) { 1472 ret_fid = FFA_MEM_FRAG_RX; 1473 ret_w3 = rc; 1474 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1475 } else { 1476 ret_fid = FFA_SUCCESS_32; 1477 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1478 } 1479 1480 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1481 } 1482 1483 static void handle_mem_reclaim(struct thread_smc_1_2_regs *args) 1484 { 1485 int rc = FFA_INVALID_PARAMETERS; 1486 uint64_t cookie = 0; 1487 1488 if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) 1489 goto out; 1490 1491 cookie = reg_pair_to_64(args->a2, args->a1); 1492 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1493 uint16_t guest_id = 0; 1494 1495 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 1496 guest_id = virt_find_guest_by_cookie(cookie); 1497 } else { 1498 guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & 1499 FFA_MEMORY_HANDLE_PRTN_MASK; 1500 } 1501 if (!guest_id) 1502 goto out; 1503 if (virt_set_guest(guest_id)) { 1504 if (!virt_reclaim_cookie_from_destroyed_guest(guest_id, 1505 cookie)) 1506 rc = FFA_OK; 1507 goto out; 1508 } 1509 } 1510 1511 switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { 1512 case TEE_SUCCESS: 1513 rc = FFA_OK; 1514 break; 1515 case TEE_ERROR_ITEM_NOT_FOUND: 1516 DMSG("cookie %#"PRIx64" not found", cookie); 1517 rc = FFA_INVALID_PARAMETERS; 1518 break; 1519 default: 1520 DMSG("cookie %#"PRIx64" busy", cookie); 1521 rc = FFA_DENIED; 1522 break; 1523 } 1524 1525 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1526 virt_unset_guest(); 1527 1528 out: 1529 set_simple_ret_val(args, rc); 1530 } 1531 1532 static void handle_notification_bitmap_create(struct thread_smc_1_2_regs *args) 1533 { 1534 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1535 uint32_t ret_fid = FFA_ERROR; 1536 uint32_t old_itr_status = 0; 1537 1538 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1539 !args->a5 && !args->a6 && !args->a7) { 1540 struct guest_partition *prtn = NULL; 1541 struct notif_vm_bitmap *nvb = NULL; 1542 uint16_t vm_id = args->a1; 1543 1544 prtn = virt_get_guest(vm_id); 1545 nvb = get_notif_vm_bitmap(prtn, vm_id); 1546 if (!nvb) { 1547 ret_val = FFA_INVALID_PARAMETERS; 1548 goto out_virt_put; 1549 } 1550 1551 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1552 1553 if (nvb->initialized) { 1554 ret_val = FFA_DENIED; 1555 goto out_unlock; 1556 } 1557 1558 nvb->initialized = true; 1559 nvb->do_bottom_half_value = -1; 1560 ret_val = FFA_OK; 1561 ret_fid = FFA_SUCCESS_32; 1562 out_unlock: 1563 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1564 out_virt_put: 1565 virt_put_guest(prtn); 1566 } 1567 1568 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1569 } 1570 1571 static void handle_notification_bitmap_destroy(struct thread_smc_1_2_regs *args) 1572 { 1573 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1574 uint32_t ret_fid = FFA_ERROR; 1575 uint32_t old_itr_status = 0; 1576 1577 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1578 !args->a5 && !args->a6 && !args->a7) { 1579 struct guest_partition *prtn = NULL; 1580 struct notif_vm_bitmap *nvb = NULL; 1581 uint16_t vm_id = args->a1; 1582 1583 prtn = virt_get_guest(vm_id); 1584 nvb = get_notif_vm_bitmap(prtn, vm_id); 1585 if (!nvb) { 1586 ret_val = FFA_INVALID_PARAMETERS; 1587 goto out_virt_put; 1588 } 1589 1590 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1591 1592 if (nvb->pending || nvb->bound) { 1593 ret_val = FFA_DENIED; 1594 goto out_unlock; 1595 } 1596 1597 memset(nvb, 0, sizeof(*nvb)); 1598 ret_val = FFA_OK; 1599 ret_fid = FFA_SUCCESS_32; 1600 out_unlock: 1601 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1602 out_virt_put: 1603 virt_put_guest(prtn); 1604 } 1605 1606 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1607 } 1608 1609 static void handle_notification_bind(struct thread_smc_1_2_regs *args) 1610 { 1611 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1612 struct guest_partition *prtn = NULL; 1613 struct notif_vm_bitmap *nvb = NULL; 1614 uint32_t ret_fid = FFA_ERROR; 1615 uint32_t old_itr_status = 0; 1616 uint64_t bitmap = 0; 1617 uint16_t vm_id = 0; 1618 1619 if (args->a5 || args->a6 || args->a7) 1620 goto out; 1621 if (args->a2) { 1622 /* We only deal with global notifications */ 1623 ret_val = FFA_DENIED; 1624 goto out; 1625 } 1626 1627 /* The destination of the eventual notification */ 1628 vm_id = FFA_DST(args->a1); 1629 bitmap = reg_pair_to_64(args->a4, args->a3); 1630 1631 prtn = virt_get_guest(vm_id); 1632 nvb = get_notif_vm_bitmap(prtn, vm_id); 1633 if (!nvb) { 1634 ret_val = FFA_INVALID_PARAMETERS; 1635 goto out_virt_put; 1636 } 1637 1638 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1639 1640 if ((bitmap & nvb->bound)) { 1641 ret_val = FFA_DENIED; 1642 } else { 1643 nvb->bound |= bitmap; 1644 ret_val = FFA_OK; 1645 ret_fid = FFA_SUCCESS_32; 1646 } 1647 1648 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1649 out_virt_put: 1650 virt_put_guest(prtn); 1651 out: 1652 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1653 } 1654 1655 static void handle_notification_unbind(struct thread_smc_1_2_regs *args) 1656 { 1657 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1658 struct guest_partition *prtn = NULL; 1659 struct notif_vm_bitmap *nvb = NULL; 1660 uint32_t ret_fid = FFA_ERROR; 1661 uint32_t old_itr_status = 0; 1662 uint64_t bitmap = 0; 1663 uint16_t vm_id = 0; 1664 1665 if (args->a2 || args->a5 || args->a6 || args->a7) 1666 goto out; 1667 1668 /* The destination of the eventual notification */ 1669 vm_id = FFA_DST(args->a1); 1670 bitmap = reg_pair_to_64(args->a4, args->a3); 1671 1672 prtn = virt_get_guest(vm_id); 1673 nvb = get_notif_vm_bitmap(prtn, vm_id); 1674 if (!nvb) { 1675 ret_val = FFA_INVALID_PARAMETERS; 1676 goto out_virt_put; 1677 } 1678 1679 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1680 1681 if (bitmap & nvb->pending) { 1682 ret_val = FFA_DENIED; 1683 } else { 1684 nvb->bound &= ~bitmap; 1685 ret_val = FFA_OK; 1686 ret_fid = FFA_SUCCESS_32; 1687 } 1688 1689 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1690 out_virt_put: 1691 virt_put_guest(prtn); 1692 out: 1693 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1694 } 1695 1696 static void handle_notification_get(struct thread_smc_1_2_regs *args) 1697 { 1698 uint32_t w2 = FFA_INVALID_PARAMETERS; 1699 struct guest_partition *prtn = NULL; 1700 struct notif_vm_bitmap *nvb = NULL; 1701 uint32_t ret_fid = FFA_ERROR; 1702 uint32_t old_itr_status = 0; 1703 uint16_t vm_id = 0; 1704 uint32_t w3 = 0; 1705 1706 if (args->a5 || args->a6 || args->a7) 1707 goto out; 1708 if (!(args->a2 & 0x1)) { 1709 ret_fid = FFA_SUCCESS_32; 1710 w2 = 0; 1711 goto out; 1712 } 1713 vm_id = FFA_DST(args->a1); 1714 1715 prtn = virt_get_guest(vm_id); 1716 nvb = get_notif_vm_bitmap(prtn, vm_id); 1717 if (!nvb) 1718 goto out_virt_put; 1719 1720 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1721 1722 reg_pair_from_64(nvb->pending, &w3, &w2); 1723 nvb->pending = 0; 1724 ret_fid = FFA_SUCCESS_32; 1725 1726 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1727 out_virt_put: 1728 virt_put_guest(prtn); 1729 out: 1730 spmc_set_args(args, ret_fid, 0, w2, w3, 0, 0); 1731 } 1732 1733 struct notif_info_get_state { 1734 struct thread_smc_1_2_regs *args; 1735 unsigned int ids_per_reg; 1736 unsigned int ids_count; 1737 unsigned int id_pos; 1738 unsigned int count; 1739 unsigned int max_list_count; 1740 unsigned int list_count; 1741 }; 1742 1743 static bool add_id_in_regs(struct notif_info_get_state *state, 1744 uint16_t id) 1745 { 1746 unsigned int reg_idx = state->id_pos / state->ids_per_reg + 3; 1747 unsigned int reg_shift = (state->id_pos % state->ids_per_reg) * 16; 1748 1749 if (reg_idx > 7) 1750 return false; 1751 1752 state->args->a[reg_idx] &= ~SHIFT_U64(0xffff, reg_shift); 1753 state->args->a[reg_idx] |= (unsigned long)id << reg_shift; 1754 1755 state->id_pos++; 1756 state->count++; 1757 return true; 1758 } 1759 1760 static bool add_id_count(struct notif_info_get_state *state) 1761 { 1762 assert(state->list_count < state->max_list_count && 1763 state->count >= 1 && state->count <= 4); 1764 1765 state->ids_count |= (state->count - 1) << (state->list_count * 2 + 12); 1766 state->list_count++; 1767 state->count = 0; 1768 1769 return state->list_count < state->max_list_count; 1770 } 1771 1772 static bool add_nvb_to_state(struct notif_info_get_state *state, 1773 uint16_t guest_id, struct notif_vm_bitmap *nvb) 1774 { 1775 if (!nvb->pending) 1776 return true; 1777 /* 1778 * Add only the guest_id, meaning a global notification for this 1779 * guest. 1780 * 1781 * If notifications for one or more specific vCPUs we'd add those 1782 * before calling add_id_count(), but that's not supported. 1783 */ 1784 return add_id_in_regs(state, guest_id) && add_id_count(state); 1785 } 1786 1787 static void handle_notification_info_get(struct thread_smc_1_2_regs *args) 1788 { 1789 struct notif_info_get_state state = { .args = args }; 1790 uint32_t ffa_res = FFA_INVALID_PARAMETERS; 1791 struct guest_partition *prtn = NULL; 1792 struct notif_vm_bitmap *nvb = NULL; 1793 uint32_t more_pending_flag = 0; 1794 uint32_t itr_state = 0; 1795 uint16_t guest_id = 0; 1796 1797 if (args->a1 || args->a2 || args->a3 || args->a4 || args->a5 || 1798 args->a6 || args->a7) 1799 goto err; 1800 1801 if (OPTEE_SMC_IS_64(args->a0)) { 1802 spmc_set_args(args, FFA_SUCCESS_64, 0, 0, 0, 0, 0); 1803 state.ids_per_reg = 4; 1804 state.max_list_count = 31; 1805 } else { 1806 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 1807 state.ids_per_reg = 2; 1808 state.max_list_count = 15; 1809 } 1810 1811 while (true) { 1812 /* 1813 * With NS-Virtualization we need to go through all 1814 * partitions to collect the notification bitmaps, without 1815 * we just check the only notification bitmap we have. 1816 */ 1817 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1818 prtn = virt_next_guest(prtn); 1819 if (!prtn) 1820 break; 1821 guest_id = virt_get_guest_id(prtn); 1822 } 1823 nvb = get_notif_vm_bitmap(prtn, guest_id); 1824 1825 itr_state = cpu_spin_lock_xsave(&spmc_notif_lock); 1826 if (!add_nvb_to_state(&state, guest_id, nvb)) 1827 more_pending_flag = BIT(0); 1828 cpu_spin_unlock_xrestore(&spmc_notif_lock, itr_state); 1829 1830 if (!IS_ENABLED(CFG_NS_VIRTUALIZATION) || more_pending_flag) 1831 break; 1832 } 1833 virt_put_guest(prtn); 1834 1835 if (!state.id_pos) { 1836 ffa_res = FFA_NO_DATA; 1837 goto err; 1838 } 1839 args->a2 = (state.list_count << FFA_NOTIF_INFO_GET_ID_COUNT_SHIFT) | 1840 (state.ids_count << FFA_NOTIF_INFO_GET_ID_LIST_SHIFT) | 1841 more_pending_flag; 1842 return; 1843 err: 1844 spmc_set_args(args, FFA_ERROR, 0, ffa_res, 0, 0, 0); 1845 } 1846 1847 void thread_spmc_set_async_notif_intid(int intid) 1848 { 1849 assert(interrupt_can_raise_sgi(interrupt_get_main_chip())); 1850 notif_intid = intid; 1851 spmc_notif_is_ready = true; 1852 DMSG("Asynchronous notifications are ready"); 1853 } 1854 1855 void notif_send_async(uint32_t value, uint16_t guest_id) 1856 { 1857 struct guest_partition *prtn = NULL; 1858 struct notif_vm_bitmap *nvb = NULL; 1859 uint32_t old_itr_status = 0; 1860 1861 prtn = virt_get_guest(guest_id); 1862 nvb = get_notif_vm_bitmap(prtn, guest_id); 1863 1864 if (nvb) { 1865 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1866 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && 1867 spmc_notif_is_ready && nvb->do_bottom_half_value >= 0 && 1868 notif_intid >= 0); 1869 nvb->pending |= BIT64(nvb->do_bottom_half_value); 1870 interrupt_raise_sgi(interrupt_get_main_chip(), notif_intid, 1871 ITR_CPU_MASK_TO_THIS_CPU); 1872 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1873 } 1874 1875 virt_put_guest(prtn); 1876 } 1877 #else 1878 void notif_send_async(uint32_t value, uint16_t guest_id) 1879 { 1880 struct guest_partition *prtn = NULL; 1881 struct notif_vm_bitmap *nvb = NULL; 1882 /* global notification, delay notification interrupt */ 1883 uint32_t flags = BIT32(1); 1884 int res = 0; 1885 1886 prtn = virt_get_guest(guest_id); 1887 nvb = get_notif_vm_bitmap(prtn, guest_id); 1888 1889 if (nvb) { 1890 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && 1891 spmc_notif_is_ready && nvb->do_bottom_half_value >= 0); 1892 res = ffa_set_notification(guest_id, optee_endpoint_id, flags, 1893 BIT64(nvb->do_bottom_half_value)); 1894 if (res) { 1895 EMSG("notification set failed with error %d", res); 1896 panic(); 1897 } 1898 } 1899 1900 virt_put_guest(prtn); 1901 } 1902 #endif 1903 1904 /* Only called from assembly */ 1905 void thread_spmc_msg_recv(struct thread_smc_1_2_regs *args); 1906 void thread_spmc_msg_recv(struct thread_smc_1_2_regs *args) 1907 { 1908 assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); 1909 switch (args->a0) { 1910 #if defined(CFG_CORE_SEL1_SPMC) 1911 case FFA_FEATURES: 1912 handle_features(args); 1913 break; 1914 case FFA_SPM_ID_GET: 1915 spmc_handle_spm_id_get(args); 1916 break; 1917 #ifdef ARM64 1918 case FFA_RXTX_MAP_64: 1919 #endif 1920 case FFA_RXTX_MAP_32: 1921 spmc_handle_rxtx_map(args, &my_rxtx); 1922 break; 1923 case FFA_RXTX_UNMAP: 1924 spmc_handle_rxtx_unmap(args, &my_rxtx); 1925 break; 1926 case FFA_RX_RELEASE: 1927 spmc_handle_rx_release(args, &my_rxtx); 1928 break; 1929 case FFA_PARTITION_INFO_GET: 1930 spmc_handle_partition_info_get(args, &my_rxtx); 1931 break; 1932 case FFA_RUN: 1933 spmc_handle_run(args); 1934 break; 1935 #endif /*CFG_CORE_SEL1_SPMC*/ 1936 case FFA_INTERRUPT: 1937 if (IS_ENABLED(CFG_CORE_SEL1_SPMC)) 1938 spmc_set_args(args, FFA_NORMAL_WORLD_RESUME, 0, 0, 0, 1939 0, 0); 1940 else 1941 spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); 1942 break; 1943 #ifdef ARM64 1944 case FFA_MSG_SEND_DIRECT_REQ_64: 1945 #endif 1946 case FFA_MSG_SEND_DIRECT_REQ_32: 1947 handle_direct_request(args, &my_rxtx); 1948 break; 1949 #if defined(CFG_CORE_SEL1_SPMC) 1950 #ifdef ARM64 1951 case FFA_MEM_SHARE_64: 1952 #endif 1953 case FFA_MEM_SHARE_32: 1954 handle_mem_share(args, &my_rxtx); 1955 break; 1956 case FFA_MEM_RECLAIM: 1957 if (!IS_ENABLED(CFG_SECURE_PARTITION) || 1958 !ffa_mem_reclaim(args, NULL)) 1959 handle_mem_reclaim(args); 1960 break; 1961 case FFA_MEM_FRAG_TX: 1962 handle_mem_frag_tx(args, &my_rxtx); 1963 break; 1964 case FFA_NOTIFICATION_BITMAP_CREATE: 1965 handle_notification_bitmap_create(args); 1966 break; 1967 case FFA_NOTIFICATION_BITMAP_DESTROY: 1968 handle_notification_bitmap_destroy(args); 1969 break; 1970 case FFA_NOTIFICATION_BIND: 1971 handle_notification_bind(args); 1972 break; 1973 case FFA_NOTIFICATION_UNBIND: 1974 handle_notification_unbind(args); 1975 break; 1976 case FFA_NOTIFICATION_GET: 1977 handle_notification_get(args); 1978 break; 1979 #ifdef ARM64 1980 case FFA_NOTIFICATION_INFO_GET_64: 1981 #endif 1982 case FFA_NOTIFICATION_INFO_GET_32: 1983 handle_notification_info_get(args); 1984 break; 1985 #endif /*CFG_CORE_SEL1_SPMC*/ 1986 case FFA_ERROR: 1987 EMSG("Cannot handle FFA_ERROR(%d)", (int)args->a2); 1988 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { 1989 /* 1990 * The SPMC will return an FFA_ERROR back so better 1991 * panic() now than flooding the log. 1992 */ 1993 panic("FFA_ERROR from SPMC is fatal"); 1994 } 1995 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, 1996 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 1997 break; 1998 default: 1999 EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); 2000 set_simple_ret_val(args, FFA_NOT_SUPPORTED); 2001 } 2002 } 2003 2004 static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) 2005 { 2006 size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 2007 struct thread_ctx *thr = threads + thread_get_id(); 2008 TEE_Result res = TEE_ERROR_BAD_PARAMETERS; 2009 struct optee_msg_arg *arg = NULL; 2010 struct mobj *mobj = NULL; 2011 uint32_t num_params = 0; 2012 size_t sz = 0; 2013 2014 mobj = mobj_ffa_get_by_cookie(cookie, 0); 2015 if (!mobj) { 2016 EMSG("Can't find cookie %#"PRIx64, cookie); 2017 return TEE_ERROR_BAD_PARAMETERS; 2018 } 2019 2020 res = mobj_inc_map(mobj); 2021 if (res) 2022 goto out_put_mobj; 2023 2024 res = TEE_ERROR_BAD_PARAMETERS; 2025 arg = mobj_get_va(mobj, offset, sizeof(*arg)); 2026 if (!arg) 2027 goto out_dec_map; 2028 2029 num_params = READ_ONCE(arg->num_params); 2030 if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) 2031 goto out_dec_map; 2032 2033 sz = OPTEE_MSG_GET_ARG_SIZE(num_params); 2034 2035 thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); 2036 if (!thr->rpc_arg) 2037 goto out_dec_map; 2038 2039 virt_on_stdcall(); 2040 res = tee_entry_std(arg, num_params); 2041 2042 thread_rpc_shm_cache_clear(&thr->shm_cache); 2043 thr->rpc_arg = NULL; 2044 2045 out_dec_map: 2046 mobj_dec_map(mobj); 2047 out_put_mobj: 2048 mobj_put(mobj); 2049 return res; 2050 } 2051 2052 /* 2053 * Helper routine for the assembly function thread_std_smc_entry() 2054 * 2055 * Note: this function is weak just to make link_dummies_paged.c happy. 2056 */ 2057 uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, 2058 uint32_t a2, uint32_t a3, 2059 uint32_t a4, uint32_t a5 __unused) 2060 { 2061 /* 2062 * Arguments are supplied from handle_yielding_call() as: 2063 * a0 <- w1 2064 * a1 <- w3 2065 * a2 <- w4 2066 * a3 <- w5 2067 * a4 <- w6 2068 * a5 <- w7 2069 */ 2070 thread_get_tsd()->rpc_target_info = swap_src_dst(a0); 2071 if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) 2072 return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); 2073 return FFA_DENIED; 2074 } 2075 2076 static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) 2077 { 2078 uint64_t offs = tpm->u.memref.offs; 2079 2080 param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + 2081 OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; 2082 2083 param->u.fmem.offs_low = offs; 2084 param->u.fmem.offs_high = offs >> 32; 2085 if (param->u.fmem.offs_high != offs >> 32) 2086 return false; 2087 2088 param->u.fmem.size = tpm->u.memref.size; 2089 if (tpm->u.memref.mobj) { 2090 uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); 2091 2092 /* If a mobj is passed it better be one with a valid cookie. */ 2093 if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) 2094 return false; 2095 param->u.fmem.global_id = cookie; 2096 } else { 2097 param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 2098 } 2099 2100 return true; 2101 } 2102 2103 static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, 2104 struct thread_param *params, 2105 struct optee_msg_arg **arg_ret) 2106 { 2107 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 2108 struct thread_ctx *thr = threads + thread_get_id(); 2109 struct optee_msg_arg *arg = thr->rpc_arg; 2110 2111 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 2112 return TEE_ERROR_BAD_PARAMETERS; 2113 2114 if (!arg) { 2115 EMSG("rpc_arg not set"); 2116 return TEE_ERROR_GENERIC; 2117 } 2118 2119 memset(arg, 0, sz); 2120 arg->cmd = cmd; 2121 arg->num_params = num_params; 2122 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 2123 2124 for (size_t n = 0; n < num_params; n++) { 2125 switch (params[n].attr) { 2126 case THREAD_PARAM_ATTR_NONE: 2127 arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; 2128 break; 2129 case THREAD_PARAM_ATTR_VALUE_IN: 2130 case THREAD_PARAM_ATTR_VALUE_OUT: 2131 case THREAD_PARAM_ATTR_VALUE_INOUT: 2132 arg->params[n].attr = params[n].attr - 2133 THREAD_PARAM_ATTR_VALUE_IN + 2134 OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 2135 arg->params[n].u.value.a = params[n].u.value.a; 2136 arg->params[n].u.value.b = params[n].u.value.b; 2137 arg->params[n].u.value.c = params[n].u.value.c; 2138 break; 2139 case THREAD_PARAM_ATTR_MEMREF_IN: 2140 case THREAD_PARAM_ATTR_MEMREF_OUT: 2141 case THREAD_PARAM_ATTR_MEMREF_INOUT: 2142 if (!set_fmem(arg->params + n, params + n)) 2143 return TEE_ERROR_BAD_PARAMETERS; 2144 break; 2145 default: 2146 return TEE_ERROR_BAD_PARAMETERS; 2147 } 2148 } 2149 2150 if (arg_ret) 2151 *arg_ret = arg; 2152 2153 return TEE_SUCCESS; 2154 } 2155 2156 static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, 2157 struct thread_param *params) 2158 { 2159 for (size_t n = 0; n < num_params; n++) { 2160 switch (params[n].attr) { 2161 case THREAD_PARAM_ATTR_VALUE_OUT: 2162 case THREAD_PARAM_ATTR_VALUE_INOUT: 2163 params[n].u.value.a = arg->params[n].u.value.a; 2164 params[n].u.value.b = arg->params[n].u.value.b; 2165 params[n].u.value.c = arg->params[n].u.value.c; 2166 break; 2167 case THREAD_PARAM_ATTR_MEMREF_OUT: 2168 case THREAD_PARAM_ATTR_MEMREF_INOUT: 2169 params[n].u.memref.size = arg->params[n].u.fmem.size; 2170 break; 2171 default: 2172 break; 2173 } 2174 } 2175 2176 return arg->ret; 2177 } 2178 2179 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 2180 struct thread_param *params) 2181 { 2182 struct thread_rpc_arg rpc_arg = { .call = { 2183 .w1 = thread_get_tsd()->rpc_target_info, 2184 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2185 }, 2186 }; 2187 struct optee_msg_arg *arg = NULL; 2188 uint32_t ret = 0; 2189 2190 ret = get_rpc_arg(cmd, num_params, params, &arg); 2191 if (ret) 2192 return ret; 2193 2194 thread_rpc(&rpc_arg); 2195 2196 return get_rpc_arg_res(arg, num_params, params); 2197 } 2198 2199 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 2200 { 2201 struct thread_rpc_arg rpc_arg = { .call = { 2202 .w1 = thread_get_tsd()->rpc_target_info, 2203 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2204 }, 2205 }; 2206 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); 2207 uint32_t res2 = 0; 2208 uint32_t res = 0; 2209 2210 DMSG("freeing cookie %#"PRIx64, cookie); 2211 2212 res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); 2213 2214 mobj_put(mobj); 2215 res2 = mobj_ffa_unregister_by_cookie(cookie); 2216 if (res2) 2217 DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, 2218 cookie, res2); 2219 if (!res) 2220 thread_rpc(&rpc_arg); 2221 } 2222 2223 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) 2224 { 2225 struct thread_rpc_arg rpc_arg = { .call = { 2226 .w1 = thread_get_tsd()->rpc_target_info, 2227 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2228 }, 2229 }; 2230 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); 2231 struct optee_msg_arg *arg = NULL; 2232 unsigned int internal_offset = 0; 2233 struct mobj *mobj = NULL; 2234 uint64_t cookie = 0; 2235 2236 if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) 2237 return NULL; 2238 2239 thread_rpc(&rpc_arg); 2240 2241 if (arg->num_params != 1 || 2242 arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) 2243 return NULL; 2244 2245 internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); 2246 cookie = READ_ONCE(arg->params->u.fmem.global_id); 2247 mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); 2248 if (!mobj) { 2249 DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", 2250 cookie, internal_offset); 2251 return NULL; 2252 } 2253 2254 assert(mobj_is_nonsec(mobj)); 2255 2256 if (mobj->size < size) { 2257 DMSG("Mobj %#"PRIx64": wrong size", cookie); 2258 mobj_put(mobj); 2259 return NULL; 2260 } 2261 2262 if (mobj_inc_map(mobj)) { 2263 DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); 2264 mobj_put(mobj); 2265 return NULL; 2266 } 2267 2268 return mobj; 2269 } 2270 2271 struct mobj *thread_rpc_alloc_payload(size_t size) 2272 { 2273 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); 2274 } 2275 2276 struct mobj *thread_rpc_alloc_kernel_payload(size_t size) 2277 { 2278 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); 2279 } 2280 2281 void thread_rpc_free_kernel_payload(struct mobj *mobj) 2282 { 2283 if (mobj) 2284 thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, 2285 mobj_get_cookie(mobj), mobj); 2286 } 2287 2288 void thread_rpc_free_payload(struct mobj *mobj) 2289 { 2290 if (mobj) 2291 thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), 2292 mobj); 2293 } 2294 2295 struct mobj *thread_rpc_alloc_global_payload(size_t size) 2296 { 2297 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); 2298 } 2299 2300 void thread_rpc_free_global_payload(struct mobj *mobj) 2301 { 2302 if (mobj) 2303 thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, 2304 mobj_get_cookie(mobj), mobj); 2305 } 2306 2307 void thread_spmc_register_secondary_ep(vaddr_t ep) 2308 { 2309 unsigned long ret = 0; 2310 2311 /* Let the SPM know the entry point for secondary CPUs */ 2312 ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); 2313 2314 if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) 2315 EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); 2316 } 2317 2318 static uint16_t ffa_id_get(void) 2319 { 2320 /* 2321 * Ask the SPM component running at a higher EL to return our FF-A ID. 2322 * This can either be the SPMC ID (if the SPMC is enabled in OP-TEE) or 2323 * the partition ID (if not). 2324 */ 2325 struct thread_smc_args args = { 2326 .a0 = FFA_ID_GET, 2327 }; 2328 2329 thread_smccc(&args); 2330 if (!is_ffa_success(args.a0)) { 2331 if (args.a0 == FFA_ERROR) 2332 EMSG("Get id failed with error %ld", args.a2); 2333 else 2334 EMSG("Get id failed"); 2335 panic(); 2336 } 2337 2338 return args.a2; 2339 } 2340 2341 static uint16_t ffa_spm_id_get(void) 2342 { 2343 /* 2344 * Ask the SPM component running at a higher EL to return its ID. 2345 * If OP-TEE implements the S-EL1 SPMC, this will get the SPMD ID. 2346 * If not, the ID of the SPMC will be returned. 2347 */ 2348 struct thread_smc_args args = { 2349 .a0 = FFA_SPM_ID_GET, 2350 }; 2351 2352 thread_smccc(&args); 2353 if (!is_ffa_success(args.a0)) { 2354 if (args.a0 == FFA_ERROR) 2355 EMSG("Get spm id failed with error %ld", args.a2); 2356 else 2357 EMSG("Get spm id failed"); 2358 panic(); 2359 } 2360 2361 return args.a2; 2362 } 2363 2364 #if defined(CFG_CORE_SEL1_SPMC) 2365 static TEE_Result spmc_init(void) 2366 { 2367 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 2368 virt_add_guest_spec_data(¬if_vm_bitmap_id, 2369 sizeof(struct notif_vm_bitmap), NULL)) 2370 panic("virt_add_guest_spec_data"); 2371 spmd_id = ffa_spm_id_get(); 2372 DMSG("SPMD ID %#"PRIx16, spmd_id); 2373 2374 spmc_id = ffa_id_get(); 2375 DMSG("SPMC ID %#"PRIx16, spmc_id); 2376 2377 optee_endpoint_id = FFA_SWD_ID_MIN; 2378 while (spmc_is_reserved_id(optee_endpoint_id)) 2379 optee_endpoint_id++; 2380 2381 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2382 2383 /* 2384 * If SPMD think we are version 1.0 it will report version 1.0 to 2385 * normal world regardless of what version we query the SPM with. 2386 * However, if SPMD think we are version 1.1 it will forward 2387 * queries from normal world to let us negotiate version. So by 2388 * setting version 1.0 here we should be compatible. 2389 * 2390 * Note that disagreement on negotiated version means that we'll 2391 * have communication problems with normal world. 2392 */ 2393 my_rxtx.ffa_vers = FFA_VERSION_1_0; 2394 2395 return TEE_SUCCESS; 2396 } 2397 #else /* !defined(CFG_CORE_SEL1_SPMC) */ 2398 static void spmc_rxtx_map(struct ffa_rxtx *rxtx) 2399 { 2400 struct thread_smc_args args = { 2401 #ifdef ARM64 2402 .a0 = FFA_RXTX_MAP_64, 2403 #else 2404 .a0 = FFA_RXTX_MAP_32, 2405 #endif 2406 .a1 = virt_to_phys(rxtx->tx), 2407 .a2 = virt_to_phys(rxtx->rx), 2408 .a3 = 1, 2409 }; 2410 2411 thread_smccc(&args); 2412 if (!is_ffa_success(args.a0)) { 2413 if (args.a0 == FFA_ERROR) 2414 EMSG("rxtx map failed with error %ld", args.a2); 2415 else 2416 EMSG("rxtx map failed"); 2417 panic(); 2418 } 2419 } 2420 2421 static uint32_t get_ffa_version(uint32_t my_version) 2422 { 2423 struct thread_smc_args args = { 2424 .a0 = FFA_VERSION, 2425 .a1 = my_version, 2426 }; 2427 2428 thread_smccc(&args); 2429 if (args.a0 & BIT(31)) { 2430 EMSG("FF-A version failed with error %ld", args.a0); 2431 panic(); 2432 } 2433 2434 return args.a0; 2435 } 2436 2437 static void *spmc_retrieve_req(uint64_t cookie, 2438 struct ffa_mem_transaction_x *trans) 2439 { 2440 struct ffa_mem_access *acc_descr_array = NULL; 2441 struct ffa_mem_access_perm *perm_descr = NULL; 2442 struct thread_smc_args args = { 2443 .a0 = FFA_MEM_RETRIEVE_REQ_32, 2444 .a3 = 0, /* Address, Using TX -> MBZ */ 2445 .a4 = 0, /* Using TX -> MBZ */ 2446 }; 2447 size_t size = 0; 2448 int rc = 0; 2449 2450 if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { 2451 struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; 2452 2453 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2454 memset(trans_descr, 0, size); 2455 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2456 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2457 trans_descr->global_handle = cookie; 2458 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2459 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2460 trans_descr->mem_access_count = 1; 2461 acc_descr_array = trans_descr->mem_access_array; 2462 } else { 2463 struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; 2464 2465 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2466 memset(trans_descr, 0, size); 2467 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2468 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2469 trans_descr->global_handle = cookie; 2470 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2471 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2472 trans_descr->mem_access_count = 1; 2473 trans_descr->mem_access_offs = sizeof(*trans_descr); 2474 trans_descr->mem_access_size = sizeof(struct ffa_mem_access); 2475 acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + 2476 sizeof(*trans_descr)); 2477 } 2478 acc_descr_array->region_offs = 0; 2479 acc_descr_array->reserved = 0; 2480 perm_descr = &acc_descr_array->access_perm; 2481 perm_descr->endpoint_id = optee_endpoint_id; 2482 perm_descr->perm = FFA_MEM_ACC_RW; 2483 perm_descr->flags = 0; 2484 2485 args.a1 = size; /* Total Length */ 2486 args.a2 = size; /* Frag Length == Total length */ 2487 thread_smccc(&args); 2488 if (args.a0 != FFA_MEM_RETRIEVE_RESP) { 2489 if (args.a0 == FFA_ERROR) 2490 EMSG("Failed to fetch cookie %#"PRIx64" error code %d", 2491 cookie, (int)args.a2); 2492 else 2493 EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, 2494 cookie, args.a0); 2495 return NULL; 2496 } 2497 rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.rx, 2498 my_rxtx.size, trans); 2499 if (rc) { 2500 EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", 2501 cookie, rc); 2502 return NULL; 2503 } 2504 2505 return my_rxtx.rx; 2506 } 2507 2508 void thread_spmc_relinquish(uint64_t cookie) 2509 { 2510 struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; 2511 struct thread_smc_args args = { 2512 .a0 = FFA_MEM_RELINQUISH, 2513 }; 2514 2515 memset(relinquish_desc, 0, sizeof(*relinquish_desc)); 2516 relinquish_desc->handle = cookie; 2517 relinquish_desc->flags = 0; 2518 relinquish_desc->endpoint_count = 1; 2519 relinquish_desc->endpoint_id_array[0] = optee_endpoint_id; 2520 thread_smccc(&args); 2521 if (!is_ffa_success(args.a0)) 2522 EMSG("Failed to relinquish cookie %#"PRIx64, cookie); 2523 } 2524 2525 static int set_pages(struct ffa_address_range *regions, 2526 unsigned int num_regions, unsigned int num_pages, 2527 struct mobj_ffa *mf) 2528 { 2529 unsigned int n = 0; 2530 unsigned int idx = 0; 2531 2532 for (n = 0; n < num_regions; n++) { 2533 unsigned int page_count = READ_ONCE(regions[n].page_count); 2534 uint64_t addr = READ_ONCE(regions[n].address); 2535 2536 if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) 2537 return FFA_INVALID_PARAMETERS; 2538 } 2539 2540 if (idx != num_pages) 2541 return FFA_INVALID_PARAMETERS; 2542 2543 return 0; 2544 } 2545 2546 struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) 2547 { 2548 struct mobj_ffa *ret = NULL; 2549 struct ffa_mem_transaction_x retrieve_desc = { }; 2550 struct ffa_mem_access *descr_array = NULL; 2551 struct ffa_mem_region *descr = NULL; 2552 struct mobj_ffa *mf = NULL; 2553 unsigned int num_pages = 0; 2554 unsigned int offs = 0; 2555 void *buf = NULL; 2556 struct thread_smc_args ffa_rx_release_args = { 2557 .a0 = FFA_RX_RELEASE 2558 }; 2559 2560 /* 2561 * OP-TEE is only supporting a single mem_region while the 2562 * specification allows for more than one. 2563 */ 2564 buf = spmc_retrieve_req(cookie, &retrieve_desc); 2565 if (!buf) { 2566 EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, 2567 cookie); 2568 return NULL; 2569 } 2570 2571 descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); 2572 offs = READ_ONCE(descr_array->region_offs); 2573 descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); 2574 2575 num_pages = READ_ONCE(descr->total_page_count); 2576 mf = mobj_ffa_spmc_new(cookie, num_pages); 2577 if (!mf) 2578 goto out; 2579 2580 if (set_pages(descr->address_range_array, 2581 READ_ONCE(descr->address_range_count), num_pages, mf)) { 2582 mobj_ffa_spmc_delete(mf); 2583 goto out; 2584 } 2585 2586 ret = mf; 2587 2588 out: 2589 /* Release RX buffer after the mem retrieve request. */ 2590 thread_smccc(&ffa_rx_release_args); 2591 2592 return ret; 2593 } 2594 2595 static uint32_t get_ffa_version_from_manifest(void *fdt) 2596 { 2597 int ret = 0; 2598 uint32_t vers = 0; 2599 2600 ret = fdt_node_check_compatible(fdt, 0, "arm,ffa-manifest-1.0"); 2601 if (ret < 0) { 2602 EMSG("Invalid FF-A manifest at %p: error %d", fdt, ret); 2603 panic(); 2604 } 2605 2606 ret = fdt_read_uint32(fdt, 0, "ffa-version", &vers); 2607 if (ret < 0) { 2608 EMSG("Can't read \"ffa-version\" from FF-A manifest at %p: error %d", 2609 fdt, ret); 2610 panic(); 2611 } 2612 2613 return vers; 2614 } 2615 2616 static TEE_Result spmc_init(void) 2617 { 2618 uint32_t my_vers = 0; 2619 uint32_t vers = 0; 2620 2621 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 2622 virt_add_guest_spec_data(¬if_vm_bitmap_id, 2623 sizeof(struct notif_vm_bitmap), NULL)) 2624 panic("virt_add_guest_spec_data"); 2625 2626 my_vers = get_ffa_version_from_manifest(get_manifest_dt()); 2627 if (my_vers < FFA_VERSION_1_0 || my_vers > FFA_VERSION_1_2) { 2628 EMSG("Unsupported version %"PRIu32".%"PRIu32" from manifest", 2629 FFA_GET_MAJOR_VERSION(my_vers), 2630 FFA_GET_MINOR_VERSION(my_vers)); 2631 panic(); 2632 } 2633 vers = get_ffa_version(my_vers); 2634 DMSG("SPMC reported version %"PRIu32".%"PRIu32, 2635 FFA_GET_MAJOR_VERSION(vers), FFA_GET_MINOR_VERSION(vers)); 2636 if (FFA_GET_MAJOR_VERSION(vers) != FFA_GET_MAJOR_VERSION(my_vers)) { 2637 EMSG("Incompatible major version %"PRIu32", expected %"PRIu32"", 2638 FFA_GET_MAJOR_VERSION(vers), 2639 FFA_GET_MAJOR_VERSION(my_vers)); 2640 panic(); 2641 } 2642 if (vers < my_vers) 2643 my_vers = vers; 2644 DMSG("Using version %"PRIu32".%"PRIu32"", 2645 FFA_GET_MAJOR_VERSION(my_vers), FFA_GET_MINOR_VERSION(my_vers)); 2646 my_rxtx.ffa_vers = my_vers; 2647 2648 spmc_rxtx_map(&my_rxtx); 2649 2650 spmc_id = ffa_spm_id_get(); 2651 DMSG("SPMC ID %#"PRIx16, spmc_id); 2652 2653 optee_endpoint_id = ffa_id_get(); 2654 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2655 2656 if (!ffa_features(FFA_NOTIFICATION_SET)) { 2657 spmc_notif_is_ready = true; 2658 DMSG("Asynchronous notifications are ready"); 2659 } 2660 2661 return TEE_SUCCESS; 2662 } 2663 #endif /* !defined(CFG_CORE_SEL1_SPMC) */ 2664 2665 nex_service_init(spmc_init); 2666