1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2020-2025, Linaro Limited. 4 * Copyright (c) 2019-2024, Arm Limited. All rights reserved. 5 */ 6 7 #include <assert.h> 8 #include <ffa.h> 9 #include <initcall.h> 10 #include <io.h> 11 #include <kernel/dt.h> 12 #include <kernel/interrupt.h> 13 #include <kernel/notif.h> 14 #include <kernel/panic.h> 15 #include <kernel/secure_partition.h> 16 #include <kernel/spinlock.h> 17 #include <kernel/spmc_sp_handler.h> 18 #include <kernel/tee_misc.h> 19 #include <kernel/thread.h> 20 #include <kernel/thread_private.h> 21 #include <kernel/thread_spmc.h> 22 #include <kernel/virtualization.h> 23 #include <libfdt.h> 24 #include <mm/core_mmu.h> 25 #include <mm/mobj.h> 26 #include <optee_ffa.h> 27 #include <optee_msg.h> 28 #include <optee_rpc_cmd.h> 29 #include <sm/optee_smc.h> 30 #include <string.h> 31 #include <sys/queue.h> 32 #include <tee/entry_std.h> 33 #include <tee/uuid.h> 34 #include <util.h> 35 36 #if defined(CFG_CORE_SEL1_SPMC) 37 struct mem_share_state { 38 struct mobj_ffa *mf; 39 unsigned int page_count; 40 unsigned int region_count; 41 unsigned int current_page_idx; 42 }; 43 44 struct mem_frag_state { 45 struct mem_share_state share; 46 tee_mm_entry_t *mm; 47 unsigned int frag_offset; 48 SLIST_ENTRY(mem_frag_state) link; 49 }; 50 #endif 51 52 struct notif_vm_bitmap { 53 bool initialized; 54 int do_bottom_half_value; 55 uint64_t pending; 56 uint64_t bound; 57 }; 58 59 static unsigned int spmc_notif_lock __nex_data = SPINLOCK_UNLOCK; 60 static bool spmc_notif_is_ready __nex_bss; 61 static int notif_intid __nex_data __maybe_unused = -1; 62 63 /* Id used to look up the guest specific struct notif_vm_bitmap */ 64 static unsigned int notif_vm_bitmap_id __nex_bss; 65 /* Notification state when ns-virtualization isn't enabled */ 66 static struct notif_vm_bitmap default_notif_vm_bitmap; 67 68 /* Initialized in spmc_init() below */ 69 uint16_t optee_endpoint_id __nex_bss; 70 static uint16_t spmc_id __nex_bss; 71 #ifdef CFG_CORE_SEL1_SPMC 72 static uint16_t spmd_id __nex_bss; 73 static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | 74 FFA_PART_PROP_DIRECT_REQ_SEND | 75 #ifdef CFG_NS_VIRTUALIZATION 76 FFA_PART_PROP_NOTIF_CREATED | 77 FFA_PART_PROP_NOTIF_DESTROYED | 78 #endif 79 #ifdef ARM64 80 FFA_PART_PROP_AARCH64_STATE | 81 #endif 82 FFA_PART_PROP_IS_PE_ID; 83 84 static uint32_t my_uuid_words[] = { 85 /* 86 * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 87 * SP, or 88 * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a 89 * logical partition, residing in the same exception level as the 90 * SPMC 91 * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 92 */ 93 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, 94 }; 95 96 /* 97 * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. 98 * 99 * struct ffa_rxtx::spin_lock protects the variables below from concurrent 100 * access this includes the use of content of struct ffa_rxtx::rx and 101 * @frag_state_head. 102 * 103 * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct 104 * ffa_rxtx::tx and false when it is owned by normal world. 105 * 106 * Note that we can't prevent normal world from updating the content of 107 * these buffers so we must always be careful when reading. while we hold 108 * the lock. 109 */ 110 111 static struct ffa_rxtx my_rxtx __nex_bss; 112 113 static bool is_nw_buf(struct ffa_rxtx *rxtx) 114 { 115 return rxtx == &my_rxtx; 116 } 117 118 static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = 119 SLIST_HEAD_INITIALIZER(&frag_state_head); 120 121 #else 122 static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE) __nex_bss; 123 static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE) __nex_bss; 124 static struct ffa_rxtx my_rxtx __nex_data = { 125 .rx = __rx_buf, 126 .tx = __tx_buf, 127 .size = sizeof(__rx_buf), 128 }; 129 #endif 130 131 bool spmc_is_reserved_id(uint16_t id) 132 { 133 #ifdef CFG_CORE_SEL1_SPMC 134 if (id == spmd_id) 135 return true; 136 #endif 137 return id == spmc_id; 138 } 139 140 static uint32_t swap_src_dst(uint32_t src_dst) 141 { 142 return (src_dst >> 16) | (src_dst << 16); 143 } 144 145 static uint16_t get_sender_id(uint32_t src_dst) 146 { 147 return src_dst >> 16; 148 } 149 150 void spmc_set_args(struct thread_smc_1_2_regs *args, uint32_t fid, 151 uint32_t src_dst, uint32_t w2, uint32_t w3, uint32_t w4, 152 uint32_t w5) 153 { 154 *args = (struct thread_smc_1_2_regs){ 155 .a0 = fid, 156 .a1 = src_dst, 157 .a2 = w2, 158 .a3 = w3, 159 .a4 = w4, 160 .a5 = w5, 161 }; 162 } 163 164 static void set_simple_ret_val(struct thread_smc_1_2_regs *args, int ffa_ret) 165 { 166 if (ffa_ret) 167 spmc_set_args(args, FFA_ERROR, 0, ffa_ret, 0, 0, 0); 168 else 169 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 170 } 171 172 uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) 173 { 174 uint32_t major_vers = FFA_GET_MAJOR_VERSION(vers); 175 uint32_t minor_vers = FFA_GET_MINOR_VERSION(vers); 176 uint32_t my_vers = FFA_VERSION_1_2; 177 uint32_t my_major_vers = 0; 178 uint32_t my_minor_vers = 0; 179 180 my_major_vers = FFA_GET_MAJOR_VERSION(my_vers); 181 my_minor_vers = FFA_GET_MINOR_VERSION(my_vers); 182 183 /* 184 * No locking, if the caller does concurrent calls to this it's 185 * only making a mess for itself. We must be able to renegotiate 186 * the FF-A version in order to support differing versions between 187 * the loader and the driver. 188 * 189 * Callers should use the version requested if we return a matching 190 * major version and a matching or larger minor version. The caller 191 * should downgrade to our minor version if our minor version is 192 * smaller. Regardless, always return our version as recommended by 193 * the specification. 194 */ 195 if (major_vers == my_major_vers) { 196 if (minor_vers > my_minor_vers) 197 rxtx->ffa_vers = my_vers; 198 else 199 rxtx->ffa_vers = vers; 200 } 201 202 return my_vers; 203 } 204 205 static bool is_ffa_success(uint32_t fid) 206 { 207 #ifdef ARM64 208 if (fid == FFA_SUCCESS_64) 209 return true; 210 #endif 211 return fid == FFA_SUCCESS_32; 212 } 213 214 static int32_t get_ffa_ret_code(const struct thread_smc_args *args) 215 { 216 if (is_ffa_success(args->a0)) 217 return FFA_OK; 218 if (args->a0 == FFA_ERROR && args->a2) 219 return args->a2; 220 return FFA_NOT_SUPPORTED; 221 } 222 223 static int ffa_simple_call(uint32_t fid, unsigned long a1, unsigned long a2, 224 unsigned long a3, unsigned long a4) 225 { 226 struct thread_smc_args args = { 227 .a0 = fid, 228 .a1 = a1, 229 .a2 = a2, 230 .a3 = a3, 231 .a4 = a4, 232 }; 233 234 thread_smccc(&args); 235 236 return get_ffa_ret_code(&args); 237 } 238 239 static int __maybe_unused ffa_features(uint32_t id) 240 { 241 return ffa_simple_call(FFA_FEATURES, id, 0, 0, 0); 242 } 243 244 static int __maybe_unused ffa_set_notification(uint16_t dst, uint16_t src, 245 uint32_t flags, uint64_t bitmap) 246 { 247 return ffa_simple_call(FFA_NOTIFICATION_SET, 248 SHIFT_U32(src, 16) | dst, flags, 249 low32_from_64(bitmap), high32_from_64(bitmap)); 250 } 251 252 #if defined(CFG_CORE_SEL1_SPMC) 253 static void handle_features(struct thread_smc_1_2_regs *args) 254 { 255 uint32_t ret_fid = FFA_ERROR; 256 uint32_t ret_w2 = FFA_NOT_SUPPORTED; 257 258 switch (args->a1) { 259 case FFA_FEATURE_SCHEDULE_RECV_INTR: 260 if (spmc_notif_is_ready) { 261 ret_fid = FFA_SUCCESS_32; 262 ret_w2 = notif_intid; 263 } 264 break; 265 266 #ifdef ARM64 267 case FFA_RXTX_MAP_64: 268 #endif 269 case FFA_RXTX_MAP_32: 270 ret_fid = FFA_SUCCESS_32; 271 ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ 272 break; 273 #ifdef ARM64 274 case FFA_MEM_SHARE_64: 275 #endif 276 case FFA_MEM_SHARE_32: 277 ret_fid = FFA_SUCCESS_32; 278 /* 279 * Partition manager supports transmission of a memory 280 * transaction descriptor in a buffer dynamically allocated 281 * by the endpoint. 282 */ 283 ret_w2 = BIT(0); 284 break; 285 286 case FFA_ERROR: 287 case FFA_VERSION: 288 case FFA_SUCCESS_32: 289 #ifdef ARM64 290 case FFA_SUCCESS_64: 291 #endif 292 case FFA_FEATURES: 293 case FFA_SPM_ID_GET: 294 case FFA_MEM_FRAG_TX: 295 case FFA_MEM_RECLAIM: 296 case FFA_MSG_SEND_DIRECT_REQ_64: 297 case FFA_MSG_SEND_DIRECT_REQ_32: 298 case FFA_INTERRUPT: 299 case FFA_PARTITION_INFO_GET: 300 case FFA_RXTX_UNMAP: 301 case FFA_RX_RELEASE: 302 case FFA_FEATURE_MANAGED_EXIT_INTR: 303 case FFA_NOTIFICATION_BITMAP_CREATE: 304 case FFA_NOTIFICATION_BITMAP_DESTROY: 305 case FFA_NOTIFICATION_BIND: 306 case FFA_NOTIFICATION_UNBIND: 307 case FFA_NOTIFICATION_SET: 308 case FFA_NOTIFICATION_GET: 309 case FFA_NOTIFICATION_INFO_GET_32: 310 #ifdef ARM64 311 case FFA_NOTIFICATION_INFO_GET_64: 312 #endif 313 ret_fid = FFA_SUCCESS_32; 314 ret_w2 = FFA_PARAM_MBZ; 315 break; 316 default: 317 break; 318 } 319 320 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, 321 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 322 } 323 324 static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) 325 { 326 tee_mm_entry_t *mm = NULL; 327 328 if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) 329 return FFA_INVALID_PARAMETERS; 330 331 mm = tee_mm_alloc(&core_virt_shm_pool, sz); 332 if (!mm) 333 return FFA_NO_MEMORY; 334 335 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, 336 sz / SMALL_PAGE_SIZE, 337 MEM_AREA_NSEC_SHM)) { 338 tee_mm_free(mm); 339 return FFA_INVALID_PARAMETERS; 340 } 341 342 *va_ret = (void *)tee_mm_get_smem(mm); 343 return 0; 344 } 345 346 void spmc_handle_spm_id_get(struct thread_smc_1_2_regs *args) 347 { 348 spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, spmc_id, 349 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 350 } 351 352 static void unmap_buf(void *va, size_t sz) 353 { 354 tee_mm_entry_t *mm = tee_mm_find(&core_virt_shm_pool, (vaddr_t)va); 355 356 assert(mm); 357 core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); 358 tee_mm_free(mm); 359 } 360 361 void spmc_handle_rxtx_map(struct thread_smc_1_2_regs *args, 362 struct ffa_rxtx *rxtx) 363 { 364 int rc = 0; 365 unsigned int sz = 0; 366 paddr_t rx_pa = 0; 367 paddr_t tx_pa = 0; 368 void *rx = NULL; 369 void *tx = NULL; 370 371 cpu_spin_lock(&rxtx->spinlock); 372 373 if (args->a3 & GENMASK_64(63, 6)) { 374 rc = FFA_INVALID_PARAMETERS; 375 goto out; 376 } 377 378 sz = args->a3 * SMALL_PAGE_SIZE; 379 if (!sz) { 380 rc = FFA_INVALID_PARAMETERS; 381 goto out; 382 } 383 /* TX/RX are swapped compared to the caller */ 384 tx_pa = args->a2; 385 rx_pa = args->a1; 386 387 if (rxtx->size) { 388 rc = FFA_DENIED; 389 goto out; 390 } 391 392 /* 393 * If the buffer comes from a SP the address is virtual and already 394 * mapped. 395 */ 396 if (is_nw_buf(rxtx)) { 397 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 398 enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; 399 bool tx_alloced = false; 400 401 /* 402 * With virtualization we establish this mapping in 403 * the nexus mapping which then is replicated to 404 * each partition. 405 * 406 * This means that this mapping must be done before 407 * any partition is created and then must not be 408 * changed. 409 */ 410 411 /* 412 * core_mmu_add_mapping() may reuse previous 413 * mappings. First check if there's any mappings to 414 * reuse so we know how to clean up in case of 415 * failure. 416 */ 417 tx = phys_to_virt(tx_pa, mt, sz); 418 rx = phys_to_virt(rx_pa, mt, sz); 419 if (!tx) { 420 tx = core_mmu_add_mapping(mt, tx_pa, sz); 421 if (!tx) { 422 rc = FFA_NO_MEMORY; 423 goto out; 424 } 425 tx_alloced = true; 426 } 427 if (!rx) 428 rx = core_mmu_add_mapping(mt, rx_pa, sz); 429 430 if (!rx) { 431 if (tx_alloced && tx) 432 core_mmu_remove_mapping(mt, tx, sz); 433 rc = FFA_NO_MEMORY; 434 goto out; 435 } 436 } else { 437 rc = map_buf(tx_pa, sz, &tx); 438 if (rc) 439 goto out; 440 rc = map_buf(rx_pa, sz, &rx); 441 if (rc) { 442 unmap_buf(tx, sz); 443 goto out; 444 } 445 } 446 rxtx->tx = tx; 447 rxtx->rx = rx; 448 } else { 449 if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { 450 rc = FFA_INVALID_PARAMETERS; 451 goto out; 452 } 453 454 if (!virt_to_phys((void *)tx_pa) || 455 !virt_to_phys((void *)rx_pa)) { 456 rc = FFA_INVALID_PARAMETERS; 457 goto out; 458 } 459 460 rxtx->tx = (void *)tx_pa; 461 rxtx->rx = (void *)rx_pa; 462 } 463 464 rxtx->size = sz; 465 rxtx->tx_is_mine = true; 466 DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); 467 DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); 468 out: 469 cpu_spin_unlock(&rxtx->spinlock); 470 set_simple_ret_val(args, rc); 471 } 472 473 void spmc_handle_rxtx_unmap(struct thread_smc_1_2_regs *args, 474 struct ffa_rxtx *rxtx) 475 { 476 int rc = FFA_INVALID_PARAMETERS; 477 478 cpu_spin_lock(&rxtx->spinlock); 479 480 if (!rxtx->size) 481 goto out; 482 483 /* 484 * We don't unmap the SP memory as the SP might still use it. 485 * We avoid to make changes to nexus mappings at this stage since 486 * there currently isn't a way to replicate those changes to all 487 * partitions. 488 */ 489 if (is_nw_buf(rxtx) && !IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 490 unmap_buf(rxtx->rx, rxtx->size); 491 unmap_buf(rxtx->tx, rxtx->size); 492 } 493 rxtx->size = 0; 494 rxtx->rx = NULL; 495 rxtx->tx = NULL; 496 rc = 0; 497 out: 498 cpu_spin_unlock(&rxtx->spinlock); 499 set_simple_ret_val(args, rc); 500 } 501 502 void spmc_handle_rx_release(struct thread_smc_1_2_regs *args, 503 struct ffa_rxtx *rxtx) 504 { 505 int rc = 0; 506 507 cpu_spin_lock(&rxtx->spinlock); 508 /* The senders RX is our TX */ 509 if (!rxtx->size || rxtx->tx_is_mine) { 510 rc = FFA_DENIED; 511 } else { 512 rc = 0; 513 rxtx->tx_is_mine = true; 514 } 515 cpu_spin_unlock(&rxtx->spinlock); 516 517 set_simple_ret_val(args, rc); 518 } 519 520 static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 521 { 522 return !w0 && !w1 && !w2 && !w3; 523 } 524 525 static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 526 { 527 /* 528 * This depends on which UUID we have been assigned. 529 * TODO add a generic mechanism to obtain our UUID. 530 * 531 * The test below is for the hard coded UUID 532 * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 533 */ 534 return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && 535 w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; 536 } 537 538 TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, 539 size_t idx, uint16_t endpoint_id, 540 uint16_t execution_context, 541 uint32_t part_props, 542 const uint32_t uuid_words[4]) 543 { 544 struct ffa_partition_info_x *fpi = NULL; 545 size_t fpi_size = sizeof(*fpi); 546 547 if (ffa_vers >= FFA_VERSION_1_1) 548 fpi_size += FFA_UUID_SIZE; 549 550 if ((idx + 1) * fpi_size > blen) 551 return TEE_ERROR_OUT_OF_MEMORY; 552 553 fpi = (void *)((vaddr_t)buf + idx * fpi_size); 554 fpi->id = endpoint_id; 555 /* Number of execution contexts implemented by this partition */ 556 fpi->execution_context = execution_context; 557 558 fpi->partition_properties = part_props; 559 560 /* In FF-A 1.0 only bits [2:0] are defined, let's mask others */ 561 if (ffa_vers < FFA_VERSION_1_1) 562 fpi->partition_properties &= FFA_PART_PROP_DIRECT_REQ_RECV | 563 FFA_PART_PROP_DIRECT_REQ_SEND | 564 FFA_PART_PROP_INDIRECT_MSGS; 565 566 if (ffa_vers >= FFA_VERSION_1_1) { 567 if (uuid_words) 568 memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); 569 else 570 memset(fpi->uuid, 0, FFA_UUID_SIZE); 571 } 572 573 return TEE_SUCCESS; 574 } 575 576 static int handle_partition_info_get_all(size_t *elem_count, 577 struct ffa_rxtx *rxtx, bool count_only) 578 { 579 if (!count_only) { 580 /* Add OP-TEE SP */ 581 if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, 582 rxtx->size, 0, optee_endpoint_id, 583 CFG_TEE_CORE_NB_CORE, 584 my_part_props, my_uuid_words)) 585 return FFA_NO_MEMORY; 586 } 587 *elem_count = 1; 588 589 if (IS_ENABLED(CFG_SECURE_PARTITION)) { 590 if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, 591 NULL, elem_count, count_only)) 592 return FFA_NO_MEMORY; 593 } 594 595 return FFA_OK; 596 } 597 598 void spmc_handle_partition_info_get(struct thread_smc_1_2_regs *args, 599 struct ffa_rxtx *rxtx) 600 { 601 TEE_Result res = TEE_SUCCESS; 602 uint32_t ret_fid = FFA_ERROR; 603 uint32_t fpi_size = 0; 604 uint32_t rc = 0; 605 bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; 606 607 if (!count_only) { 608 cpu_spin_lock(&rxtx->spinlock); 609 610 if (!rxtx->size || !rxtx->tx_is_mine) { 611 rc = FFA_BUSY; 612 goto out; 613 } 614 } 615 616 if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { 617 size_t elem_count = 0; 618 619 ret_fid = handle_partition_info_get_all(&elem_count, rxtx, 620 count_only); 621 622 if (ret_fid) { 623 rc = ret_fid; 624 ret_fid = FFA_ERROR; 625 } else { 626 ret_fid = FFA_SUCCESS_32; 627 rc = elem_count; 628 } 629 630 goto out; 631 } 632 633 if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { 634 if (!count_only) { 635 res = spmc_fill_partition_entry(rxtx->ffa_vers, 636 rxtx->tx, rxtx->size, 0, 637 optee_endpoint_id, 638 CFG_TEE_CORE_NB_CORE, 639 my_part_props, 640 my_uuid_words); 641 if (res) { 642 ret_fid = FFA_ERROR; 643 rc = FFA_INVALID_PARAMETERS; 644 goto out; 645 } 646 } 647 rc = 1; 648 } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { 649 uint32_t uuid_array[4] = { 0 }; 650 size_t count = 0; 651 652 uuid_array[0] = args->a1; 653 uuid_array[1] = args->a2; 654 uuid_array[2] = args->a3; 655 uuid_array[3] = args->a4; 656 657 res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, 658 rxtx->size, uuid_array, &count, 659 count_only); 660 if (res != TEE_SUCCESS) { 661 ret_fid = FFA_ERROR; 662 rc = FFA_INVALID_PARAMETERS; 663 goto out; 664 } 665 rc = count; 666 } else { 667 ret_fid = FFA_ERROR; 668 rc = FFA_INVALID_PARAMETERS; 669 goto out; 670 } 671 672 ret_fid = FFA_SUCCESS_32; 673 674 out: 675 if (ret_fid == FFA_SUCCESS_32 && !count_only && 676 rxtx->ffa_vers >= FFA_VERSION_1_1) 677 fpi_size = sizeof(struct ffa_partition_info_x) + FFA_UUID_SIZE; 678 679 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, fpi_size, 680 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 681 if (!count_only) { 682 rxtx->tx_is_mine = false; 683 cpu_spin_unlock(&rxtx->spinlock); 684 } 685 } 686 687 static void spmc_handle_run(struct thread_smc_1_2_regs *args) 688 { 689 uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); 690 uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); 691 uint32_t rc = FFA_INVALID_PARAMETERS; 692 693 /* 694 * OP-TEE core threads are only preemted using controlled exit so 695 * FFA_RUN mustn't be used to resume such threads. 696 */ 697 if (endpoint == optee_endpoint_id) 698 goto out; 699 700 /* 701 * The endpoint should be a S-EL0 SP, try to resume the SP from 702 * preempted into busy state. 703 */ 704 rc = spmc_sp_resume_from_preempted(endpoint); 705 if (rc) 706 goto out; 707 thread_resume_from_rpc(thread_id, 0, 0, 0, 0); 708 /* 709 * thread_resume_from_rpc() only returns if the thread_id 710 * is invalid. 711 */ 712 rc = FFA_INVALID_PARAMETERS; 713 714 out: 715 set_simple_ret_val(args, rc); 716 } 717 #endif /*CFG_CORE_SEL1_SPMC*/ 718 719 static struct notif_vm_bitmap *get_notif_vm_bitmap(struct guest_partition *prtn, 720 uint16_t vm_id) 721 { 722 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 723 if (!prtn) 724 return NULL; 725 assert(vm_id == virt_get_guest_id(prtn)); 726 return virt_get_guest_spec_data(prtn, notif_vm_bitmap_id); 727 } 728 if (vm_id) 729 return NULL; 730 return &default_notif_vm_bitmap; 731 } 732 733 static uint32_t spmc_enable_async_notif(uint32_t bottom_half_value, 734 uint16_t vm_id) 735 { 736 struct guest_partition *prtn = NULL; 737 struct notif_vm_bitmap *nvb = NULL; 738 uint32_t old_itr_status = 0; 739 uint32_t res = 0; 740 741 if (!spmc_notif_is_ready) { 742 /* 743 * This should never happen, not if normal world respects the 744 * exchanged capabilities. 745 */ 746 EMSG("Asynchronous notifications are not ready"); 747 return TEE_ERROR_NOT_IMPLEMENTED; 748 } 749 750 if (bottom_half_value >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE) { 751 EMSG("Invalid bottom half value %"PRIu32, bottom_half_value); 752 return TEE_ERROR_BAD_PARAMETERS; 753 } 754 755 prtn = virt_get_guest(vm_id); 756 nvb = get_notif_vm_bitmap(prtn, vm_id); 757 if (!nvb) { 758 res = TEE_ERROR_BAD_PARAMETERS; 759 goto out; 760 } 761 762 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 763 nvb->do_bottom_half_value = bottom_half_value; 764 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 765 766 notif_deliver_atomic_event(NOTIF_EVENT_STARTED, vm_id); 767 res = TEE_SUCCESS; 768 out: 769 virt_put_guest(prtn); 770 return res; 771 } 772 773 static void handle_yielding_call(struct thread_smc_1_2_regs *args, 774 uint32_t direct_resp_fid) 775 { 776 TEE_Result res = 0; 777 778 thread_check_canaries(); 779 780 #ifdef ARM64 781 /* Saving this for an eventual RPC */ 782 thread_get_core_local()->direct_resp_fid = direct_resp_fid; 783 #endif 784 785 if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { 786 /* Note connection to struct thread_rpc_arg::ret */ 787 thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, 788 0); 789 res = TEE_ERROR_BAD_PARAMETERS; 790 } else { 791 thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, 792 args->a6, args->a7); 793 res = TEE_ERROR_BUSY; 794 } 795 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 796 0, res, 0, 0); 797 } 798 799 static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) 800 { 801 uint64_t cookie = reg_pair_to_64(a5, a4); 802 uint32_t res = 0; 803 804 res = mobj_ffa_unregister_by_cookie(cookie); 805 switch (res) { 806 case TEE_SUCCESS: 807 case TEE_ERROR_ITEM_NOT_FOUND: 808 return 0; 809 case TEE_ERROR_BUSY: 810 EMSG("res %#"PRIx32, res); 811 return FFA_BUSY; 812 default: 813 EMSG("res %#"PRIx32, res); 814 return FFA_INVALID_PARAMETERS; 815 } 816 } 817 818 static void handle_blocking_call(struct thread_smc_1_2_regs *args, 819 uint32_t direct_resp_fid) 820 { 821 uint32_t sec_caps = 0; 822 823 switch (args->a3) { 824 case OPTEE_FFA_GET_API_VERSION: 825 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 826 OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, 827 0); 828 break; 829 case OPTEE_FFA_GET_OS_VERSION: 830 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 831 CFG_OPTEE_REVISION_MAJOR, 832 CFG_OPTEE_REVISION_MINOR, 833 TEE_IMPL_GIT_SHA1 >> 32); 834 break; 835 case OPTEE_FFA_EXCHANGE_CAPABILITIES: 836 sec_caps = OPTEE_FFA_SEC_CAP_ARG_OFFSET; 837 if (spmc_notif_is_ready) 838 sec_caps |= OPTEE_FFA_SEC_CAP_ASYNC_NOTIF; 839 if (IS_ENABLED(CFG_RPMB_ANNOUNCE_PROBE_CAP)) 840 sec_caps |= OPTEE_FFA_SEC_CAP_RPMB_PROBE; 841 spmc_set_args(args, direct_resp_fid, 842 swap_src_dst(args->a1), 0, 0, 843 THREAD_RPC_MAX_NUM_PARAMS, sec_caps); 844 break; 845 case OPTEE_FFA_UNREGISTER_SHM: 846 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 847 handle_unregister_shm(args->a4, args->a5), 0, 0); 848 break; 849 case OPTEE_FFA_ENABLE_ASYNC_NOTIF: 850 spmc_set_args(args, direct_resp_fid, 851 swap_src_dst(args->a1), 0, 852 spmc_enable_async_notif(args->a4, 853 FFA_SRC(args->a1)), 854 0, 0); 855 break; 856 default: 857 EMSG("Unhandled blocking service ID %#"PRIx32, 858 (uint32_t)args->a3); 859 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 860 TEE_ERROR_BAD_PARAMETERS, 0, 0); 861 } 862 } 863 864 static void handle_framework_direct_request(struct thread_smc_1_2_regs *args, 865 struct ffa_rxtx *rxtx, 866 uint32_t direct_resp_fid) 867 { 868 uint32_t w0 = FFA_ERROR; 869 uint32_t w1 = FFA_PARAM_MBZ; 870 uint32_t w2 = FFA_NOT_SUPPORTED; 871 uint32_t w3 = FFA_PARAM_MBZ; 872 873 switch (args->a2 & FFA_MSG_TYPE_MASK) { 874 case FFA_MSG_SEND_VM_CREATED: 875 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 876 uint16_t guest_id = args->a5; 877 TEE_Result res = virt_guest_created(guest_id); 878 879 w0 = direct_resp_fid; 880 w1 = swap_src_dst(args->a1); 881 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; 882 if (res == TEE_SUCCESS) 883 w3 = FFA_OK; 884 else if (res == TEE_ERROR_OUT_OF_MEMORY) 885 w3 = FFA_DENIED; 886 else 887 w3 = FFA_INVALID_PARAMETERS; 888 } 889 break; 890 case FFA_MSG_SEND_VM_DESTROYED: 891 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 892 uint16_t guest_id = args->a5; 893 TEE_Result res = virt_guest_destroyed(guest_id); 894 895 w0 = direct_resp_fid; 896 w1 = swap_src_dst(args->a1); 897 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_DESTROYED; 898 if (res == TEE_SUCCESS) 899 w3 = FFA_OK; 900 else 901 w3 = FFA_INVALID_PARAMETERS; 902 } 903 break; 904 case FFA_MSG_VERSION_REQ: 905 w0 = direct_resp_fid; 906 w1 = swap_src_dst(args->a1); 907 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; 908 w3 = spmc_exchange_version(args->a3, rxtx); 909 break; 910 default: 911 break; 912 } 913 spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 914 } 915 916 static void handle_direct_request(struct thread_smc_1_2_regs *args, 917 struct ffa_rxtx *rxtx) 918 { 919 uint32_t direct_resp_fid = 0; 920 921 if (IS_ENABLED(CFG_SECURE_PARTITION) && 922 FFA_DST(args->a1) != spmc_id && 923 FFA_DST(args->a1) != optee_endpoint_id) { 924 spmc_sp_start_thread(args); 925 return; 926 } 927 928 if (OPTEE_SMC_IS_64(args->a0)) 929 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_64; 930 else 931 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_32; 932 933 if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { 934 handle_framework_direct_request(args, rxtx, direct_resp_fid); 935 return; 936 } 937 938 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 939 virt_set_guest(get_sender_id(args->a1))) { 940 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 941 TEE_ERROR_ITEM_NOT_FOUND, 0, 0); 942 return; 943 } 944 945 if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) 946 handle_yielding_call(args, direct_resp_fid); 947 else 948 handle_blocking_call(args, direct_resp_fid); 949 950 /* 951 * Note that handle_yielding_call() typically only returns if a 952 * thread cannot be allocated or found. virt_unset_guest() is also 953 * called from thread_state_suspend() and thread_state_free(). 954 */ 955 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 956 virt_unset_guest(); 957 } 958 959 int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, 960 struct ffa_mem_transaction_x *trans) 961 { 962 uint16_t mem_reg_attr = 0; 963 uint32_t flags = 0; 964 uint32_t count = 0; 965 uint32_t offs = 0; 966 uint32_t size = 0; 967 size_t n = 0; 968 969 if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) 970 return FFA_INVALID_PARAMETERS; 971 972 if (ffa_vers >= FFA_VERSION_1_1) { 973 struct ffa_mem_transaction_1_1 *descr = NULL; 974 975 if (blen < sizeof(*descr)) 976 return FFA_INVALID_PARAMETERS; 977 978 descr = buf; 979 trans->sender_id = READ_ONCE(descr->sender_id); 980 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 981 flags = READ_ONCE(descr->flags); 982 trans->global_handle = READ_ONCE(descr->global_handle); 983 trans->tag = READ_ONCE(descr->tag); 984 985 count = READ_ONCE(descr->mem_access_count); 986 size = READ_ONCE(descr->mem_access_size); 987 offs = READ_ONCE(descr->mem_access_offs); 988 } else { 989 struct ffa_mem_transaction_1_0 *descr = NULL; 990 991 if (blen < sizeof(*descr)) 992 return FFA_INVALID_PARAMETERS; 993 994 descr = buf; 995 trans->sender_id = READ_ONCE(descr->sender_id); 996 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 997 flags = READ_ONCE(descr->flags); 998 trans->global_handle = READ_ONCE(descr->global_handle); 999 trans->tag = READ_ONCE(descr->tag); 1000 1001 count = READ_ONCE(descr->mem_access_count); 1002 size = sizeof(struct ffa_mem_access); 1003 offs = offsetof(struct ffa_mem_transaction_1_0, 1004 mem_access_array); 1005 } 1006 1007 if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || 1008 size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) 1009 return FFA_INVALID_PARAMETERS; 1010 1011 /* Check that the endpoint memory access descriptor array fits */ 1012 if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || 1013 n > blen) 1014 return FFA_INVALID_PARAMETERS; 1015 1016 trans->mem_reg_attr = mem_reg_attr; 1017 trans->flags = flags; 1018 trans->mem_access_size = size; 1019 trans->mem_access_count = count; 1020 trans->mem_access_offs = offs; 1021 return 0; 1022 } 1023 1024 #if defined(CFG_CORE_SEL1_SPMC) 1025 static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, 1026 unsigned int mem_access_count, uint8_t *acc_perms, 1027 unsigned int *region_offs) 1028 { 1029 struct ffa_mem_access_perm *descr = NULL; 1030 struct ffa_mem_access *mem_acc = NULL; 1031 unsigned int n = 0; 1032 1033 for (n = 0; n < mem_access_count; n++) { 1034 mem_acc = (void *)(mem_acc_base + mem_access_size * n); 1035 descr = &mem_acc->access_perm; 1036 if (READ_ONCE(descr->endpoint_id) == optee_endpoint_id) { 1037 *acc_perms = READ_ONCE(descr->perm); 1038 *region_offs = READ_ONCE(mem_acc[n].region_offs); 1039 return 0; 1040 } 1041 } 1042 1043 return FFA_INVALID_PARAMETERS; 1044 } 1045 1046 static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, 1047 size_t blen, unsigned int *page_count, 1048 unsigned int *region_count, size_t *addr_range_offs) 1049 { 1050 const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 1051 const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; 1052 struct ffa_mem_region *region_descr = NULL; 1053 unsigned int region_descr_offs = 0; 1054 uint8_t mem_acc_perm = 0; 1055 size_t n = 0; 1056 1057 if (mem_trans->mem_reg_attr != exp_mem_reg_attr) 1058 return FFA_INVALID_PARAMETERS; 1059 1060 /* Check that the access permissions matches what's expected */ 1061 if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, 1062 mem_trans->mem_access_size, 1063 mem_trans->mem_access_count, 1064 &mem_acc_perm, ®ion_descr_offs) || 1065 mem_acc_perm != exp_mem_acc_perm) 1066 return FFA_INVALID_PARAMETERS; 1067 1068 /* Check that the Composite memory region descriptor fits */ 1069 if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || 1070 n > blen) 1071 return FFA_INVALID_PARAMETERS; 1072 1073 if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, 1074 struct ffa_mem_region)) 1075 return FFA_INVALID_PARAMETERS; 1076 1077 region_descr = (struct ffa_mem_region *)((vaddr_t)buf + 1078 region_descr_offs); 1079 *page_count = READ_ONCE(region_descr->total_page_count); 1080 *region_count = READ_ONCE(region_descr->address_range_count); 1081 *addr_range_offs = n; 1082 return 0; 1083 } 1084 1085 static int add_mem_share_helper(struct mem_share_state *s, void *buf, 1086 size_t flen) 1087 { 1088 unsigned int region_count = flen / sizeof(struct ffa_address_range); 1089 struct ffa_address_range *arange = NULL; 1090 unsigned int n = 0; 1091 1092 if (region_count > s->region_count) 1093 region_count = s->region_count; 1094 1095 if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) 1096 return FFA_INVALID_PARAMETERS; 1097 arange = buf; 1098 1099 for (n = 0; n < region_count; n++) { 1100 unsigned int page_count = READ_ONCE(arange[n].page_count); 1101 uint64_t addr = READ_ONCE(arange[n].address); 1102 1103 if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, 1104 addr, page_count)) 1105 return FFA_INVALID_PARAMETERS; 1106 } 1107 1108 s->region_count -= region_count; 1109 if (s->region_count) 1110 return region_count * sizeof(*arange); 1111 1112 if (s->current_page_idx != s->page_count) 1113 return FFA_INVALID_PARAMETERS; 1114 1115 return 0; 1116 } 1117 1118 static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) 1119 { 1120 int rc = 0; 1121 1122 rc = add_mem_share_helper(&s->share, buf, flen); 1123 if (rc >= 0) { 1124 if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { 1125 /* We're not at the end of the descriptor yet */ 1126 if (s->share.region_count) 1127 return s->frag_offset; 1128 1129 /* We're done */ 1130 rc = 0; 1131 } else { 1132 rc = FFA_INVALID_PARAMETERS; 1133 } 1134 } 1135 1136 SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); 1137 if (rc < 0) 1138 mobj_ffa_sel1_spmc_delete(s->share.mf); 1139 else 1140 mobj_ffa_push_to_inactive(s->share.mf); 1141 free(s); 1142 1143 return rc; 1144 } 1145 1146 static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, 1147 void *buf) 1148 { 1149 struct ffa_mem_access_perm *perm = NULL; 1150 struct ffa_mem_access *mem_acc = NULL; 1151 1152 if (!IS_ENABLED(CFG_SECURE_PARTITION)) 1153 return false; 1154 1155 if (mem_trans->mem_access_count < 1) 1156 return false; 1157 1158 mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); 1159 perm = &mem_acc->access_perm; 1160 1161 /* 1162 * perm->endpoint_id is read here only to check if the endpoint is 1163 * OP-TEE. We do read it later on again, but there are some additional 1164 * checks there to make sure that the data is correct. 1165 */ 1166 return READ_ONCE(perm->endpoint_id) != optee_endpoint_id; 1167 } 1168 1169 static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, 1170 tee_mm_entry_t *mm, void *buf, size_t blen, 1171 size_t flen, uint64_t *global_handle) 1172 { 1173 int rc = 0; 1174 struct mem_share_state share = { }; 1175 size_t addr_range_offs = 0; 1176 uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1177 size_t n = 0; 1178 1179 rc = mem_share_init(mem_trans, buf, flen, &share.page_count, 1180 &share.region_count, &addr_range_offs); 1181 if (rc) 1182 return rc; 1183 1184 if (!share.page_count || !share.region_count) 1185 return FFA_INVALID_PARAMETERS; 1186 1187 if (MUL_OVERFLOW(share.region_count, 1188 sizeof(struct ffa_address_range), &n) || 1189 ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) 1190 return FFA_INVALID_PARAMETERS; 1191 1192 if (mem_trans->global_handle) 1193 cookie = mem_trans->global_handle; 1194 share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); 1195 if (!share.mf) 1196 return FFA_NO_MEMORY; 1197 1198 if (flen != blen) { 1199 struct mem_frag_state *s = calloc(1, sizeof(*s)); 1200 1201 if (!s) { 1202 rc = FFA_NO_MEMORY; 1203 goto err; 1204 } 1205 s->share = share; 1206 s->mm = mm; 1207 s->frag_offset = addr_range_offs; 1208 1209 SLIST_INSERT_HEAD(&frag_state_head, s, link); 1210 rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, 1211 flen - addr_range_offs); 1212 1213 if (rc >= 0) 1214 *global_handle = mobj_ffa_get_cookie(share.mf); 1215 1216 return rc; 1217 } 1218 1219 rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, 1220 flen - addr_range_offs); 1221 if (rc) { 1222 /* 1223 * Number of consumed bytes may be returned instead of 0 for 1224 * done. 1225 */ 1226 rc = FFA_INVALID_PARAMETERS; 1227 goto err; 1228 } 1229 1230 *global_handle = mobj_ffa_push_to_inactive(share.mf); 1231 1232 return 0; 1233 err: 1234 mobj_ffa_sel1_spmc_delete(share.mf); 1235 return rc; 1236 } 1237 1238 static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, 1239 unsigned int page_count, 1240 uint64_t *global_handle, struct ffa_rxtx *rxtx) 1241 { 1242 struct ffa_mem_transaction_x mem_trans = { }; 1243 int rc = 0; 1244 size_t len = 0; 1245 void *buf = NULL; 1246 tee_mm_entry_t *mm = NULL; 1247 vaddr_t offs = pbuf & SMALL_PAGE_MASK; 1248 1249 if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) 1250 return FFA_INVALID_PARAMETERS; 1251 if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) 1252 return FFA_INVALID_PARAMETERS; 1253 1254 /* 1255 * Check that the length reported in flen is covered by len even 1256 * if the offset is taken into account. 1257 */ 1258 if (len < flen || len - offs < flen) 1259 return FFA_INVALID_PARAMETERS; 1260 1261 mm = tee_mm_alloc(&core_virt_shm_pool, len); 1262 if (!mm) 1263 return FFA_NO_MEMORY; 1264 1265 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, 1266 page_count, MEM_AREA_NSEC_SHM)) { 1267 rc = FFA_INVALID_PARAMETERS; 1268 goto out; 1269 } 1270 buf = (void *)(tee_mm_get_smem(mm) + offs); 1271 1272 cpu_spin_lock(&rxtx->spinlock); 1273 rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); 1274 if (rc) 1275 goto unlock; 1276 1277 if (is_sp_share(&mem_trans, buf)) { 1278 rc = spmc_sp_add_share(&mem_trans, buf, blen, flen, 1279 global_handle, NULL); 1280 goto unlock; 1281 } 1282 1283 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1284 virt_set_guest(mem_trans.sender_id)) { 1285 rc = FFA_DENIED; 1286 goto unlock; 1287 } 1288 1289 rc = add_mem_share(&mem_trans, mm, buf, blen, flen, global_handle); 1290 1291 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1292 virt_unset_guest(); 1293 1294 unlock: 1295 cpu_spin_unlock(&rxtx->spinlock); 1296 if (rc > 0) 1297 return rc; 1298 1299 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1300 out: 1301 tee_mm_free(mm); 1302 return rc; 1303 } 1304 1305 static int handle_mem_share_rxbuf(size_t blen, size_t flen, 1306 uint64_t *global_handle, 1307 struct ffa_rxtx *rxtx) 1308 { 1309 struct ffa_mem_transaction_x mem_trans = { }; 1310 int rc = FFA_DENIED; 1311 1312 cpu_spin_lock(&rxtx->spinlock); 1313 1314 if (!rxtx->rx || flen > rxtx->size) 1315 goto out; 1316 1317 rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, 1318 &mem_trans); 1319 if (rc) 1320 goto out; 1321 if (is_sp_share(&mem_trans, rxtx->rx)) { 1322 rc = spmc_sp_add_share(&mem_trans, rxtx, blen, flen, 1323 global_handle, NULL); 1324 goto out; 1325 } 1326 1327 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1328 virt_set_guest(mem_trans.sender_id)) 1329 goto out; 1330 1331 rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, 1332 global_handle); 1333 1334 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1335 virt_unset_guest(); 1336 1337 out: 1338 cpu_spin_unlock(&rxtx->spinlock); 1339 1340 return rc; 1341 } 1342 1343 static void handle_mem_share(struct thread_smc_1_2_regs *args, 1344 struct ffa_rxtx *rxtx) 1345 { 1346 uint32_t tot_len = args->a1; 1347 uint32_t frag_len = args->a2; 1348 uint64_t addr = args->a3; 1349 uint32_t page_count = args->a4; 1350 uint32_t ret_w1 = 0; 1351 uint32_t ret_w2 = FFA_INVALID_PARAMETERS; 1352 uint32_t ret_w3 = 0; 1353 uint32_t ret_fid = FFA_ERROR; 1354 uint64_t global_handle = 0; 1355 int rc = 0; 1356 1357 /* Check that the MBZs are indeed 0 */ 1358 if (args->a5 || args->a6 || args->a7) 1359 goto out; 1360 1361 /* Check that fragment length doesn't exceed total length */ 1362 if (frag_len > tot_len) 1363 goto out; 1364 1365 /* Check for 32-bit calling convention */ 1366 if (args->a0 == FFA_MEM_SHARE_32) 1367 addr &= UINT32_MAX; 1368 1369 if (!addr) { 1370 /* 1371 * The memory transaction descriptor is passed via our rx 1372 * buffer. 1373 */ 1374 if (page_count) 1375 goto out; 1376 rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, 1377 rxtx); 1378 } else { 1379 rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, 1380 &global_handle, rxtx); 1381 } 1382 if (rc < 0) { 1383 ret_w2 = rc; 1384 } else if (rc > 0) { 1385 ret_fid = FFA_MEM_FRAG_RX; 1386 ret_w3 = rc; 1387 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1388 } else { 1389 ret_fid = FFA_SUCCESS_32; 1390 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1391 } 1392 out: 1393 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1394 } 1395 1396 static struct mem_frag_state *get_frag_state(uint64_t global_handle) 1397 { 1398 struct mem_frag_state *s = NULL; 1399 1400 SLIST_FOREACH(s, &frag_state_head, link) 1401 if (mobj_ffa_get_cookie(s->share.mf) == global_handle) 1402 return s; 1403 1404 return NULL; 1405 } 1406 1407 static void handle_mem_frag_tx(struct thread_smc_1_2_regs *args, 1408 struct ffa_rxtx *rxtx) 1409 { 1410 uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); 1411 size_t flen = args->a3; 1412 uint32_t endpoint_id = args->a4; 1413 struct mem_frag_state *s = NULL; 1414 tee_mm_entry_t *mm = NULL; 1415 unsigned int page_count = 0; 1416 void *buf = NULL; 1417 uint32_t ret_w1 = 0; 1418 uint32_t ret_w2 = 0; 1419 uint32_t ret_w3 = 0; 1420 uint32_t ret_fid = 0; 1421 int rc = 0; 1422 1423 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1424 uint16_t guest_id = endpoint_id >> 16; 1425 1426 if (!guest_id || virt_set_guest(guest_id)) { 1427 rc = FFA_INVALID_PARAMETERS; 1428 goto out_set_rc; 1429 } 1430 } 1431 1432 /* 1433 * Currently we're only doing this for fragmented FFA_MEM_SHARE_* 1434 * requests. 1435 */ 1436 1437 cpu_spin_lock(&rxtx->spinlock); 1438 1439 s = get_frag_state(global_handle); 1440 if (!s) { 1441 rc = FFA_INVALID_PARAMETERS; 1442 goto out; 1443 } 1444 1445 mm = s->mm; 1446 if (mm) { 1447 if (flen > tee_mm_get_bytes(mm)) { 1448 rc = FFA_INVALID_PARAMETERS; 1449 goto out; 1450 } 1451 page_count = s->share.page_count; 1452 buf = (void *)tee_mm_get_smem(mm); 1453 } else { 1454 if (flen > rxtx->size) { 1455 rc = FFA_INVALID_PARAMETERS; 1456 goto out; 1457 } 1458 buf = rxtx->rx; 1459 } 1460 1461 rc = add_mem_share_frag(s, buf, flen); 1462 out: 1463 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1464 virt_unset_guest(); 1465 1466 cpu_spin_unlock(&rxtx->spinlock); 1467 1468 if (rc <= 0 && mm) { 1469 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1470 tee_mm_free(mm); 1471 } 1472 1473 out_set_rc: 1474 if (rc < 0) { 1475 ret_fid = FFA_ERROR; 1476 ret_w2 = rc; 1477 } else if (rc > 0) { 1478 ret_fid = FFA_MEM_FRAG_RX; 1479 ret_w3 = rc; 1480 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1481 } else { 1482 ret_fid = FFA_SUCCESS_32; 1483 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1484 } 1485 1486 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1487 } 1488 1489 static void handle_mem_reclaim(struct thread_smc_1_2_regs *args) 1490 { 1491 int rc = FFA_INVALID_PARAMETERS; 1492 uint64_t cookie = 0; 1493 1494 if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) 1495 goto out; 1496 1497 cookie = reg_pair_to_64(args->a2, args->a1); 1498 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1499 uint16_t guest_id = 0; 1500 1501 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 1502 guest_id = virt_find_guest_by_cookie(cookie); 1503 } else { 1504 guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & 1505 FFA_MEMORY_HANDLE_PRTN_MASK; 1506 } 1507 if (!guest_id) 1508 goto out; 1509 if (virt_set_guest(guest_id)) { 1510 if (!virt_reclaim_cookie_from_destroyed_guest(guest_id, 1511 cookie)) 1512 rc = FFA_OK; 1513 goto out; 1514 } 1515 } 1516 1517 switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { 1518 case TEE_SUCCESS: 1519 rc = FFA_OK; 1520 break; 1521 case TEE_ERROR_ITEM_NOT_FOUND: 1522 DMSG("cookie %#"PRIx64" not found", cookie); 1523 rc = FFA_INVALID_PARAMETERS; 1524 break; 1525 default: 1526 DMSG("cookie %#"PRIx64" busy", cookie); 1527 rc = FFA_DENIED; 1528 break; 1529 } 1530 1531 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1532 virt_unset_guest(); 1533 1534 out: 1535 set_simple_ret_val(args, rc); 1536 } 1537 1538 static void handle_notification_bitmap_create(struct thread_smc_1_2_regs *args) 1539 { 1540 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1541 uint32_t ret_fid = FFA_ERROR; 1542 uint32_t old_itr_status = 0; 1543 1544 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1545 !args->a5 && !args->a6 && !args->a7) { 1546 struct guest_partition *prtn = NULL; 1547 struct notif_vm_bitmap *nvb = NULL; 1548 uint16_t vm_id = args->a1; 1549 1550 prtn = virt_get_guest(vm_id); 1551 nvb = get_notif_vm_bitmap(prtn, vm_id); 1552 if (!nvb) { 1553 ret_val = FFA_INVALID_PARAMETERS; 1554 goto out_virt_put; 1555 } 1556 1557 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1558 1559 if (nvb->initialized) { 1560 ret_val = FFA_DENIED; 1561 goto out_unlock; 1562 } 1563 1564 nvb->initialized = true; 1565 nvb->do_bottom_half_value = -1; 1566 ret_val = FFA_OK; 1567 ret_fid = FFA_SUCCESS_32; 1568 out_unlock: 1569 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1570 out_virt_put: 1571 virt_put_guest(prtn); 1572 } 1573 1574 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1575 } 1576 1577 static void handle_notification_bitmap_destroy(struct thread_smc_1_2_regs *args) 1578 { 1579 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1580 uint32_t ret_fid = FFA_ERROR; 1581 uint32_t old_itr_status = 0; 1582 1583 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1584 !args->a5 && !args->a6 && !args->a7) { 1585 struct guest_partition *prtn = NULL; 1586 struct notif_vm_bitmap *nvb = NULL; 1587 uint16_t vm_id = args->a1; 1588 1589 prtn = virt_get_guest(vm_id); 1590 nvb = get_notif_vm_bitmap(prtn, vm_id); 1591 if (!nvb) { 1592 ret_val = FFA_INVALID_PARAMETERS; 1593 goto out_virt_put; 1594 } 1595 1596 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1597 1598 if (nvb->pending || nvb->bound) { 1599 ret_val = FFA_DENIED; 1600 goto out_unlock; 1601 } 1602 1603 memset(nvb, 0, sizeof(*nvb)); 1604 ret_val = FFA_OK; 1605 ret_fid = FFA_SUCCESS_32; 1606 out_unlock: 1607 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1608 out_virt_put: 1609 virt_put_guest(prtn); 1610 } 1611 1612 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1613 } 1614 1615 static void handle_notification_bind(struct thread_smc_1_2_regs *args) 1616 { 1617 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1618 struct guest_partition *prtn = NULL; 1619 struct notif_vm_bitmap *nvb = NULL; 1620 uint32_t ret_fid = FFA_ERROR; 1621 uint32_t old_itr_status = 0; 1622 uint64_t bitmap = 0; 1623 uint16_t vm_id = 0; 1624 1625 if (args->a5 || args->a6 || args->a7) 1626 goto out; 1627 if (args->a2) { 1628 /* We only deal with global notifications */ 1629 ret_val = FFA_DENIED; 1630 goto out; 1631 } 1632 1633 /* The destination of the eventual notification */ 1634 vm_id = FFA_DST(args->a1); 1635 bitmap = reg_pair_to_64(args->a4, args->a3); 1636 1637 prtn = virt_get_guest(vm_id); 1638 nvb = get_notif_vm_bitmap(prtn, vm_id); 1639 if (!nvb) { 1640 ret_val = FFA_INVALID_PARAMETERS; 1641 goto out_virt_put; 1642 } 1643 1644 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1645 1646 if ((bitmap & nvb->bound)) { 1647 ret_val = FFA_DENIED; 1648 } else { 1649 nvb->bound |= bitmap; 1650 ret_val = FFA_OK; 1651 ret_fid = FFA_SUCCESS_32; 1652 } 1653 1654 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1655 out_virt_put: 1656 virt_put_guest(prtn); 1657 out: 1658 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1659 } 1660 1661 static void handle_notification_unbind(struct thread_smc_1_2_regs *args) 1662 { 1663 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1664 struct guest_partition *prtn = NULL; 1665 struct notif_vm_bitmap *nvb = NULL; 1666 uint32_t ret_fid = FFA_ERROR; 1667 uint32_t old_itr_status = 0; 1668 uint64_t bitmap = 0; 1669 uint16_t vm_id = 0; 1670 1671 if (args->a2 || args->a5 || args->a6 || args->a7) 1672 goto out; 1673 1674 /* The destination of the eventual notification */ 1675 vm_id = FFA_DST(args->a1); 1676 bitmap = reg_pair_to_64(args->a4, args->a3); 1677 1678 prtn = virt_get_guest(vm_id); 1679 nvb = get_notif_vm_bitmap(prtn, vm_id); 1680 if (!nvb) { 1681 ret_val = FFA_INVALID_PARAMETERS; 1682 goto out_virt_put; 1683 } 1684 1685 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1686 1687 if (bitmap & nvb->pending) { 1688 ret_val = FFA_DENIED; 1689 } else { 1690 nvb->bound &= ~bitmap; 1691 ret_val = FFA_OK; 1692 ret_fid = FFA_SUCCESS_32; 1693 } 1694 1695 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1696 out_virt_put: 1697 virt_put_guest(prtn); 1698 out: 1699 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1700 } 1701 1702 static void handle_notification_get(struct thread_smc_1_2_regs *args) 1703 { 1704 uint32_t w2 = FFA_INVALID_PARAMETERS; 1705 struct guest_partition *prtn = NULL; 1706 struct notif_vm_bitmap *nvb = NULL; 1707 uint32_t ret_fid = FFA_ERROR; 1708 uint32_t old_itr_status = 0; 1709 uint16_t vm_id = 0; 1710 uint32_t w3 = 0; 1711 1712 if (args->a5 || args->a6 || args->a7) 1713 goto out; 1714 if (!(args->a2 & 0x1)) { 1715 ret_fid = FFA_SUCCESS_32; 1716 w2 = 0; 1717 goto out; 1718 } 1719 vm_id = FFA_DST(args->a1); 1720 1721 prtn = virt_get_guest(vm_id); 1722 nvb = get_notif_vm_bitmap(prtn, vm_id); 1723 if (!nvb) 1724 goto out_virt_put; 1725 1726 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1727 1728 reg_pair_from_64(nvb->pending, &w3, &w2); 1729 nvb->pending = 0; 1730 ret_fid = FFA_SUCCESS_32; 1731 1732 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1733 out_virt_put: 1734 virt_put_guest(prtn); 1735 out: 1736 spmc_set_args(args, ret_fid, 0, w2, w3, 0, 0); 1737 } 1738 1739 struct notif_info_get_state { 1740 struct thread_smc_1_2_regs *args; 1741 unsigned int ids_per_reg; 1742 unsigned int ids_count; 1743 unsigned int id_pos; 1744 unsigned int count; 1745 unsigned int max_list_count; 1746 unsigned int list_count; 1747 }; 1748 1749 static bool add_id_in_regs(struct notif_info_get_state *state, 1750 uint16_t id) 1751 { 1752 unsigned int reg_idx = state->id_pos / state->ids_per_reg + 3; 1753 unsigned int reg_shift = (state->id_pos % state->ids_per_reg) * 16; 1754 1755 if (reg_idx > 7) 1756 return false; 1757 1758 state->args->a[reg_idx] &= ~SHIFT_U64(0xffff, reg_shift); 1759 state->args->a[reg_idx] |= (unsigned long)id << reg_shift; 1760 1761 state->id_pos++; 1762 state->count++; 1763 return true; 1764 } 1765 1766 static bool add_id_count(struct notif_info_get_state *state) 1767 { 1768 assert(state->list_count < state->max_list_count && 1769 state->count >= 1 && state->count <= 4); 1770 1771 state->ids_count |= (state->count - 1) << (state->list_count * 2 + 12); 1772 state->list_count++; 1773 state->count = 0; 1774 1775 return state->list_count < state->max_list_count; 1776 } 1777 1778 static bool add_nvb_to_state(struct notif_info_get_state *state, 1779 uint16_t guest_id, struct notif_vm_bitmap *nvb) 1780 { 1781 if (!nvb->pending) 1782 return true; 1783 /* 1784 * Add only the guest_id, meaning a global notification for this 1785 * guest. 1786 * 1787 * If notifications for one or more specific vCPUs we'd add those 1788 * before calling add_id_count(), but that's not supported. 1789 */ 1790 return add_id_in_regs(state, guest_id) && add_id_count(state); 1791 } 1792 1793 static void handle_notification_info_get(struct thread_smc_1_2_regs *args) 1794 { 1795 struct notif_info_get_state state = { .args = args }; 1796 uint32_t ffa_res = FFA_INVALID_PARAMETERS; 1797 struct guest_partition *prtn = NULL; 1798 struct notif_vm_bitmap *nvb = NULL; 1799 uint32_t more_pending_flag = 0; 1800 uint32_t itr_state = 0; 1801 uint16_t guest_id = 0; 1802 1803 if (args->a1 || args->a2 || args->a3 || args->a4 || args->a5 || 1804 args->a6 || args->a7) 1805 goto err; 1806 1807 if (OPTEE_SMC_IS_64(args->a0)) { 1808 spmc_set_args(args, FFA_SUCCESS_64, 0, 0, 0, 0, 0); 1809 state.ids_per_reg = 4; 1810 state.max_list_count = 31; 1811 } else { 1812 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 1813 state.ids_per_reg = 2; 1814 state.max_list_count = 15; 1815 } 1816 1817 while (true) { 1818 /* 1819 * With NS-Virtualization we need to go through all 1820 * partitions to collect the notification bitmaps, without 1821 * we just check the only notification bitmap we have. 1822 */ 1823 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1824 prtn = virt_next_guest(prtn); 1825 if (!prtn) 1826 break; 1827 guest_id = virt_get_guest_id(prtn); 1828 } 1829 nvb = get_notif_vm_bitmap(prtn, guest_id); 1830 1831 itr_state = cpu_spin_lock_xsave(&spmc_notif_lock); 1832 if (!add_nvb_to_state(&state, guest_id, nvb)) 1833 more_pending_flag = BIT(0); 1834 cpu_spin_unlock_xrestore(&spmc_notif_lock, itr_state); 1835 1836 if (!IS_ENABLED(CFG_NS_VIRTUALIZATION) || more_pending_flag) 1837 break; 1838 } 1839 virt_put_guest(prtn); 1840 1841 if (!state.id_pos) { 1842 ffa_res = FFA_NO_DATA; 1843 goto err; 1844 } 1845 args->a2 = (state.list_count << FFA_NOTIF_INFO_GET_ID_COUNT_SHIFT) | 1846 (state.ids_count << FFA_NOTIF_INFO_GET_ID_LIST_SHIFT) | 1847 more_pending_flag; 1848 return; 1849 err: 1850 spmc_set_args(args, FFA_ERROR, 0, ffa_res, 0, 0, 0); 1851 } 1852 1853 void thread_spmc_set_async_notif_intid(int intid) 1854 { 1855 assert(interrupt_can_raise_sgi(interrupt_get_main_chip())); 1856 notif_intid = intid; 1857 spmc_notif_is_ready = true; 1858 DMSG("Asynchronous notifications are ready"); 1859 } 1860 1861 void notif_send_async(uint32_t value, uint16_t guest_id) 1862 { 1863 struct guest_partition *prtn = NULL; 1864 struct notif_vm_bitmap *nvb = NULL; 1865 uint32_t old_itr_status = 0; 1866 1867 prtn = virt_get_guest(guest_id); 1868 nvb = get_notif_vm_bitmap(prtn, guest_id); 1869 1870 if (nvb) { 1871 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1872 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && 1873 spmc_notif_is_ready && nvb->do_bottom_half_value >= 0 && 1874 notif_intid >= 0); 1875 nvb->pending |= BIT64(nvb->do_bottom_half_value); 1876 interrupt_raise_sgi(interrupt_get_main_chip(), notif_intid, 1877 ITR_CPU_MASK_TO_THIS_CPU); 1878 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1879 } 1880 1881 virt_put_guest(prtn); 1882 } 1883 #else 1884 void notif_send_async(uint32_t value, uint16_t guest_id) 1885 { 1886 struct guest_partition *prtn = NULL; 1887 struct notif_vm_bitmap *nvb = NULL; 1888 /* global notification, delay notification interrupt */ 1889 uint32_t flags = BIT32(1); 1890 int res = 0; 1891 1892 prtn = virt_get_guest(guest_id); 1893 nvb = get_notif_vm_bitmap(prtn, guest_id); 1894 1895 if (nvb) { 1896 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && 1897 spmc_notif_is_ready && nvb->do_bottom_half_value >= 0); 1898 res = ffa_set_notification(guest_id, optee_endpoint_id, flags, 1899 BIT64(nvb->do_bottom_half_value)); 1900 if (res) { 1901 EMSG("notification set failed with error %d", res); 1902 panic(); 1903 } 1904 } 1905 1906 virt_put_guest(prtn); 1907 } 1908 #endif 1909 1910 /* Only called from assembly */ 1911 void thread_spmc_msg_recv(struct thread_smc_1_2_regs *args); 1912 void thread_spmc_msg_recv(struct thread_smc_1_2_regs *args) 1913 { 1914 assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); 1915 switch (args->a0) { 1916 #if defined(CFG_CORE_SEL1_SPMC) 1917 case FFA_FEATURES: 1918 handle_features(args); 1919 break; 1920 case FFA_SPM_ID_GET: 1921 spmc_handle_spm_id_get(args); 1922 break; 1923 #ifdef ARM64 1924 case FFA_RXTX_MAP_64: 1925 #endif 1926 case FFA_RXTX_MAP_32: 1927 spmc_handle_rxtx_map(args, &my_rxtx); 1928 break; 1929 case FFA_RXTX_UNMAP: 1930 spmc_handle_rxtx_unmap(args, &my_rxtx); 1931 break; 1932 case FFA_RX_RELEASE: 1933 spmc_handle_rx_release(args, &my_rxtx); 1934 break; 1935 case FFA_PARTITION_INFO_GET: 1936 spmc_handle_partition_info_get(args, &my_rxtx); 1937 break; 1938 case FFA_RUN: 1939 spmc_handle_run(args); 1940 break; 1941 #endif /*CFG_CORE_SEL1_SPMC*/ 1942 case FFA_INTERRUPT: 1943 if (IS_ENABLED(CFG_CORE_SEL1_SPMC)) 1944 spmc_set_args(args, FFA_NORMAL_WORLD_RESUME, 0, 0, 0, 1945 0, 0); 1946 else 1947 spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); 1948 break; 1949 #ifdef ARM64 1950 case FFA_MSG_SEND_DIRECT_REQ_64: 1951 #endif 1952 case FFA_MSG_SEND_DIRECT_REQ_32: 1953 handle_direct_request(args, &my_rxtx); 1954 break; 1955 #if defined(CFG_CORE_SEL1_SPMC) 1956 #ifdef ARM64 1957 case FFA_MEM_SHARE_64: 1958 #endif 1959 case FFA_MEM_SHARE_32: 1960 handle_mem_share(args, &my_rxtx); 1961 break; 1962 case FFA_MEM_RECLAIM: 1963 if (!IS_ENABLED(CFG_SECURE_PARTITION) || 1964 !ffa_mem_reclaim(args, NULL)) 1965 handle_mem_reclaim(args); 1966 break; 1967 case FFA_MEM_FRAG_TX: 1968 handle_mem_frag_tx(args, &my_rxtx); 1969 break; 1970 case FFA_NOTIFICATION_BITMAP_CREATE: 1971 handle_notification_bitmap_create(args); 1972 break; 1973 case FFA_NOTIFICATION_BITMAP_DESTROY: 1974 handle_notification_bitmap_destroy(args); 1975 break; 1976 case FFA_NOTIFICATION_BIND: 1977 handle_notification_bind(args); 1978 break; 1979 case FFA_NOTIFICATION_UNBIND: 1980 handle_notification_unbind(args); 1981 break; 1982 case FFA_NOTIFICATION_GET: 1983 handle_notification_get(args); 1984 break; 1985 #ifdef ARM64 1986 case FFA_NOTIFICATION_INFO_GET_64: 1987 #endif 1988 case FFA_NOTIFICATION_INFO_GET_32: 1989 handle_notification_info_get(args); 1990 break; 1991 #endif /*CFG_CORE_SEL1_SPMC*/ 1992 case FFA_ERROR: 1993 EMSG("Cannot handle FFA_ERROR(%d)", (int)args->a2); 1994 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { 1995 /* 1996 * The SPMC will return an FFA_ERROR back so better 1997 * panic() now than flooding the log. 1998 */ 1999 panic("FFA_ERROR from SPMC is fatal"); 2000 } 2001 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, 2002 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 2003 break; 2004 default: 2005 EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); 2006 set_simple_ret_val(args, FFA_NOT_SUPPORTED); 2007 } 2008 } 2009 2010 static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) 2011 { 2012 size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 2013 struct thread_ctx *thr = threads + thread_get_id(); 2014 TEE_Result res = TEE_ERROR_BAD_PARAMETERS; 2015 struct optee_msg_arg *arg = NULL; 2016 struct mobj *mobj = NULL; 2017 uint32_t num_params = 0; 2018 size_t sz = 0; 2019 2020 mobj = mobj_ffa_get_by_cookie(cookie, 0); 2021 if (!mobj) { 2022 EMSG("Can't find cookie %#"PRIx64, cookie); 2023 return TEE_ERROR_BAD_PARAMETERS; 2024 } 2025 2026 res = mobj_inc_map(mobj); 2027 if (res) 2028 goto out_put_mobj; 2029 2030 res = TEE_ERROR_BAD_PARAMETERS; 2031 arg = mobj_get_va(mobj, offset, sizeof(*arg)); 2032 if (!arg) 2033 goto out_dec_map; 2034 2035 num_params = READ_ONCE(arg->num_params); 2036 if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) 2037 goto out_dec_map; 2038 2039 sz = OPTEE_MSG_GET_ARG_SIZE(num_params); 2040 2041 thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); 2042 if (!thr->rpc_arg) 2043 goto out_dec_map; 2044 2045 virt_on_stdcall(); 2046 res = tee_entry_std(arg, num_params); 2047 2048 thread_rpc_shm_cache_clear(&thr->shm_cache); 2049 thr->rpc_arg = NULL; 2050 2051 out_dec_map: 2052 mobj_dec_map(mobj); 2053 out_put_mobj: 2054 mobj_put(mobj); 2055 return res; 2056 } 2057 2058 /* 2059 * Helper routine for the assembly function thread_std_smc_entry() 2060 * 2061 * Note: this function is weak just to make link_dummies_paged.c happy. 2062 */ 2063 uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, 2064 uint32_t a2, uint32_t a3, 2065 uint32_t a4, uint32_t a5 __unused) 2066 { 2067 /* 2068 * Arguments are supplied from handle_yielding_call() as: 2069 * a0 <- w1 2070 * a1 <- w3 2071 * a2 <- w4 2072 * a3 <- w5 2073 * a4 <- w6 2074 * a5 <- w7 2075 */ 2076 thread_get_tsd()->rpc_target_info = swap_src_dst(a0); 2077 if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) 2078 return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); 2079 return FFA_DENIED; 2080 } 2081 2082 static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) 2083 { 2084 uint64_t offs = tpm->u.memref.offs; 2085 2086 param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + 2087 OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; 2088 2089 param->u.fmem.offs_low = offs; 2090 param->u.fmem.offs_high = offs >> 32; 2091 if (param->u.fmem.offs_high != offs >> 32) 2092 return false; 2093 2094 param->u.fmem.size = tpm->u.memref.size; 2095 if (tpm->u.memref.mobj) { 2096 uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); 2097 2098 /* If a mobj is passed it better be one with a valid cookie. */ 2099 if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) 2100 return false; 2101 param->u.fmem.global_id = cookie; 2102 } else { 2103 param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 2104 } 2105 2106 return true; 2107 } 2108 2109 static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, 2110 struct thread_param *params, 2111 struct optee_msg_arg **arg_ret) 2112 { 2113 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 2114 struct thread_ctx *thr = threads + thread_get_id(); 2115 struct optee_msg_arg *arg = thr->rpc_arg; 2116 2117 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 2118 return TEE_ERROR_BAD_PARAMETERS; 2119 2120 if (!arg) { 2121 EMSG("rpc_arg not set"); 2122 return TEE_ERROR_GENERIC; 2123 } 2124 2125 memset(arg, 0, sz); 2126 arg->cmd = cmd; 2127 arg->num_params = num_params; 2128 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 2129 2130 for (size_t n = 0; n < num_params; n++) { 2131 switch (params[n].attr) { 2132 case THREAD_PARAM_ATTR_NONE: 2133 arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; 2134 break; 2135 case THREAD_PARAM_ATTR_VALUE_IN: 2136 case THREAD_PARAM_ATTR_VALUE_OUT: 2137 case THREAD_PARAM_ATTR_VALUE_INOUT: 2138 arg->params[n].attr = params[n].attr - 2139 THREAD_PARAM_ATTR_VALUE_IN + 2140 OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 2141 arg->params[n].u.value.a = params[n].u.value.a; 2142 arg->params[n].u.value.b = params[n].u.value.b; 2143 arg->params[n].u.value.c = params[n].u.value.c; 2144 break; 2145 case THREAD_PARAM_ATTR_MEMREF_IN: 2146 case THREAD_PARAM_ATTR_MEMREF_OUT: 2147 case THREAD_PARAM_ATTR_MEMREF_INOUT: 2148 if (!set_fmem(arg->params + n, params + n)) 2149 return TEE_ERROR_BAD_PARAMETERS; 2150 break; 2151 default: 2152 return TEE_ERROR_BAD_PARAMETERS; 2153 } 2154 } 2155 2156 if (arg_ret) 2157 *arg_ret = arg; 2158 2159 return TEE_SUCCESS; 2160 } 2161 2162 static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, 2163 struct thread_param *params) 2164 { 2165 for (size_t n = 0; n < num_params; n++) { 2166 switch (params[n].attr) { 2167 case THREAD_PARAM_ATTR_VALUE_OUT: 2168 case THREAD_PARAM_ATTR_VALUE_INOUT: 2169 params[n].u.value.a = arg->params[n].u.value.a; 2170 params[n].u.value.b = arg->params[n].u.value.b; 2171 params[n].u.value.c = arg->params[n].u.value.c; 2172 break; 2173 case THREAD_PARAM_ATTR_MEMREF_OUT: 2174 case THREAD_PARAM_ATTR_MEMREF_INOUT: 2175 params[n].u.memref.size = arg->params[n].u.fmem.size; 2176 break; 2177 default: 2178 break; 2179 } 2180 } 2181 2182 return arg->ret; 2183 } 2184 2185 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 2186 struct thread_param *params) 2187 { 2188 struct thread_rpc_arg rpc_arg = { .call = { 2189 .w1 = thread_get_tsd()->rpc_target_info, 2190 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2191 }, 2192 }; 2193 struct optee_msg_arg *arg = NULL; 2194 uint32_t ret = 0; 2195 2196 ret = get_rpc_arg(cmd, num_params, params, &arg); 2197 if (ret) 2198 return ret; 2199 2200 thread_rpc(&rpc_arg); 2201 2202 return get_rpc_arg_res(arg, num_params, params); 2203 } 2204 2205 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 2206 { 2207 struct thread_rpc_arg rpc_arg = { .call = { 2208 .w1 = thread_get_tsd()->rpc_target_info, 2209 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2210 }, 2211 }; 2212 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); 2213 uint32_t res2 = 0; 2214 uint32_t res = 0; 2215 2216 DMSG("freeing cookie %#"PRIx64, cookie); 2217 2218 res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); 2219 2220 mobj_put(mobj); 2221 res2 = mobj_ffa_unregister_by_cookie(cookie); 2222 if (res2) 2223 DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, 2224 cookie, res2); 2225 if (!res) 2226 thread_rpc(&rpc_arg); 2227 } 2228 2229 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) 2230 { 2231 struct thread_rpc_arg rpc_arg = { .call = { 2232 .w1 = thread_get_tsd()->rpc_target_info, 2233 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2234 }, 2235 }; 2236 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); 2237 struct optee_msg_arg *arg = NULL; 2238 unsigned int internal_offset = 0; 2239 struct mobj *mobj = NULL; 2240 uint64_t cookie = 0; 2241 2242 if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) 2243 return NULL; 2244 2245 thread_rpc(&rpc_arg); 2246 2247 if (arg->num_params != 1 || 2248 arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) 2249 return NULL; 2250 2251 internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); 2252 cookie = READ_ONCE(arg->params->u.fmem.global_id); 2253 mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); 2254 if (!mobj) { 2255 DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", 2256 cookie, internal_offset); 2257 return NULL; 2258 } 2259 2260 assert(mobj_is_nonsec(mobj)); 2261 2262 if (mobj->size < size) { 2263 DMSG("Mobj %#"PRIx64": wrong size", cookie); 2264 mobj_put(mobj); 2265 return NULL; 2266 } 2267 2268 if (mobj_inc_map(mobj)) { 2269 DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); 2270 mobj_put(mobj); 2271 return NULL; 2272 } 2273 2274 return mobj; 2275 } 2276 2277 struct mobj *thread_rpc_alloc_payload(size_t size) 2278 { 2279 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); 2280 } 2281 2282 struct mobj *thread_rpc_alloc_kernel_payload(size_t size) 2283 { 2284 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); 2285 } 2286 2287 void thread_rpc_free_kernel_payload(struct mobj *mobj) 2288 { 2289 if (mobj) 2290 thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, 2291 mobj_get_cookie(mobj), mobj); 2292 } 2293 2294 void thread_rpc_free_payload(struct mobj *mobj) 2295 { 2296 if (mobj) 2297 thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), 2298 mobj); 2299 } 2300 2301 struct mobj *thread_rpc_alloc_global_payload(size_t size) 2302 { 2303 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); 2304 } 2305 2306 void thread_rpc_free_global_payload(struct mobj *mobj) 2307 { 2308 if (mobj) 2309 thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, 2310 mobj_get_cookie(mobj), mobj); 2311 } 2312 2313 void thread_spmc_register_secondary_ep(vaddr_t ep) 2314 { 2315 unsigned long ret = 0; 2316 2317 /* Let the SPM know the entry point for secondary CPUs */ 2318 ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); 2319 2320 if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) 2321 EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); 2322 } 2323 2324 static uint16_t ffa_id_get(void) 2325 { 2326 /* 2327 * Ask the SPM component running at a higher EL to return our FF-A ID. 2328 * This can either be the SPMC ID (if the SPMC is enabled in OP-TEE) or 2329 * the partition ID (if not). 2330 */ 2331 struct thread_smc_args args = { 2332 .a0 = FFA_ID_GET, 2333 }; 2334 2335 thread_smccc(&args); 2336 if (!is_ffa_success(args.a0)) { 2337 if (args.a0 == FFA_ERROR) 2338 EMSG("Get id failed with error %ld", args.a2); 2339 else 2340 EMSG("Get id failed"); 2341 panic(); 2342 } 2343 2344 return args.a2; 2345 } 2346 2347 static uint16_t ffa_spm_id_get(void) 2348 { 2349 /* 2350 * Ask the SPM component running at a higher EL to return its ID. 2351 * If OP-TEE implements the S-EL1 SPMC, this will get the SPMD ID. 2352 * If not, the ID of the SPMC will be returned. 2353 */ 2354 struct thread_smc_args args = { 2355 .a0 = FFA_SPM_ID_GET, 2356 }; 2357 2358 thread_smccc(&args); 2359 if (!is_ffa_success(args.a0)) { 2360 if (args.a0 == FFA_ERROR) 2361 EMSG("Get spm id failed with error %ld", args.a2); 2362 else 2363 EMSG("Get spm id failed"); 2364 panic(); 2365 } 2366 2367 return args.a2; 2368 } 2369 2370 #if defined(CFG_CORE_SEL1_SPMC) 2371 static TEE_Result spmc_init(void) 2372 { 2373 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 2374 virt_add_guest_spec_data(¬if_vm_bitmap_id, 2375 sizeof(struct notif_vm_bitmap), NULL)) 2376 panic("virt_add_guest_spec_data"); 2377 spmd_id = ffa_spm_id_get(); 2378 DMSG("SPMD ID %#"PRIx16, spmd_id); 2379 2380 spmc_id = ffa_id_get(); 2381 DMSG("SPMC ID %#"PRIx16, spmc_id); 2382 2383 optee_endpoint_id = FFA_SWD_ID_MIN; 2384 while (spmc_is_reserved_id(optee_endpoint_id)) 2385 optee_endpoint_id++; 2386 2387 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2388 2389 /* 2390 * If SPMD think we are version 1.0 it will report version 1.0 to 2391 * normal world regardless of what version we query the SPM with. 2392 * However, if SPMD think we are version 1.1 it will forward 2393 * queries from normal world to let us negotiate version. So by 2394 * setting version 1.0 here we should be compatible. 2395 * 2396 * Note that disagreement on negotiated version means that we'll 2397 * have communication problems with normal world. 2398 */ 2399 my_rxtx.ffa_vers = FFA_VERSION_1_0; 2400 2401 return TEE_SUCCESS; 2402 } 2403 #else /* !defined(CFG_CORE_SEL1_SPMC) */ 2404 static void spmc_rxtx_map(struct ffa_rxtx *rxtx) 2405 { 2406 struct thread_smc_args args = { 2407 #ifdef ARM64 2408 .a0 = FFA_RXTX_MAP_64, 2409 #else 2410 .a0 = FFA_RXTX_MAP_32, 2411 #endif 2412 .a1 = virt_to_phys(rxtx->tx), 2413 .a2 = virt_to_phys(rxtx->rx), 2414 .a3 = 1, 2415 }; 2416 2417 thread_smccc(&args); 2418 if (!is_ffa_success(args.a0)) { 2419 if (args.a0 == FFA_ERROR) 2420 EMSG("rxtx map failed with error %ld", args.a2); 2421 else 2422 EMSG("rxtx map failed"); 2423 panic(); 2424 } 2425 } 2426 2427 static uint32_t get_ffa_version(uint32_t my_version) 2428 { 2429 struct thread_smc_args args = { 2430 .a0 = FFA_VERSION, 2431 .a1 = my_version, 2432 }; 2433 2434 thread_smccc(&args); 2435 if (args.a0 & BIT(31)) { 2436 EMSG("FF-A version failed with error %ld", args.a0); 2437 panic(); 2438 } 2439 2440 return args.a0; 2441 } 2442 2443 static void *spmc_retrieve_req(uint64_t cookie, 2444 struct ffa_mem_transaction_x *trans) 2445 { 2446 struct ffa_mem_access *acc_descr_array = NULL; 2447 struct ffa_mem_access_perm *perm_descr = NULL; 2448 struct thread_smc_args args = { 2449 .a0 = FFA_MEM_RETRIEVE_REQ_32, 2450 .a3 = 0, /* Address, Using TX -> MBZ */ 2451 .a4 = 0, /* Using TX -> MBZ */ 2452 }; 2453 size_t size = 0; 2454 int rc = 0; 2455 2456 if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { 2457 struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; 2458 2459 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2460 memset(trans_descr, 0, size); 2461 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2462 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2463 trans_descr->global_handle = cookie; 2464 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2465 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2466 trans_descr->mem_access_count = 1; 2467 acc_descr_array = trans_descr->mem_access_array; 2468 } else { 2469 struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; 2470 2471 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2472 memset(trans_descr, 0, size); 2473 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2474 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2475 trans_descr->global_handle = cookie; 2476 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2477 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2478 trans_descr->mem_access_count = 1; 2479 trans_descr->mem_access_offs = sizeof(*trans_descr); 2480 trans_descr->mem_access_size = sizeof(struct ffa_mem_access); 2481 acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + 2482 sizeof(*trans_descr)); 2483 } 2484 acc_descr_array->region_offs = 0; 2485 acc_descr_array->reserved = 0; 2486 perm_descr = &acc_descr_array->access_perm; 2487 perm_descr->endpoint_id = optee_endpoint_id; 2488 perm_descr->perm = FFA_MEM_ACC_RW; 2489 perm_descr->flags = 0; 2490 2491 args.a1 = size; /* Total Length */ 2492 args.a2 = size; /* Frag Length == Total length */ 2493 thread_smccc(&args); 2494 if (args.a0 != FFA_MEM_RETRIEVE_RESP) { 2495 if (args.a0 == FFA_ERROR) 2496 EMSG("Failed to fetch cookie %#"PRIx64" error code %d", 2497 cookie, (int)args.a2); 2498 else 2499 EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, 2500 cookie, args.a0); 2501 return NULL; 2502 } 2503 rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.rx, 2504 my_rxtx.size, trans); 2505 if (rc) { 2506 EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", 2507 cookie, rc); 2508 return NULL; 2509 } 2510 2511 return my_rxtx.rx; 2512 } 2513 2514 void thread_spmc_relinquish(uint64_t cookie) 2515 { 2516 struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; 2517 struct thread_smc_args args = { 2518 .a0 = FFA_MEM_RELINQUISH, 2519 }; 2520 2521 memset(relinquish_desc, 0, sizeof(*relinquish_desc)); 2522 relinquish_desc->handle = cookie; 2523 relinquish_desc->flags = 0; 2524 relinquish_desc->endpoint_count = 1; 2525 relinquish_desc->endpoint_id_array[0] = optee_endpoint_id; 2526 thread_smccc(&args); 2527 if (!is_ffa_success(args.a0)) 2528 EMSG("Failed to relinquish cookie %#"PRIx64, cookie); 2529 } 2530 2531 static int set_pages(struct ffa_address_range *regions, 2532 unsigned int num_regions, unsigned int num_pages, 2533 struct mobj_ffa *mf) 2534 { 2535 unsigned int n = 0; 2536 unsigned int idx = 0; 2537 2538 for (n = 0; n < num_regions; n++) { 2539 unsigned int page_count = READ_ONCE(regions[n].page_count); 2540 uint64_t addr = READ_ONCE(regions[n].address); 2541 2542 if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) 2543 return FFA_INVALID_PARAMETERS; 2544 } 2545 2546 if (idx != num_pages) 2547 return FFA_INVALID_PARAMETERS; 2548 2549 return 0; 2550 } 2551 2552 struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) 2553 { 2554 struct mobj_ffa *ret = NULL; 2555 struct ffa_mem_transaction_x retrieve_desc = { }; 2556 struct ffa_mem_access *descr_array = NULL; 2557 struct ffa_mem_region *descr = NULL; 2558 struct mobj_ffa *mf = NULL; 2559 unsigned int num_pages = 0; 2560 unsigned int offs = 0; 2561 void *buf = NULL; 2562 struct thread_smc_args ffa_rx_release_args = { 2563 .a0 = FFA_RX_RELEASE 2564 }; 2565 2566 /* 2567 * OP-TEE is only supporting a single mem_region while the 2568 * specification allows for more than one. 2569 */ 2570 buf = spmc_retrieve_req(cookie, &retrieve_desc); 2571 if (!buf) { 2572 EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, 2573 cookie); 2574 return NULL; 2575 } 2576 2577 descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); 2578 offs = READ_ONCE(descr_array->region_offs); 2579 descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); 2580 2581 num_pages = READ_ONCE(descr->total_page_count); 2582 mf = mobj_ffa_spmc_new(cookie, num_pages); 2583 if (!mf) 2584 goto out; 2585 2586 if (set_pages(descr->address_range_array, 2587 READ_ONCE(descr->address_range_count), num_pages, mf)) { 2588 mobj_ffa_spmc_delete(mf); 2589 goto out; 2590 } 2591 2592 ret = mf; 2593 2594 out: 2595 /* Release RX buffer after the mem retrieve request. */ 2596 thread_smccc(&ffa_rx_release_args); 2597 2598 return ret; 2599 } 2600 2601 static uint32_t get_ffa_version_from_manifest(void *fdt) 2602 { 2603 int ret = 0; 2604 uint32_t vers = 0; 2605 2606 ret = fdt_node_check_compatible(fdt, 0, "arm,ffa-manifest-1.0"); 2607 if (ret < 0) { 2608 EMSG("Invalid FF-A manifest at %p: error %d", fdt, ret); 2609 panic(); 2610 } 2611 2612 ret = fdt_read_uint32(fdt, 0, "ffa-version", &vers); 2613 if (ret < 0) { 2614 EMSG("Can't read \"ffa-version\" from FF-A manifest at %p: error %d", 2615 fdt, ret); 2616 panic(); 2617 } 2618 2619 return vers; 2620 } 2621 2622 static TEE_Result spmc_init(void) 2623 { 2624 uint32_t my_vers = 0; 2625 uint32_t vers = 0; 2626 2627 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 2628 virt_add_guest_spec_data(¬if_vm_bitmap_id, 2629 sizeof(struct notif_vm_bitmap), NULL)) 2630 panic("virt_add_guest_spec_data"); 2631 2632 my_vers = get_ffa_version_from_manifest(get_manifest_dt()); 2633 if (my_vers < FFA_VERSION_1_0 || my_vers > FFA_VERSION_1_2) { 2634 EMSG("Unsupported version %"PRIu32".%"PRIu32" from manifest", 2635 FFA_GET_MAJOR_VERSION(my_vers), 2636 FFA_GET_MINOR_VERSION(my_vers)); 2637 panic(); 2638 } 2639 vers = get_ffa_version(my_vers); 2640 DMSG("SPMC reported version %"PRIu32".%"PRIu32, 2641 FFA_GET_MAJOR_VERSION(vers), FFA_GET_MINOR_VERSION(vers)); 2642 if (FFA_GET_MAJOR_VERSION(vers) != FFA_GET_MAJOR_VERSION(my_vers)) { 2643 EMSG("Incompatible major version %"PRIu32", expected %"PRIu32"", 2644 FFA_GET_MAJOR_VERSION(vers), 2645 FFA_GET_MAJOR_VERSION(my_vers)); 2646 panic(); 2647 } 2648 if (vers < my_vers) 2649 my_vers = vers; 2650 DMSG("Using version %"PRIu32".%"PRIu32"", 2651 FFA_GET_MAJOR_VERSION(my_vers), FFA_GET_MINOR_VERSION(my_vers)); 2652 my_rxtx.ffa_vers = my_vers; 2653 2654 spmc_rxtx_map(&my_rxtx); 2655 2656 spmc_id = ffa_spm_id_get(); 2657 DMSG("SPMC ID %#"PRIx16, spmc_id); 2658 2659 optee_endpoint_id = ffa_id_get(); 2660 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2661 2662 if (!ffa_features(FFA_NOTIFICATION_SET)) { 2663 spmc_notif_is_ready = true; 2664 DMSG("Asynchronous notifications are ready"); 2665 } 2666 2667 return TEE_SUCCESS; 2668 } 2669 #endif /* !defined(CFG_CORE_SEL1_SPMC) */ 2670 2671 nex_service_init(spmc_init); 2672