1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2020-2023, Linaro Limited. 4 * Copyright (c) 2019-2024, Arm Limited. All rights reserved. 5 */ 6 7 #include <assert.h> 8 #include <ffa.h> 9 #include <initcall.h> 10 #include <io.h> 11 #include <kernel/interrupt.h> 12 #include <kernel/notif.h> 13 #include <kernel/panic.h> 14 #include <kernel/secure_partition.h> 15 #include <kernel/spinlock.h> 16 #include <kernel/spmc_sp_handler.h> 17 #include <kernel/tee_misc.h> 18 #include <kernel/thread.h> 19 #include <kernel/thread_private.h> 20 #include <kernel/thread_spmc.h> 21 #include <kernel/virtualization.h> 22 #include <mm/core_mmu.h> 23 #include <mm/mobj.h> 24 #include <optee_ffa.h> 25 #include <optee_msg.h> 26 #include <optee_rpc_cmd.h> 27 #include <sm/optee_smc.h> 28 #include <string.h> 29 #include <sys/queue.h> 30 #include <tee/entry_std.h> 31 #include <tee/uuid.h> 32 #include <util.h> 33 34 #if defined(CFG_CORE_SEL1_SPMC) 35 struct mem_share_state { 36 struct mobj_ffa *mf; 37 unsigned int page_count; 38 unsigned int region_count; 39 unsigned int current_page_idx; 40 }; 41 42 struct mem_frag_state { 43 struct mem_share_state share; 44 tee_mm_entry_t *mm; 45 unsigned int frag_offset; 46 SLIST_ENTRY(mem_frag_state) link; 47 }; 48 #endif 49 50 struct notif_vm_bitmap { 51 bool initialized; 52 int do_bottom_half_value; 53 uint64_t pending; 54 uint64_t bound; 55 }; 56 57 static unsigned int spmc_notif_lock __nex_data = SPINLOCK_UNLOCK; 58 static bool spmc_notif_is_ready __nex_bss; 59 static int notif_intid __nex_data __maybe_unused = -1; 60 61 /* Id used to look up the guest specific struct notif_vm_bitmap */ 62 static unsigned int notif_vm_bitmap_id __nex_bss; 63 /* Notification state when ns-virtualization isn't enabled */ 64 static struct notif_vm_bitmap default_notif_vm_bitmap; 65 66 /* Initialized in spmc_init() below */ 67 uint16_t optee_endpoint_id __nex_bss; 68 uint16_t spmc_id __nex_bss; 69 #ifdef CFG_CORE_SEL1_SPMC 70 uint16_t spmd_id __nex_bss; 71 static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | 72 FFA_PART_PROP_DIRECT_REQ_SEND | 73 #ifdef CFG_NS_VIRTUALIZATION 74 FFA_PART_PROP_NOTIF_CREATED | 75 FFA_PART_PROP_NOTIF_DESTROYED | 76 #endif 77 #ifdef ARM64 78 FFA_PART_PROP_AARCH64_STATE | 79 #endif 80 FFA_PART_PROP_IS_PE_ID; 81 82 static uint32_t my_uuid_words[] = { 83 /* 84 * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 85 * SP, or 86 * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a 87 * logical partition, residing in the same exception level as the 88 * SPMC 89 * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 90 */ 91 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, 92 }; 93 94 /* 95 * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. 96 * 97 * struct ffa_rxtx::spin_lock protects the variables below from concurrent 98 * access this includes the use of content of struct ffa_rxtx::rx and 99 * @frag_state_head. 100 * 101 * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct 102 * ffa_rxtx::tx and false when it is owned by normal world. 103 * 104 * Note that we can't prevent normal world from updating the content of 105 * these buffers so we must always be careful when reading. while we hold 106 * the lock. 107 */ 108 109 static struct ffa_rxtx my_rxtx __nex_bss; 110 111 static bool is_nw_buf(struct ffa_rxtx *rxtx) 112 { 113 return rxtx == &my_rxtx; 114 } 115 116 static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = 117 SLIST_HEAD_INITIALIZER(&frag_state_head); 118 119 #else 120 static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 121 static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 122 static struct ffa_rxtx my_rxtx = { 123 .rx = __rx_buf, 124 .tx = __tx_buf, 125 .size = sizeof(__rx_buf), 126 }; 127 #endif 128 129 static uint32_t swap_src_dst(uint32_t src_dst) 130 { 131 return (src_dst >> 16) | (src_dst << 16); 132 } 133 134 static uint16_t get_sender_id(uint32_t src_dst) 135 { 136 return src_dst >> 16; 137 } 138 139 void spmc_set_args(struct thread_smc_args *args, uint32_t fid, uint32_t src_dst, 140 uint32_t w2, uint32_t w3, uint32_t w4, uint32_t w5) 141 { 142 *args = (struct thread_smc_args){ .a0 = fid, 143 .a1 = src_dst, 144 .a2 = w2, 145 .a3 = w3, 146 .a4 = w4, 147 .a5 = w5, }; 148 } 149 150 static void set_simple_ret_val(struct thread_smc_args *args, int ffa_ret) 151 { 152 if (ffa_ret) 153 spmc_set_args(args, FFA_ERROR, 0, ffa_ret, 0, 0, 0); 154 else 155 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 156 } 157 158 uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) 159 { 160 /* 161 * No locking, if the caller does concurrent calls to this it's 162 * only making a mess for itself. We must be able to renegotiate 163 * the FF-A version in order to support differing versions between 164 * the loader and the driver. 165 */ 166 if (vers < FFA_VERSION_1_1) 167 rxtx->ffa_vers = FFA_VERSION_1_0; 168 else 169 rxtx->ffa_vers = FFA_VERSION_1_1; 170 171 return rxtx->ffa_vers; 172 } 173 174 static bool is_ffa_success(uint32_t fid) 175 { 176 #ifdef ARM64 177 if (fid == FFA_SUCCESS_64) 178 return true; 179 #endif 180 return fid == FFA_SUCCESS_32; 181 } 182 183 static int32_t get_ffa_ret_code(const struct thread_smc_args *args) 184 { 185 if (is_ffa_success(args->a0)) 186 return FFA_OK; 187 if (args->a0 == FFA_ERROR && args->a2) 188 return args->a2; 189 return FFA_NOT_SUPPORTED; 190 } 191 192 static int ffa_simple_call(uint32_t fid, unsigned long a1, unsigned long a2, 193 unsigned long a3, unsigned long a4) 194 { 195 struct thread_smc_args args = { 196 .a0 = fid, 197 .a1 = a1, 198 .a2 = a2, 199 .a3 = a3, 200 .a4 = a4, 201 }; 202 203 thread_smccc(&args); 204 205 return get_ffa_ret_code(&args); 206 } 207 208 static int __maybe_unused ffa_features(uint32_t id) 209 { 210 return ffa_simple_call(FFA_FEATURES, id, 0, 0, 0); 211 } 212 213 static int __maybe_unused ffa_set_notification(uint16_t dst, uint16_t src, 214 uint32_t flags, uint64_t bitmap) 215 { 216 return ffa_simple_call(FFA_NOTIFICATION_SET, 217 SHIFT_U32(src, 16) | dst, flags, 218 low32_from_64(bitmap), high32_from_64(bitmap)); 219 } 220 221 #if defined(CFG_CORE_SEL1_SPMC) 222 static void handle_features(struct thread_smc_args *args) 223 { 224 uint32_t ret_fid = FFA_ERROR; 225 uint32_t ret_w2 = FFA_NOT_SUPPORTED; 226 227 switch (args->a1) { 228 case FFA_FEATURE_SCHEDULE_RECV_INTR: 229 if (spmc_notif_is_ready) { 230 ret_fid = FFA_SUCCESS_32; 231 ret_w2 = notif_intid; 232 } 233 break; 234 235 #ifdef ARM64 236 case FFA_RXTX_MAP_64: 237 #endif 238 case FFA_RXTX_MAP_32: 239 ret_fid = FFA_SUCCESS_32; 240 ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ 241 break; 242 #ifdef ARM64 243 case FFA_MEM_SHARE_64: 244 #endif 245 case FFA_MEM_SHARE_32: 246 ret_fid = FFA_SUCCESS_32; 247 /* 248 * Partition manager supports transmission of a memory 249 * transaction descriptor in a buffer dynamically allocated 250 * by the endpoint. 251 */ 252 ret_w2 = BIT(0); 253 break; 254 255 case FFA_ERROR: 256 case FFA_VERSION: 257 case FFA_SUCCESS_32: 258 #ifdef ARM64 259 case FFA_SUCCESS_64: 260 #endif 261 case FFA_FEATURES: 262 case FFA_SPM_ID_GET: 263 case FFA_MEM_FRAG_TX: 264 case FFA_MEM_RECLAIM: 265 case FFA_MSG_SEND_DIRECT_REQ_64: 266 case FFA_MSG_SEND_DIRECT_REQ_32: 267 case FFA_INTERRUPT: 268 case FFA_PARTITION_INFO_GET: 269 case FFA_RXTX_UNMAP: 270 case FFA_RX_RELEASE: 271 case FFA_FEATURE_MANAGED_EXIT_INTR: 272 case FFA_NOTIFICATION_BITMAP_CREATE: 273 case FFA_NOTIFICATION_BITMAP_DESTROY: 274 case FFA_NOTIFICATION_BIND: 275 case FFA_NOTIFICATION_UNBIND: 276 case FFA_NOTIFICATION_SET: 277 case FFA_NOTIFICATION_GET: 278 case FFA_NOTIFICATION_INFO_GET_32: 279 #ifdef ARM64 280 case FFA_NOTIFICATION_INFO_GET_64: 281 #endif 282 ret_fid = FFA_SUCCESS_32; 283 ret_w2 = FFA_PARAM_MBZ; 284 break; 285 default: 286 break; 287 } 288 289 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, 290 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 291 } 292 293 static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) 294 { 295 tee_mm_entry_t *mm = NULL; 296 297 if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) 298 return FFA_INVALID_PARAMETERS; 299 300 mm = tee_mm_alloc(&core_virt_shm_pool, sz); 301 if (!mm) 302 return FFA_NO_MEMORY; 303 304 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, 305 sz / SMALL_PAGE_SIZE, 306 MEM_AREA_NSEC_SHM)) { 307 tee_mm_free(mm); 308 return FFA_INVALID_PARAMETERS; 309 } 310 311 *va_ret = (void *)tee_mm_get_smem(mm); 312 return 0; 313 } 314 315 void spmc_handle_spm_id_get(struct thread_smc_args *args) 316 { 317 spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, spmc_id, 318 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 319 } 320 321 static void unmap_buf(void *va, size_t sz) 322 { 323 tee_mm_entry_t *mm = tee_mm_find(&core_virt_shm_pool, (vaddr_t)va); 324 325 assert(mm); 326 core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); 327 tee_mm_free(mm); 328 } 329 330 void spmc_handle_rxtx_map(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 331 { 332 int rc = 0; 333 unsigned int sz = 0; 334 paddr_t rx_pa = 0; 335 paddr_t tx_pa = 0; 336 void *rx = NULL; 337 void *tx = NULL; 338 339 cpu_spin_lock(&rxtx->spinlock); 340 341 if (args->a3 & GENMASK_64(63, 6)) { 342 rc = FFA_INVALID_PARAMETERS; 343 goto out; 344 } 345 346 sz = args->a3 * SMALL_PAGE_SIZE; 347 if (!sz) { 348 rc = FFA_INVALID_PARAMETERS; 349 goto out; 350 } 351 /* TX/RX are swapped compared to the caller */ 352 tx_pa = args->a2; 353 rx_pa = args->a1; 354 355 if (rxtx->size) { 356 rc = FFA_DENIED; 357 goto out; 358 } 359 360 /* 361 * If the buffer comes from a SP the address is virtual and already 362 * mapped. 363 */ 364 if (is_nw_buf(rxtx)) { 365 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 366 enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; 367 bool tx_alloced = false; 368 369 /* 370 * With virtualization we establish this mapping in 371 * the nexus mapping which then is replicated to 372 * each partition. 373 * 374 * This means that this mapping must be done before 375 * any partition is created and then must not be 376 * changed. 377 */ 378 379 /* 380 * core_mmu_add_mapping() may reuse previous 381 * mappings. First check if there's any mappings to 382 * reuse so we know how to clean up in case of 383 * failure. 384 */ 385 tx = phys_to_virt(tx_pa, mt, sz); 386 rx = phys_to_virt(rx_pa, mt, sz); 387 if (!tx) { 388 tx = core_mmu_add_mapping(mt, tx_pa, sz); 389 if (!tx) { 390 rc = FFA_NO_MEMORY; 391 goto out; 392 } 393 tx_alloced = true; 394 } 395 if (!rx) 396 rx = core_mmu_add_mapping(mt, rx_pa, sz); 397 398 if (!rx) { 399 if (tx_alloced && tx) 400 core_mmu_remove_mapping(mt, tx, sz); 401 rc = FFA_NO_MEMORY; 402 goto out; 403 } 404 } else { 405 rc = map_buf(tx_pa, sz, &tx); 406 if (rc) 407 goto out; 408 rc = map_buf(rx_pa, sz, &rx); 409 if (rc) { 410 unmap_buf(tx, sz); 411 goto out; 412 } 413 } 414 rxtx->tx = tx; 415 rxtx->rx = rx; 416 } else { 417 if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { 418 rc = FFA_INVALID_PARAMETERS; 419 goto out; 420 } 421 422 if (!virt_to_phys((void *)tx_pa) || 423 !virt_to_phys((void *)rx_pa)) { 424 rc = FFA_INVALID_PARAMETERS; 425 goto out; 426 } 427 428 rxtx->tx = (void *)tx_pa; 429 rxtx->rx = (void *)rx_pa; 430 } 431 432 rxtx->size = sz; 433 rxtx->tx_is_mine = true; 434 DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); 435 DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); 436 out: 437 cpu_spin_unlock(&rxtx->spinlock); 438 set_simple_ret_val(args, rc); 439 } 440 441 void spmc_handle_rxtx_unmap(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 442 { 443 int rc = FFA_INVALID_PARAMETERS; 444 445 cpu_spin_lock(&rxtx->spinlock); 446 447 if (!rxtx->size) 448 goto out; 449 450 /* 451 * We don't unmap the SP memory as the SP might still use it. 452 * We avoid to make changes to nexus mappings at this stage since 453 * there currently isn't a way to replicate those changes to all 454 * partitions. 455 */ 456 if (is_nw_buf(rxtx) && !IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 457 unmap_buf(rxtx->rx, rxtx->size); 458 unmap_buf(rxtx->tx, rxtx->size); 459 } 460 rxtx->size = 0; 461 rxtx->rx = NULL; 462 rxtx->tx = NULL; 463 rc = 0; 464 out: 465 cpu_spin_unlock(&rxtx->spinlock); 466 set_simple_ret_val(args, rc); 467 } 468 469 void spmc_handle_rx_release(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 470 { 471 int rc = 0; 472 473 cpu_spin_lock(&rxtx->spinlock); 474 /* The senders RX is our TX */ 475 if (!rxtx->size || rxtx->tx_is_mine) { 476 rc = FFA_DENIED; 477 } else { 478 rc = 0; 479 rxtx->tx_is_mine = true; 480 } 481 cpu_spin_unlock(&rxtx->spinlock); 482 483 set_simple_ret_val(args, rc); 484 } 485 486 static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 487 { 488 return !w0 && !w1 && !w2 && !w3; 489 } 490 491 static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 492 { 493 /* 494 * This depends on which UUID we have been assigned. 495 * TODO add a generic mechanism to obtain our UUID. 496 * 497 * The test below is for the hard coded UUID 498 * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 499 */ 500 return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && 501 w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; 502 } 503 504 TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, 505 size_t idx, uint16_t endpoint_id, 506 uint16_t execution_context, 507 uint32_t part_props, 508 const uint32_t uuid_words[4]) 509 { 510 struct ffa_partition_info_x *fpi = NULL; 511 size_t fpi_size = sizeof(*fpi); 512 513 if (ffa_vers >= FFA_VERSION_1_1) 514 fpi_size += FFA_UUID_SIZE; 515 516 if ((idx + 1) * fpi_size > blen) 517 return TEE_ERROR_OUT_OF_MEMORY; 518 519 fpi = (void *)((vaddr_t)buf + idx * fpi_size); 520 fpi->id = endpoint_id; 521 /* Number of execution contexts implemented by this partition */ 522 fpi->execution_context = execution_context; 523 524 fpi->partition_properties = part_props; 525 526 /* In FF-A 1.0 only bits [2:0] are defined, let's mask others */ 527 if (ffa_vers < FFA_VERSION_1_1) 528 fpi->partition_properties &= FFA_PART_PROP_DIRECT_REQ_RECV | 529 FFA_PART_PROP_DIRECT_REQ_SEND | 530 FFA_PART_PROP_INDIRECT_MSGS; 531 532 if (ffa_vers >= FFA_VERSION_1_1) { 533 if (uuid_words) 534 memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); 535 else 536 memset(fpi->uuid, 0, FFA_UUID_SIZE); 537 } 538 539 return TEE_SUCCESS; 540 } 541 542 static int handle_partition_info_get_all(size_t *elem_count, 543 struct ffa_rxtx *rxtx, bool count_only) 544 { 545 if (!count_only) { 546 /* Add OP-TEE SP */ 547 if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, 548 rxtx->size, 0, optee_endpoint_id, 549 CFG_TEE_CORE_NB_CORE, 550 my_part_props, my_uuid_words)) 551 return FFA_NO_MEMORY; 552 } 553 *elem_count = 1; 554 555 if (IS_ENABLED(CFG_SECURE_PARTITION)) { 556 if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, 557 NULL, elem_count, count_only)) 558 return FFA_NO_MEMORY; 559 } 560 561 return FFA_OK; 562 } 563 564 void spmc_handle_partition_info_get(struct thread_smc_args *args, 565 struct ffa_rxtx *rxtx) 566 { 567 TEE_Result res = TEE_SUCCESS; 568 uint32_t ret_fid = FFA_ERROR; 569 uint32_t fpi_size = 0; 570 uint32_t rc = 0; 571 bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; 572 573 if (!count_only) { 574 cpu_spin_lock(&rxtx->spinlock); 575 576 if (!rxtx->size || !rxtx->tx_is_mine) { 577 rc = FFA_BUSY; 578 goto out; 579 } 580 } 581 582 if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { 583 size_t elem_count = 0; 584 585 ret_fid = handle_partition_info_get_all(&elem_count, rxtx, 586 count_only); 587 588 if (ret_fid) { 589 rc = ret_fid; 590 ret_fid = FFA_ERROR; 591 } else { 592 ret_fid = FFA_SUCCESS_32; 593 rc = elem_count; 594 } 595 596 goto out; 597 } 598 599 if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { 600 if (!count_only) { 601 res = spmc_fill_partition_entry(rxtx->ffa_vers, 602 rxtx->tx, rxtx->size, 0, 603 optee_endpoint_id, 604 CFG_TEE_CORE_NB_CORE, 605 my_part_props, 606 my_uuid_words); 607 if (res) { 608 ret_fid = FFA_ERROR; 609 rc = FFA_INVALID_PARAMETERS; 610 goto out; 611 } 612 } 613 rc = 1; 614 } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { 615 uint32_t uuid_array[4] = { 0 }; 616 TEE_UUID uuid = { }; 617 size_t count = 0; 618 619 uuid_array[0] = args->a1; 620 uuid_array[1] = args->a2; 621 uuid_array[2] = args->a3; 622 uuid_array[3] = args->a4; 623 tee_uuid_from_octets(&uuid, (uint8_t *)uuid_array); 624 625 res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, 626 rxtx->size, &uuid, &count, 627 count_only); 628 if (res != TEE_SUCCESS) { 629 ret_fid = FFA_ERROR; 630 rc = FFA_INVALID_PARAMETERS; 631 goto out; 632 } 633 rc = count; 634 } else { 635 ret_fid = FFA_ERROR; 636 rc = FFA_INVALID_PARAMETERS; 637 goto out; 638 } 639 640 ret_fid = FFA_SUCCESS_32; 641 642 out: 643 if (ret_fid == FFA_SUCCESS_32 && !count_only && 644 rxtx->ffa_vers >= FFA_VERSION_1_1) 645 fpi_size = sizeof(struct ffa_partition_info_x) + FFA_UUID_SIZE; 646 647 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, fpi_size, 648 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 649 if (!count_only) { 650 rxtx->tx_is_mine = false; 651 cpu_spin_unlock(&rxtx->spinlock); 652 } 653 } 654 655 static void spmc_handle_run(struct thread_smc_args *args) 656 { 657 uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); 658 uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); 659 uint32_t rc = FFA_OK; 660 661 if (endpoint != optee_endpoint_id) { 662 /* 663 * The endpoint should be an SP, try to resume the SP from 664 * preempted into busy state. 665 */ 666 rc = spmc_sp_resume_from_preempted(endpoint); 667 if (rc) 668 goto out; 669 } 670 671 thread_resume_from_rpc(thread_id, 0, 0, 0, 0); 672 673 /* thread_resume_from_rpc return only of the thread_id is invalid */ 674 rc = FFA_INVALID_PARAMETERS; 675 676 out: 677 set_simple_ret_val(args, rc); 678 } 679 #endif /*CFG_CORE_SEL1_SPMC*/ 680 681 static struct notif_vm_bitmap *get_notif_vm_bitmap(struct guest_partition *prtn, 682 uint16_t vm_id) 683 { 684 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 685 if (!prtn) 686 return NULL; 687 assert(vm_id == virt_get_guest_id(prtn)); 688 return virt_get_guest_spec_data(prtn, notif_vm_bitmap_id); 689 } 690 if (vm_id) 691 return NULL; 692 return &default_notif_vm_bitmap; 693 } 694 695 static uint32_t spmc_enable_async_notif(uint32_t bottom_half_value, 696 uint16_t vm_id) 697 { 698 struct guest_partition *prtn = NULL; 699 struct notif_vm_bitmap *nvb = NULL; 700 uint32_t old_itr_status = 0; 701 uint32_t res = 0; 702 703 if (!spmc_notif_is_ready) { 704 /* 705 * This should never happen, not if normal world respects the 706 * exchanged capabilities. 707 */ 708 EMSG("Asynchronous notifications are not ready"); 709 return TEE_ERROR_NOT_IMPLEMENTED; 710 } 711 712 if (bottom_half_value >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE) { 713 EMSG("Invalid bottom half value %"PRIu32, bottom_half_value); 714 return TEE_ERROR_BAD_PARAMETERS; 715 } 716 717 prtn = virt_get_guest(vm_id); 718 nvb = get_notif_vm_bitmap(prtn, vm_id); 719 if (!nvb) { 720 res = TEE_ERROR_BAD_PARAMETERS; 721 goto out; 722 } 723 724 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 725 nvb->do_bottom_half_value = bottom_half_value; 726 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 727 728 notif_deliver_atomic_event(NOTIF_EVENT_STARTED, vm_id); 729 res = TEE_SUCCESS; 730 out: 731 virt_put_guest(prtn); 732 return res; 733 } 734 735 static void handle_yielding_call(struct thread_smc_args *args, 736 uint32_t direct_resp_fid) 737 { 738 TEE_Result res = 0; 739 740 thread_check_canaries(); 741 742 #ifdef ARM64 743 /* Saving this for an eventual RPC */ 744 thread_get_core_local()->direct_resp_fid = direct_resp_fid; 745 #endif 746 747 if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { 748 /* Note connection to struct thread_rpc_arg::ret */ 749 thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, 750 0); 751 res = TEE_ERROR_BAD_PARAMETERS; 752 } else { 753 thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, 754 args->a6, args->a7); 755 res = TEE_ERROR_BUSY; 756 } 757 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 758 0, res, 0, 0); 759 } 760 761 static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) 762 { 763 uint64_t cookie = reg_pair_to_64(a5, a4); 764 uint32_t res = 0; 765 766 res = mobj_ffa_unregister_by_cookie(cookie); 767 switch (res) { 768 case TEE_SUCCESS: 769 case TEE_ERROR_ITEM_NOT_FOUND: 770 return 0; 771 case TEE_ERROR_BUSY: 772 EMSG("res %#"PRIx32, res); 773 return FFA_BUSY; 774 default: 775 EMSG("res %#"PRIx32, res); 776 return FFA_INVALID_PARAMETERS; 777 } 778 } 779 780 static void handle_blocking_call(struct thread_smc_args *args, 781 uint32_t direct_resp_fid) 782 { 783 uint32_t sec_caps = 0; 784 785 switch (args->a3) { 786 case OPTEE_FFA_GET_API_VERSION: 787 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 788 OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, 789 0); 790 break; 791 case OPTEE_FFA_GET_OS_VERSION: 792 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 793 CFG_OPTEE_REVISION_MAJOR, 794 CFG_OPTEE_REVISION_MINOR, 795 TEE_IMPL_GIT_SHA1 >> 32); 796 break; 797 case OPTEE_FFA_EXCHANGE_CAPABILITIES: 798 sec_caps = OPTEE_FFA_SEC_CAP_ARG_OFFSET; 799 if (spmc_notif_is_ready) 800 sec_caps |= OPTEE_FFA_SEC_CAP_ASYNC_NOTIF; 801 sec_caps |= OPTEE_FFA_SEC_CAP_RPMB_PROBE; 802 spmc_set_args(args, direct_resp_fid, 803 swap_src_dst(args->a1), 0, 0, 804 THREAD_RPC_MAX_NUM_PARAMS, sec_caps); 805 break; 806 case OPTEE_FFA_UNREGISTER_SHM: 807 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 808 handle_unregister_shm(args->a4, args->a5), 0, 0); 809 break; 810 case OPTEE_FFA_ENABLE_ASYNC_NOTIF: 811 spmc_set_args(args, direct_resp_fid, 812 swap_src_dst(args->a1), 0, 813 spmc_enable_async_notif(args->a4, 814 FFA_SRC(args->a1)), 815 0, 0); 816 break; 817 default: 818 EMSG("Unhandled blocking service ID %#"PRIx32, 819 (uint32_t)args->a3); 820 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 821 TEE_ERROR_BAD_PARAMETERS, 0, 0); 822 } 823 } 824 825 static void handle_framework_direct_request(struct thread_smc_args *args, 826 struct ffa_rxtx *rxtx, 827 uint32_t direct_resp_fid) 828 { 829 uint32_t w0 = FFA_ERROR; 830 uint32_t w1 = FFA_PARAM_MBZ; 831 uint32_t w2 = FFA_NOT_SUPPORTED; 832 uint32_t w3 = FFA_PARAM_MBZ; 833 834 switch (args->a2 & FFA_MSG_TYPE_MASK) { 835 case FFA_MSG_SEND_VM_CREATED: 836 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 837 uint16_t guest_id = args->a5; 838 TEE_Result res = virt_guest_created(guest_id); 839 840 w0 = direct_resp_fid; 841 w1 = swap_src_dst(args->a1); 842 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; 843 if (res == TEE_SUCCESS) 844 w3 = FFA_OK; 845 else if (res == TEE_ERROR_OUT_OF_MEMORY) 846 w3 = FFA_DENIED; 847 else 848 w3 = FFA_INVALID_PARAMETERS; 849 } 850 break; 851 case FFA_MSG_SEND_VM_DESTROYED: 852 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 853 uint16_t guest_id = args->a5; 854 TEE_Result res = virt_guest_destroyed(guest_id); 855 856 w0 = direct_resp_fid; 857 w1 = swap_src_dst(args->a1); 858 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_DESTROYED; 859 if (res == TEE_SUCCESS) 860 w3 = FFA_OK; 861 else 862 w3 = FFA_INVALID_PARAMETERS; 863 } 864 break; 865 case FFA_MSG_VERSION_REQ: 866 w0 = direct_resp_fid; 867 w1 = swap_src_dst(args->a1); 868 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; 869 w3 = spmc_exchange_version(args->a3, rxtx); 870 break; 871 default: 872 break; 873 } 874 spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 875 } 876 877 static void handle_direct_request(struct thread_smc_args *args, 878 struct ffa_rxtx *rxtx) 879 { 880 uint32_t direct_resp_fid = 0; 881 882 if (IS_ENABLED(CFG_SECURE_PARTITION) && 883 FFA_DST(args->a1) != spmc_id && 884 FFA_DST(args->a1) != optee_endpoint_id) { 885 spmc_sp_start_thread(args); 886 return; 887 } 888 889 if (OPTEE_SMC_IS_64(args->a0)) 890 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_64; 891 else 892 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_32; 893 894 if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { 895 handle_framework_direct_request(args, rxtx, direct_resp_fid); 896 return; 897 } 898 899 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 900 virt_set_guest(get_sender_id(args->a1))) { 901 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 902 TEE_ERROR_ITEM_NOT_FOUND, 0, 0); 903 return; 904 } 905 906 if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) 907 handle_yielding_call(args, direct_resp_fid); 908 else 909 handle_blocking_call(args, direct_resp_fid); 910 911 /* 912 * Note that handle_yielding_call() typically only returns if a 913 * thread cannot be allocated or found. virt_unset_guest() is also 914 * called from thread_state_suspend() and thread_state_free(). 915 */ 916 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 917 virt_unset_guest(); 918 } 919 920 int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, 921 struct ffa_mem_transaction_x *trans) 922 { 923 uint16_t mem_reg_attr = 0; 924 uint32_t flags = 0; 925 uint32_t count = 0; 926 uint32_t offs = 0; 927 uint32_t size = 0; 928 size_t n = 0; 929 930 if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) 931 return FFA_INVALID_PARAMETERS; 932 933 if (ffa_vers >= FFA_VERSION_1_1) { 934 struct ffa_mem_transaction_1_1 *descr = NULL; 935 936 if (blen < sizeof(*descr)) 937 return FFA_INVALID_PARAMETERS; 938 939 descr = buf; 940 trans->sender_id = READ_ONCE(descr->sender_id); 941 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 942 flags = READ_ONCE(descr->flags); 943 trans->global_handle = READ_ONCE(descr->global_handle); 944 trans->tag = READ_ONCE(descr->tag); 945 946 count = READ_ONCE(descr->mem_access_count); 947 size = READ_ONCE(descr->mem_access_size); 948 offs = READ_ONCE(descr->mem_access_offs); 949 } else { 950 struct ffa_mem_transaction_1_0 *descr = NULL; 951 952 if (blen < sizeof(*descr)) 953 return FFA_INVALID_PARAMETERS; 954 955 descr = buf; 956 trans->sender_id = READ_ONCE(descr->sender_id); 957 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 958 flags = READ_ONCE(descr->flags); 959 trans->global_handle = READ_ONCE(descr->global_handle); 960 trans->tag = READ_ONCE(descr->tag); 961 962 count = READ_ONCE(descr->mem_access_count); 963 size = sizeof(struct ffa_mem_access); 964 offs = offsetof(struct ffa_mem_transaction_1_0, 965 mem_access_array); 966 } 967 968 if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || 969 size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) 970 return FFA_INVALID_PARAMETERS; 971 972 /* Check that the endpoint memory access descriptor array fits */ 973 if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || 974 n > blen) 975 return FFA_INVALID_PARAMETERS; 976 977 trans->mem_reg_attr = mem_reg_attr; 978 trans->flags = flags; 979 trans->mem_access_size = size; 980 trans->mem_access_count = count; 981 trans->mem_access_offs = offs; 982 return 0; 983 } 984 985 #if defined(CFG_CORE_SEL1_SPMC) 986 static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, 987 unsigned int mem_access_count, uint8_t *acc_perms, 988 unsigned int *region_offs) 989 { 990 struct ffa_mem_access_perm *descr = NULL; 991 struct ffa_mem_access *mem_acc = NULL; 992 unsigned int n = 0; 993 994 for (n = 0; n < mem_access_count; n++) { 995 mem_acc = (void *)(mem_acc_base + mem_access_size * n); 996 descr = &mem_acc->access_perm; 997 if (READ_ONCE(descr->endpoint_id) == optee_endpoint_id) { 998 *acc_perms = READ_ONCE(descr->perm); 999 *region_offs = READ_ONCE(mem_acc[n].region_offs); 1000 return 0; 1001 } 1002 } 1003 1004 return FFA_INVALID_PARAMETERS; 1005 } 1006 1007 static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, 1008 size_t blen, unsigned int *page_count, 1009 unsigned int *region_count, size_t *addr_range_offs) 1010 { 1011 const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 1012 const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; 1013 struct ffa_mem_region *region_descr = NULL; 1014 unsigned int region_descr_offs = 0; 1015 uint8_t mem_acc_perm = 0; 1016 size_t n = 0; 1017 1018 if (mem_trans->mem_reg_attr != exp_mem_reg_attr) 1019 return FFA_INVALID_PARAMETERS; 1020 1021 /* Check that the access permissions matches what's expected */ 1022 if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, 1023 mem_trans->mem_access_size, 1024 mem_trans->mem_access_count, 1025 &mem_acc_perm, ®ion_descr_offs) || 1026 mem_acc_perm != exp_mem_acc_perm) 1027 return FFA_INVALID_PARAMETERS; 1028 1029 /* Check that the Composite memory region descriptor fits */ 1030 if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || 1031 n > blen) 1032 return FFA_INVALID_PARAMETERS; 1033 1034 if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, 1035 struct ffa_mem_region)) 1036 return FFA_INVALID_PARAMETERS; 1037 1038 region_descr = (struct ffa_mem_region *)((vaddr_t)buf + 1039 region_descr_offs); 1040 *page_count = READ_ONCE(region_descr->total_page_count); 1041 *region_count = READ_ONCE(region_descr->address_range_count); 1042 *addr_range_offs = n; 1043 return 0; 1044 } 1045 1046 static int add_mem_share_helper(struct mem_share_state *s, void *buf, 1047 size_t flen) 1048 { 1049 unsigned int region_count = flen / sizeof(struct ffa_address_range); 1050 struct ffa_address_range *arange = NULL; 1051 unsigned int n = 0; 1052 1053 if (region_count > s->region_count) 1054 region_count = s->region_count; 1055 1056 if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) 1057 return FFA_INVALID_PARAMETERS; 1058 arange = buf; 1059 1060 for (n = 0; n < region_count; n++) { 1061 unsigned int page_count = READ_ONCE(arange[n].page_count); 1062 uint64_t addr = READ_ONCE(arange[n].address); 1063 1064 if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, 1065 addr, page_count)) 1066 return FFA_INVALID_PARAMETERS; 1067 } 1068 1069 s->region_count -= region_count; 1070 if (s->region_count) 1071 return region_count * sizeof(*arange); 1072 1073 if (s->current_page_idx != s->page_count) 1074 return FFA_INVALID_PARAMETERS; 1075 1076 return 0; 1077 } 1078 1079 static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) 1080 { 1081 int rc = 0; 1082 1083 rc = add_mem_share_helper(&s->share, buf, flen); 1084 if (rc >= 0) { 1085 if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { 1086 /* We're not at the end of the descriptor yet */ 1087 if (s->share.region_count) 1088 return s->frag_offset; 1089 1090 /* We're done */ 1091 rc = 0; 1092 } else { 1093 rc = FFA_INVALID_PARAMETERS; 1094 } 1095 } 1096 1097 SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); 1098 if (rc < 0) 1099 mobj_ffa_sel1_spmc_delete(s->share.mf); 1100 else 1101 mobj_ffa_push_to_inactive(s->share.mf); 1102 free(s); 1103 1104 return rc; 1105 } 1106 1107 static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, 1108 void *buf) 1109 { 1110 struct ffa_mem_access_perm *perm = NULL; 1111 struct ffa_mem_access *mem_acc = NULL; 1112 1113 if (!IS_ENABLED(CFG_SECURE_PARTITION)) 1114 return false; 1115 1116 if (mem_trans->mem_access_count < 1) 1117 return false; 1118 1119 mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); 1120 perm = &mem_acc->access_perm; 1121 1122 /* 1123 * perm->endpoint_id is read here only to check if the endpoint is 1124 * OP-TEE. We do read it later on again, but there are some additional 1125 * checks there to make sure that the data is correct. 1126 */ 1127 return READ_ONCE(perm->endpoint_id) != optee_endpoint_id; 1128 } 1129 1130 static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, 1131 tee_mm_entry_t *mm, void *buf, size_t blen, 1132 size_t flen, uint64_t *global_handle) 1133 { 1134 int rc = 0; 1135 struct mem_share_state share = { }; 1136 size_t addr_range_offs = 0; 1137 uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1138 size_t n = 0; 1139 1140 rc = mem_share_init(mem_trans, buf, flen, &share.page_count, 1141 &share.region_count, &addr_range_offs); 1142 if (rc) 1143 return rc; 1144 1145 if (!share.page_count || !share.region_count) 1146 return FFA_INVALID_PARAMETERS; 1147 1148 if (MUL_OVERFLOW(share.region_count, 1149 sizeof(struct ffa_address_range), &n) || 1150 ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) 1151 return FFA_INVALID_PARAMETERS; 1152 1153 if (mem_trans->global_handle) 1154 cookie = mem_trans->global_handle; 1155 share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); 1156 if (!share.mf) 1157 return FFA_NO_MEMORY; 1158 1159 if (flen != blen) { 1160 struct mem_frag_state *s = calloc(1, sizeof(*s)); 1161 1162 if (!s) { 1163 rc = FFA_NO_MEMORY; 1164 goto err; 1165 } 1166 s->share = share; 1167 s->mm = mm; 1168 s->frag_offset = addr_range_offs; 1169 1170 SLIST_INSERT_HEAD(&frag_state_head, s, link); 1171 rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, 1172 flen - addr_range_offs); 1173 1174 if (rc >= 0) 1175 *global_handle = mobj_ffa_get_cookie(share.mf); 1176 1177 return rc; 1178 } 1179 1180 rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, 1181 flen - addr_range_offs); 1182 if (rc) { 1183 /* 1184 * Number of consumed bytes may be returned instead of 0 for 1185 * done. 1186 */ 1187 rc = FFA_INVALID_PARAMETERS; 1188 goto err; 1189 } 1190 1191 *global_handle = mobj_ffa_push_to_inactive(share.mf); 1192 1193 return 0; 1194 err: 1195 mobj_ffa_sel1_spmc_delete(share.mf); 1196 return rc; 1197 } 1198 1199 static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, 1200 unsigned int page_count, 1201 uint64_t *global_handle, struct ffa_rxtx *rxtx) 1202 { 1203 struct ffa_mem_transaction_x mem_trans = { }; 1204 int rc = 0; 1205 size_t len = 0; 1206 void *buf = NULL; 1207 tee_mm_entry_t *mm = NULL; 1208 vaddr_t offs = pbuf & SMALL_PAGE_MASK; 1209 1210 if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) 1211 return FFA_INVALID_PARAMETERS; 1212 if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) 1213 return FFA_INVALID_PARAMETERS; 1214 1215 /* 1216 * Check that the length reported in flen is covered by len even 1217 * if the offset is taken into account. 1218 */ 1219 if (len < flen || len - offs < flen) 1220 return FFA_INVALID_PARAMETERS; 1221 1222 mm = tee_mm_alloc(&core_virt_shm_pool, len); 1223 if (!mm) 1224 return FFA_NO_MEMORY; 1225 1226 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, 1227 page_count, MEM_AREA_NSEC_SHM)) { 1228 rc = FFA_INVALID_PARAMETERS; 1229 goto out; 1230 } 1231 buf = (void *)(tee_mm_get_smem(mm) + offs); 1232 1233 cpu_spin_lock(&rxtx->spinlock); 1234 rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); 1235 if (rc) 1236 goto unlock; 1237 1238 if (is_sp_share(&mem_trans, buf)) { 1239 rc = spmc_sp_add_share(&mem_trans, buf, blen, flen, 1240 global_handle, NULL); 1241 goto unlock; 1242 } 1243 1244 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1245 virt_set_guest(mem_trans.sender_id)) { 1246 rc = FFA_DENIED; 1247 goto unlock; 1248 } 1249 1250 rc = add_mem_share(&mem_trans, mm, buf, blen, flen, global_handle); 1251 1252 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1253 virt_unset_guest(); 1254 1255 unlock: 1256 cpu_spin_unlock(&rxtx->spinlock); 1257 if (rc > 0) 1258 return rc; 1259 1260 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1261 out: 1262 tee_mm_free(mm); 1263 return rc; 1264 } 1265 1266 static int handle_mem_share_rxbuf(size_t blen, size_t flen, 1267 uint64_t *global_handle, 1268 struct ffa_rxtx *rxtx) 1269 { 1270 struct ffa_mem_transaction_x mem_trans = { }; 1271 int rc = FFA_DENIED; 1272 1273 cpu_spin_lock(&rxtx->spinlock); 1274 1275 if (!rxtx->rx || flen > rxtx->size) 1276 goto out; 1277 1278 rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, 1279 &mem_trans); 1280 if (rc) 1281 goto out; 1282 if (is_sp_share(&mem_trans, rxtx->rx)) { 1283 rc = spmc_sp_add_share(&mem_trans, rxtx, blen, flen, 1284 global_handle, NULL); 1285 goto out; 1286 } 1287 1288 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1289 virt_set_guest(mem_trans.sender_id)) 1290 goto out; 1291 1292 rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, 1293 global_handle); 1294 1295 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1296 virt_unset_guest(); 1297 1298 out: 1299 cpu_spin_unlock(&rxtx->spinlock); 1300 1301 return rc; 1302 } 1303 1304 static void handle_mem_share(struct thread_smc_args *args, 1305 struct ffa_rxtx *rxtx) 1306 { 1307 uint32_t tot_len = args->a1; 1308 uint32_t frag_len = args->a2; 1309 uint64_t addr = args->a3; 1310 uint32_t page_count = args->a4; 1311 uint32_t ret_w1 = 0; 1312 uint32_t ret_w2 = FFA_INVALID_PARAMETERS; 1313 uint32_t ret_w3 = 0; 1314 uint32_t ret_fid = FFA_ERROR; 1315 uint64_t global_handle = 0; 1316 int rc = 0; 1317 1318 /* Check that the MBZs are indeed 0 */ 1319 if (args->a5 || args->a6 || args->a7) 1320 goto out; 1321 1322 /* Check that fragment length doesn't exceed total length */ 1323 if (frag_len > tot_len) 1324 goto out; 1325 1326 /* Check for 32-bit calling convention */ 1327 if (args->a0 == FFA_MEM_SHARE_32) 1328 addr &= UINT32_MAX; 1329 1330 if (!addr) { 1331 /* 1332 * The memory transaction descriptor is passed via our rx 1333 * buffer. 1334 */ 1335 if (page_count) 1336 goto out; 1337 rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, 1338 rxtx); 1339 } else { 1340 rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, 1341 &global_handle, rxtx); 1342 } 1343 if (rc < 0) { 1344 ret_w2 = rc; 1345 } else if (rc > 0) { 1346 ret_fid = FFA_MEM_FRAG_RX; 1347 ret_w3 = rc; 1348 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1349 } else { 1350 ret_fid = FFA_SUCCESS_32; 1351 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1352 } 1353 out: 1354 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1355 } 1356 1357 static struct mem_frag_state *get_frag_state(uint64_t global_handle) 1358 { 1359 struct mem_frag_state *s = NULL; 1360 1361 SLIST_FOREACH(s, &frag_state_head, link) 1362 if (mobj_ffa_get_cookie(s->share.mf) == global_handle) 1363 return s; 1364 1365 return NULL; 1366 } 1367 1368 static void handle_mem_frag_tx(struct thread_smc_args *args, 1369 struct ffa_rxtx *rxtx) 1370 { 1371 uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); 1372 size_t flen = args->a3; 1373 uint32_t endpoint_id = args->a4; 1374 struct mem_frag_state *s = NULL; 1375 tee_mm_entry_t *mm = NULL; 1376 unsigned int page_count = 0; 1377 void *buf = NULL; 1378 uint32_t ret_w1 = 0; 1379 uint32_t ret_w2 = 0; 1380 uint32_t ret_w3 = 0; 1381 uint32_t ret_fid = 0; 1382 int rc = 0; 1383 1384 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1385 uint16_t guest_id = endpoint_id >> 16; 1386 1387 if (!guest_id || virt_set_guest(guest_id)) { 1388 rc = FFA_INVALID_PARAMETERS; 1389 goto out_set_rc; 1390 } 1391 } 1392 1393 /* 1394 * Currently we're only doing this for fragmented FFA_MEM_SHARE_* 1395 * requests. 1396 */ 1397 1398 cpu_spin_lock(&rxtx->spinlock); 1399 1400 s = get_frag_state(global_handle); 1401 if (!s) { 1402 rc = FFA_INVALID_PARAMETERS; 1403 goto out; 1404 } 1405 1406 mm = s->mm; 1407 if (mm) { 1408 if (flen > tee_mm_get_bytes(mm)) { 1409 rc = FFA_INVALID_PARAMETERS; 1410 goto out; 1411 } 1412 page_count = s->share.page_count; 1413 buf = (void *)tee_mm_get_smem(mm); 1414 } else { 1415 if (flen > rxtx->size) { 1416 rc = FFA_INVALID_PARAMETERS; 1417 goto out; 1418 } 1419 buf = rxtx->rx; 1420 } 1421 1422 rc = add_mem_share_frag(s, buf, flen); 1423 out: 1424 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1425 virt_unset_guest(); 1426 1427 cpu_spin_unlock(&rxtx->spinlock); 1428 1429 if (rc <= 0 && mm) { 1430 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1431 tee_mm_free(mm); 1432 } 1433 1434 out_set_rc: 1435 if (rc < 0) { 1436 ret_fid = FFA_ERROR; 1437 ret_w2 = rc; 1438 } else if (rc > 0) { 1439 ret_fid = FFA_MEM_FRAG_RX; 1440 ret_w3 = rc; 1441 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1442 } else { 1443 ret_fid = FFA_SUCCESS_32; 1444 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1445 } 1446 1447 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1448 } 1449 1450 static void handle_mem_reclaim(struct thread_smc_args *args) 1451 { 1452 int rc = FFA_INVALID_PARAMETERS; 1453 uint64_t cookie = 0; 1454 1455 if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) 1456 goto out; 1457 1458 cookie = reg_pair_to_64(args->a2, args->a1); 1459 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1460 uint16_t guest_id = 0; 1461 1462 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 1463 guest_id = virt_find_guest_by_cookie(cookie); 1464 } else { 1465 guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & 1466 FFA_MEMORY_HANDLE_PRTN_MASK; 1467 } 1468 if (!guest_id) 1469 goto out; 1470 if (virt_set_guest(guest_id)) { 1471 if (!virt_reclaim_cookie_from_destroyed_guest(guest_id, 1472 cookie)) 1473 rc = FFA_OK; 1474 goto out; 1475 } 1476 } 1477 1478 switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { 1479 case TEE_SUCCESS: 1480 rc = FFA_OK; 1481 break; 1482 case TEE_ERROR_ITEM_NOT_FOUND: 1483 DMSG("cookie %#"PRIx64" not found", cookie); 1484 rc = FFA_INVALID_PARAMETERS; 1485 break; 1486 default: 1487 DMSG("cookie %#"PRIx64" busy", cookie); 1488 rc = FFA_DENIED; 1489 break; 1490 } 1491 1492 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1493 virt_unset_guest(); 1494 1495 out: 1496 set_simple_ret_val(args, rc); 1497 } 1498 1499 static void handle_notification_bitmap_create(struct thread_smc_args *args) 1500 { 1501 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1502 uint32_t ret_fid = FFA_ERROR; 1503 uint32_t old_itr_status = 0; 1504 1505 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1506 !args->a5 && !args->a6 && !args->a7) { 1507 struct guest_partition *prtn = NULL; 1508 struct notif_vm_bitmap *nvb = NULL; 1509 uint16_t vm_id = args->a1; 1510 1511 prtn = virt_get_guest(vm_id); 1512 nvb = get_notif_vm_bitmap(prtn, vm_id); 1513 if (!nvb) { 1514 ret_val = FFA_INVALID_PARAMETERS; 1515 goto out_virt_put; 1516 } 1517 1518 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1519 1520 if (nvb->initialized) { 1521 ret_val = FFA_DENIED; 1522 goto out_unlock; 1523 } 1524 1525 nvb->initialized = true; 1526 nvb->do_bottom_half_value = -1; 1527 ret_val = FFA_OK; 1528 ret_fid = FFA_SUCCESS_32; 1529 out_unlock: 1530 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1531 out_virt_put: 1532 virt_put_guest(prtn); 1533 } 1534 1535 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1536 } 1537 1538 static void handle_notification_bitmap_destroy(struct thread_smc_args *args) 1539 { 1540 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1541 uint32_t ret_fid = FFA_ERROR; 1542 uint32_t old_itr_status = 0; 1543 1544 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1545 !args->a5 && !args->a6 && !args->a7) { 1546 struct guest_partition *prtn = NULL; 1547 struct notif_vm_bitmap *nvb = NULL; 1548 uint16_t vm_id = args->a1; 1549 1550 prtn = virt_get_guest(vm_id); 1551 nvb = get_notif_vm_bitmap(prtn, vm_id); 1552 if (!nvb) { 1553 ret_val = FFA_INVALID_PARAMETERS; 1554 goto out_virt_put; 1555 } 1556 1557 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1558 1559 if (nvb->pending || nvb->bound) { 1560 ret_val = FFA_DENIED; 1561 goto out_unlock; 1562 } 1563 1564 memset(nvb, 0, sizeof(*nvb)); 1565 ret_val = FFA_OK; 1566 ret_fid = FFA_SUCCESS_32; 1567 out_unlock: 1568 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1569 out_virt_put: 1570 virt_put_guest(prtn); 1571 } 1572 1573 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1574 } 1575 1576 static void handle_notification_bind(struct thread_smc_args *args) 1577 { 1578 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1579 struct guest_partition *prtn = NULL; 1580 struct notif_vm_bitmap *nvb = NULL; 1581 uint32_t ret_fid = FFA_ERROR; 1582 uint32_t old_itr_status = 0; 1583 uint64_t bitmap = 0; 1584 uint16_t vm_id = 0; 1585 1586 if (args->a5 || args->a6 || args->a7) 1587 goto out; 1588 if (args->a2) { 1589 /* We only deal with global notifications */ 1590 ret_val = FFA_DENIED; 1591 goto out; 1592 } 1593 1594 /* The destination of the eventual notification */ 1595 vm_id = FFA_DST(args->a1); 1596 bitmap = reg_pair_to_64(args->a4, args->a3); 1597 1598 prtn = virt_get_guest(vm_id); 1599 nvb = get_notif_vm_bitmap(prtn, vm_id); 1600 if (!nvb) { 1601 ret_val = FFA_INVALID_PARAMETERS; 1602 goto out_virt_put; 1603 } 1604 1605 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1606 1607 if ((bitmap & nvb->bound)) { 1608 ret_val = FFA_DENIED; 1609 } else { 1610 nvb->bound |= bitmap; 1611 ret_val = FFA_OK; 1612 ret_fid = FFA_SUCCESS_32; 1613 } 1614 1615 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1616 out_virt_put: 1617 virt_put_guest(prtn); 1618 out: 1619 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1620 } 1621 1622 static void handle_notification_unbind(struct thread_smc_args *args) 1623 { 1624 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1625 struct guest_partition *prtn = NULL; 1626 struct notif_vm_bitmap *nvb = NULL; 1627 uint32_t ret_fid = FFA_ERROR; 1628 uint32_t old_itr_status = 0; 1629 uint64_t bitmap = 0; 1630 uint16_t vm_id = 0; 1631 1632 if (args->a2 || args->a5 || args->a6 || args->a7) 1633 goto out; 1634 1635 /* The destination of the eventual notification */ 1636 vm_id = FFA_DST(args->a1); 1637 bitmap = reg_pair_to_64(args->a4, args->a3); 1638 1639 prtn = virt_get_guest(vm_id); 1640 nvb = get_notif_vm_bitmap(prtn, vm_id); 1641 if (!nvb) { 1642 ret_val = FFA_INVALID_PARAMETERS; 1643 goto out_virt_put; 1644 } 1645 1646 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1647 1648 if (bitmap & nvb->pending) { 1649 ret_val = FFA_DENIED; 1650 } else { 1651 nvb->bound &= ~bitmap; 1652 ret_val = FFA_OK; 1653 ret_fid = FFA_SUCCESS_32; 1654 } 1655 1656 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1657 out_virt_put: 1658 virt_put_guest(prtn); 1659 out: 1660 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1661 } 1662 1663 static void handle_notification_get(struct thread_smc_args *args) 1664 { 1665 uint32_t w2 = FFA_INVALID_PARAMETERS; 1666 struct guest_partition *prtn = NULL; 1667 struct notif_vm_bitmap *nvb = NULL; 1668 uint32_t ret_fid = FFA_ERROR; 1669 uint32_t old_itr_status = 0; 1670 uint16_t vm_id = 0; 1671 uint32_t w3 = 0; 1672 1673 if (args->a5 || args->a6 || args->a7) 1674 goto out; 1675 if (!(args->a2 & 0x1)) { 1676 ret_fid = FFA_SUCCESS_32; 1677 w2 = 0; 1678 goto out; 1679 } 1680 vm_id = FFA_DST(args->a1); 1681 1682 prtn = virt_get_guest(vm_id); 1683 nvb = get_notif_vm_bitmap(prtn, vm_id); 1684 if (!nvb) 1685 goto out_virt_put; 1686 1687 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1688 1689 reg_pair_from_64(nvb->pending, &w3, &w2); 1690 nvb->pending = 0; 1691 ret_fid = FFA_SUCCESS_32; 1692 1693 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1694 out_virt_put: 1695 virt_put_guest(prtn); 1696 out: 1697 spmc_set_args(args, ret_fid, 0, w2, w3, 0, 0); 1698 } 1699 1700 struct notif_info_get_state { 1701 struct thread_smc_args *args; 1702 unsigned int ids_per_reg; 1703 unsigned int ids_count; 1704 unsigned int id_pos; 1705 unsigned int count; 1706 unsigned int max_list_count; 1707 unsigned int list_count; 1708 }; 1709 1710 static unsigned long get_smc_arg(struct thread_smc_args *args, unsigned int idx) 1711 { 1712 switch (idx) { 1713 case 0: 1714 return args->a0; 1715 case 1: 1716 return args->a1; 1717 case 2: 1718 return args->a2; 1719 case 3: 1720 return args->a3; 1721 case 4: 1722 return args->a4; 1723 case 5: 1724 return args->a5; 1725 case 6: 1726 return args->a6; 1727 case 7: 1728 return args->a7; 1729 default: 1730 assert(0); 1731 return 0; 1732 } 1733 } 1734 1735 static void set_smc_arg(struct thread_smc_args *args, unsigned int idx, 1736 unsigned long val) 1737 { 1738 switch (idx) { 1739 case 0: 1740 args->a0 = val; 1741 break; 1742 case 1: 1743 args->a1 = val; 1744 break; 1745 case 2: 1746 args->a2 = val; 1747 break; 1748 case 3: 1749 args->a3 = val; 1750 break; 1751 case 4: 1752 args->a4 = val; 1753 break; 1754 case 5: 1755 args->a5 = val; 1756 break; 1757 case 6: 1758 args->a6 = val; 1759 break; 1760 case 7: 1761 args->a7 = val; 1762 break; 1763 default: 1764 assert(0); 1765 } 1766 } 1767 1768 static bool add_id_in_regs(struct notif_info_get_state *state, 1769 uint16_t id) 1770 { 1771 unsigned int reg_idx = state->id_pos / state->ids_per_reg + 3; 1772 unsigned int reg_shift = (state->id_pos % state->ids_per_reg) * 16; 1773 unsigned long v; 1774 1775 if (reg_idx > 7) 1776 return false; 1777 1778 v = get_smc_arg(state->args, reg_idx); 1779 v &= ~(0xffffUL << reg_shift); 1780 v |= (unsigned long)id << reg_shift; 1781 set_smc_arg(state->args, reg_idx, v); 1782 1783 state->id_pos++; 1784 state->count++; 1785 return true; 1786 } 1787 1788 static bool add_id_count(struct notif_info_get_state *state) 1789 { 1790 assert(state->list_count < state->max_list_count && 1791 state->count >= 1 && state->count <= 4); 1792 1793 state->ids_count |= (state->count - 1) << (state->list_count * 2 + 12); 1794 state->list_count++; 1795 state->count = 0; 1796 1797 return state->list_count < state->max_list_count; 1798 } 1799 1800 static bool add_nvb_to_state(struct notif_info_get_state *state, 1801 uint16_t guest_id, struct notif_vm_bitmap *nvb) 1802 { 1803 if (!nvb->pending) 1804 return true; 1805 /* 1806 * Add only the guest_id, meaning a global notification for this 1807 * guest. 1808 * 1809 * If notifications for one or more specific vCPUs we'd add those 1810 * before calling add_id_count(), but that's not supported. 1811 */ 1812 return add_id_in_regs(state, guest_id) && add_id_count(state); 1813 } 1814 1815 static void handle_notification_info_get(struct thread_smc_args *args) 1816 { 1817 struct notif_info_get_state state = { .args = args }; 1818 uint32_t ffa_res = FFA_INVALID_PARAMETERS; 1819 struct guest_partition *prtn = NULL; 1820 struct notif_vm_bitmap *nvb = NULL; 1821 uint32_t more_pending_flag = 0; 1822 uint32_t itr_state = 0; 1823 uint16_t guest_id = 0; 1824 1825 if (args->a1 || args->a2 || args->a3 || args->a4 || args->a5 || 1826 args->a6 || args->a7) 1827 goto err; 1828 1829 if (OPTEE_SMC_IS_64(args->a0)) { 1830 spmc_set_args(args, FFA_SUCCESS_64, 0, 0, 0, 0, 0); 1831 state.ids_per_reg = 4; 1832 state.max_list_count = 31; 1833 } else { 1834 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 1835 state.ids_per_reg = 2; 1836 state.max_list_count = 15; 1837 } 1838 1839 while (true) { 1840 /* 1841 * With NS-Virtualization we need to go through all 1842 * partitions to collect the notification bitmaps, without 1843 * we just check the only notification bitmap we have. 1844 */ 1845 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1846 prtn = virt_next_guest(prtn); 1847 if (!prtn) 1848 break; 1849 guest_id = virt_get_guest_id(prtn); 1850 } 1851 nvb = get_notif_vm_bitmap(prtn, guest_id); 1852 1853 itr_state = cpu_spin_lock_xsave(&spmc_notif_lock); 1854 if (!add_nvb_to_state(&state, guest_id, nvb)) 1855 more_pending_flag = BIT(0); 1856 cpu_spin_unlock_xrestore(&spmc_notif_lock, itr_state); 1857 1858 if (!IS_ENABLED(CFG_NS_VIRTUALIZATION) || more_pending_flag) 1859 break; 1860 } 1861 virt_put_guest(prtn); 1862 1863 if (!state.id_pos) { 1864 ffa_res = FFA_NO_DATA; 1865 goto err; 1866 } 1867 args->a2 = (state.list_count << FFA_NOTIF_INFO_GET_ID_COUNT_SHIFT) | 1868 (state.ids_count << FFA_NOTIF_INFO_GET_ID_LIST_SHIFT) | 1869 more_pending_flag; 1870 return; 1871 err: 1872 spmc_set_args(args, FFA_ERROR, 0, ffa_res, 0, 0, 0); 1873 } 1874 1875 void thread_spmc_set_async_notif_intid(int intid) 1876 { 1877 assert(interrupt_can_raise_sgi(interrupt_get_main_chip())); 1878 notif_intid = intid; 1879 spmc_notif_is_ready = true; 1880 DMSG("Asynchronous notifications are ready"); 1881 } 1882 1883 void notif_send_async(uint32_t value, uint16_t guest_id) 1884 { 1885 struct guest_partition *prtn = NULL; 1886 struct notif_vm_bitmap *nvb = NULL; 1887 uint32_t old_itr_status = 0; 1888 1889 prtn = virt_get_guest(guest_id); 1890 nvb = get_notif_vm_bitmap(prtn, guest_id); 1891 1892 if (nvb) { 1893 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1894 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && 1895 spmc_notif_is_ready && nvb->do_bottom_half_value >= 0 && 1896 notif_intid >= 0); 1897 nvb->pending |= BIT64(nvb->do_bottom_half_value); 1898 interrupt_raise_sgi(interrupt_get_main_chip(), notif_intid, 1899 ITR_CPU_MASK_TO_THIS_CPU); 1900 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1901 } 1902 1903 virt_put_guest(prtn); 1904 } 1905 #else 1906 void notif_send_async(uint32_t value, uint16_t guest_id) 1907 { 1908 struct guest_partition *prtn = NULL; 1909 struct notif_vm_bitmap *nvb = NULL; 1910 /* global notification, delay notification interrupt */ 1911 uint32_t flags = BIT32(1); 1912 int res = 0; 1913 1914 prtn = virt_get_guest(guest_id); 1915 nvb = get_notif_vm_bitmap(prtn, guest_id); 1916 1917 if (nvb) { 1918 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && 1919 spmc_notif_is_ready && nvb->do_bottom_half_value >= 0); 1920 res = ffa_set_notification(guest_id, optee_endpoint_id, flags, 1921 BIT64(nvb->do_bottom_half_value)); 1922 if (res) { 1923 EMSG("notification set failed with error %d", res); 1924 panic(); 1925 } 1926 } 1927 1928 virt_put_guest(prtn); 1929 } 1930 #endif 1931 1932 /* Only called from assembly */ 1933 void thread_spmc_msg_recv(struct thread_smc_args *args); 1934 void thread_spmc_msg_recv(struct thread_smc_args *args) 1935 { 1936 assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); 1937 switch (args->a0) { 1938 #if defined(CFG_CORE_SEL1_SPMC) 1939 case FFA_FEATURES: 1940 handle_features(args); 1941 break; 1942 case FFA_SPM_ID_GET: 1943 spmc_handle_spm_id_get(args); 1944 break; 1945 #ifdef ARM64 1946 case FFA_RXTX_MAP_64: 1947 #endif 1948 case FFA_RXTX_MAP_32: 1949 spmc_handle_rxtx_map(args, &my_rxtx); 1950 break; 1951 case FFA_RXTX_UNMAP: 1952 spmc_handle_rxtx_unmap(args, &my_rxtx); 1953 break; 1954 case FFA_RX_RELEASE: 1955 spmc_handle_rx_release(args, &my_rxtx); 1956 break; 1957 case FFA_PARTITION_INFO_GET: 1958 spmc_handle_partition_info_get(args, &my_rxtx); 1959 break; 1960 case FFA_RUN: 1961 spmc_handle_run(args); 1962 break; 1963 #endif /*CFG_CORE_SEL1_SPMC*/ 1964 case FFA_INTERRUPT: 1965 if (IS_ENABLED(CFG_CORE_SEL1_SPMC)) 1966 spmc_set_args(args, FFA_NORMAL_WORLD_RESUME, 0, 0, 0, 1967 0, 0); 1968 else 1969 spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); 1970 break; 1971 #ifdef ARM64 1972 case FFA_MSG_SEND_DIRECT_REQ_64: 1973 #endif 1974 case FFA_MSG_SEND_DIRECT_REQ_32: 1975 handle_direct_request(args, &my_rxtx); 1976 break; 1977 #if defined(CFG_CORE_SEL1_SPMC) 1978 #ifdef ARM64 1979 case FFA_MEM_SHARE_64: 1980 #endif 1981 case FFA_MEM_SHARE_32: 1982 handle_mem_share(args, &my_rxtx); 1983 break; 1984 case FFA_MEM_RECLAIM: 1985 if (!IS_ENABLED(CFG_SECURE_PARTITION) || 1986 !ffa_mem_reclaim(args, NULL)) 1987 handle_mem_reclaim(args); 1988 break; 1989 case FFA_MEM_FRAG_TX: 1990 handle_mem_frag_tx(args, &my_rxtx); 1991 break; 1992 case FFA_NOTIFICATION_BITMAP_CREATE: 1993 handle_notification_bitmap_create(args); 1994 break; 1995 case FFA_NOTIFICATION_BITMAP_DESTROY: 1996 handle_notification_bitmap_destroy(args); 1997 break; 1998 case FFA_NOTIFICATION_BIND: 1999 handle_notification_bind(args); 2000 break; 2001 case FFA_NOTIFICATION_UNBIND: 2002 handle_notification_unbind(args); 2003 break; 2004 case FFA_NOTIFICATION_GET: 2005 handle_notification_get(args); 2006 break; 2007 #ifdef ARM64 2008 case FFA_NOTIFICATION_INFO_GET_64: 2009 #endif 2010 case FFA_NOTIFICATION_INFO_GET_32: 2011 handle_notification_info_get(args); 2012 break; 2013 #endif /*CFG_CORE_SEL1_SPMC*/ 2014 case FFA_ERROR: 2015 EMSG("Cannot handle FFA_ERROR(%d)", (int)args->a2); 2016 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { 2017 /* 2018 * The SPMC will return an FFA_ERROR back so better 2019 * panic() now than flooding the log. 2020 */ 2021 panic("FFA_ERROR from SPMC is fatal"); 2022 } 2023 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, 2024 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 2025 break; 2026 default: 2027 EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); 2028 set_simple_ret_val(args, FFA_NOT_SUPPORTED); 2029 } 2030 } 2031 2032 static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) 2033 { 2034 size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 2035 struct thread_ctx *thr = threads + thread_get_id(); 2036 TEE_Result res = TEE_ERROR_BAD_PARAMETERS; 2037 struct optee_msg_arg *arg = NULL; 2038 struct mobj *mobj = NULL; 2039 uint32_t num_params = 0; 2040 size_t sz = 0; 2041 2042 mobj = mobj_ffa_get_by_cookie(cookie, 0); 2043 if (!mobj) { 2044 EMSG("Can't find cookie %#"PRIx64, cookie); 2045 return TEE_ERROR_BAD_PARAMETERS; 2046 } 2047 2048 res = mobj_inc_map(mobj); 2049 if (res) 2050 goto out_put_mobj; 2051 2052 res = TEE_ERROR_BAD_PARAMETERS; 2053 arg = mobj_get_va(mobj, offset, sizeof(*arg)); 2054 if (!arg) 2055 goto out_dec_map; 2056 2057 num_params = READ_ONCE(arg->num_params); 2058 if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) 2059 goto out_dec_map; 2060 2061 sz = OPTEE_MSG_GET_ARG_SIZE(num_params); 2062 2063 thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); 2064 if (!thr->rpc_arg) 2065 goto out_dec_map; 2066 2067 virt_on_stdcall(); 2068 res = tee_entry_std(arg, num_params); 2069 2070 thread_rpc_shm_cache_clear(&thr->shm_cache); 2071 thr->rpc_arg = NULL; 2072 2073 out_dec_map: 2074 mobj_dec_map(mobj); 2075 out_put_mobj: 2076 mobj_put(mobj); 2077 return res; 2078 } 2079 2080 /* 2081 * Helper routine for the assembly function thread_std_smc_entry() 2082 * 2083 * Note: this function is weak just to make link_dummies_paged.c happy. 2084 */ 2085 uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, 2086 uint32_t a2, uint32_t a3, 2087 uint32_t a4, uint32_t a5 __unused) 2088 { 2089 /* 2090 * Arguments are supplied from handle_yielding_call() as: 2091 * a0 <- w1 2092 * a1 <- w3 2093 * a2 <- w4 2094 * a3 <- w5 2095 * a4 <- w6 2096 * a5 <- w7 2097 */ 2098 thread_get_tsd()->rpc_target_info = swap_src_dst(a0); 2099 if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) 2100 return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); 2101 return FFA_DENIED; 2102 } 2103 2104 static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) 2105 { 2106 uint64_t offs = tpm->u.memref.offs; 2107 2108 param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + 2109 OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; 2110 2111 param->u.fmem.offs_low = offs; 2112 param->u.fmem.offs_high = offs >> 32; 2113 if (param->u.fmem.offs_high != offs >> 32) 2114 return false; 2115 2116 param->u.fmem.size = tpm->u.memref.size; 2117 if (tpm->u.memref.mobj) { 2118 uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); 2119 2120 /* If a mobj is passed it better be one with a valid cookie. */ 2121 if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) 2122 return false; 2123 param->u.fmem.global_id = cookie; 2124 } else { 2125 param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 2126 } 2127 2128 return true; 2129 } 2130 2131 static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, 2132 struct thread_param *params, 2133 struct optee_msg_arg **arg_ret) 2134 { 2135 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 2136 struct thread_ctx *thr = threads + thread_get_id(); 2137 struct optee_msg_arg *arg = thr->rpc_arg; 2138 2139 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 2140 return TEE_ERROR_BAD_PARAMETERS; 2141 2142 if (!arg) { 2143 EMSG("rpc_arg not set"); 2144 return TEE_ERROR_GENERIC; 2145 } 2146 2147 memset(arg, 0, sz); 2148 arg->cmd = cmd; 2149 arg->num_params = num_params; 2150 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 2151 2152 for (size_t n = 0; n < num_params; n++) { 2153 switch (params[n].attr) { 2154 case THREAD_PARAM_ATTR_NONE: 2155 arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; 2156 break; 2157 case THREAD_PARAM_ATTR_VALUE_IN: 2158 case THREAD_PARAM_ATTR_VALUE_OUT: 2159 case THREAD_PARAM_ATTR_VALUE_INOUT: 2160 arg->params[n].attr = params[n].attr - 2161 THREAD_PARAM_ATTR_VALUE_IN + 2162 OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 2163 arg->params[n].u.value.a = params[n].u.value.a; 2164 arg->params[n].u.value.b = params[n].u.value.b; 2165 arg->params[n].u.value.c = params[n].u.value.c; 2166 break; 2167 case THREAD_PARAM_ATTR_MEMREF_IN: 2168 case THREAD_PARAM_ATTR_MEMREF_OUT: 2169 case THREAD_PARAM_ATTR_MEMREF_INOUT: 2170 if (!set_fmem(arg->params + n, params + n)) 2171 return TEE_ERROR_BAD_PARAMETERS; 2172 break; 2173 default: 2174 return TEE_ERROR_BAD_PARAMETERS; 2175 } 2176 } 2177 2178 if (arg_ret) 2179 *arg_ret = arg; 2180 2181 return TEE_SUCCESS; 2182 } 2183 2184 static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, 2185 struct thread_param *params) 2186 { 2187 for (size_t n = 0; n < num_params; n++) { 2188 switch (params[n].attr) { 2189 case THREAD_PARAM_ATTR_VALUE_OUT: 2190 case THREAD_PARAM_ATTR_VALUE_INOUT: 2191 params[n].u.value.a = arg->params[n].u.value.a; 2192 params[n].u.value.b = arg->params[n].u.value.b; 2193 params[n].u.value.c = arg->params[n].u.value.c; 2194 break; 2195 case THREAD_PARAM_ATTR_MEMREF_OUT: 2196 case THREAD_PARAM_ATTR_MEMREF_INOUT: 2197 params[n].u.memref.size = arg->params[n].u.fmem.size; 2198 break; 2199 default: 2200 break; 2201 } 2202 } 2203 2204 return arg->ret; 2205 } 2206 2207 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 2208 struct thread_param *params) 2209 { 2210 struct thread_rpc_arg rpc_arg = { .call = { 2211 .w1 = thread_get_tsd()->rpc_target_info, 2212 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2213 }, 2214 }; 2215 struct optee_msg_arg *arg = NULL; 2216 uint32_t ret = 0; 2217 2218 ret = get_rpc_arg(cmd, num_params, params, &arg); 2219 if (ret) 2220 return ret; 2221 2222 thread_rpc(&rpc_arg); 2223 2224 return get_rpc_arg_res(arg, num_params, params); 2225 } 2226 2227 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 2228 { 2229 struct thread_rpc_arg rpc_arg = { .call = { 2230 .w1 = thread_get_tsd()->rpc_target_info, 2231 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2232 }, 2233 }; 2234 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); 2235 uint32_t res2 = 0; 2236 uint32_t res = 0; 2237 2238 DMSG("freeing cookie %#"PRIx64, cookie); 2239 2240 res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); 2241 2242 mobj_put(mobj); 2243 res2 = mobj_ffa_unregister_by_cookie(cookie); 2244 if (res2) 2245 DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, 2246 cookie, res2); 2247 if (!res) 2248 thread_rpc(&rpc_arg); 2249 } 2250 2251 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) 2252 { 2253 struct thread_rpc_arg rpc_arg = { .call = { 2254 .w1 = thread_get_tsd()->rpc_target_info, 2255 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2256 }, 2257 }; 2258 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); 2259 struct optee_msg_arg *arg = NULL; 2260 unsigned int internal_offset = 0; 2261 struct mobj *mobj = NULL; 2262 uint64_t cookie = 0; 2263 2264 if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) 2265 return NULL; 2266 2267 thread_rpc(&rpc_arg); 2268 2269 if (arg->num_params != 1 || 2270 arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) 2271 return NULL; 2272 2273 internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); 2274 cookie = READ_ONCE(arg->params->u.fmem.global_id); 2275 mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); 2276 if (!mobj) { 2277 DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", 2278 cookie, internal_offset); 2279 return NULL; 2280 } 2281 2282 assert(mobj_is_nonsec(mobj)); 2283 2284 if (mobj->size < size) { 2285 DMSG("Mobj %#"PRIx64": wrong size", cookie); 2286 mobj_put(mobj); 2287 return NULL; 2288 } 2289 2290 if (mobj_inc_map(mobj)) { 2291 DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); 2292 mobj_put(mobj); 2293 return NULL; 2294 } 2295 2296 return mobj; 2297 } 2298 2299 struct mobj *thread_rpc_alloc_payload(size_t size) 2300 { 2301 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); 2302 } 2303 2304 struct mobj *thread_rpc_alloc_kernel_payload(size_t size) 2305 { 2306 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); 2307 } 2308 2309 void thread_rpc_free_kernel_payload(struct mobj *mobj) 2310 { 2311 if (mobj) 2312 thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, 2313 mobj_get_cookie(mobj), mobj); 2314 } 2315 2316 void thread_rpc_free_payload(struct mobj *mobj) 2317 { 2318 if (mobj) 2319 thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), 2320 mobj); 2321 } 2322 2323 struct mobj *thread_rpc_alloc_global_payload(size_t size) 2324 { 2325 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); 2326 } 2327 2328 void thread_rpc_free_global_payload(struct mobj *mobj) 2329 { 2330 if (mobj) 2331 thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, 2332 mobj_get_cookie(mobj), mobj); 2333 } 2334 2335 void thread_spmc_register_secondary_ep(vaddr_t ep) 2336 { 2337 unsigned long ret = 0; 2338 2339 /* Let the SPM know the entry point for secondary CPUs */ 2340 ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); 2341 2342 if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) 2343 EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); 2344 } 2345 2346 static uint16_t ffa_id_get(void) 2347 { 2348 /* 2349 * Ask the SPM component running at a higher EL to return our FF-A ID. 2350 * This can either be the SPMC ID (if the SPMC is enabled in OP-TEE) or 2351 * the partition ID (if not). 2352 */ 2353 struct thread_smc_args args = { 2354 .a0 = FFA_ID_GET, 2355 }; 2356 2357 thread_smccc(&args); 2358 if (!is_ffa_success(args.a0)) { 2359 if (args.a0 == FFA_ERROR) 2360 EMSG("Get id failed with error %ld", args.a2); 2361 else 2362 EMSG("Get id failed"); 2363 panic(); 2364 } 2365 2366 return args.a2; 2367 } 2368 2369 static uint16_t ffa_spm_id_get(void) 2370 { 2371 /* 2372 * Ask the SPM component running at a higher EL to return its ID. 2373 * If OP-TEE implements the S-EL1 SPMC, this will get the SPMD ID. 2374 * If not, the ID of the SPMC will be returned. 2375 */ 2376 struct thread_smc_args args = { 2377 .a0 = FFA_SPM_ID_GET, 2378 }; 2379 2380 thread_smccc(&args); 2381 if (!is_ffa_success(args.a0)) { 2382 if (args.a0 == FFA_ERROR) 2383 EMSG("Get spm id failed with error %ld", args.a2); 2384 else 2385 EMSG("Get spm id failed"); 2386 panic(); 2387 } 2388 2389 return args.a2; 2390 } 2391 2392 #if defined(CFG_CORE_SEL1_SPMC) 2393 static TEE_Result spmc_init(void) 2394 { 2395 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 2396 virt_add_guest_spec_data(¬if_vm_bitmap_id, 2397 sizeof(struct notif_vm_bitmap), NULL)) 2398 panic("virt_add_guest_spec_data"); 2399 spmd_id = ffa_spm_id_get(); 2400 DMSG("SPMD ID %#"PRIx16, spmd_id); 2401 2402 spmc_id = ffa_id_get(); 2403 DMSG("SPMC ID %#"PRIx16, spmc_id); 2404 2405 optee_endpoint_id = FFA_SWD_ID_MIN; 2406 while (optee_endpoint_id == spmd_id || optee_endpoint_id == spmc_id) 2407 optee_endpoint_id++; 2408 2409 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2410 2411 /* 2412 * If SPMD think we are version 1.0 it will report version 1.0 to 2413 * normal world regardless of what version we query the SPM with. 2414 * However, if SPMD think we are version 1.1 it will forward 2415 * queries from normal world to let us negotiate version. So by 2416 * setting version 1.0 here we should be compatible. 2417 * 2418 * Note that disagreement on negotiated version means that we'll 2419 * have communication problems with normal world. 2420 */ 2421 my_rxtx.ffa_vers = FFA_VERSION_1_0; 2422 2423 return TEE_SUCCESS; 2424 } 2425 #else /* !defined(CFG_CORE_SEL1_SPMC) */ 2426 static void spmc_rxtx_map(struct ffa_rxtx *rxtx) 2427 { 2428 struct thread_smc_args args = { 2429 #ifdef ARM64 2430 .a0 = FFA_RXTX_MAP_64, 2431 #else 2432 .a0 = FFA_RXTX_MAP_32, 2433 #endif 2434 .a1 = virt_to_phys(rxtx->tx), 2435 .a2 = virt_to_phys(rxtx->rx), 2436 .a3 = 1, 2437 }; 2438 2439 thread_smccc(&args); 2440 if (!is_ffa_success(args.a0)) { 2441 if (args.a0 == FFA_ERROR) 2442 EMSG("rxtx map failed with error %ld", args.a2); 2443 else 2444 EMSG("rxtx map failed"); 2445 panic(); 2446 } 2447 } 2448 2449 static uint32_t get_ffa_version(uint32_t my_version) 2450 { 2451 struct thread_smc_args args = { 2452 .a0 = FFA_VERSION, 2453 .a1 = my_version, 2454 }; 2455 2456 thread_smccc(&args); 2457 if (args.a0 & BIT(31)) { 2458 EMSG("FF-A version failed with error %ld", args.a0); 2459 panic(); 2460 } 2461 2462 return args.a0; 2463 } 2464 2465 static void *spmc_retrieve_req(uint64_t cookie, 2466 struct ffa_mem_transaction_x *trans) 2467 { 2468 struct ffa_mem_access *acc_descr_array = NULL; 2469 struct ffa_mem_access_perm *perm_descr = NULL; 2470 struct thread_smc_args args = { 2471 .a0 = FFA_MEM_RETRIEVE_REQ_32, 2472 .a3 = 0, /* Address, Using TX -> MBZ */ 2473 .a4 = 0, /* Using TX -> MBZ */ 2474 }; 2475 size_t size = 0; 2476 int rc = 0; 2477 2478 if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { 2479 struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; 2480 2481 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2482 memset(trans_descr, 0, size); 2483 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2484 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2485 trans_descr->global_handle = cookie; 2486 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2487 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2488 trans_descr->mem_access_count = 1; 2489 acc_descr_array = trans_descr->mem_access_array; 2490 } else { 2491 struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; 2492 2493 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2494 memset(trans_descr, 0, size); 2495 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2496 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2497 trans_descr->global_handle = cookie; 2498 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2499 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2500 trans_descr->mem_access_count = 1; 2501 trans_descr->mem_access_offs = sizeof(*trans_descr); 2502 trans_descr->mem_access_size = sizeof(struct ffa_mem_access); 2503 acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + 2504 sizeof(*trans_descr)); 2505 } 2506 acc_descr_array->region_offs = 0; 2507 acc_descr_array->reserved = 0; 2508 perm_descr = &acc_descr_array->access_perm; 2509 perm_descr->endpoint_id = optee_endpoint_id; 2510 perm_descr->perm = FFA_MEM_ACC_RW; 2511 perm_descr->flags = 0; 2512 2513 args.a1 = size; /* Total Length */ 2514 args.a2 = size; /* Frag Length == Total length */ 2515 thread_smccc(&args); 2516 if (args.a0 != FFA_MEM_RETRIEVE_RESP) { 2517 if (args.a0 == FFA_ERROR) 2518 EMSG("Failed to fetch cookie %#"PRIx64" error code %d", 2519 cookie, (int)args.a2); 2520 else 2521 EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, 2522 cookie, args.a0); 2523 return NULL; 2524 } 2525 rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.rx, 2526 my_rxtx.size, trans); 2527 if (rc) { 2528 EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", 2529 cookie, rc); 2530 return NULL; 2531 } 2532 2533 return my_rxtx.rx; 2534 } 2535 2536 void thread_spmc_relinquish(uint64_t cookie) 2537 { 2538 struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; 2539 struct thread_smc_args args = { 2540 .a0 = FFA_MEM_RELINQUISH, 2541 }; 2542 2543 memset(relinquish_desc, 0, sizeof(*relinquish_desc)); 2544 relinquish_desc->handle = cookie; 2545 relinquish_desc->flags = 0; 2546 relinquish_desc->endpoint_count = 1; 2547 relinquish_desc->endpoint_id_array[0] = optee_endpoint_id; 2548 thread_smccc(&args); 2549 if (!is_ffa_success(args.a0)) 2550 EMSG("Failed to relinquish cookie %#"PRIx64, cookie); 2551 } 2552 2553 static int set_pages(struct ffa_address_range *regions, 2554 unsigned int num_regions, unsigned int num_pages, 2555 struct mobj_ffa *mf) 2556 { 2557 unsigned int n = 0; 2558 unsigned int idx = 0; 2559 2560 for (n = 0; n < num_regions; n++) { 2561 unsigned int page_count = READ_ONCE(regions[n].page_count); 2562 uint64_t addr = READ_ONCE(regions[n].address); 2563 2564 if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) 2565 return FFA_INVALID_PARAMETERS; 2566 } 2567 2568 if (idx != num_pages) 2569 return FFA_INVALID_PARAMETERS; 2570 2571 return 0; 2572 } 2573 2574 struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) 2575 { 2576 struct mobj_ffa *ret = NULL; 2577 struct ffa_mem_transaction_x retrieve_desc = { }; 2578 struct ffa_mem_access *descr_array = NULL; 2579 struct ffa_mem_region *descr = NULL; 2580 struct mobj_ffa *mf = NULL; 2581 unsigned int num_pages = 0; 2582 unsigned int offs = 0; 2583 void *buf = NULL; 2584 struct thread_smc_args ffa_rx_release_args = { 2585 .a0 = FFA_RX_RELEASE 2586 }; 2587 2588 /* 2589 * OP-TEE is only supporting a single mem_region while the 2590 * specification allows for more than one. 2591 */ 2592 buf = spmc_retrieve_req(cookie, &retrieve_desc); 2593 if (!buf) { 2594 EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, 2595 cookie); 2596 return NULL; 2597 } 2598 2599 descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); 2600 offs = READ_ONCE(descr_array->region_offs); 2601 descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); 2602 2603 num_pages = READ_ONCE(descr->total_page_count); 2604 mf = mobj_ffa_spmc_new(cookie, num_pages); 2605 if (!mf) 2606 goto out; 2607 2608 if (set_pages(descr->address_range_array, 2609 READ_ONCE(descr->address_range_count), num_pages, mf)) { 2610 mobj_ffa_spmc_delete(mf); 2611 goto out; 2612 } 2613 2614 ret = mf; 2615 2616 out: 2617 /* Release RX buffer after the mem retrieve request. */ 2618 thread_smccc(&ffa_rx_release_args); 2619 2620 return ret; 2621 } 2622 2623 static TEE_Result spmc_init(void) 2624 { 2625 unsigned int major = 0; 2626 unsigned int minor __maybe_unused = 0; 2627 uint32_t my_vers = 0; 2628 uint32_t vers = 0; 2629 2630 my_vers = MAKE_FFA_VERSION(FFA_VERSION_MAJOR, FFA_VERSION_MINOR); 2631 vers = get_ffa_version(my_vers); 2632 major = (vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK; 2633 minor = (vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK; 2634 DMSG("SPMC reported version %u.%u", major, minor); 2635 if (major != FFA_VERSION_MAJOR) { 2636 EMSG("Incompatible major version %u, expected %u", 2637 major, FFA_VERSION_MAJOR); 2638 panic(); 2639 } 2640 if (vers < my_vers) 2641 my_vers = vers; 2642 DMSG("Using version %u.%u", 2643 (my_vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK, 2644 (my_vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK); 2645 my_rxtx.ffa_vers = my_vers; 2646 2647 spmc_rxtx_map(&my_rxtx); 2648 2649 spmc_id = ffa_spm_id_get(); 2650 DMSG("SPMC ID %#"PRIx16, spmc_id); 2651 2652 optee_endpoint_id = ffa_id_get(); 2653 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2654 2655 if (!ffa_features(FFA_NOTIFICATION_SET)) { 2656 spmc_notif_is_ready = true; 2657 DMSG("Asynchronous notifications are ready"); 2658 } 2659 2660 return TEE_SUCCESS; 2661 } 2662 #endif /* !defined(CFG_CORE_SEL1_SPMC) */ 2663 2664 nex_service_init(spmc_init); 2665