1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2020-2023, Linaro Limited. 4 * Copyright (c) 2019-2021, Arm Limited. All rights reserved. 5 */ 6 7 #include <assert.h> 8 #include <ffa.h> 9 #include <initcall.h> 10 #include <io.h> 11 #include <kernel/interrupt.h> 12 #include <kernel/notif.h> 13 #include <kernel/panic.h> 14 #include <kernel/secure_partition.h> 15 #include <kernel/spinlock.h> 16 #include <kernel/spmc_sp_handler.h> 17 #include <kernel/tee_misc.h> 18 #include <kernel/thread.h> 19 #include <kernel/thread_private.h> 20 #include <kernel/thread_spmc.h> 21 #include <kernel/virtualization.h> 22 #include <mm/core_mmu.h> 23 #include <mm/mobj.h> 24 #include <optee_ffa.h> 25 #include <optee_msg.h> 26 #include <optee_rpc_cmd.h> 27 #include <sm/optee_smc.h> 28 #include <string.h> 29 #include <sys/queue.h> 30 #include <tee/entry_std.h> 31 #include <tee/uuid.h> 32 #include <util.h> 33 34 #if defined(CFG_CORE_SEL1_SPMC) 35 struct mem_share_state { 36 struct mobj_ffa *mf; 37 unsigned int page_count; 38 unsigned int region_count; 39 unsigned int current_page_idx; 40 }; 41 42 struct mem_frag_state { 43 struct mem_share_state share; 44 tee_mm_entry_t *mm; 45 unsigned int frag_offset; 46 SLIST_ENTRY(mem_frag_state) link; 47 }; 48 #endif 49 50 static unsigned int spmc_notif_lock = SPINLOCK_UNLOCK; 51 static int do_bottom_half_value = -1; 52 static uint16_t notif_vm_id; 53 static bool spmc_notif_is_ready; 54 55 /* Initialized in spmc_init() below */ 56 static uint16_t my_endpoint_id __nex_bss; 57 #ifdef CFG_CORE_SEL1_SPMC 58 static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | 59 FFA_PART_PROP_DIRECT_REQ_SEND | 60 #ifdef CFG_NS_VIRTUALIZATION 61 FFA_PART_PROP_NOTIF_CREATED | 62 FFA_PART_PROP_NOTIF_DESTROYED | 63 #endif 64 #ifdef ARM64 65 FFA_PART_PROP_AARCH64_STATE | 66 #endif 67 FFA_PART_PROP_IS_PE_ID; 68 69 static uint32_t my_uuid_words[] = { 70 /* 71 * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 72 * SP, or 73 * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a 74 * logical partition, residing in the same exception level as the 75 * SPMC 76 * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 77 */ 78 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, 79 }; 80 81 /* 82 * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. 83 * 84 * struct ffa_rxtx::spin_lock protects the variables below from concurrent 85 * access this includes the use of content of struct ffa_rxtx::rx and 86 * @frag_state_head. 87 * 88 * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct 89 * ffa_rxtx::tx and false when it is owned by normal world. 90 * 91 * Note that we can't prevent normal world from updating the content of 92 * these buffers so we must always be careful when reading. while we hold 93 * the lock. 94 */ 95 96 static struct ffa_rxtx my_rxtx __nex_bss; 97 98 static bool is_nw_buf(struct ffa_rxtx *rxtx) 99 { 100 return rxtx == &my_rxtx; 101 } 102 103 static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = 104 SLIST_HEAD_INITIALIZER(&frag_state_head); 105 106 static uint64_t notif_pending_bitmap; 107 static uint64_t notif_bound_bitmap; 108 static bool notif_vm_id_valid; 109 static int notif_intid = -1; 110 #else 111 static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 112 static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 113 static struct ffa_rxtx my_rxtx = { 114 .rx = __rx_buf, 115 .tx = __tx_buf, 116 .size = sizeof(__rx_buf), 117 }; 118 #endif 119 120 static uint32_t swap_src_dst(uint32_t src_dst) 121 { 122 return (src_dst >> 16) | (src_dst << 16); 123 } 124 125 static uint16_t get_sender_id(uint32_t src_dst) 126 { 127 return src_dst >> 16; 128 } 129 130 void spmc_set_args(struct thread_smc_args *args, uint32_t fid, uint32_t src_dst, 131 uint32_t w2, uint32_t w3, uint32_t w4, uint32_t w5) 132 { 133 *args = (struct thread_smc_args){ .a0 = fid, 134 .a1 = src_dst, 135 .a2 = w2, 136 .a3 = w3, 137 .a4 = w4, 138 .a5 = w5, }; 139 } 140 141 static void set_simple_ret_val(struct thread_smc_args *args, int ffa_ret) 142 { 143 if (ffa_ret) 144 spmc_set_args(args, FFA_ERROR, 0, ffa_ret, 0, 0, 0); 145 else 146 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 147 } 148 149 uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) 150 { 151 /* 152 * No locking, if the caller does concurrent calls to this it's 153 * only making a mess for itself. We must be able to renegotiate 154 * the FF-A version in order to support differing versions between 155 * the loader and the driver. 156 */ 157 if (vers < FFA_VERSION_1_1) 158 rxtx->ffa_vers = FFA_VERSION_1_0; 159 else 160 rxtx->ffa_vers = FFA_VERSION_1_1; 161 162 return rxtx->ffa_vers; 163 } 164 165 static bool is_ffa_success(uint32_t fid) 166 { 167 #ifdef ARM64 168 if (fid == FFA_SUCCESS_64) 169 return true; 170 #endif 171 return fid == FFA_SUCCESS_32; 172 } 173 174 static int32_t get_ffa_ret_code(const struct thread_smc_args *args) 175 { 176 if (is_ffa_success(args->a0)) 177 return FFA_OK; 178 if (args->a0 == FFA_ERROR && args->a2) 179 return args->a2; 180 return FFA_NOT_SUPPORTED; 181 } 182 183 static int ffa_simple_call(uint32_t fid, unsigned long a1, unsigned long a2, 184 unsigned long a3, unsigned long a4) 185 { 186 struct thread_smc_args args = { 187 .a0 = fid, 188 .a1 = a1, 189 .a2 = a2, 190 .a3 = a3, 191 .a4 = a4, 192 }; 193 194 thread_smccc(&args); 195 196 return get_ffa_ret_code(&args); 197 } 198 199 static int __maybe_unused ffa_features(uint32_t id) 200 { 201 return ffa_simple_call(FFA_FEATURES, id, 0, 0, 0); 202 } 203 204 static int __maybe_unused ffa_set_notification(uint16_t dst, uint16_t src, 205 uint32_t flags, uint64_t bitmap) 206 { 207 return ffa_simple_call(FFA_NOTIFICATION_SET, 208 SHIFT_U32(src, 16) | dst, flags, 209 low32_from_64(bitmap), high32_from_64(bitmap)); 210 } 211 212 #if defined(CFG_CORE_SEL1_SPMC) 213 static void handle_features(struct thread_smc_args *args) 214 { 215 uint32_t ret_fid = FFA_ERROR; 216 uint32_t ret_w2 = FFA_NOT_SUPPORTED; 217 218 switch (args->a1) { 219 case FFA_FEATURE_SCHEDULE_RECV_INTR: 220 if (spmc_notif_is_ready) { 221 ret_fid = FFA_SUCCESS_32; 222 ret_w2 = notif_intid; 223 } 224 break; 225 226 #ifdef ARM64 227 case FFA_RXTX_MAP_64: 228 #endif 229 case FFA_RXTX_MAP_32: 230 ret_fid = FFA_SUCCESS_32; 231 ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ 232 break; 233 #ifdef ARM64 234 case FFA_MEM_SHARE_64: 235 #endif 236 case FFA_MEM_SHARE_32: 237 ret_fid = FFA_SUCCESS_32; 238 /* 239 * Partition manager supports transmission of a memory 240 * transaction descriptor in a buffer dynamically allocated 241 * by the endpoint. 242 */ 243 ret_w2 = BIT(0); 244 break; 245 246 case FFA_ERROR: 247 case FFA_VERSION: 248 case FFA_SUCCESS_32: 249 #ifdef ARM64 250 case FFA_SUCCESS_64: 251 #endif 252 case FFA_FEATURES: 253 case FFA_SPM_ID_GET: 254 case FFA_MEM_FRAG_TX: 255 case FFA_MEM_RECLAIM: 256 case FFA_MSG_SEND_DIRECT_REQ_64: 257 case FFA_MSG_SEND_DIRECT_REQ_32: 258 case FFA_INTERRUPT: 259 case FFA_PARTITION_INFO_GET: 260 case FFA_RXTX_UNMAP: 261 case FFA_RX_RELEASE: 262 case FFA_FEATURE_MANAGED_EXIT_INTR: 263 case FFA_NOTIFICATION_BITMAP_CREATE: 264 case FFA_NOTIFICATION_BITMAP_DESTROY: 265 case FFA_NOTIFICATION_BIND: 266 case FFA_NOTIFICATION_UNBIND: 267 case FFA_NOTIFICATION_SET: 268 case FFA_NOTIFICATION_GET: 269 case FFA_NOTIFICATION_INFO_GET_32: 270 #ifdef ARM64 271 case FFA_NOTIFICATION_INFO_GET_64: 272 #endif 273 ret_fid = FFA_SUCCESS_32; 274 ret_w2 = FFA_PARAM_MBZ; 275 break; 276 default: 277 break; 278 } 279 280 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, 281 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 282 } 283 284 static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) 285 { 286 tee_mm_entry_t *mm = NULL; 287 288 if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) 289 return FFA_INVALID_PARAMETERS; 290 291 mm = tee_mm_alloc(&tee_mm_shm, sz); 292 if (!mm) 293 return FFA_NO_MEMORY; 294 295 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, 296 sz / SMALL_PAGE_SIZE, 297 MEM_AREA_NSEC_SHM)) { 298 tee_mm_free(mm); 299 return FFA_INVALID_PARAMETERS; 300 } 301 302 *va_ret = (void *)tee_mm_get_smem(mm); 303 return 0; 304 } 305 306 static void handle_spm_id_get(struct thread_smc_args *args) 307 { 308 spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, my_endpoint_id, 309 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 310 } 311 312 static void unmap_buf(void *va, size_t sz) 313 { 314 tee_mm_entry_t *mm = tee_mm_find(&tee_mm_shm, (vaddr_t)va); 315 316 assert(mm); 317 core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); 318 tee_mm_free(mm); 319 } 320 321 void spmc_handle_rxtx_map(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 322 { 323 int rc = 0; 324 unsigned int sz = 0; 325 paddr_t rx_pa = 0; 326 paddr_t tx_pa = 0; 327 void *rx = NULL; 328 void *tx = NULL; 329 330 cpu_spin_lock(&rxtx->spinlock); 331 332 if (args->a3 & GENMASK_64(63, 6)) { 333 rc = FFA_INVALID_PARAMETERS; 334 goto out; 335 } 336 337 sz = args->a3 * SMALL_PAGE_SIZE; 338 if (!sz) { 339 rc = FFA_INVALID_PARAMETERS; 340 goto out; 341 } 342 /* TX/RX are swapped compared to the caller */ 343 tx_pa = args->a2; 344 rx_pa = args->a1; 345 346 if (rxtx->size) { 347 rc = FFA_DENIED; 348 goto out; 349 } 350 351 /* 352 * If the buffer comes from a SP the address is virtual and already 353 * mapped. 354 */ 355 if (is_nw_buf(rxtx)) { 356 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 357 enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; 358 bool tx_alloced = false; 359 360 /* 361 * With virtualization we establish this mapping in 362 * the nexus mapping which then is replicated to 363 * each partition. 364 * 365 * This means that this mapping must be done before 366 * any partition is created and then must not be 367 * changed. 368 */ 369 370 /* 371 * core_mmu_add_mapping() may reuse previous 372 * mappings. First check if there's any mappings to 373 * reuse so we know how to clean up in case of 374 * failure. 375 */ 376 tx = phys_to_virt(tx_pa, mt, sz); 377 rx = phys_to_virt(rx_pa, mt, sz); 378 if (!tx) { 379 tx = core_mmu_add_mapping(mt, tx_pa, sz); 380 if (!tx) { 381 rc = FFA_NO_MEMORY; 382 goto out; 383 } 384 tx_alloced = true; 385 } 386 if (!rx) 387 rx = core_mmu_add_mapping(mt, rx_pa, sz); 388 389 if (!rx) { 390 if (tx_alloced && tx) 391 core_mmu_remove_mapping(mt, tx, sz); 392 rc = FFA_NO_MEMORY; 393 goto out; 394 } 395 } else { 396 rc = map_buf(tx_pa, sz, &tx); 397 if (rc) 398 goto out; 399 rc = map_buf(rx_pa, sz, &rx); 400 if (rc) { 401 unmap_buf(tx, sz); 402 goto out; 403 } 404 } 405 rxtx->tx = tx; 406 rxtx->rx = rx; 407 } else { 408 if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { 409 rc = FFA_INVALID_PARAMETERS; 410 goto out; 411 } 412 413 if (!virt_to_phys((void *)tx_pa) || 414 !virt_to_phys((void *)rx_pa)) { 415 rc = FFA_INVALID_PARAMETERS; 416 goto out; 417 } 418 419 rxtx->tx = (void *)tx_pa; 420 rxtx->rx = (void *)rx_pa; 421 } 422 423 rxtx->size = sz; 424 rxtx->tx_is_mine = true; 425 DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); 426 DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); 427 out: 428 cpu_spin_unlock(&rxtx->spinlock); 429 set_simple_ret_val(args, rc); 430 } 431 432 void spmc_handle_rxtx_unmap(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 433 { 434 int rc = FFA_INVALID_PARAMETERS; 435 436 cpu_spin_lock(&rxtx->spinlock); 437 438 if (!rxtx->size) 439 goto out; 440 441 /* We don't unmap the SP memory as the SP might still use it */ 442 if (is_nw_buf(rxtx)) { 443 unmap_buf(rxtx->rx, rxtx->size); 444 unmap_buf(rxtx->tx, rxtx->size); 445 } 446 rxtx->size = 0; 447 rxtx->rx = NULL; 448 rxtx->tx = NULL; 449 rc = 0; 450 out: 451 cpu_spin_unlock(&rxtx->spinlock); 452 set_simple_ret_val(args, rc); 453 } 454 455 void spmc_handle_rx_release(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 456 { 457 int rc = 0; 458 459 cpu_spin_lock(&rxtx->spinlock); 460 /* The senders RX is our TX */ 461 if (!rxtx->size || rxtx->tx_is_mine) { 462 rc = FFA_DENIED; 463 } else { 464 rc = 0; 465 rxtx->tx_is_mine = true; 466 } 467 cpu_spin_unlock(&rxtx->spinlock); 468 469 set_simple_ret_val(args, rc); 470 } 471 472 static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 473 { 474 return !w0 && !w1 && !w2 && !w3; 475 } 476 477 static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 478 { 479 /* 480 * This depends on which UUID we have been assigned. 481 * TODO add a generic mechanism to obtain our UUID. 482 * 483 * The test below is for the hard coded UUID 484 * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 485 */ 486 return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && 487 w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; 488 } 489 490 TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, 491 size_t idx, uint16_t endpoint_id, 492 uint16_t execution_context, 493 uint32_t part_props, 494 const uint32_t uuid_words[4]) 495 { 496 struct ffa_partition_info_x *fpi = NULL; 497 size_t fpi_size = sizeof(*fpi); 498 499 if (ffa_vers >= FFA_VERSION_1_1) 500 fpi_size += FFA_UUID_SIZE; 501 502 if ((idx + 1) * fpi_size > blen) 503 return TEE_ERROR_OUT_OF_MEMORY; 504 505 fpi = (void *)((vaddr_t)buf + idx * fpi_size); 506 fpi->id = endpoint_id; 507 /* Number of execution contexts implemented by this partition */ 508 fpi->execution_context = execution_context; 509 510 fpi->partition_properties = part_props; 511 512 if (ffa_vers >= FFA_VERSION_1_1) { 513 if (uuid_words) 514 memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); 515 else 516 memset(fpi->uuid, 0, FFA_UUID_SIZE); 517 } 518 519 return TEE_SUCCESS; 520 } 521 522 static int handle_partition_info_get_all(size_t *elem_count, 523 struct ffa_rxtx *rxtx, bool count_only) 524 { 525 if (!count_only) { 526 /* Add OP-TEE SP */ 527 if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, 528 rxtx->size, 0, my_endpoint_id, 529 CFG_TEE_CORE_NB_CORE, 530 my_part_props, my_uuid_words)) 531 return FFA_NO_MEMORY; 532 } 533 *elem_count = 1; 534 535 if (IS_ENABLED(CFG_SECURE_PARTITION)) { 536 if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, 537 NULL, elem_count, count_only)) 538 return FFA_NO_MEMORY; 539 } 540 541 return FFA_OK; 542 } 543 544 void spmc_handle_partition_info_get(struct thread_smc_args *args, 545 struct ffa_rxtx *rxtx) 546 { 547 TEE_Result res = TEE_SUCCESS; 548 uint32_t ret_fid = FFA_ERROR; 549 uint32_t fpi_size = 0; 550 uint32_t rc = 0; 551 bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; 552 553 if (!count_only) { 554 cpu_spin_lock(&rxtx->spinlock); 555 556 if (!rxtx->size || !rxtx->tx_is_mine) { 557 rc = FFA_BUSY; 558 goto out; 559 } 560 } 561 562 if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { 563 size_t elem_count = 0; 564 565 ret_fid = handle_partition_info_get_all(&elem_count, rxtx, 566 count_only); 567 568 if (ret_fid) { 569 rc = ret_fid; 570 ret_fid = FFA_ERROR; 571 } else { 572 ret_fid = FFA_SUCCESS_32; 573 rc = elem_count; 574 } 575 576 goto out; 577 } 578 579 if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { 580 if (!count_only) { 581 res = spmc_fill_partition_entry(rxtx->ffa_vers, 582 rxtx->tx, rxtx->size, 0, 583 my_endpoint_id, 584 CFG_TEE_CORE_NB_CORE, 585 my_part_props, 586 my_uuid_words); 587 if (res) { 588 ret_fid = FFA_ERROR; 589 rc = FFA_INVALID_PARAMETERS; 590 goto out; 591 } 592 } 593 rc = 1; 594 } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { 595 uint32_t uuid_array[4] = { 0 }; 596 TEE_UUID uuid = { }; 597 size_t count = 0; 598 599 uuid_array[0] = args->a1; 600 uuid_array[1] = args->a2; 601 uuid_array[2] = args->a3; 602 uuid_array[3] = args->a4; 603 tee_uuid_from_octets(&uuid, (uint8_t *)uuid_array); 604 605 res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, 606 rxtx->size, &uuid, &count, 607 count_only); 608 if (res != TEE_SUCCESS) { 609 ret_fid = FFA_ERROR; 610 rc = FFA_INVALID_PARAMETERS; 611 goto out; 612 } 613 rc = count; 614 } else { 615 ret_fid = FFA_ERROR; 616 rc = FFA_INVALID_PARAMETERS; 617 goto out; 618 } 619 620 ret_fid = FFA_SUCCESS_32; 621 622 out: 623 if (ret_fid == FFA_SUCCESS_32 && !count_only && 624 rxtx->ffa_vers >= FFA_VERSION_1_1) 625 fpi_size = sizeof(struct ffa_partition_info_x) + FFA_UUID_SIZE; 626 627 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, fpi_size, 628 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 629 if (!count_only) { 630 rxtx->tx_is_mine = false; 631 cpu_spin_unlock(&rxtx->spinlock); 632 } 633 } 634 635 static void spmc_handle_run(struct thread_smc_args *args) 636 { 637 uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); 638 uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); 639 uint32_t rc = FFA_OK; 640 641 if (endpoint != my_endpoint_id) { 642 /* 643 * The endpoint should be an SP, try to resume the SP from 644 * preempted into busy state. 645 */ 646 rc = spmc_sp_resume_from_preempted(endpoint); 647 if (rc) 648 goto out; 649 } 650 651 thread_resume_from_rpc(thread_id, 0, 0, 0, 0); 652 653 /* thread_resume_from_rpc return only of the thread_id is invalid */ 654 rc = FFA_INVALID_PARAMETERS; 655 656 out: 657 set_simple_ret_val(args, rc); 658 } 659 #endif /*CFG_CORE_SEL1_SPMC*/ 660 661 static uint32_t spmc_enable_async_notif(uint32_t bottom_half_value, 662 uint16_t vm_id) 663 { 664 uint32_t old_itr_status = 0; 665 666 if (!spmc_notif_is_ready) { 667 /* 668 * This should never happen, not if normal world respects the 669 * exchanged capabilities. 670 */ 671 EMSG("Asynchronous notifications are not ready"); 672 return TEE_ERROR_NOT_IMPLEMENTED; 673 } 674 675 if (bottom_half_value >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE) { 676 EMSG("Invalid bottom half value %"PRIu32, bottom_half_value); 677 return TEE_ERROR_BAD_PARAMETERS; 678 } 679 680 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 681 do_bottom_half_value = bottom_half_value; 682 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) 683 notif_vm_id = vm_id; 684 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 685 686 notif_deliver_atomic_event(NOTIF_EVENT_STARTED); 687 return TEE_SUCCESS; 688 } 689 690 static void handle_yielding_call(struct thread_smc_args *args, 691 uint32_t direct_resp_fid) 692 { 693 TEE_Result res = 0; 694 695 thread_check_canaries(); 696 697 #ifdef ARM64 698 /* Saving this for an eventual RPC */ 699 thread_get_core_local()->direct_resp_fid = direct_resp_fid; 700 #endif 701 702 if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { 703 /* Note connection to struct thread_rpc_arg::ret */ 704 thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, 705 0); 706 res = TEE_ERROR_BAD_PARAMETERS; 707 } else { 708 thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, 709 args->a6, args->a7); 710 res = TEE_ERROR_BUSY; 711 } 712 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 713 0, res, 0, 0); 714 } 715 716 static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) 717 { 718 uint64_t cookie = reg_pair_to_64(a5, a4); 719 uint32_t res = 0; 720 721 res = mobj_ffa_unregister_by_cookie(cookie); 722 switch (res) { 723 case TEE_SUCCESS: 724 case TEE_ERROR_ITEM_NOT_FOUND: 725 return 0; 726 case TEE_ERROR_BUSY: 727 EMSG("res %#"PRIx32, res); 728 return FFA_BUSY; 729 default: 730 EMSG("res %#"PRIx32, res); 731 return FFA_INVALID_PARAMETERS; 732 } 733 } 734 735 static void handle_blocking_call(struct thread_smc_args *args, 736 uint32_t direct_resp_fid) 737 { 738 uint32_t sec_caps = 0; 739 740 switch (args->a3) { 741 case OPTEE_FFA_GET_API_VERSION: 742 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 743 OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, 744 0); 745 break; 746 case OPTEE_FFA_GET_OS_VERSION: 747 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 748 CFG_OPTEE_REVISION_MAJOR, 749 CFG_OPTEE_REVISION_MINOR, TEE_IMPL_GIT_SHA1); 750 break; 751 case OPTEE_FFA_EXCHANGE_CAPABILITIES: 752 sec_caps = OPTEE_FFA_SEC_CAP_ARG_OFFSET; 753 if (spmc_notif_is_ready) 754 sec_caps |= OPTEE_FFA_SEC_CAP_ASYNC_NOTIF; 755 spmc_set_args(args, direct_resp_fid, 756 swap_src_dst(args->a1), 0, 0, 757 THREAD_RPC_MAX_NUM_PARAMS, sec_caps); 758 break; 759 case OPTEE_FFA_UNREGISTER_SHM: 760 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 761 handle_unregister_shm(args->a4, args->a5), 0, 0); 762 break; 763 case OPTEE_FFA_ENABLE_ASYNC_NOTIF: 764 spmc_set_args(args, direct_resp_fid, 765 swap_src_dst(args->a1), 0, 766 spmc_enable_async_notif(args->a4, 767 FFA_SRC(args->a1)), 768 0, 0); 769 break; 770 default: 771 EMSG("Unhandled blocking service ID %#"PRIx32, 772 (uint32_t)args->a3); 773 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 774 TEE_ERROR_BAD_PARAMETERS, 0, 0); 775 } 776 } 777 778 static void handle_framework_direct_request(struct thread_smc_args *args, 779 struct ffa_rxtx *rxtx, 780 uint32_t direct_resp_fid) 781 { 782 uint32_t w0 = FFA_ERROR; 783 uint32_t w1 = FFA_PARAM_MBZ; 784 uint32_t w2 = FFA_NOT_SUPPORTED; 785 uint32_t w3 = FFA_PARAM_MBZ; 786 787 switch (args->a2 & FFA_MSG_TYPE_MASK) { 788 case FFA_MSG_SEND_VM_CREATED: 789 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 790 uint16_t guest_id = args->a5; 791 TEE_Result res = virt_guest_created(guest_id); 792 793 w0 = direct_resp_fid; 794 w1 = swap_src_dst(args->a1); 795 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; 796 if (res == TEE_SUCCESS) 797 w3 = FFA_OK; 798 else if (res == TEE_ERROR_OUT_OF_MEMORY) 799 w3 = FFA_DENIED; 800 else 801 w3 = FFA_INVALID_PARAMETERS; 802 } 803 break; 804 case FFA_MSG_SEND_VM_DESTROYED: 805 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 806 uint16_t guest_id = args->a5; 807 TEE_Result res = virt_guest_destroyed(guest_id); 808 809 w0 = direct_resp_fid; 810 w1 = swap_src_dst(args->a1); 811 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_DESTROYED; 812 if (res == TEE_SUCCESS) 813 w3 = FFA_OK; 814 else 815 w3 = FFA_INVALID_PARAMETERS; 816 } 817 break; 818 case FFA_MSG_VERSION_REQ: 819 w0 = direct_resp_fid; 820 w1 = swap_src_dst(args->a1); 821 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; 822 w3 = spmc_exchange_version(args->a3, rxtx); 823 break; 824 default: 825 break; 826 } 827 spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 828 } 829 830 static void handle_direct_request(struct thread_smc_args *args, 831 struct ffa_rxtx *rxtx) 832 { 833 uint32_t direct_resp_fid = 0; 834 835 if (IS_ENABLED(CFG_SECURE_PARTITION) && 836 FFA_DST(args->a1) != my_endpoint_id) { 837 spmc_sp_start_thread(args); 838 return; 839 } 840 841 if (OPTEE_SMC_IS_64(args->a0)) 842 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_64; 843 else 844 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_32; 845 846 if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { 847 handle_framework_direct_request(args, rxtx, direct_resp_fid); 848 return; 849 } 850 851 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 852 virt_set_guest(get_sender_id(args->a1))) { 853 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 854 TEE_ERROR_ITEM_NOT_FOUND, 0, 0); 855 return; 856 } 857 858 if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) 859 handle_yielding_call(args, direct_resp_fid); 860 else 861 handle_blocking_call(args, direct_resp_fid); 862 863 /* 864 * Note that handle_yielding_call() typically only returns if a 865 * thread cannot be allocated or found. virt_unset_guest() is also 866 * called from thread_state_suspend() and thread_state_free(). 867 */ 868 virt_unset_guest(); 869 } 870 871 int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, 872 struct ffa_mem_transaction_x *trans) 873 { 874 uint16_t mem_reg_attr = 0; 875 uint32_t flags = 0; 876 uint32_t count = 0; 877 uint32_t offs = 0; 878 uint32_t size = 0; 879 size_t n = 0; 880 881 if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) 882 return FFA_INVALID_PARAMETERS; 883 884 if (ffa_vers >= FFA_VERSION_1_1) { 885 struct ffa_mem_transaction_1_1 *descr = NULL; 886 887 if (blen < sizeof(*descr)) 888 return FFA_INVALID_PARAMETERS; 889 890 descr = buf; 891 trans->sender_id = READ_ONCE(descr->sender_id); 892 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 893 flags = READ_ONCE(descr->flags); 894 trans->global_handle = READ_ONCE(descr->global_handle); 895 trans->tag = READ_ONCE(descr->tag); 896 897 count = READ_ONCE(descr->mem_access_count); 898 size = READ_ONCE(descr->mem_access_size); 899 offs = READ_ONCE(descr->mem_access_offs); 900 } else { 901 struct ffa_mem_transaction_1_0 *descr = NULL; 902 903 if (blen < sizeof(*descr)) 904 return FFA_INVALID_PARAMETERS; 905 906 descr = buf; 907 trans->sender_id = READ_ONCE(descr->sender_id); 908 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 909 flags = READ_ONCE(descr->flags); 910 trans->global_handle = READ_ONCE(descr->global_handle); 911 trans->tag = READ_ONCE(descr->tag); 912 913 count = READ_ONCE(descr->mem_access_count); 914 size = sizeof(struct ffa_mem_access); 915 offs = offsetof(struct ffa_mem_transaction_1_0, 916 mem_access_array); 917 } 918 919 if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || 920 size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) 921 return FFA_INVALID_PARAMETERS; 922 923 /* Check that the endpoint memory access descriptor array fits */ 924 if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || 925 n > blen) 926 return FFA_INVALID_PARAMETERS; 927 928 trans->mem_reg_attr = mem_reg_attr; 929 trans->flags = flags; 930 trans->mem_access_size = size; 931 trans->mem_access_count = count; 932 trans->mem_access_offs = offs; 933 return 0; 934 } 935 936 #if defined(CFG_CORE_SEL1_SPMC) 937 static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, 938 unsigned int mem_access_count, uint8_t *acc_perms, 939 unsigned int *region_offs) 940 { 941 struct ffa_mem_access_perm *descr = NULL; 942 struct ffa_mem_access *mem_acc = NULL; 943 unsigned int n = 0; 944 945 for (n = 0; n < mem_access_count; n++) { 946 mem_acc = (void *)(mem_acc_base + mem_access_size * n); 947 descr = &mem_acc->access_perm; 948 if (READ_ONCE(descr->endpoint_id) == my_endpoint_id) { 949 *acc_perms = READ_ONCE(descr->perm); 950 *region_offs = READ_ONCE(mem_acc[n].region_offs); 951 return 0; 952 } 953 } 954 955 return FFA_INVALID_PARAMETERS; 956 } 957 958 static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, 959 size_t blen, unsigned int *page_count, 960 unsigned int *region_count, size_t *addr_range_offs) 961 { 962 const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 963 const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; 964 struct ffa_mem_region *region_descr = NULL; 965 unsigned int region_descr_offs = 0; 966 uint8_t mem_acc_perm = 0; 967 size_t n = 0; 968 969 if (mem_trans->mem_reg_attr != exp_mem_reg_attr) 970 return FFA_INVALID_PARAMETERS; 971 972 /* Check that the access permissions matches what's expected */ 973 if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, 974 mem_trans->mem_access_size, 975 mem_trans->mem_access_count, 976 &mem_acc_perm, ®ion_descr_offs) || 977 mem_acc_perm != exp_mem_acc_perm) 978 return FFA_INVALID_PARAMETERS; 979 980 /* Check that the Composite memory region descriptor fits */ 981 if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || 982 n > blen) 983 return FFA_INVALID_PARAMETERS; 984 985 if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, 986 struct ffa_mem_region)) 987 return FFA_INVALID_PARAMETERS; 988 989 region_descr = (struct ffa_mem_region *)((vaddr_t)buf + 990 region_descr_offs); 991 *page_count = READ_ONCE(region_descr->total_page_count); 992 *region_count = READ_ONCE(region_descr->address_range_count); 993 *addr_range_offs = n; 994 return 0; 995 } 996 997 static int add_mem_share_helper(struct mem_share_state *s, void *buf, 998 size_t flen) 999 { 1000 unsigned int region_count = flen / sizeof(struct ffa_address_range); 1001 struct ffa_address_range *arange = NULL; 1002 unsigned int n = 0; 1003 1004 if (region_count > s->region_count) 1005 region_count = s->region_count; 1006 1007 if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) 1008 return FFA_INVALID_PARAMETERS; 1009 arange = buf; 1010 1011 for (n = 0; n < region_count; n++) { 1012 unsigned int page_count = READ_ONCE(arange[n].page_count); 1013 uint64_t addr = READ_ONCE(arange[n].address); 1014 1015 if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, 1016 addr, page_count)) 1017 return FFA_INVALID_PARAMETERS; 1018 } 1019 1020 s->region_count -= region_count; 1021 if (s->region_count) 1022 return region_count * sizeof(*arange); 1023 1024 if (s->current_page_idx != s->page_count) 1025 return FFA_INVALID_PARAMETERS; 1026 1027 return 0; 1028 } 1029 1030 static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) 1031 { 1032 int rc = 0; 1033 1034 rc = add_mem_share_helper(&s->share, buf, flen); 1035 if (rc >= 0) { 1036 if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { 1037 /* We're not at the end of the descriptor yet */ 1038 if (s->share.region_count) 1039 return s->frag_offset; 1040 1041 /* We're done */ 1042 rc = 0; 1043 } else { 1044 rc = FFA_INVALID_PARAMETERS; 1045 } 1046 } 1047 1048 SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); 1049 if (rc < 0) 1050 mobj_ffa_sel1_spmc_delete(s->share.mf); 1051 else 1052 mobj_ffa_push_to_inactive(s->share.mf); 1053 free(s); 1054 1055 return rc; 1056 } 1057 1058 static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, 1059 void *buf) 1060 { 1061 struct ffa_mem_access_perm *perm = NULL; 1062 struct ffa_mem_access *mem_acc = NULL; 1063 1064 if (!IS_ENABLED(CFG_SECURE_PARTITION)) 1065 return false; 1066 1067 if (mem_trans->mem_access_count < 1) 1068 return false; 1069 1070 mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); 1071 perm = &mem_acc->access_perm; 1072 1073 /* 1074 * perm->endpoint_id is read here only to check if the endpoint is 1075 * OP-TEE. We do read it later on again, but there are some additional 1076 * checks there to make sure that the data is correct. 1077 */ 1078 return READ_ONCE(perm->endpoint_id) != my_endpoint_id; 1079 } 1080 1081 static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, 1082 tee_mm_entry_t *mm, void *buf, size_t blen, 1083 size_t flen, uint64_t *global_handle) 1084 { 1085 int rc = 0; 1086 struct mem_share_state share = { }; 1087 size_t addr_range_offs = 0; 1088 uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1089 size_t n = 0; 1090 1091 rc = mem_share_init(mem_trans, buf, flen, &share.page_count, 1092 &share.region_count, &addr_range_offs); 1093 if (rc) 1094 return rc; 1095 1096 if (MUL_OVERFLOW(share.region_count, 1097 sizeof(struct ffa_address_range), &n) || 1098 ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) 1099 return FFA_INVALID_PARAMETERS; 1100 1101 if (mem_trans->global_handle) 1102 cookie = mem_trans->global_handle; 1103 share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); 1104 if (!share.mf) 1105 return FFA_NO_MEMORY; 1106 1107 if (flen != blen) { 1108 struct mem_frag_state *s = calloc(sizeof(*s), 1); 1109 1110 if (!s) { 1111 rc = FFA_NO_MEMORY; 1112 goto err; 1113 } 1114 s->share = share; 1115 s->mm = mm; 1116 s->frag_offset = addr_range_offs; 1117 1118 SLIST_INSERT_HEAD(&frag_state_head, s, link); 1119 rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, 1120 flen - addr_range_offs); 1121 1122 if (rc >= 0) 1123 *global_handle = mobj_ffa_get_cookie(share.mf); 1124 1125 return rc; 1126 } 1127 1128 rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, 1129 flen - addr_range_offs); 1130 if (rc) { 1131 /* 1132 * Number of consumed bytes may be returned instead of 0 for 1133 * done. 1134 */ 1135 rc = FFA_INVALID_PARAMETERS; 1136 goto err; 1137 } 1138 1139 *global_handle = mobj_ffa_push_to_inactive(share.mf); 1140 1141 return 0; 1142 err: 1143 mobj_ffa_sel1_spmc_delete(share.mf); 1144 return rc; 1145 } 1146 1147 static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, 1148 unsigned int page_count, 1149 uint64_t *global_handle, struct ffa_rxtx *rxtx) 1150 { 1151 struct ffa_mem_transaction_x mem_trans = { }; 1152 int rc = 0; 1153 size_t len = 0; 1154 void *buf = NULL; 1155 tee_mm_entry_t *mm = NULL; 1156 vaddr_t offs = pbuf & SMALL_PAGE_MASK; 1157 1158 if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) 1159 return FFA_INVALID_PARAMETERS; 1160 if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) 1161 return FFA_INVALID_PARAMETERS; 1162 1163 /* 1164 * Check that the length reported in flen is covered by len even 1165 * if the offset is taken into account. 1166 */ 1167 if (len < flen || len - offs < flen) 1168 return FFA_INVALID_PARAMETERS; 1169 1170 mm = tee_mm_alloc(&tee_mm_shm, len); 1171 if (!mm) 1172 return FFA_NO_MEMORY; 1173 1174 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, 1175 page_count, MEM_AREA_NSEC_SHM)) { 1176 rc = FFA_INVALID_PARAMETERS; 1177 goto out; 1178 } 1179 buf = (void *)(tee_mm_get_smem(mm) + offs); 1180 1181 cpu_spin_lock(&rxtx->spinlock); 1182 rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); 1183 if (!rc && IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1184 virt_set_guest(mem_trans.sender_id)) 1185 rc = FFA_DENIED; 1186 if (!rc) 1187 rc = add_mem_share(&mem_trans, mm, buf, blen, flen, 1188 global_handle); 1189 virt_unset_guest(); 1190 cpu_spin_unlock(&rxtx->spinlock); 1191 if (rc > 0) 1192 return rc; 1193 1194 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1195 out: 1196 tee_mm_free(mm); 1197 return rc; 1198 } 1199 1200 static int handle_mem_share_rxbuf(size_t blen, size_t flen, 1201 uint64_t *global_handle, 1202 struct ffa_rxtx *rxtx) 1203 { 1204 struct ffa_mem_transaction_x mem_trans = { }; 1205 int rc = FFA_DENIED; 1206 1207 cpu_spin_lock(&rxtx->spinlock); 1208 1209 if (!rxtx->rx || flen > rxtx->size) 1210 goto out; 1211 1212 rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, 1213 &mem_trans); 1214 if (rc) 1215 goto out; 1216 if (is_sp_share(&mem_trans, rxtx->rx)) { 1217 rc = spmc_sp_add_share(&mem_trans, rxtx, blen, 1218 global_handle, NULL); 1219 goto out; 1220 } 1221 1222 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1223 virt_set_guest(mem_trans.sender_id)) 1224 goto out; 1225 1226 rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, 1227 global_handle); 1228 1229 virt_unset_guest(); 1230 1231 out: 1232 cpu_spin_unlock(&rxtx->spinlock); 1233 1234 return rc; 1235 } 1236 1237 static void handle_mem_share(struct thread_smc_args *args, 1238 struct ffa_rxtx *rxtx) 1239 { 1240 uint32_t tot_len = args->a1; 1241 uint32_t frag_len = args->a2; 1242 uint64_t addr = args->a3; 1243 uint32_t page_count = args->a4; 1244 uint32_t ret_w1 = 0; 1245 uint32_t ret_w2 = FFA_INVALID_PARAMETERS; 1246 uint32_t ret_w3 = 0; 1247 uint32_t ret_fid = FFA_ERROR; 1248 uint64_t global_handle = 0; 1249 int rc = 0; 1250 1251 /* Check that the MBZs are indeed 0 */ 1252 if (args->a5 || args->a6 || args->a7) 1253 goto out; 1254 1255 /* Check that fragment length doesn't exceed total length */ 1256 if (frag_len > tot_len) 1257 goto out; 1258 1259 /* Check for 32-bit calling convention */ 1260 if (args->a0 == FFA_MEM_SHARE_32) 1261 addr &= UINT32_MAX; 1262 1263 if (!addr) { 1264 /* 1265 * The memory transaction descriptor is passed via our rx 1266 * buffer. 1267 */ 1268 if (page_count) 1269 goto out; 1270 rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, 1271 rxtx); 1272 } else { 1273 rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, 1274 &global_handle, rxtx); 1275 } 1276 if (rc < 0) { 1277 ret_w2 = rc; 1278 } else if (rc > 0) { 1279 ret_fid = FFA_MEM_FRAG_RX; 1280 ret_w3 = rc; 1281 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1282 } else { 1283 ret_fid = FFA_SUCCESS_32; 1284 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1285 } 1286 out: 1287 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1288 } 1289 1290 static struct mem_frag_state *get_frag_state(uint64_t global_handle) 1291 { 1292 struct mem_frag_state *s = NULL; 1293 1294 SLIST_FOREACH(s, &frag_state_head, link) 1295 if (mobj_ffa_get_cookie(s->share.mf) == global_handle) 1296 return s; 1297 1298 return NULL; 1299 } 1300 1301 static void handle_mem_frag_tx(struct thread_smc_args *args, 1302 struct ffa_rxtx *rxtx) 1303 { 1304 uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); 1305 size_t flen = args->a3; 1306 uint32_t endpoint_id = args->a4; 1307 struct mem_frag_state *s = NULL; 1308 tee_mm_entry_t *mm = NULL; 1309 unsigned int page_count = 0; 1310 void *buf = NULL; 1311 uint32_t ret_w1 = 0; 1312 uint32_t ret_w2 = 0; 1313 uint32_t ret_w3 = 0; 1314 uint32_t ret_fid = 0; 1315 int rc = 0; 1316 1317 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1318 uint16_t guest_id = endpoint_id >> 16; 1319 1320 if (!guest_id || virt_set_guest(guest_id)) { 1321 rc = FFA_INVALID_PARAMETERS; 1322 goto out_set_rc; 1323 } 1324 } 1325 1326 /* 1327 * Currently we're only doing this for fragmented FFA_MEM_SHARE_* 1328 * requests. 1329 */ 1330 1331 cpu_spin_lock(&rxtx->spinlock); 1332 1333 s = get_frag_state(global_handle); 1334 if (!s) { 1335 rc = FFA_INVALID_PARAMETERS; 1336 goto out; 1337 } 1338 1339 mm = s->mm; 1340 if (mm) { 1341 if (flen > tee_mm_get_bytes(mm)) { 1342 rc = FFA_INVALID_PARAMETERS; 1343 goto out; 1344 } 1345 page_count = s->share.page_count; 1346 buf = (void *)tee_mm_get_smem(mm); 1347 } else { 1348 if (flen > rxtx->size) { 1349 rc = FFA_INVALID_PARAMETERS; 1350 goto out; 1351 } 1352 buf = rxtx->rx; 1353 } 1354 1355 rc = add_mem_share_frag(s, buf, flen); 1356 out: 1357 virt_unset_guest(); 1358 cpu_spin_unlock(&rxtx->spinlock); 1359 1360 if (rc <= 0 && mm) { 1361 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1362 tee_mm_free(mm); 1363 } 1364 1365 out_set_rc: 1366 if (rc < 0) { 1367 ret_fid = FFA_ERROR; 1368 ret_w2 = rc; 1369 } else if (rc > 0) { 1370 ret_fid = FFA_MEM_FRAG_RX; 1371 ret_w3 = rc; 1372 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1373 } else { 1374 ret_fid = FFA_SUCCESS_32; 1375 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1376 } 1377 1378 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1379 } 1380 1381 static void handle_mem_reclaim(struct thread_smc_args *args) 1382 { 1383 int rc = FFA_INVALID_PARAMETERS; 1384 uint64_t cookie = 0; 1385 1386 if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) 1387 goto out; 1388 1389 cookie = reg_pair_to_64(args->a2, args->a1); 1390 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1391 uint16_t guest_id = 0; 1392 1393 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 1394 guest_id = virt_find_guest_by_cookie(cookie); 1395 } else { 1396 guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & 1397 FFA_MEMORY_HANDLE_PRTN_MASK; 1398 } 1399 if (!guest_id) 1400 goto out; 1401 if (virt_set_guest(guest_id)) { 1402 if (!virt_reclaim_cookie_from_destroyed_guest(guest_id, 1403 cookie)) 1404 rc = FFA_OK; 1405 goto out; 1406 } 1407 } 1408 1409 switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { 1410 case TEE_SUCCESS: 1411 rc = FFA_OK; 1412 break; 1413 case TEE_ERROR_ITEM_NOT_FOUND: 1414 DMSG("cookie %#"PRIx64" not found", cookie); 1415 rc = FFA_INVALID_PARAMETERS; 1416 break; 1417 default: 1418 DMSG("cookie %#"PRIx64" busy", cookie); 1419 rc = FFA_DENIED; 1420 break; 1421 } 1422 1423 virt_unset_guest(); 1424 1425 out: 1426 set_simple_ret_val(args, rc); 1427 } 1428 1429 static void handle_notification_bitmap_create(struct thread_smc_args *args) 1430 { 1431 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1432 uint32_t ret_fid = FFA_ERROR; 1433 uint32_t old_itr_status = 0; 1434 1435 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1436 !args->a5 && !args->a6 && !args->a7) { 1437 uint16_t vm_id = args->a1; 1438 1439 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1440 1441 if (notif_vm_id_valid) { 1442 if (vm_id == notif_vm_id) 1443 ret_val = FFA_DENIED; 1444 else 1445 ret_val = FFA_NO_MEMORY; 1446 } else { 1447 notif_vm_id = vm_id; 1448 notif_vm_id_valid = true; 1449 ret_val = FFA_OK; 1450 ret_fid = FFA_SUCCESS_32; 1451 } 1452 1453 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1454 } 1455 1456 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1457 } 1458 1459 static void handle_notification_bitmap_destroy(struct thread_smc_args *args) 1460 { 1461 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1462 uint32_t ret_fid = FFA_ERROR; 1463 uint32_t old_itr_status = 0; 1464 1465 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1466 !args->a5 && !args->a6 && !args->a7) { 1467 uint16_t vm_id = args->a1; 1468 1469 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1470 1471 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1472 if (notif_pending_bitmap || notif_bound_bitmap) { 1473 ret_val = FFA_DENIED; 1474 } else { 1475 notif_vm_id_valid = false; 1476 ret_val = FFA_OK; 1477 ret_fid = FFA_SUCCESS_32; 1478 } 1479 } 1480 1481 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1482 } 1483 1484 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1485 } 1486 1487 static void handle_notification_bind(struct thread_smc_args *args) 1488 { 1489 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1490 uint32_t ret_fid = FFA_ERROR; 1491 uint32_t old_itr_status = 0; 1492 uint64_t bitmap = 0; 1493 uint16_t vm_id = 0; 1494 1495 if (args->a5 || args->a6 || args->a7) 1496 goto out; 1497 if (args->a2) { 1498 /* We only deal with global notifications for now */ 1499 ret_val = FFA_NOT_SUPPORTED; 1500 goto out; 1501 } 1502 1503 /* The destination of the eventual notification */ 1504 vm_id = FFA_DST(args->a1); 1505 bitmap = reg_pair_to_64(args->a4, args->a3); 1506 1507 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1508 1509 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1510 if (bitmap & notif_bound_bitmap) { 1511 ret_val = FFA_DENIED; 1512 } else { 1513 notif_bound_bitmap |= bitmap; 1514 ret_val = FFA_OK; 1515 ret_fid = FFA_SUCCESS_32; 1516 } 1517 } 1518 1519 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1520 out: 1521 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1522 } 1523 1524 static void handle_notification_unbind(struct thread_smc_args *args) 1525 { 1526 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1527 uint32_t ret_fid = FFA_ERROR; 1528 uint32_t old_itr_status = 0; 1529 uint64_t bitmap = 0; 1530 uint16_t vm_id = 0; 1531 1532 if (args->a2 || args->a5 || args->a6 || args->a7) 1533 goto out; 1534 1535 /* The destination of the eventual notification */ 1536 vm_id = FFA_DST(args->a1); 1537 bitmap = reg_pair_to_64(args->a4, args->a3); 1538 1539 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1540 1541 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1542 /* 1543 * Spec says: 1544 * At least one notification is bound to another Sender or 1545 * is currently pending. 1546 * 1547 * Not sure what the intention is. 1548 */ 1549 if (bitmap & notif_pending_bitmap) { 1550 ret_val = FFA_DENIED; 1551 } else { 1552 notif_bound_bitmap &= ~bitmap; 1553 ret_val = FFA_OK; 1554 ret_fid = FFA_SUCCESS_32; 1555 } 1556 } 1557 1558 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1559 out: 1560 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1561 } 1562 1563 static void handle_notification_get(struct thread_smc_args *args) 1564 { 1565 uint32_t w2 = FFA_INVALID_PARAMETERS; 1566 uint32_t ret_fid = FFA_ERROR; 1567 uint32_t old_itr_status = 0; 1568 uint16_t vm_id = 0; 1569 uint32_t w3 = 0; 1570 1571 if (args->a5 || args->a6 || args->a7) 1572 goto out; 1573 if (!(args->a2 & 0x1)) { 1574 ret_fid = FFA_SUCCESS_32; 1575 w2 = 0; 1576 goto out; 1577 } 1578 vm_id = FFA_DST(args->a1); 1579 1580 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1581 1582 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1583 reg_pair_from_64(notif_pending_bitmap, &w3, &w2); 1584 notif_pending_bitmap = 0; 1585 ret_fid = FFA_SUCCESS_32; 1586 } 1587 1588 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1589 out: 1590 spmc_set_args(args, ret_fid, 0, w2, w3, 0, 0); 1591 } 1592 1593 static void handle_notification_info_get(struct thread_smc_args *args) 1594 { 1595 uint32_t w2 = FFA_INVALID_PARAMETERS; 1596 uint32_t ret_fid = FFA_ERROR; 1597 1598 if (args->a1 || args->a2 || args->a3 || args->a4 || args->a5 || 1599 args->a6 || args->a7) 1600 goto out; 1601 1602 if (OPTEE_SMC_IS_64(args->a0)) 1603 ret_fid = FFA_SUCCESS_64; 1604 else 1605 ret_fid = FFA_SUCCESS_32; 1606 1607 /* 1608 * Note, we're only supporting physical OS kernel in normal world 1609 * with Global Notifications. 1610 * So one list of ID list registers (BIT[11:7]) 1611 * and one count of IDs (BIT[13:12] + 1) 1612 * and the VM is always 0. 1613 */ 1614 w2 = SHIFT_U32(1, 7); 1615 out: 1616 spmc_set_args(args, ret_fid, 0, w2, 0, 0, 0); 1617 } 1618 1619 void thread_spmc_set_async_notif_intid(int intid) 1620 { 1621 assert(interrupt_can_raise_sgi(interrupt_get_main_chip())); 1622 notif_intid = intid; 1623 spmc_notif_is_ready = true; 1624 DMSG("Asynchronous notifications are ready"); 1625 } 1626 1627 void notif_send_async(uint32_t value) 1628 { 1629 uint32_t old_itr_status = 0; 1630 1631 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1632 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && spmc_notif_is_ready && 1633 do_bottom_half_value >= 0 && notif_intid >= 0); 1634 notif_pending_bitmap |= BIT64(do_bottom_half_value); 1635 interrupt_raise_sgi(interrupt_get_main_chip(), notif_intid, 1636 ITR_CPU_MASK_TO_THIS_CPU); 1637 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1638 } 1639 #else 1640 void notif_send_async(uint32_t value) 1641 { 1642 /* global notification, delay notification interrupt */ 1643 uint32_t flags = BIT32(1); 1644 int res = 0; 1645 1646 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && spmc_notif_is_ready && 1647 do_bottom_half_value >= 0); 1648 res = ffa_set_notification(notif_vm_id, my_endpoint_id, flags, 1649 BIT64(do_bottom_half_value)); 1650 if (res) { 1651 EMSG("notification set failed with error %d", res); 1652 panic(); 1653 } 1654 } 1655 #endif 1656 1657 /* Only called from assembly */ 1658 void thread_spmc_msg_recv(struct thread_smc_args *args); 1659 void thread_spmc_msg_recv(struct thread_smc_args *args) 1660 { 1661 assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); 1662 switch (args->a0) { 1663 #if defined(CFG_CORE_SEL1_SPMC) 1664 case FFA_FEATURES: 1665 handle_features(args); 1666 break; 1667 case FFA_SPM_ID_GET: 1668 handle_spm_id_get(args); 1669 break; 1670 #ifdef ARM64 1671 case FFA_RXTX_MAP_64: 1672 #endif 1673 case FFA_RXTX_MAP_32: 1674 spmc_handle_rxtx_map(args, &my_rxtx); 1675 break; 1676 case FFA_RXTX_UNMAP: 1677 spmc_handle_rxtx_unmap(args, &my_rxtx); 1678 break; 1679 case FFA_RX_RELEASE: 1680 spmc_handle_rx_release(args, &my_rxtx); 1681 break; 1682 case FFA_PARTITION_INFO_GET: 1683 spmc_handle_partition_info_get(args, &my_rxtx); 1684 break; 1685 case FFA_RUN: 1686 spmc_handle_run(args); 1687 break; 1688 #endif /*CFG_CORE_SEL1_SPMC*/ 1689 case FFA_INTERRUPT: 1690 if (IS_ENABLED(CFG_CORE_SEL1_SPMC)) 1691 spmc_set_args(args, FFA_NORMAL_WORLD_RESUME, 0, 0, 0, 1692 0, 0); 1693 else 1694 spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); 1695 break; 1696 #ifdef ARM64 1697 case FFA_MSG_SEND_DIRECT_REQ_64: 1698 #endif 1699 case FFA_MSG_SEND_DIRECT_REQ_32: 1700 handle_direct_request(args, &my_rxtx); 1701 break; 1702 #if defined(CFG_CORE_SEL1_SPMC) 1703 #ifdef ARM64 1704 case FFA_MEM_SHARE_64: 1705 #endif 1706 case FFA_MEM_SHARE_32: 1707 handle_mem_share(args, &my_rxtx); 1708 break; 1709 case FFA_MEM_RECLAIM: 1710 if (!IS_ENABLED(CFG_SECURE_PARTITION) || 1711 !ffa_mem_reclaim(args, NULL)) 1712 handle_mem_reclaim(args); 1713 break; 1714 case FFA_MEM_FRAG_TX: 1715 handle_mem_frag_tx(args, &my_rxtx); 1716 break; 1717 case FFA_NOTIFICATION_BITMAP_CREATE: 1718 handle_notification_bitmap_create(args); 1719 break; 1720 case FFA_NOTIFICATION_BITMAP_DESTROY: 1721 handle_notification_bitmap_destroy(args); 1722 break; 1723 case FFA_NOTIFICATION_BIND: 1724 handle_notification_bind(args); 1725 break; 1726 case FFA_NOTIFICATION_UNBIND: 1727 handle_notification_unbind(args); 1728 break; 1729 case FFA_NOTIFICATION_GET: 1730 handle_notification_get(args); 1731 break; 1732 #ifdef ARM64 1733 case FFA_NOTIFICATION_INFO_GET_64: 1734 #endif 1735 case FFA_NOTIFICATION_INFO_GET_32: 1736 handle_notification_info_get(args); 1737 break; 1738 #endif /*CFG_CORE_SEL1_SPMC*/ 1739 case FFA_ERROR: 1740 EMSG("Cannot handle FFA_ERROR(%d)", (int)args->a2); 1741 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { 1742 /* 1743 * The SPMC will return an FFA_ERROR back so better 1744 * panic() now than flooding the log. 1745 */ 1746 panic("FFA_ERROR from SPMC is fatal"); 1747 } 1748 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, 1749 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 1750 break; 1751 default: 1752 EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); 1753 set_simple_ret_val(args, FFA_NOT_SUPPORTED); 1754 } 1755 } 1756 1757 static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) 1758 { 1759 size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1760 struct thread_ctx *thr = threads + thread_get_id(); 1761 TEE_Result res = TEE_ERROR_BAD_PARAMETERS; 1762 struct optee_msg_arg *arg = NULL; 1763 struct mobj *mobj = NULL; 1764 uint32_t num_params = 0; 1765 size_t sz = 0; 1766 1767 mobj = mobj_ffa_get_by_cookie(cookie, 0); 1768 if (!mobj) { 1769 EMSG("Can't find cookie %#"PRIx64, cookie); 1770 return TEE_ERROR_BAD_PARAMETERS; 1771 } 1772 1773 res = mobj_inc_map(mobj); 1774 if (res) 1775 goto out_put_mobj; 1776 1777 res = TEE_ERROR_BAD_PARAMETERS; 1778 arg = mobj_get_va(mobj, offset, sizeof(*arg)); 1779 if (!arg) 1780 goto out_dec_map; 1781 1782 num_params = READ_ONCE(arg->num_params); 1783 if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) 1784 goto out_dec_map; 1785 1786 sz = OPTEE_MSG_GET_ARG_SIZE(num_params); 1787 1788 thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); 1789 if (!thr->rpc_arg) 1790 goto out_dec_map; 1791 1792 virt_on_stdcall(); 1793 res = tee_entry_std(arg, num_params); 1794 1795 thread_rpc_shm_cache_clear(&thr->shm_cache); 1796 thr->rpc_arg = NULL; 1797 1798 out_dec_map: 1799 mobj_dec_map(mobj); 1800 out_put_mobj: 1801 mobj_put(mobj); 1802 return res; 1803 } 1804 1805 /* 1806 * Helper routine for the assembly function thread_std_smc_entry() 1807 * 1808 * Note: this function is weak just to make link_dummies_paged.c happy. 1809 */ 1810 uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, 1811 uint32_t a2, uint32_t a3, 1812 uint32_t a4, uint32_t a5 __unused) 1813 { 1814 /* 1815 * Arguments are supplied from handle_yielding_call() as: 1816 * a0 <- w1 1817 * a1 <- w3 1818 * a2 <- w4 1819 * a3 <- w5 1820 * a4 <- w6 1821 * a5 <- w7 1822 */ 1823 thread_get_tsd()->rpc_target_info = swap_src_dst(a0); 1824 if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) 1825 return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); 1826 return FFA_DENIED; 1827 } 1828 1829 static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) 1830 { 1831 uint64_t offs = tpm->u.memref.offs; 1832 1833 param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + 1834 OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; 1835 1836 param->u.fmem.offs_low = offs; 1837 param->u.fmem.offs_high = offs >> 32; 1838 if (param->u.fmem.offs_high != offs >> 32) 1839 return false; 1840 1841 param->u.fmem.size = tpm->u.memref.size; 1842 if (tpm->u.memref.mobj) { 1843 uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); 1844 1845 /* If a mobj is passed it better be one with a valid cookie. */ 1846 if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) 1847 return false; 1848 param->u.fmem.global_id = cookie; 1849 } else { 1850 param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1851 } 1852 1853 return true; 1854 } 1855 1856 static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, 1857 struct thread_param *params, 1858 struct optee_msg_arg **arg_ret) 1859 { 1860 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1861 struct thread_ctx *thr = threads + thread_get_id(); 1862 struct optee_msg_arg *arg = thr->rpc_arg; 1863 1864 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 1865 return TEE_ERROR_BAD_PARAMETERS; 1866 1867 if (!arg) { 1868 EMSG("rpc_arg not set"); 1869 return TEE_ERROR_GENERIC; 1870 } 1871 1872 memset(arg, 0, sz); 1873 arg->cmd = cmd; 1874 arg->num_params = num_params; 1875 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 1876 1877 for (size_t n = 0; n < num_params; n++) { 1878 switch (params[n].attr) { 1879 case THREAD_PARAM_ATTR_NONE: 1880 arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; 1881 break; 1882 case THREAD_PARAM_ATTR_VALUE_IN: 1883 case THREAD_PARAM_ATTR_VALUE_OUT: 1884 case THREAD_PARAM_ATTR_VALUE_INOUT: 1885 arg->params[n].attr = params[n].attr - 1886 THREAD_PARAM_ATTR_VALUE_IN + 1887 OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 1888 arg->params[n].u.value.a = params[n].u.value.a; 1889 arg->params[n].u.value.b = params[n].u.value.b; 1890 arg->params[n].u.value.c = params[n].u.value.c; 1891 break; 1892 case THREAD_PARAM_ATTR_MEMREF_IN: 1893 case THREAD_PARAM_ATTR_MEMREF_OUT: 1894 case THREAD_PARAM_ATTR_MEMREF_INOUT: 1895 if (!set_fmem(arg->params + n, params + n)) 1896 return TEE_ERROR_BAD_PARAMETERS; 1897 break; 1898 default: 1899 return TEE_ERROR_BAD_PARAMETERS; 1900 } 1901 } 1902 1903 if (arg_ret) 1904 *arg_ret = arg; 1905 1906 return TEE_SUCCESS; 1907 } 1908 1909 static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, 1910 struct thread_param *params) 1911 { 1912 for (size_t n = 0; n < num_params; n++) { 1913 switch (params[n].attr) { 1914 case THREAD_PARAM_ATTR_VALUE_OUT: 1915 case THREAD_PARAM_ATTR_VALUE_INOUT: 1916 params[n].u.value.a = arg->params[n].u.value.a; 1917 params[n].u.value.b = arg->params[n].u.value.b; 1918 params[n].u.value.c = arg->params[n].u.value.c; 1919 break; 1920 case THREAD_PARAM_ATTR_MEMREF_OUT: 1921 case THREAD_PARAM_ATTR_MEMREF_INOUT: 1922 params[n].u.memref.size = arg->params[n].u.fmem.size; 1923 break; 1924 default: 1925 break; 1926 } 1927 } 1928 1929 return arg->ret; 1930 } 1931 1932 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 1933 struct thread_param *params) 1934 { 1935 struct thread_rpc_arg rpc_arg = { .call = { 1936 .w1 = thread_get_tsd()->rpc_target_info, 1937 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1938 }, 1939 }; 1940 struct optee_msg_arg *arg = NULL; 1941 uint32_t ret = 0; 1942 1943 ret = get_rpc_arg(cmd, num_params, params, &arg); 1944 if (ret) 1945 return ret; 1946 1947 thread_rpc(&rpc_arg); 1948 1949 return get_rpc_arg_res(arg, num_params, params); 1950 } 1951 1952 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 1953 { 1954 struct thread_rpc_arg rpc_arg = { .call = { 1955 .w1 = thread_get_tsd()->rpc_target_info, 1956 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1957 }, 1958 }; 1959 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); 1960 uint32_t res2 = 0; 1961 uint32_t res = 0; 1962 1963 DMSG("freeing cookie %#"PRIx64, cookie); 1964 1965 res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); 1966 1967 mobj_put(mobj); 1968 res2 = mobj_ffa_unregister_by_cookie(cookie); 1969 if (res2) 1970 DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, 1971 cookie, res2); 1972 if (!res) 1973 thread_rpc(&rpc_arg); 1974 } 1975 1976 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) 1977 { 1978 struct thread_rpc_arg rpc_arg = { .call = { 1979 .w1 = thread_get_tsd()->rpc_target_info, 1980 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1981 }, 1982 }; 1983 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); 1984 struct optee_msg_arg *arg = NULL; 1985 unsigned int internal_offset = 0; 1986 struct mobj *mobj = NULL; 1987 uint64_t cookie = 0; 1988 1989 if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) 1990 return NULL; 1991 1992 thread_rpc(&rpc_arg); 1993 1994 if (arg->num_params != 1 || 1995 arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) 1996 return NULL; 1997 1998 internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); 1999 cookie = READ_ONCE(arg->params->u.fmem.global_id); 2000 mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); 2001 if (!mobj) { 2002 DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", 2003 cookie, internal_offset); 2004 return NULL; 2005 } 2006 2007 assert(mobj_is_nonsec(mobj)); 2008 2009 if (mobj->size < size) { 2010 DMSG("Mobj %#"PRIx64": wrong size", cookie); 2011 mobj_put(mobj); 2012 return NULL; 2013 } 2014 2015 if (mobj_inc_map(mobj)) { 2016 DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); 2017 mobj_put(mobj); 2018 return NULL; 2019 } 2020 2021 return mobj; 2022 } 2023 2024 struct mobj *thread_rpc_alloc_payload(size_t size) 2025 { 2026 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); 2027 } 2028 2029 struct mobj *thread_rpc_alloc_kernel_payload(size_t size) 2030 { 2031 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); 2032 } 2033 2034 void thread_rpc_free_kernel_payload(struct mobj *mobj) 2035 { 2036 if (mobj) 2037 thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, 2038 mobj_get_cookie(mobj), mobj); 2039 } 2040 2041 void thread_rpc_free_payload(struct mobj *mobj) 2042 { 2043 if (mobj) 2044 thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), 2045 mobj); 2046 } 2047 2048 struct mobj *thread_rpc_alloc_global_payload(size_t size) 2049 { 2050 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); 2051 } 2052 2053 void thread_rpc_free_global_payload(struct mobj *mobj) 2054 { 2055 if (mobj) 2056 thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, 2057 mobj_get_cookie(mobj), mobj); 2058 } 2059 2060 void thread_spmc_register_secondary_ep(vaddr_t ep) 2061 { 2062 unsigned long ret = 0; 2063 2064 /* Let the SPM know the entry point for secondary CPUs */ 2065 ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); 2066 2067 if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) 2068 EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); 2069 } 2070 2071 #if defined(CFG_CORE_SEL1_SPMC) 2072 static TEE_Result spmc_init(void) 2073 { 2074 my_endpoint_id = SPMC_ENDPOINT_ID; 2075 DMSG("My endpoint ID %#x", my_endpoint_id); 2076 2077 /* 2078 * If SPMD think we are version 1.0 it will report version 1.0 to 2079 * normal world regardless of what version we query the SPM with. 2080 * However, if SPMD think we are version 1.1 it will forward 2081 * queries from normal world to let us negotiate version. So by 2082 * setting version 1.0 here we should be compatible. 2083 * 2084 * Note that disagreement on negotiated version means that we'll 2085 * have communication problems with normal world. 2086 */ 2087 my_rxtx.ffa_vers = FFA_VERSION_1_0; 2088 2089 return TEE_SUCCESS; 2090 } 2091 #else /* !defined(CFG_CORE_SEL1_SPMC) */ 2092 static void spmc_rxtx_map(struct ffa_rxtx *rxtx) 2093 { 2094 struct thread_smc_args args = { 2095 #ifdef ARM64 2096 .a0 = FFA_RXTX_MAP_64, 2097 #else 2098 .a0 = FFA_RXTX_MAP_32, 2099 #endif 2100 .a1 = virt_to_phys(rxtx->tx), 2101 .a2 = virt_to_phys(rxtx->rx), 2102 .a3 = 1, 2103 }; 2104 2105 thread_smccc(&args); 2106 if (!is_ffa_success(args.a0)) { 2107 if (args.a0 == FFA_ERROR) 2108 EMSG("rxtx map failed with error %ld", args.a2); 2109 else 2110 EMSG("rxtx map failed"); 2111 panic(); 2112 } 2113 } 2114 2115 static uint16_t get_my_id(void) 2116 { 2117 struct thread_smc_args args = { 2118 .a0 = FFA_ID_GET, 2119 }; 2120 2121 thread_smccc(&args); 2122 if (!is_ffa_success(args.a0)) { 2123 if (args.a0 == FFA_ERROR) 2124 EMSG("Get id failed with error %ld", args.a2); 2125 else 2126 EMSG("Get id failed"); 2127 panic(); 2128 } 2129 2130 return args.a2; 2131 } 2132 2133 static uint32_t get_ffa_version(uint32_t my_version) 2134 { 2135 struct thread_smc_args args = { 2136 .a0 = FFA_VERSION, 2137 .a1 = my_version, 2138 }; 2139 2140 thread_smccc(&args); 2141 if (args.a0 & BIT(31)) { 2142 EMSG("FF-A version failed with error %ld", args.a0); 2143 panic(); 2144 } 2145 2146 return args.a0; 2147 } 2148 2149 static void *spmc_retrieve_req(uint64_t cookie, 2150 struct ffa_mem_transaction_x *trans) 2151 { 2152 struct ffa_mem_access *acc_descr_array = NULL; 2153 struct ffa_mem_access_perm *perm_descr = NULL; 2154 struct thread_smc_args args = { 2155 .a0 = FFA_MEM_RETRIEVE_REQ_32, 2156 .a3 = 0, /* Address, Using TX -> MBZ */ 2157 .a4 = 0, /* Using TX -> MBZ */ 2158 }; 2159 size_t size = 0; 2160 int rc = 0; 2161 2162 if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { 2163 struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; 2164 2165 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2166 memset(trans_descr, 0, size); 2167 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2168 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2169 trans_descr->global_handle = cookie; 2170 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2171 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2172 trans_descr->mem_access_count = 1; 2173 acc_descr_array = trans_descr->mem_access_array; 2174 } else { 2175 struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; 2176 2177 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2178 memset(trans_descr, 0, size); 2179 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2180 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2181 trans_descr->global_handle = cookie; 2182 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2183 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2184 trans_descr->mem_access_count = 1; 2185 trans_descr->mem_access_offs = sizeof(*trans_descr); 2186 trans_descr->mem_access_size = sizeof(struct ffa_mem_access); 2187 acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + 2188 sizeof(*trans_descr)); 2189 } 2190 acc_descr_array->region_offs = 0; 2191 acc_descr_array->reserved = 0; 2192 perm_descr = &acc_descr_array->access_perm; 2193 perm_descr->endpoint_id = my_endpoint_id; 2194 perm_descr->perm = FFA_MEM_ACC_RW; 2195 perm_descr->flags = 0; 2196 2197 args.a1 = size; /* Total Length */ 2198 args.a2 = size; /* Frag Length == Total length */ 2199 thread_smccc(&args); 2200 if (args.a0 != FFA_MEM_RETRIEVE_RESP) { 2201 if (args.a0 == FFA_ERROR) 2202 EMSG("Failed to fetch cookie %#"PRIx64" error code %d", 2203 cookie, (int)args.a2); 2204 else 2205 EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, 2206 cookie, args.a0); 2207 return NULL; 2208 } 2209 rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.rx, 2210 my_rxtx.size, trans); 2211 if (rc) { 2212 EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", 2213 cookie, rc); 2214 return NULL; 2215 } 2216 2217 return my_rxtx.rx; 2218 } 2219 2220 void thread_spmc_relinquish(uint64_t cookie) 2221 { 2222 struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; 2223 struct thread_smc_args args = { 2224 .a0 = FFA_MEM_RELINQUISH, 2225 }; 2226 2227 memset(relinquish_desc, 0, sizeof(*relinquish_desc)); 2228 relinquish_desc->handle = cookie; 2229 relinquish_desc->flags = 0; 2230 relinquish_desc->endpoint_count = 1; 2231 relinquish_desc->endpoint_id_array[0] = my_endpoint_id; 2232 thread_smccc(&args); 2233 if (!is_ffa_success(args.a0)) 2234 EMSG("Failed to relinquish cookie %#"PRIx64, cookie); 2235 } 2236 2237 static int set_pages(struct ffa_address_range *regions, 2238 unsigned int num_regions, unsigned int num_pages, 2239 struct mobj_ffa *mf) 2240 { 2241 unsigned int n = 0; 2242 unsigned int idx = 0; 2243 2244 for (n = 0; n < num_regions; n++) { 2245 unsigned int page_count = READ_ONCE(regions[n].page_count); 2246 uint64_t addr = READ_ONCE(regions[n].address); 2247 2248 if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) 2249 return FFA_INVALID_PARAMETERS; 2250 } 2251 2252 if (idx != num_pages) 2253 return FFA_INVALID_PARAMETERS; 2254 2255 return 0; 2256 } 2257 2258 struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) 2259 { 2260 struct mobj_ffa *ret = NULL; 2261 struct ffa_mem_transaction_x retrieve_desc = { }; 2262 struct ffa_mem_access *descr_array = NULL; 2263 struct ffa_mem_region *descr = NULL; 2264 struct mobj_ffa *mf = NULL; 2265 unsigned int num_pages = 0; 2266 unsigned int offs = 0; 2267 void *buf = NULL; 2268 struct thread_smc_args ffa_rx_release_args = { 2269 .a0 = FFA_RX_RELEASE 2270 }; 2271 2272 /* 2273 * OP-TEE is only supporting a single mem_region while the 2274 * specification allows for more than one. 2275 */ 2276 buf = spmc_retrieve_req(cookie, &retrieve_desc); 2277 if (!buf) { 2278 EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, 2279 cookie); 2280 return NULL; 2281 } 2282 2283 descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); 2284 offs = READ_ONCE(descr_array->region_offs); 2285 descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); 2286 2287 num_pages = READ_ONCE(descr->total_page_count); 2288 mf = mobj_ffa_spmc_new(cookie, num_pages); 2289 if (!mf) 2290 goto out; 2291 2292 if (set_pages(descr->address_range_array, 2293 READ_ONCE(descr->address_range_count), num_pages, mf)) { 2294 mobj_ffa_spmc_delete(mf); 2295 goto out; 2296 } 2297 2298 ret = mf; 2299 2300 out: 2301 /* Release RX buffer after the mem retrieve request. */ 2302 thread_smccc(&ffa_rx_release_args); 2303 2304 return ret; 2305 } 2306 2307 static TEE_Result spmc_init(void) 2308 { 2309 unsigned int major = 0; 2310 unsigned int minor __maybe_unused = 0; 2311 uint32_t my_vers = 0; 2312 uint32_t vers = 0; 2313 2314 my_vers = MAKE_FFA_VERSION(FFA_VERSION_MAJOR, FFA_VERSION_MINOR); 2315 vers = get_ffa_version(my_vers); 2316 major = (vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK; 2317 minor = (vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK; 2318 DMSG("SPMC reported version %u.%u", major, minor); 2319 if (major != FFA_VERSION_MAJOR) { 2320 EMSG("Incompatible major version %u, expected %u", 2321 major, FFA_VERSION_MAJOR); 2322 panic(); 2323 } 2324 if (vers < my_vers) 2325 my_vers = vers; 2326 DMSG("Using version %u.%u", 2327 (my_vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK, 2328 (my_vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK); 2329 my_rxtx.ffa_vers = my_vers; 2330 2331 spmc_rxtx_map(&my_rxtx); 2332 my_endpoint_id = get_my_id(); 2333 DMSG("My endpoint ID %#x", my_endpoint_id); 2334 2335 if (!ffa_features(FFA_NOTIFICATION_SET)) { 2336 spmc_notif_is_ready = true; 2337 DMSG("Asynchronous notifications are ready"); 2338 } 2339 2340 return TEE_SUCCESS; 2341 } 2342 #endif /* !defined(CFG_CORE_SEL1_SPMC) */ 2343 2344 /* 2345 * boot_final() is always done before exiting at end of boot 2346 * initialization. In case of virtualization the init-calls are done only 2347 * once a OP-TEE partition has been created. So with virtualization we have 2348 * to initialize via boot_final() to make sure we have a value assigned 2349 * before it's used the first time. 2350 */ 2351 #ifdef CFG_NS_VIRTUALIZATION 2352 boot_final(spmc_init); 2353 #else 2354 service_init(spmc_init); 2355 #endif 2356