1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2020-2023, Linaro Limited. 4 * Copyright (c) 2019-2024, Arm Limited. All rights reserved. 5 */ 6 7 #include <assert.h> 8 #include <ffa.h> 9 #include <initcall.h> 10 #include <io.h> 11 #include <kernel/interrupt.h> 12 #include <kernel/notif.h> 13 #include <kernel/panic.h> 14 #include <kernel/secure_partition.h> 15 #include <kernel/spinlock.h> 16 #include <kernel/spmc_sp_handler.h> 17 #include <kernel/tee_misc.h> 18 #include <kernel/thread.h> 19 #include <kernel/thread_private.h> 20 #include <kernel/thread_spmc.h> 21 #include <kernel/virtualization.h> 22 #include <mm/core_mmu.h> 23 #include <mm/mobj.h> 24 #include <optee_ffa.h> 25 #include <optee_msg.h> 26 #include <optee_rpc_cmd.h> 27 #include <sm/optee_smc.h> 28 #include <string.h> 29 #include <sys/queue.h> 30 #include <tee/entry_std.h> 31 #include <tee/uuid.h> 32 #include <util.h> 33 34 #if defined(CFG_CORE_SEL1_SPMC) 35 struct mem_share_state { 36 struct mobj_ffa *mf; 37 unsigned int page_count; 38 unsigned int region_count; 39 unsigned int current_page_idx; 40 }; 41 42 struct mem_frag_state { 43 struct mem_share_state share; 44 tee_mm_entry_t *mm; 45 unsigned int frag_offset; 46 SLIST_ENTRY(mem_frag_state) link; 47 }; 48 #endif 49 50 static unsigned int spmc_notif_lock = SPINLOCK_UNLOCK; 51 static int do_bottom_half_value = -1; 52 static uint16_t notif_vm_id; 53 static bool spmc_notif_is_ready; 54 55 /* Initialized in spmc_init() below */ 56 uint16_t optee_endpoint_id __nex_bss; 57 uint16_t spmc_id __nex_bss; 58 #ifdef CFG_CORE_SEL1_SPMC 59 uint16_t spmd_id __nex_bss; 60 static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | 61 FFA_PART_PROP_DIRECT_REQ_SEND | 62 #ifdef CFG_NS_VIRTUALIZATION 63 FFA_PART_PROP_NOTIF_CREATED | 64 FFA_PART_PROP_NOTIF_DESTROYED | 65 #endif 66 #ifdef ARM64 67 FFA_PART_PROP_AARCH64_STATE | 68 #endif 69 FFA_PART_PROP_IS_PE_ID; 70 71 static uint32_t my_uuid_words[] = { 72 /* 73 * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 74 * SP, or 75 * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a 76 * logical partition, residing in the same exception level as the 77 * SPMC 78 * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 79 */ 80 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, 81 }; 82 83 /* 84 * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. 85 * 86 * struct ffa_rxtx::spin_lock protects the variables below from concurrent 87 * access this includes the use of content of struct ffa_rxtx::rx and 88 * @frag_state_head. 89 * 90 * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct 91 * ffa_rxtx::tx and false when it is owned by normal world. 92 * 93 * Note that we can't prevent normal world from updating the content of 94 * these buffers so we must always be careful when reading. while we hold 95 * the lock. 96 */ 97 98 static struct ffa_rxtx my_rxtx __nex_bss; 99 100 static bool is_nw_buf(struct ffa_rxtx *rxtx) 101 { 102 return rxtx == &my_rxtx; 103 } 104 105 static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = 106 SLIST_HEAD_INITIALIZER(&frag_state_head); 107 108 static uint64_t notif_pending_bitmap; 109 static uint64_t notif_bound_bitmap; 110 static bool notif_vm_id_valid; 111 static int notif_intid = -1; 112 #else 113 static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 114 static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 115 static struct ffa_rxtx my_rxtx = { 116 .rx = __rx_buf, 117 .tx = __tx_buf, 118 .size = sizeof(__rx_buf), 119 }; 120 #endif 121 122 static uint32_t swap_src_dst(uint32_t src_dst) 123 { 124 return (src_dst >> 16) | (src_dst << 16); 125 } 126 127 static uint16_t get_sender_id(uint32_t src_dst) 128 { 129 return src_dst >> 16; 130 } 131 132 void spmc_set_args(struct thread_smc_args *args, uint32_t fid, uint32_t src_dst, 133 uint32_t w2, uint32_t w3, uint32_t w4, uint32_t w5) 134 { 135 *args = (struct thread_smc_args){ .a0 = fid, 136 .a1 = src_dst, 137 .a2 = w2, 138 .a3 = w3, 139 .a4 = w4, 140 .a5 = w5, }; 141 } 142 143 static void set_simple_ret_val(struct thread_smc_args *args, int ffa_ret) 144 { 145 if (ffa_ret) 146 spmc_set_args(args, FFA_ERROR, 0, ffa_ret, 0, 0, 0); 147 else 148 spmc_set_args(args, FFA_SUCCESS_32, 0, 0, 0, 0, 0); 149 } 150 151 uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) 152 { 153 /* 154 * No locking, if the caller does concurrent calls to this it's 155 * only making a mess for itself. We must be able to renegotiate 156 * the FF-A version in order to support differing versions between 157 * the loader and the driver. 158 */ 159 if (vers < FFA_VERSION_1_1) 160 rxtx->ffa_vers = FFA_VERSION_1_0; 161 else 162 rxtx->ffa_vers = FFA_VERSION_1_1; 163 164 return rxtx->ffa_vers; 165 } 166 167 static bool is_ffa_success(uint32_t fid) 168 { 169 #ifdef ARM64 170 if (fid == FFA_SUCCESS_64) 171 return true; 172 #endif 173 return fid == FFA_SUCCESS_32; 174 } 175 176 static int32_t get_ffa_ret_code(const struct thread_smc_args *args) 177 { 178 if (is_ffa_success(args->a0)) 179 return FFA_OK; 180 if (args->a0 == FFA_ERROR && args->a2) 181 return args->a2; 182 return FFA_NOT_SUPPORTED; 183 } 184 185 static int ffa_simple_call(uint32_t fid, unsigned long a1, unsigned long a2, 186 unsigned long a3, unsigned long a4) 187 { 188 struct thread_smc_args args = { 189 .a0 = fid, 190 .a1 = a1, 191 .a2 = a2, 192 .a3 = a3, 193 .a4 = a4, 194 }; 195 196 thread_smccc(&args); 197 198 return get_ffa_ret_code(&args); 199 } 200 201 static int __maybe_unused ffa_features(uint32_t id) 202 { 203 return ffa_simple_call(FFA_FEATURES, id, 0, 0, 0); 204 } 205 206 static int __maybe_unused ffa_set_notification(uint16_t dst, uint16_t src, 207 uint32_t flags, uint64_t bitmap) 208 { 209 return ffa_simple_call(FFA_NOTIFICATION_SET, 210 SHIFT_U32(src, 16) | dst, flags, 211 low32_from_64(bitmap), high32_from_64(bitmap)); 212 } 213 214 #if defined(CFG_CORE_SEL1_SPMC) 215 static void handle_features(struct thread_smc_args *args) 216 { 217 uint32_t ret_fid = FFA_ERROR; 218 uint32_t ret_w2 = FFA_NOT_SUPPORTED; 219 220 switch (args->a1) { 221 case FFA_FEATURE_SCHEDULE_RECV_INTR: 222 if (spmc_notif_is_ready) { 223 ret_fid = FFA_SUCCESS_32; 224 ret_w2 = notif_intid; 225 } 226 break; 227 228 #ifdef ARM64 229 case FFA_RXTX_MAP_64: 230 #endif 231 case FFA_RXTX_MAP_32: 232 ret_fid = FFA_SUCCESS_32; 233 ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ 234 break; 235 #ifdef ARM64 236 case FFA_MEM_SHARE_64: 237 #endif 238 case FFA_MEM_SHARE_32: 239 ret_fid = FFA_SUCCESS_32; 240 /* 241 * Partition manager supports transmission of a memory 242 * transaction descriptor in a buffer dynamically allocated 243 * by the endpoint. 244 */ 245 ret_w2 = BIT(0); 246 break; 247 248 case FFA_ERROR: 249 case FFA_VERSION: 250 case FFA_SUCCESS_32: 251 #ifdef ARM64 252 case FFA_SUCCESS_64: 253 #endif 254 case FFA_FEATURES: 255 case FFA_SPM_ID_GET: 256 case FFA_MEM_FRAG_TX: 257 case FFA_MEM_RECLAIM: 258 case FFA_MSG_SEND_DIRECT_REQ_64: 259 case FFA_MSG_SEND_DIRECT_REQ_32: 260 case FFA_INTERRUPT: 261 case FFA_PARTITION_INFO_GET: 262 case FFA_RXTX_UNMAP: 263 case FFA_RX_RELEASE: 264 case FFA_FEATURE_MANAGED_EXIT_INTR: 265 case FFA_NOTIFICATION_BITMAP_CREATE: 266 case FFA_NOTIFICATION_BITMAP_DESTROY: 267 case FFA_NOTIFICATION_BIND: 268 case FFA_NOTIFICATION_UNBIND: 269 case FFA_NOTIFICATION_SET: 270 case FFA_NOTIFICATION_GET: 271 case FFA_NOTIFICATION_INFO_GET_32: 272 #ifdef ARM64 273 case FFA_NOTIFICATION_INFO_GET_64: 274 #endif 275 ret_fid = FFA_SUCCESS_32; 276 ret_w2 = FFA_PARAM_MBZ; 277 break; 278 default: 279 break; 280 } 281 282 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, 283 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 284 } 285 286 static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) 287 { 288 tee_mm_entry_t *mm = NULL; 289 290 if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) 291 return FFA_INVALID_PARAMETERS; 292 293 mm = tee_mm_alloc(&core_virt_shm_pool, sz); 294 if (!mm) 295 return FFA_NO_MEMORY; 296 297 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, 298 sz / SMALL_PAGE_SIZE, 299 MEM_AREA_NSEC_SHM)) { 300 tee_mm_free(mm); 301 return FFA_INVALID_PARAMETERS; 302 } 303 304 *va_ret = (void *)tee_mm_get_smem(mm); 305 return 0; 306 } 307 308 void spmc_handle_spm_id_get(struct thread_smc_args *args) 309 { 310 spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, spmc_id, 311 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 312 } 313 314 static void unmap_buf(void *va, size_t sz) 315 { 316 tee_mm_entry_t *mm = tee_mm_find(&core_virt_shm_pool, (vaddr_t)va); 317 318 assert(mm); 319 core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); 320 tee_mm_free(mm); 321 } 322 323 void spmc_handle_rxtx_map(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 324 { 325 int rc = 0; 326 unsigned int sz = 0; 327 paddr_t rx_pa = 0; 328 paddr_t tx_pa = 0; 329 void *rx = NULL; 330 void *tx = NULL; 331 332 cpu_spin_lock(&rxtx->spinlock); 333 334 if (args->a3 & GENMASK_64(63, 6)) { 335 rc = FFA_INVALID_PARAMETERS; 336 goto out; 337 } 338 339 sz = args->a3 * SMALL_PAGE_SIZE; 340 if (!sz) { 341 rc = FFA_INVALID_PARAMETERS; 342 goto out; 343 } 344 /* TX/RX are swapped compared to the caller */ 345 tx_pa = args->a2; 346 rx_pa = args->a1; 347 348 if (rxtx->size) { 349 rc = FFA_DENIED; 350 goto out; 351 } 352 353 /* 354 * If the buffer comes from a SP the address is virtual and already 355 * mapped. 356 */ 357 if (is_nw_buf(rxtx)) { 358 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 359 enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; 360 bool tx_alloced = false; 361 362 /* 363 * With virtualization we establish this mapping in 364 * the nexus mapping which then is replicated to 365 * each partition. 366 * 367 * This means that this mapping must be done before 368 * any partition is created and then must not be 369 * changed. 370 */ 371 372 /* 373 * core_mmu_add_mapping() may reuse previous 374 * mappings. First check if there's any mappings to 375 * reuse so we know how to clean up in case of 376 * failure. 377 */ 378 tx = phys_to_virt(tx_pa, mt, sz); 379 rx = phys_to_virt(rx_pa, mt, sz); 380 if (!tx) { 381 tx = core_mmu_add_mapping(mt, tx_pa, sz); 382 if (!tx) { 383 rc = FFA_NO_MEMORY; 384 goto out; 385 } 386 tx_alloced = true; 387 } 388 if (!rx) 389 rx = core_mmu_add_mapping(mt, rx_pa, sz); 390 391 if (!rx) { 392 if (tx_alloced && tx) 393 core_mmu_remove_mapping(mt, tx, sz); 394 rc = FFA_NO_MEMORY; 395 goto out; 396 } 397 } else { 398 rc = map_buf(tx_pa, sz, &tx); 399 if (rc) 400 goto out; 401 rc = map_buf(rx_pa, sz, &rx); 402 if (rc) { 403 unmap_buf(tx, sz); 404 goto out; 405 } 406 } 407 rxtx->tx = tx; 408 rxtx->rx = rx; 409 } else { 410 if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { 411 rc = FFA_INVALID_PARAMETERS; 412 goto out; 413 } 414 415 if (!virt_to_phys((void *)tx_pa) || 416 !virt_to_phys((void *)rx_pa)) { 417 rc = FFA_INVALID_PARAMETERS; 418 goto out; 419 } 420 421 rxtx->tx = (void *)tx_pa; 422 rxtx->rx = (void *)rx_pa; 423 } 424 425 rxtx->size = sz; 426 rxtx->tx_is_mine = true; 427 DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); 428 DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); 429 out: 430 cpu_spin_unlock(&rxtx->spinlock); 431 set_simple_ret_val(args, rc); 432 } 433 434 void spmc_handle_rxtx_unmap(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 435 { 436 int rc = FFA_INVALID_PARAMETERS; 437 438 cpu_spin_lock(&rxtx->spinlock); 439 440 if (!rxtx->size) 441 goto out; 442 443 /* We don't unmap the SP memory as the SP might still use it */ 444 if (is_nw_buf(rxtx)) { 445 unmap_buf(rxtx->rx, rxtx->size); 446 unmap_buf(rxtx->tx, rxtx->size); 447 } 448 rxtx->size = 0; 449 rxtx->rx = NULL; 450 rxtx->tx = NULL; 451 rc = 0; 452 out: 453 cpu_spin_unlock(&rxtx->spinlock); 454 set_simple_ret_val(args, rc); 455 } 456 457 void spmc_handle_rx_release(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 458 { 459 int rc = 0; 460 461 cpu_spin_lock(&rxtx->spinlock); 462 /* The senders RX is our TX */ 463 if (!rxtx->size || rxtx->tx_is_mine) { 464 rc = FFA_DENIED; 465 } else { 466 rc = 0; 467 rxtx->tx_is_mine = true; 468 } 469 cpu_spin_unlock(&rxtx->spinlock); 470 471 set_simple_ret_val(args, rc); 472 } 473 474 static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 475 { 476 return !w0 && !w1 && !w2 && !w3; 477 } 478 479 static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 480 { 481 /* 482 * This depends on which UUID we have been assigned. 483 * TODO add a generic mechanism to obtain our UUID. 484 * 485 * The test below is for the hard coded UUID 486 * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 487 */ 488 return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && 489 w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; 490 } 491 492 TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, 493 size_t idx, uint16_t endpoint_id, 494 uint16_t execution_context, 495 uint32_t part_props, 496 const uint32_t uuid_words[4]) 497 { 498 struct ffa_partition_info_x *fpi = NULL; 499 size_t fpi_size = sizeof(*fpi); 500 501 if (ffa_vers >= FFA_VERSION_1_1) 502 fpi_size += FFA_UUID_SIZE; 503 504 if ((idx + 1) * fpi_size > blen) 505 return TEE_ERROR_OUT_OF_MEMORY; 506 507 fpi = (void *)((vaddr_t)buf + idx * fpi_size); 508 fpi->id = endpoint_id; 509 /* Number of execution contexts implemented by this partition */ 510 fpi->execution_context = execution_context; 511 512 fpi->partition_properties = part_props; 513 514 if (ffa_vers >= FFA_VERSION_1_1) { 515 if (uuid_words) 516 memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); 517 else 518 memset(fpi->uuid, 0, FFA_UUID_SIZE); 519 } 520 521 return TEE_SUCCESS; 522 } 523 524 static int handle_partition_info_get_all(size_t *elem_count, 525 struct ffa_rxtx *rxtx, bool count_only) 526 { 527 if (!count_only) { 528 /* Add OP-TEE SP */ 529 if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, 530 rxtx->size, 0, optee_endpoint_id, 531 CFG_TEE_CORE_NB_CORE, 532 my_part_props, my_uuid_words)) 533 return FFA_NO_MEMORY; 534 } 535 *elem_count = 1; 536 537 if (IS_ENABLED(CFG_SECURE_PARTITION)) { 538 if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, 539 NULL, elem_count, count_only)) 540 return FFA_NO_MEMORY; 541 } 542 543 return FFA_OK; 544 } 545 546 void spmc_handle_partition_info_get(struct thread_smc_args *args, 547 struct ffa_rxtx *rxtx) 548 { 549 TEE_Result res = TEE_SUCCESS; 550 uint32_t ret_fid = FFA_ERROR; 551 uint32_t fpi_size = 0; 552 uint32_t rc = 0; 553 bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; 554 555 if (!count_only) { 556 cpu_spin_lock(&rxtx->spinlock); 557 558 if (!rxtx->size || !rxtx->tx_is_mine) { 559 rc = FFA_BUSY; 560 goto out; 561 } 562 } 563 564 if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { 565 size_t elem_count = 0; 566 567 ret_fid = handle_partition_info_get_all(&elem_count, rxtx, 568 count_only); 569 570 if (ret_fid) { 571 rc = ret_fid; 572 ret_fid = FFA_ERROR; 573 } else { 574 ret_fid = FFA_SUCCESS_32; 575 rc = elem_count; 576 } 577 578 goto out; 579 } 580 581 if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { 582 if (!count_only) { 583 res = spmc_fill_partition_entry(rxtx->ffa_vers, 584 rxtx->tx, rxtx->size, 0, 585 optee_endpoint_id, 586 CFG_TEE_CORE_NB_CORE, 587 my_part_props, 588 my_uuid_words); 589 if (res) { 590 ret_fid = FFA_ERROR; 591 rc = FFA_INVALID_PARAMETERS; 592 goto out; 593 } 594 } 595 rc = 1; 596 } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { 597 uint32_t uuid_array[4] = { 0 }; 598 TEE_UUID uuid = { }; 599 size_t count = 0; 600 601 uuid_array[0] = args->a1; 602 uuid_array[1] = args->a2; 603 uuid_array[2] = args->a3; 604 uuid_array[3] = args->a4; 605 tee_uuid_from_octets(&uuid, (uint8_t *)uuid_array); 606 607 res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, 608 rxtx->size, &uuid, &count, 609 count_only); 610 if (res != TEE_SUCCESS) { 611 ret_fid = FFA_ERROR; 612 rc = FFA_INVALID_PARAMETERS; 613 goto out; 614 } 615 rc = count; 616 } else { 617 ret_fid = FFA_ERROR; 618 rc = FFA_INVALID_PARAMETERS; 619 goto out; 620 } 621 622 ret_fid = FFA_SUCCESS_32; 623 624 out: 625 if (ret_fid == FFA_SUCCESS_32 && !count_only && 626 rxtx->ffa_vers >= FFA_VERSION_1_1) 627 fpi_size = sizeof(struct ffa_partition_info_x) + FFA_UUID_SIZE; 628 629 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, fpi_size, 630 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 631 if (!count_only) { 632 rxtx->tx_is_mine = false; 633 cpu_spin_unlock(&rxtx->spinlock); 634 } 635 } 636 637 static void spmc_handle_run(struct thread_smc_args *args) 638 { 639 uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); 640 uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); 641 uint32_t rc = FFA_OK; 642 643 if (endpoint != optee_endpoint_id) { 644 /* 645 * The endpoint should be an SP, try to resume the SP from 646 * preempted into busy state. 647 */ 648 rc = spmc_sp_resume_from_preempted(endpoint); 649 if (rc) 650 goto out; 651 } 652 653 thread_resume_from_rpc(thread_id, 0, 0, 0, 0); 654 655 /* thread_resume_from_rpc return only of the thread_id is invalid */ 656 rc = FFA_INVALID_PARAMETERS; 657 658 out: 659 set_simple_ret_val(args, rc); 660 } 661 #endif /*CFG_CORE_SEL1_SPMC*/ 662 663 static uint32_t spmc_enable_async_notif(uint32_t bottom_half_value, 664 uint16_t vm_id) 665 { 666 uint32_t old_itr_status = 0; 667 668 if (!spmc_notif_is_ready) { 669 /* 670 * This should never happen, not if normal world respects the 671 * exchanged capabilities. 672 */ 673 EMSG("Asynchronous notifications are not ready"); 674 return TEE_ERROR_NOT_IMPLEMENTED; 675 } 676 677 if (bottom_half_value >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE) { 678 EMSG("Invalid bottom half value %"PRIu32, bottom_half_value); 679 return TEE_ERROR_BAD_PARAMETERS; 680 } 681 682 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 683 do_bottom_half_value = bottom_half_value; 684 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) 685 notif_vm_id = vm_id; 686 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 687 688 notif_deliver_atomic_event(NOTIF_EVENT_STARTED); 689 return TEE_SUCCESS; 690 } 691 692 static void handle_yielding_call(struct thread_smc_args *args, 693 uint32_t direct_resp_fid) 694 { 695 TEE_Result res = 0; 696 697 thread_check_canaries(); 698 699 #ifdef ARM64 700 /* Saving this for an eventual RPC */ 701 thread_get_core_local()->direct_resp_fid = direct_resp_fid; 702 #endif 703 704 if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { 705 /* Note connection to struct thread_rpc_arg::ret */ 706 thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, 707 0); 708 res = TEE_ERROR_BAD_PARAMETERS; 709 } else { 710 thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, 711 args->a6, args->a7); 712 res = TEE_ERROR_BUSY; 713 } 714 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 715 0, res, 0, 0); 716 } 717 718 static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) 719 { 720 uint64_t cookie = reg_pair_to_64(a5, a4); 721 uint32_t res = 0; 722 723 res = mobj_ffa_unregister_by_cookie(cookie); 724 switch (res) { 725 case TEE_SUCCESS: 726 case TEE_ERROR_ITEM_NOT_FOUND: 727 return 0; 728 case TEE_ERROR_BUSY: 729 EMSG("res %#"PRIx32, res); 730 return FFA_BUSY; 731 default: 732 EMSG("res %#"PRIx32, res); 733 return FFA_INVALID_PARAMETERS; 734 } 735 } 736 737 static void handle_blocking_call(struct thread_smc_args *args, 738 uint32_t direct_resp_fid) 739 { 740 uint32_t sec_caps = 0; 741 742 switch (args->a3) { 743 case OPTEE_FFA_GET_API_VERSION: 744 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 745 OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, 746 0); 747 break; 748 case OPTEE_FFA_GET_OS_VERSION: 749 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 750 CFG_OPTEE_REVISION_MAJOR, 751 CFG_OPTEE_REVISION_MINOR, 752 TEE_IMPL_GIT_SHA1 >> 32); 753 break; 754 case OPTEE_FFA_EXCHANGE_CAPABILITIES: 755 sec_caps = OPTEE_FFA_SEC_CAP_ARG_OFFSET; 756 if (spmc_notif_is_ready) 757 sec_caps |= OPTEE_FFA_SEC_CAP_ASYNC_NOTIF; 758 spmc_set_args(args, direct_resp_fid, 759 swap_src_dst(args->a1), 0, 0, 760 THREAD_RPC_MAX_NUM_PARAMS, sec_caps); 761 break; 762 case OPTEE_FFA_UNREGISTER_SHM: 763 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 764 handle_unregister_shm(args->a4, args->a5), 0, 0); 765 break; 766 case OPTEE_FFA_ENABLE_ASYNC_NOTIF: 767 spmc_set_args(args, direct_resp_fid, 768 swap_src_dst(args->a1), 0, 769 spmc_enable_async_notif(args->a4, 770 FFA_SRC(args->a1)), 771 0, 0); 772 break; 773 default: 774 EMSG("Unhandled blocking service ID %#"PRIx32, 775 (uint32_t)args->a3); 776 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 777 TEE_ERROR_BAD_PARAMETERS, 0, 0); 778 } 779 } 780 781 static void handle_framework_direct_request(struct thread_smc_args *args, 782 struct ffa_rxtx *rxtx, 783 uint32_t direct_resp_fid) 784 { 785 uint32_t w0 = FFA_ERROR; 786 uint32_t w1 = FFA_PARAM_MBZ; 787 uint32_t w2 = FFA_NOT_SUPPORTED; 788 uint32_t w3 = FFA_PARAM_MBZ; 789 790 switch (args->a2 & FFA_MSG_TYPE_MASK) { 791 case FFA_MSG_SEND_VM_CREATED: 792 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 793 uint16_t guest_id = args->a5; 794 TEE_Result res = virt_guest_created(guest_id); 795 796 w0 = direct_resp_fid; 797 w1 = swap_src_dst(args->a1); 798 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; 799 if (res == TEE_SUCCESS) 800 w3 = FFA_OK; 801 else if (res == TEE_ERROR_OUT_OF_MEMORY) 802 w3 = FFA_DENIED; 803 else 804 w3 = FFA_INVALID_PARAMETERS; 805 } 806 break; 807 case FFA_MSG_SEND_VM_DESTROYED: 808 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 809 uint16_t guest_id = args->a5; 810 TEE_Result res = virt_guest_destroyed(guest_id); 811 812 w0 = direct_resp_fid; 813 w1 = swap_src_dst(args->a1); 814 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_DESTROYED; 815 if (res == TEE_SUCCESS) 816 w3 = FFA_OK; 817 else 818 w3 = FFA_INVALID_PARAMETERS; 819 } 820 break; 821 case FFA_MSG_VERSION_REQ: 822 w0 = direct_resp_fid; 823 w1 = swap_src_dst(args->a1); 824 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; 825 w3 = spmc_exchange_version(args->a3, rxtx); 826 break; 827 default: 828 break; 829 } 830 spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 831 } 832 833 static void handle_direct_request(struct thread_smc_args *args, 834 struct ffa_rxtx *rxtx) 835 { 836 uint32_t direct_resp_fid = 0; 837 838 if (IS_ENABLED(CFG_SECURE_PARTITION) && 839 FFA_DST(args->a1) != spmc_id && 840 FFA_DST(args->a1) != optee_endpoint_id) { 841 spmc_sp_start_thread(args); 842 return; 843 } 844 845 if (OPTEE_SMC_IS_64(args->a0)) 846 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_64; 847 else 848 direct_resp_fid = FFA_MSG_SEND_DIRECT_RESP_32; 849 850 if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { 851 handle_framework_direct_request(args, rxtx, direct_resp_fid); 852 return; 853 } 854 855 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 856 virt_set_guest(get_sender_id(args->a1))) { 857 spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, 858 TEE_ERROR_ITEM_NOT_FOUND, 0, 0); 859 return; 860 } 861 862 if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) 863 handle_yielding_call(args, direct_resp_fid); 864 else 865 handle_blocking_call(args, direct_resp_fid); 866 867 /* 868 * Note that handle_yielding_call() typically only returns if a 869 * thread cannot be allocated or found. virt_unset_guest() is also 870 * called from thread_state_suspend() and thread_state_free(). 871 */ 872 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 873 virt_unset_guest(); 874 } 875 876 int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, 877 struct ffa_mem_transaction_x *trans) 878 { 879 uint16_t mem_reg_attr = 0; 880 uint32_t flags = 0; 881 uint32_t count = 0; 882 uint32_t offs = 0; 883 uint32_t size = 0; 884 size_t n = 0; 885 886 if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) 887 return FFA_INVALID_PARAMETERS; 888 889 if (ffa_vers >= FFA_VERSION_1_1) { 890 struct ffa_mem_transaction_1_1 *descr = NULL; 891 892 if (blen < sizeof(*descr)) 893 return FFA_INVALID_PARAMETERS; 894 895 descr = buf; 896 trans->sender_id = READ_ONCE(descr->sender_id); 897 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 898 flags = READ_ONCE(descr->flags); 899 trans->global_handle = READ_ONCE(descr->global_handle); 900 trans->tag = READ_ONCE(descr->tag); 901 902 count = READ_ONCE(descr->mem_access_count); 903 size = READ_ONCE(descr->mem_access_size); 904 offs = READ_ONCE(descr->mem_access_offs); 905 } else { 906 struct ffa_mem_transaction_1_0 *descr = NULL; 907 908 if (blen < sizeof(*descr)) 909 return FFA_INVALID_PARAMETERS; 910 911 descr = buf; 912 trans->sender_id = READ_ONCE(descr->sender_id); 913 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 914 flags = READ_ONCE(descr->flags); 915 trans->global_handle = READ_ONCE(descr->global_handle); 916 trans->tag = READ_ONCE(descr->tag); 917 918 count = READ_ONCE(descr->mem_access_count); 919 size = sizeof(struct ffa_mem_access); 920 offs = offsetof(struct ffa_mem_transaction_1_0, 921 mem_access_array); 922 } 923 924 if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || 925 size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) 926 return FFA_INVALID_PARAMETERS; 927 928 /* Check that the endpoint memory access descriptor array fits */ 929 if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || 930 n > blen) 931 return FFA_INVALID_PARAMETERS; 932 933 trans->mem_reg_attr = mem_reg_attr; 934 trans->flags = flags; 935 trans->mem_access_size = size; 936 trans->mem_access_count = count; 937 trans->mem_access_offs = offs; 938 return 0; 939 } 940 941 #if defined(CFG_CORE_SEL1_SPMC) 942 static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, 943 unsigned int mem_access_count, uint8_t *acc_perms, 944 unsigned int *region_offs) 945 { 946 struct ffa_mem_access_perm *descr = NULL; 947 struct ffa_mem_access *mem_acc = NULL; 948 unsigned int n = 0; 949 950 for (n = 0; n < mem_access_count; n++) { 951 mem_acc = (void *)(mem_acc_base + mem_access_size * n); 952 descr = &mem_acc->access_perm; 953 if (READ_ONCE(descr->endpoint_id) == optee_endpoint_id) { 954 *acc_perms = READ_ONCE(descr->perm); 955 *region_offs = READ_ONCE(mem_acc[n].region_offs); 956 return 0; 957 } 958 } 959 960 return FFA_INVALID_PARAMETERS; 961 } 962 963 static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, 964 size_t blen, unsigned int *page_count, 965 unsigned int *region_count, size_t *addr_range_offs) 966 { 967 const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 968 const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; 969 struct ffa_mem_region *region_descr = NULL; 970 unsigned int region_descr_offs = 0; 971 uint8_t mem_acc_perm = 0; 972 size_t n = 0; 973 974 if (mem_trans->mem_reg_attr != exp_mem_reg_attr) 975 return FFA_INVALID_PARAMETERS; 976 977 /* Check that the access permissions matches what's expected */ 978 if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, 979 mem_trans->mem_access_size, 980 mem_trans->mem_access_count, 981 &mem_acc_perm, ®ion_descr_offs) || 982 mem_acc_perm != exp_mem_acc_perm) 983 return FFA_INVALID_PARAMETERS; 984 985 /* Check that the Composite memory region descriptor fits */ 986 if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || 987 n > blen) 988 return FFA_INVALID_PARAMETERS; 989 990 if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, 991 struct ffa_mem_region)) 992 return FFA_INVALID_PARAMETERS; 993 994 region_descr = (struct ffa_mem_region *)((vaddr_t)buf + 995 region_descr_offs); 996 *page_count = READ_ONCE(region_descr->total_page_count); 997 *region_count = READ_ONCE(region_descr->address_range_count); 998 *addr_range_offs = n; 999 return 0; 1000 } 1001 1002 static int add_mem_share_helper(struct mem_share_state *s, void *buf, 1003 size_t flen) 1004 { 1005 unsigned int region_count = flen / sizeof(struct ffa_address_range); 1006 struct ffa_address_range *arange = NULL; 1007 unsigned int n = 0; 1008 1009 if (region_count > s->region_count) 1010 region_count = s->region_count; 1011 1012 if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) 1013 return FFA_INVALID_PARAMETERS; 1014 arange = buf; 1015 1016 for (n = 0; n < region_count; n++) { 1017 unsigned int page_count = READ_ONCE(arange[n].page_count); 1018 uint64_t addr = READ_ONCE(arange[n].address); 1019 1020 if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, 1021 addr, page_count)) 1022 return FFA_INVALID_PARAMETERS; 1023 } 1024 1025 s->region_count -= region_count; 1026 if (s->region_count) 1027 return region_count * sizeof(*arange); 1028 1029 if (s->current_page_idx != s->page_count) 1030 return FFA_INVALID_PARAMETERS; 1031 1032 return 0; 1033 } 1034 1035 static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) 1036 { 1037 int rc = 0; 1038 1039 rc = add_mem_share_helper(&s->share, buf, flen); 1040 if (rc >= 0) { 1041 if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { 1042 /* We're not at the end of the descriptor yet */ 1043 if (s->share.region_count) 1044 return s->frag_offset; 1045 1046 /* We're done */ 1047 rc = 0; 1048 } else { 1049 rc = FFA_INVALID_PARAMETERS; 1050 } 1051 } 1052 1053 SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); 1054 if (rc < 0) 1055 mobj_ffa_sel1_spmc_delete(s->share.mf); 1056 else 1057 mobj_ffa_push_to_inactive(s->share.mf); 1058 free(s); 1059 1060 return rc; 1061 } 1062 1063 static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, 1064 void *buf) 1065 { 1066 struct ffa_mem_access_perm *perm = NULL; 1067 struct ffa_mem_access *mem_acc = NULL; 1068 1069 if (!IS_ENABLED(CFG_SECURE_PARTITION)) 1070 return false; 1071 1072 if (mem_trans->mem_access_count < 1) 1073 return false; 1074 1075 mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); 1076 perm = &mem_acc->access_perm; 1077 1078 /* 1079 * perm->endpoint_id is read here only to check if the endpoint is 1080 * OP-TEE. We do read it later on again, but there are some additional 1081 * checks there to make sure that the data is correct. 1082 */ 1083 return READ_ONCE(perm->endpoint_id) != optee_endpoint_id; 1084 } 1085 1086 static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, 1087 tee_mm_entry_t *mm, void *buf, size_t blen, 1088 size_t flen, uint64_t *global_handle) 1089 { 1090 int rc = 0; 1091 struct mem_share_state share = { }; 1092 size_t addr_range_offs = 0; 1093 uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1094 size_t n = 0; 1095 1096 rc = mem_share_init(mem_trans, buf, flen, &share.page_count, 1097 &share.region_count, &addr_range_offs); 1098 if (rc) 1099 return rc; 1100 1101 if (!share.page_count || !share.region_count) 1102 return FFA_INVALID_PARAMETERS; 1103 1104 if (MUL_OVERFLOW(share.region_count, 1105 sizeof(struct ffa_address_range), &n) || 1106 ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) 1107 return FFA_INVALID_PARAMETERS; 1108 1109 if (mem_trans->global_handle) 1110 cookie = mem_trans->global_handle; 1111 share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); 1112 if (!share.mf) 1113 return FFA_NO_MEMORY; 1114 1115 if (flen != blen) { 1116 struct mem_frag_state *s = calloc(1, sizeof(*s)); 1117 1118 if (!s) { 1119 rc = FFA_NO_MEMORY; 1120 goto err; 1121 } 1122 s->share = share; 1123 s->mm = mm; 1124 s->frag_offset = addr_range_offs; 1125 1126 SLIST_INSERT_HEAD(&frag_state_head, s, link); 1127 rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, 1128 flen - addr_range_offs); 1129 1130 if (rc >= 0) 1131 *global_handle = mobj_ffa_get_cookie(share.mf); 1132 1133 return rc; 1134 } 1135 1136 rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, 1137 flen - addr_range_offs); 1138 if (rc) { 1139 /* 1140 * Number of consumed bytes may be returned instead of 0 for 1141 * done. 1142 */ 1143 rc = FFA_INVALID_PARAMETERS; 1144 goto err; 1145 } 1146 1147 *global_handle = mobj_ffa_push_to_inactive(share.mf); 1148 1149 return 0; 1150 err: 1151 mobj_ffa_sel1_spmc_delete(share.mf); 1152 return rc; 1153 } 1154 1155 static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, 1156 unsigned int page_count, 1157 uint64_t *global_handle, struct ffa_rxtx *rxtx) 1158 { 1159 struct ffa_mem_transaction_x mem_trans = { }; 1160 int rc = 0; 1161 size_t len = 0; 1162 void *buf = NULL; 1163 tee_mm_entry_t *mm = NULL; 1164 vaddr_t offs = pbuf & SMALL_PAGE_MASK; 1165 1166 if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) 1167 return FFA_INVALID_PARAMETERS; 1168 if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) 1169 return FFA_INVALID_PARAMETERS; 1170 1171 /* 1172 * Check that the length reported in flen is covered by len even 1173 * if the offset is taken into account. 1174 */ 1175 if (len < flen || len - offs < flen) 1176 return FFA_INVALID_PARAMETERS; 1177 1178 mm = tee_mm_alloc(&core_virt_shm_pool, len); 1179 if (!mm) 1180 return FFA_NO_MEMORY; 1181 1182 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, 1183 page_count, MEM_AREA_NSEC_SHM)) { 1184 rc = FFA_INVALID_PARAMETERS; 1185 goto out; 1186 } 1187 buf = (void *)(tee_mm_get_smem(mm) + offs); 1188 1189 cpu_spin_lock(&rxtx->spinlock); 1190 rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); 1191 if (rc) 1192 goto unlock; 1193 1194 if (is_sp_share(&mem_trans, buf)) { 1195 rc = spmc_sp_add_share(&mem_trans, buf, blen, flen, 1196 global_handle, NULL); 1197 goto unlock; 1198 } 1199 1200 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1201 virt_set_guest(mem_trans.sender_id)) { 1202 rc = FFA_DENIED; 1203 goto unlock; 1204 } 1205 1206 rc = add_mem_share(&mem_trans, mm, buf, blen, flen, global_handle); 1207 1208 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1209 virt_unset_guest(); 1210 1211 unlock: 1212 cpu_spin_unlock(&rxtx->spinlock); 1213 if (rc > 0) 1214 return rc; 1215 1216 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1217 out: 1218 tee_mm_free(mm); 1219 return rc; 1220 } 1221 1222 static int handle_mem_share_rxbuf(size_t blen, size_t flen, 1223 uint64_t *global_handle, 1224 struct ffa_rxtx *rxtx) 1225 { 1226 struct ffa_mem_transaction_x mem_trans = { }; 1227 int rc = FFA_DENIED; 1228 1229 cpu_spin_lock(&rxtx->spinlock); 1230 1231 if (!rxtx->rx || flen > rxtx->size) 1232 goto out; 1233 1234 rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, 1235 &mem_trans); 1236 if (rc) 1237 goto out; 1238 if (is_sp_share(&mem_trans, rxtx->rx)) { 1239 rc = spmc_sp_add_share(&mem_trans, rxtx, blen, flen, 1240 global_handle, NULL); 1241 goto out; 1242 } 1243 1244 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1245 virt_set_guest(mem_trans.sender_id)) 1246 goto out; 1247 1248 rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, 1249 global_handle); 1250 1251 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1252 virt_unset_guest(); 1253 1254 out: 1255 cpu_spin_unlock(&rxtx->spinlock); 1256 1257 return rc; 1258 } 1259 1260 static void handle_mem_share(struct thread_smc_args *args, 1261 struct ffa_rxtx *rxtx) 1262 { 1263 uint32_t tot_len = args->a1; 1264 uint32_t frag_len = args->a2; 1265 uint64_t addr = args->a3; 1266 uint32_t page_count = args->a4; 1267 uint32_t ret_w1 = 0; 1268 uint32_t ret_w2 = FFA_INVALID_PARAMETERS; 1269 uint32_t ret_w3 = 0; 1270 uint32_t ret_fid = FFA_ERROR; 1271 uint64_t global_handle = 0; 1272 int rc = 0; 1273 1274 /* Check that the MBZs are indeed 0 */ 1275 if (args->a5 || args->a6 || args->a7) 1276 goto out; 1277 1278 /* Check that fragment length doesn't exceed total length */ 1279 if (frag_len > tot_len) 1280 goto out; 1281 1282 /* Check for 32-bit calling convention */ 1283 if (args->a0 == FFA_MEM_SHARE_32) 1284 addr &= UINT32_MAX; 1285 1286 if (!addr) { 1287 /* 1288 * The memory transaction descriptor is passed via our rx 1289 * buffer. 1290 */ 1291 if (page_count) 1292 goto out; 1293 rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, 1294 rxtx); 1295 } else { 1296 rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, 1297 &global_handle, rxtx); 1298 } 1299 if (rc < 0) { 1300 ret_w2 = rc; 1301 } else if (rc > 0) { 1302 ret_fid = FFA_MEM_FRAG_RX; 1303 ret_w3 = rc; 1304 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1305 } else { 1306 ret_fid = FFA_SUCCESS_32; 1307 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1308 } 1309 out: 1310 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1311 } 1312 1313 static struct mem_frag_state *get_frag_state(uint64_t global_handle) 1314 { 1315 struct mem_frag_state *s = NULL; 1316 1317 SLIST_FOREACH(s, &frag_state_head, link) 1318 if (mobj_ffa_get_cookie(s->share.mf) == global_handle) 1319 return s; 1320 1321 return NULL; 1322 } 1323 1324 static void handle_mem_frag_tx(struct thread_smc_args *args, 1325 struct ffa_rxtx *rxtx) 1326 { 1327 uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); 1328 size_t flen = args->a3; 1329 uint32_t endpoint_id = args->a4; 1330 struct mem_frag_state *s = NULL; 1331 tee_mm_entry_t *mm = NULL; 1332 unsigned int page_count = 0; 1333 void *buf = NULL; 1334 uint32_t ret_w1 = 0; 1335 uint32_t ret_w2 = 0; 1336 uint32_t ret_w3 = 0; 1337 uint32_t ret_fid = 0; 1338 int rc = 0; 1339 1340 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1341 uint16_t guest_id = endpoint_id >> 16; 1342 1343 if (!guest_id || virt_set_guest(guest_id)) { 1344 rc = FFA_INVALID_PARAMETERS; 1345 goto out_set_rc; 1346 } 1347 } 1348 1349 /* 1350 * Currently we're only doing this for fragmented FFA_MEM_SHARE_* 1351 * requests. 1352 */ 1353 1354 cpu_spin_lock(&rxtx->spinlock); 1355 1356 s = get_frag_state(global_handle); 1357 if (!s) { 1358 rc = FFA_INVALID_PARAMETERS; 1359 goto out; 1360 } 1361 1362 mm = s->mm; 1363 if (mm) { 1364 if (flen > tee_mm_get_bytes(mm)) { 1365 rc = FFA_INVALID_PARAMETERS; 1366 goto out; 1367 } 1368 page_count = s->share.page_count; 1369 buf = (void *)tee_mm_get_smem(mm); 1370 } else { 1371 if (flen > rxtx->size) { 1372 rc = FFA_INVALID_PARAMETERS; 1373 goto out; 1374 } 1375 buf = rxtx->rx; 1376 } 1377 1378 rc = add_mem_share_frag(s, buf, flen); 1379 out: 1380 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1381 virt_unset_guest(); 1382 1383 cpu_spin_unlock(&rxtx->spinlock); 1384 1385 if (rc <= 0 && mm) { 1386 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1387 tee_mm_free(mm); 1388 } 1389 1390 out_set_rc: 1391 if (rc < 0) { 1392 ret_fid = FFA_ERROR; 1393 ret_w2 = rc; 1394 } else if (rc > 0) { 1395 ret_fid = FFA_MEM_FRAG_RX; 1396 ret_w3 = rc; 1397 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1398 } else { 1399 ret_fid = FFA_SUCCESS_32; 1400 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1401 } 1402 1403 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1404 } 1405 1406 static void handle_mem_reclaim(struct thread_smc_args *args) 1407 { 1408 int rc = FFA_INVALID_PARAMETERS; 1409 uint64_t cookie = 0; 1410 1411 if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) 1412 goto out; 1413 1414 cookie = reg_pair_to_64(args->a2, args->a1); 1415 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1416 uint16_t guest_id = 0; 1417 1418 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 1419 guest_id = virt_find_guest_by_cookie(cookie); 1420 } else { 1421 guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & 1422 FFA_MEMORY_HANDLE_PRTN_MASK; 1423 } 1424 if (!guest_id) 1425 goto out; 1426 if (virt_set_guest(guest_id)) { 1427 if (!virt_reclaim_cookie_from_destroyed_guest(guest_id, 1428 cookie)) 1429 rc = FFA_OK; 1430 goto out; 1431 } 1432 } 1433 1434 switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { 1435 case TEE_SUCCESS: 1436 rc = FFA_OK; 1437 break; 1438 case TEE_ERROR_ITEM_NOT_FOUND: 1439 DMSG("cookie %#"PRIx64" not found", cookie); 1440 rc = FFA_INVALID_PARAMETERS; 1441 break; 1442 default: 1443 DMSG("cookie %#"PRIx64" busy", cookie); 1444 rc = FFA_DENIED; 1445 break; 1446 } 1447 1448 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 1449 virt_unset_guest(); 1450 1451 out: 1452 set_simple_ret_val(args, rc); 1453 } 1454 1455 static void handle_notification_bitmap_create(struct thread_smc_args *args) 1456 { 1457 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1458 uint32_t ret_fid = FFA_ERROR; 1459 uint32_t old_itr_status = 0; 1460 1461 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1462 !args->a5 && !args->a6 && !args->a7) { 1463 uint16_t vm_id = args->a1; 1464 1465 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1466 1467 if (notif_vm_id_valid) { 1468 if (vm_id == notif_vm_id) 1469 ret_val = FFA_DENIED; 1470 else 1471 ret_val = FFA_NO_MEMORY; 1472 } else { 1473 notif_vm_id = vm_id; 1474 notif_vm_id_valid = true; 1475 ret_val = FFA_OK; 1476 ret_fid = FFA_SUCCESS_32; 1477 } 1478 1479 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1480 } 1481 1482 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1483 } 1484 1485 static void handle_notification_bitmap_destroy(struct thread_smc_args *args) 1486 { 1487 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1488 uint32_t ret_fid = FFA_ERROR; 1489 uint32_t old_itr_status = 0; 1490 1491 if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && 1492 !args->a5 && !args->a6 && !args->a7) { 1493 uint16_t vm_id = args->a1; 1494 1495 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1496 1497 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1498 if (notif_pending_bitmap || notif_bound_bitmap) { 1499 ret_val = FFA_DENIED; 1500 } else { 1501 notif_vm_id_valid = false; 1502 ret_val = FFA_OK; 1503 ret_fid = FFA_SUCCESS_32; 1504 } 1505 } 1506 1507 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1508 } 1509 1510 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1511 } 1512 1513 static void handle_notification_bind(struct thread_smc_args *args) 1514 { 1515 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1516 uint32_t ret_fid = FFA_ERROR; 1517 uint32_t old_itr_status = 0; 1518 uint64_t bitmap = 0; 1519 uint16_t vm_id = 0; 1520 1521 if (args->a5 || args->a6 || args->a7) 1522 goto out; 1523 if (args->a2) { 1524 /* We only deal with global notifications for now */ 1525 ret_val = FFA_NOT_SUPPORTED; 1526 goto out; 1527 } 1528 1529 /* The destination of the eventual notification */ 1530 vm_id = FFA_DST(args->a1); 1531 bitmap = reg_pair_to_64(args->a4, args->a3); 1532 1533 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1534 1535 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1536 if (bitmap & notif_bound_bitmap) { 1537 ret_val = FFA_DENIED; 1538 } else { 1539 notif_bound_bitmap |= bitmap; 1540 ret_val = FFA_OK; 1541 ret_fid = FFA_SUCCESS_32; 1542 } 1543 } 1544 1545 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1546 out: 1547 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1548 } 1549 1550 static void handle_notification_unbind(struct thread_smc_args *args) 1551 { 1552 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1553 uint32_t ret_fid = FFA_ERROR; 1554 uint32_t old_itr_status = 0; 1555 uint64_t bitmap = 0; 1556 uint16_t vm_id = 0; 1557 1558 if (args->a2 || args->a5 || args->a6 || args->a7) 1559 goto out; 1560 1561 /* The destination of the eventual notification */ 1562 vm_id = FFA_DST(args->a1); 1563 bitmap = reg_pair_to_64(args->a4, args->a3); 1564 1565 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1566 1567 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1568 /* 1569 * Spec says: 1570 * At least one notification is bound to another Sender or 1571 * is currently pending. 1572 * 1573 * Not sure what the intention is. 1574 */ 1575 if (bitmap & notif_pending_bitmap) { 1576 ret_val = FFA_DENIED; 1577 } else { 1578 notif_bound_bitmap &= ~bitmap; 1579 ret_val = FFA_OK; 1580 ret_fid = FFA_SUCCESS_32; 1581 } 1582 } 1583 1584 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1585 out: 1586 spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); 1587 } 1588 1589 static void handle_notification_get(struct thread_smc_args *args) 1590 { 1591 uint32_t w2 = FFA_INVALID_PARAMETERS; 1592 uint32_t ret_fid = FFA_ERROR; 1593 uint32_t old_itr_status = 0; 1594 uint16_t vm_id = 0; 1595 uint32_t w3 = 0; 1596 1597 if (args->a5 || args->a6 || args->a7) 1598 goto out; 1599 if (!(args->a2 & 0x1)) { 1600 ret_fid = FFA_SUCCESS_32; 1601 w2 = 0; 1602 goto out; 1603 } 1604 vm_id = FFA_DST(args->a1); 1605 1606 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1607 1608 if (notif_vm_id_valid && vm_id == notif_vm_id) { 1609 reg_pair_from_64(notif_pending_bitmap, &w3, &w2); 1610 notif_pending_bitmap = 0; 1611 ret_fid = FFA_SUCCESS_32; 1612 } 1613 1614 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1615 out: 1616 spmc_set_args(args, ret_fid, 0, w2, w3, 0, 0); 1617 } 1618 1619 static void handle_notification_info_get(struct thread_smc_args *args) 1620 { 1621 uint32_t w2 = FFA_INVALID_PARAMETERS; 1622 uint32_t ret_fid = FFA_ERROR; 1623 1624 if (args->a1 || args->a2 || args->a3 || args->a4 || args->a5 || 1625 args->a6 || args->a7) 1626 goto out; 1627 1628 if (OPTEE_SMC_IS_64(args->a0)) 1629 ret_fid = FFA_SUCCESS_64; 1630 else 1631 ret_fid = FFA_SUCCESS_32; 1632 1633 /* 1634 * Note, we're only supporting physical OS kernel in normal world 1635 * with Global Notifications. 1636 * So one list of ID list registers (BIT[11:7]) 1637 * and one count of IDs (BIT[13:12] + 1) 1638 * and the VM is always 0. 1639 */ 1640 w2 = SHIFT_U32(1, 7); 1641 out: 1642 spmc_set_args(args, ret_fid, 0, w2, 0, 0, 0); 1643 } 1644 1645 void thread_spmc_set_async_notif_intid(int intid) 1646 { 1647 assert(interrupt_can_raise_sgi(interrupt_get_main_chip())); 1648 notif_intid = intid; 1649 spmc_notif_is_ready = true; 1650 DMSG("Asynchronous notifications are ready"); 1651 } 1652 1653 void notif_send_async(uint32_t value) 1654 { 1655 uint32_t old_itr_status = 0; 1656 1657 old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); 1658 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && spmc_notif_is_ready && 1659 do_bottom_half_value >= 0 && notif_intid >= 0); 1660 notif_pending_bitmap |= BIT64(do_bottom_half_value); 1661 interrupt_raise_sgi(interrupt_get_main_chip(), notif_intid, 1662 ITR_CPU_MASK_TO_THIS_CPU); 1663 cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); 1664 } 1665 #else 1666 void notif_send_async(uint32_t value) 1667 { 1668 /* global notification, delay notification interrupt */ 1669 uint32_t flags = BIT32(1); 1670 int res = 0; 1671 1672 assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && spmc_notif_is_ready && 1673 do_bottom_half_value >= 0); 1674 res = ffa_set_notification(notif_vm_id, optee_endpoint_id, flags, 1675 BIT64(do_bottom_half_value)); 1676 if (res) { 1677 EMSG("notification set failed with error %d", res); 1678 panic(); 1679 } 1680 } 1681 #endif 1682 1683 /* Only called from assembly */ 1684 void thread_spmc_msg_recv(struct thread_smc_args *args); 1685 void thread_spmc_msg_recv(struct thread_smc_args *args) 1686 { 1687 assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); 1688 switch (args->a0) { 1689 #if defined(CFG_CORE_SEL1_SPMC) 1690 case FFA_FEATURES: 1691 handle_features(args); 1692 break; 1693 case FFA_SPM_ID_GET: 1694 spmc_handle_spm_id_get(args); 1695 break; 1696 #ifdef ARM64 1697 case FFA_RXTX_MAP_64: 1698 #endif 1699 case FFA_RXTX_MAP_32: 1700 spmc_handle_rxtx_map(args, &my_rxtx); 1701 break; 1702 case FFA_RXTX_UNMAP: 1703 spmc_handle_rxtx_unmap(args, &my_rxtx); 1704 break; 1705 case FFA_RX_RELEASE: 1706 spmc_handle_rx_release(args, &my_rxtx); 1707 break; 1708 case FFA_PARTITION_INFO_GET: 1709 spmc_handle_partition_info_get(args, &my_rxtx); 1710 break; 1711 case FFA_RUN: 1712 spmc_handle_run(args); 1713 break; 1714 #endif /*CFG_CORE_SEL1_SPMC*/ 1715 case FFA_INTERRUPT: 1716 if (IS_ENABLED(CFG_CORE_SEL1_SPMC)) 1717 spmc_set_args(args, FFA_NORMAL_WORLD_RESUME, 0, 0, 0, 1718 0, 0); 1719 else 1720 spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); 1721 break; 1722 #ifdef ARM64 1723 case FFA_MSG_SEND_DIRECT_REQ_64: 1724 #endif 1725 case FFA_MSG_SEND_DIRECT_REQ_32: 1726 handle_direct_request(args, &my_rxtx); 1727 break; 1728 #if defined(CFG_CORE_SEL1_SPMC) 1729 #ifdef ARM64 1730 case FFA_MEM_SHARE_64: 1731 #endif 1732 case FFA_MEM_SHARE_32: 1733 handle_mem_share(args, &my_rxtx); 1734 break; 1735 case FFA_MEM_RECLAIM: 1736 if (!IS_ENABLED(CFG_SECURE_PARTITION) || 1737 !ffa_mem_reclaim(args, NULL)) 1738 handle_mem_reclaim(args); 1739 break; 1740 case FFA_MEM_FRAG_TX: 1741 handle_mem_frag_tx(args, &my_rxtx); 1742 break; 1743 case FFA_NOTIFICATION_BITMAP_CREATE: 1744 handle_notification_bitmap_create(args); 1745 break; 1746 case FFA_NOTIFICATION_BITMAP_DESTROY: 1747 handle_notification_bitmap_destroy(args); 1748 break; 1749 case FFA_NOTIFICATION_BIND: 1750 handle_notification_bind(args); 1751 break; 1752 case FFA_NOTIFICATION_UNBIND: 1753 handle_notification_unbind(args); 1754 break; 1755 case FFA_NOTIFICATION_GET: 1756 handle_notification_get(args); 1757 break; 1758 #ifdef ARM64 1759 case FFA_NOTIFICATION_INFO_GET_64: 1760 #endif 1761 case FFA_NOTIFICATION_INFO_GET_32: 1762 handle_notification_info_get(args); 1763 break; 1764 #endif /*CFG_CORE_SEL1_SPMC*/ 1765 case FFA_ERROR: 1766 EMSG("Cannot handle FFA_ERROR(%d)", (int)args->a2); 1767 if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { 1768 /* 1769 * The SPMC will return an FFA_ERROR back so better 1770 * panic() now than flooding the log. 1771 */ 1772 panic("FFA_ERROR from SPMC is fatal"); 1773 } 1774 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, 1775 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 1776 break; 1777 default: 1778 EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); 1779 set_simple_ret_val(args, FFA_NOT_SUPPORTED); 1780 } 1781 } 1782 1783 static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) 1784 { 1785 size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1786 struct thread_ctx *thr = threads + thread_get_id(); 1787 TEE_Result res = TEE_ERROR_BAD_PARAMETERS; 1788 struct optee_msg_arg *arg = NULL; 1789 struct mobj *mobj = NULL; 1790 uint32_t num_params = 0; 1791 size_t sz = 0; 1792 1793 mobj = mobj_ffa_get_by_cookie(cookie, 0); 1794 if (!mobj) { 1795 EMSG("Can't find cookie %#"PRIx64, cookie); 1796 return TEE_ERROR_BAD_PARAMETERS; 1797 } 1798 1799 res = mobj_inc_map(mobj); 1800 if (res) 1801 goto out_put_mobj; 1802 1803 res = TEE_ERROR_BAD_PARAMETERS; 1804 arg = mobj_get_va(mobj, offset, sizeof(*arg)); 1805 if (!arg) 1806 goto out_dec_map; 1807 1808 num_params = READ_ONCE(arg->num_params); 1809 if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) 1810 goto out_dec_map; 1811 1812 sz = OPTEE_MSG_GET_ARG_SIZE(num_params); 1813 1814 thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); 1815 if (!thr->rpc_arg) 1816 goto out_dec_map; 1817 1818 virt_on_stdcall(); 1819 res = tee_entry_std(arg, num_params); 1820 1821 thread_rpc_shm_cache_clear(&thr->shm_cache); 1822 thr->rpc_arg = NULL; 1823 1824 out_dec_map: 1825 mobj_dec_map(mobj); 1826 out_put_mobj: 1827 mobj_put(mobj); 1828 return res; 1829 } 1830 1831 /* 1832 * Helper routine for the assembly function thread_std_smc_entry() 1833 * 1834 * Note: this function is weak just to make link_dummies_paged.c happy. 1835 */ 1836 uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, 1837 uint32_t a2, uint32_t a3, 1838 uint32_t a4, uint32_t a5 __unused) 1839 { 1840 /* 1841 * Arguments are supplied from handle_yielding_call() as: 1842 * a0 <- w1 1843 * a1 <- w3 1844 * a2 <- w4 1845 * a3 <- w5 1846 * a4 <- w6 1847 * a5 <- w7 1848 */ 1849 thread_get_tsd()->rpc_target_info = swap_src_dst(a0); 1850 if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) 1851 return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); 1852 return FFA_DENIED; 1853 } 1854 1855 static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) 1856 { 1857 uint64_t offs = tpm->u.memref.offs; 1858 1859 param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + 1860 OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; 1861 1862 param->u.fmem.offs_low = offs; 1863 param->u.fmem.offs_high = offs >> 32; 1864 if (param->u.fmem.offs_high != offs >> 32) 1865 return false; 1866 1867 param->u.fmem.size = tpm->u.memref.size; 1868 if (tpm->u.memref.mobj) { 1869 uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); 1870 1871 /* If a mobj is passed it better be one with a valid cookie. */ 1872 if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) 1873 return false; 1874 param->u.fmem.global_id = cookie; 1875 } else { 1876 param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1877 } 1878 1879 return true; 1880 } 1881 1882 static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, 1883 struct thread_param *params, 1884 struct optee_msg_arg **arg_ret) 1885 { 1886 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1887 struct thread_ctx *thr = threads + thread_get_id(); 1888 struct optee_msg_arg *arg = thr->rpc_arg; 1889 1890 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 1891 return TEE_ERROR_BAD_PARAMETERS; 1892 1893 if (!arg) { 1894 EMSG("rpc_arg not set"); 1895 return TEE_ERROR_GENERIC; 1896 } 1897 1898 memset(arg, 0, sz); 1899 arg->cmd = cmd; 1900 arg->num_params = num_params; 1901 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 1902 1903 for (size_t n = 0; n < num_params; n++) { 1904 switch (params[n].attr) { 1905 case THREAD_PARAM_ATTR_NONE: 1906 arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; 1907 break; 1908 case THREAD_PARAM_ATTR_VALUE_IN: 1909 case THREAD_PARAM_ATTR_VALUE_OUT: 1910 case THREAD_PARAM_ATTR_VALUE_INOUT: 1911 arg->params[n].attr = params[n].attr - 1912 THREAD_PARAM_ATTR_VALUE_IN + 1913 OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 1914 arg->params[n].u.value.a = params[n].u.value.a; 1915 arg->params[n].u.value.b = params[n].u.value.b; 1916 arg->params[n].u.value.c = params[n].u.value.c; 1917 break; 1918 case THREAD_PARAM_ATTR_MEMREF_IN: 1919 case THREAD_PARAM_ATTR_MEMREF_OUT: 1920 case THREAD_PARAM_ATTR_MEMREF_INOUT: 1921 if (!set_fmem(arg->params + n, params + n)) 1922 return TEE_ERROR_BAD_PARAMETERS; 1923 break; 1924 default: 1925 return TEE_ERROR_BAD_PARAMETERS; 1926 } 1927 } 1928 1929 if (arg_ret) 1930 *arg_ret = arg; 1931 1932 return TEE_SUCCESS; 1933 } 1934 1935 static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, 1936 struct thread_param *params) 1937 { 1938 for (size_t n = 0; n < num_params; n++) { 1939 switch (params[n].attr) { 1940 case THREAD_PARAM_ATTR_VALUE_OUT: 1941 case THREAD_PARAM_ATTR_VALUE_INOUT: 1942 params[n].u.value.a = arg->params[n].u.value.a; 1943 params[n].u.value.b = arg->params[n].u.value.b; 1944 params[n].u.value.c = arg->params[n].u.value.c; 1945 break; 1946 case THREAD_PARAM_ATTR_MEMREF_OUT: 1947 case THREAD_PARAM_ATTR_MEMREF_INOUT: 1948 params[n].u.memref.size = arg->params[n].u.fmem.size; 1949 break; 1950 default: 1951 break; 1952 } 1953 } 1954 1955 return arg->ret; 1956 } 1957 1958 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 1959 struct thread_param *params) 1960 { 1961 struct thread_rpc_arg rpc_arg = { .call = { 1962 .w1 = thread_get_tsd()->rpc_target_info, 1963 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1964 }, 1965 }; 1966 struct optee_msg_arg *arg = NULL; 1967 uint32_t ret = 0; 1968 1969 ret = get_rpc_arg(cmd, num_params, params, &arg); 1970 if (ret) 1971 return ret; 1972 1973 thread_rpc(&rpc_arg); 1974 1975 return get_rpc_arg_res(arg, num_params, params); 1976 } 1977 1978 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 1979 { 1980 struct thread_rpc_arg rpc_arg = { .call = { 1981 .w1 = thread_get_tsd()->rpc_target_info, 1982 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1983 }, 1984 }; 1985 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); 1986 uint32_t res2 = 0; 1987 uint32_t res = 0; 1988 1989 DMSG("freeing cookie %#"PRIx64, cookie); 1990 1991 res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); 1992 1993 mobj_put(mobj); 1994 res2 = mobj_ffa_unregister_by_cookie(cookie); 1995 if (res2) 1996 DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, 1997 cookie, res2); 1998 if (!res) 1999 thread_rpc(&rpc_arg); 2000 } 2001 2002 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) 2003 { 2004 struct thread_rpc_arg rpc_arg = { .call = { 2005 .w1 = thread_get_tsd()->rpc_target_info, 2006 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 2007 }, 2008 }; 2009 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); 2010 struct optee_msg_arg *arg = NULL; 2011 unsigned int internal_offset = 0; 2012 struct mobj *mobj = NULL; 2013 uint64_t cookie = 0; 2014 2015 if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) 2016 return NULL; 2017 2018 thread_rpc(&rpc_arg); 2019 2020 if (arg->num_params != 1 || 2021 arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) 2022 return NULL; 2023 2024 internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); 2025 cookie = READ_ONCE(arg->params->u.fmem.global_id); 2026 mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); 2027 if (!mobj) { 2028 DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", 2029 cookie, internal_offset); 2030 return NULL; 2031 } 2032 2033 assert(mobj_is_nonsec(mobj)); 2034 2035 if (mobj->size < size) { 2036 DMSG("Mobj %#"PRIx64": wrong size", cookie); 2037 mobj_put(mobj); 2038 return NULL; 2039 } 2040 2041 if (mobj_inc_map(mobj)) { 2042 DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); 2043 mobj_put(mobj); 2044 return NULL; 2045 } 2046 2047 return mobj; 2048 } 2049 2050 struct mobj *thread_rpc_alloc_payload(size_t size) 2051 { 2052 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); 2053 } 2054 2055 struct mobj *thread_rpc_alloc_kernel_payload(size_t size) 2056 { 2057 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); 2058 } 2059 2060 void thread_rpc_free_kernel_payload(struct mobj *mobj) 2061 { 2062 if (mobj) 2063 thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, 2064 mobj_get_cookie(mobj), mobj); 2065 } 2066 2067 void thread_rpc_free_payload(struct mobj *mobj) 2068 { 2069 if (mobj) 2070 thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), 2071 mobj); 2072 } 2073 2074 struct mobj *thread_rpc_alloc_global_payload(size_t size) 2075 { 2076 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); 2077 } 2078 2079 void thread_rpc_free_global_payload(struct mobj *mobj) 2080 { 2081 if (mobj) 2082 thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, 2083 mobj_get_cookie(mobj), mobj); 2084 } 2085 2086 void thread_spmc_register_secondary_ep(vaddr_t ep) 2087 { 2088 unsigned long ret = 0; 2089 2090 /* Let the SPM know the entry point for secondary CPUs */ 2091 ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); 2092 2093 if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) 2094 EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); 2095 } 2096 2097 static uint16_t ffa_id_get(void) 2098 { 2099 /* 2100 * Ask the SPM component running at a higher EL to return our FF-A ID. 2101 * This can either be the SPMC ID (if the SPMC is enabled in OP-TEE) or 2102 * the partition ID (if not). 2103 */ 2104 struct thread_smc_args args = { 2105 .a0 = FFA_ID_GET, 2106 }; 2107 2108 thread_smccc(&args); 2109 if (!is_ffa_success(args.a0)) { 2110 if (args.a0 == FFA_ERROR) 2111 EMSG("Get id failed with error %ld", args.a2); 2112 else 2113 EMSG("Get id failed"); 2114 panic(); 2115 } 2116 2117 return args.a2; 2118 } 2119 2120 static uint16_t ffa_spm_id_get(void) 2121 { 2122 /* 2123 * Ask the SPM component running at a higher EL to return its ID. 2124 * If OP-TEE implements the S-EL1 SPMC, this will get the SPMD ID. 2125 * If not, the ID of the SPMC will be returned. 2126 */ 2127 struct thread_smc_args args = { 2128 .a0 = FFA_SPM_ID_GET, 2129 }; 2130 2131 thread_smccc(&args); 2132 if (!is_ffa_success(args.a0)) { 2133 if (args.a0 == FFA_ERROR) 2134 EMSG("Get spm id failed with error %ld", args.a2); 2135 else 2136 EMSG("Get spm id failed"); 2137 panic(); 2138 } 2139 2140 return args.a2; 2141 } 2142 2143 #if defined(CFG_CORE_SEL1_SPMC) 2144 static TEE_Result spmc_init(void) 2145 { 2146 spmd_id = ffa_spm_id_get(); 2147 DMSG("SPMD ID %#"PRIx16, spmd_id); 2148 2149 spmc_id = ffa_id_get(); 2150 DMSG("SPMC ID %#"PRIx16, spmc_id); 2151 2152 optee_endpoint_id = FFA_SWD_ID_MIN; 2153 while (optee_endpoint_id == spmd_id || optee_endpoint_id == spmc_id) 2154 optee_endpoint_id++; 2155 2156 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2157 2158 /* 2159 * If SPMD think we are version 1.0 it will report version 1.0 to 2160 * normal world regardless of what version we query the SPM with. 2161 * However, if SPMD think we are version 1.1 it will forward 2162 * queries from normal world to let us negotiate version. So by 2163 * setting version 1.0 here we should be compatible. 2164 * 2165 * Note that disagreement on negotiated version means that we'll 2166 * have communication problems with normal world. 2167 */ 2168 my_rxtx.ffa_vers = FFA_VERSION_1_0; 2169 2170 return TEE_SUCCESS; 2171 } 2172 #else /* !defined(CFG_CORE_SEL1_SPMC) */ 2173 static void spmc_rxtx_map(struct ffa_rxtx *rxtx) 2174 { 2175 struct thread_smc_args args = { 2176 #ifdef ARM64 2177 .a0 = FFA_RXTX_MAP_64, 2178 #else 2179 .a0 = FFA_RXTX_MAP_32, 2180 #endif 2181 .a1 = virt_to_phys(rxtx->tx), 2182 .a2 = virt_to_phys(rxtx->rx), 2183 .a3 = 1, 2184 }; 2185 2186 thread_smccc(&args); 2187 if (!is_ffa_success(args.a0)) { 2188 if (args.a0 == FFA_ERROR) 2189 EMSG("rxtx map failed with error %ld", args.a2); 2190 else 2191 EMSG("rxtx map failed"); 2192 panic(); 2193 } 2194 } 2195 2196 static uint32_t get_ffa_version(uint32_t my_version) 2197 { 2198 struct thread_smc_args args = { 2199 .a0 = FFA_VERSION, 2200 .a1 = my_version, 2201 }; 2202 2203 thread_smccc(&args); 2204 if (args.a0 & BIT(31)) { 2205 EMSG("FF-A version failed with error %ld", args.a0); 2206 panic(); 2207 } 2208 2209 return args.a0; 2210 } 2211 2212 static void *spmc_retrieve_req(uint64_t cookie, 2213 struct ffa_mem_transaction_x *trans) 2214 { 2215 struct ffa_mem_access *acc_descr_array = NULL; 2216 struct ffa_mem_access_perm *perm_descr = NULL; 2217 struct thread_smc_args args = { 2218 .a0 = FFA_MEM_RETRIEVE_REQ_32, 2219 .a3 = 0, /* Address, Using TX -> MBZ */ 2220 .a4 = 0, /* Using TX -> MBZ */ 2221 }; 2222 size_t size = 0; 2223 int rc = 0; 2224 2225 if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { 2226 struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; 2227 2228 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2229 memset(trans_descr, 0, size); 2230 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2231 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2232 trans_descr->global_handle = cookie; 2233 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2234 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2235 trans_descr->mem_access_count = 1; 2236 acc_descr_array = trans_descr->mem_access_array; 2237 } else { 2238 struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; 2239 2240 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 2241 memset(trans_descr, 0, size); 2242 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 2243 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 2244 trans_descr->global_handle = cookie; 2245 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 2246 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 2247 trans_descr->mem_access_count = 1; 2248 trans_descr->mem_access_offs = sizeof(*trans_descr); 2249 trans_descr->mem_access_size = sizeof(struct ffa_mem_access); 2250 acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + 2251 sizeof(*trans_descr)); 2252 } 2253 acc_descr_array->region_offs = 0; 2254 acc_descr_array->reserved = 0; 2255 perm_descr = &acc_descr_array->access_perm; 2256 perm_descr->endpoint_id = optee_endpoint_id; 2257 perm_descr->perm = FFA_MEM_ACC_RW; 2258 perm_descr->flags = 0; 2259 2260 args.a1 = size; /* Total Length */ 2261 args.a2 = size; /* Frag Length == Total length */ 2262 thread_smccc(&args); 2263 if (args.a0 != FFA_MEM_RETRIEVE_RESP) { 2264 if (args.a0 == FFA_ERROR) 2265 EMSG("Failed to fetch cookie %#"PRIx64" error code %d", 2266 cookie, (int)args.a2); 2267 else 2268 EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, 2269 cookie, args.a0); 2270 return NULL; 2271 } 2272 rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.rx, 2273 my_rxtx.size, trans); 2274 if (rc) { 2275 EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", 2276 cookie, rc); 2277 return NULL; 2278 } 2279 2280 return my_rxtx.rx; 2281 } 2282 2283 void thread_spmc_relinquish(uint64_t cookie) 2284 { 2285 struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; 2286 struct thread_smc_args args = { 2287 .a0 = FFA_MEM_RELINQUISH, 2288 }; 2289 2290 memset(relinquish_desc, 0, sizeof(*relinquish_desc)); 2291 relinquish_desc->handle = cookie; 2292 relinquish_desc->flags = 0; 2293 relinquish_desc->endpoint_count = 1; 2294 relinquish_desc->endpoint_id_array[0] = optee_endpoint_id; 2295 thread_smccc(&args); 2296 if (!is_ffa_success(args.a0)) 2297 EMSG("Failed to relinquish cookie %#"PRIx64, cookie); 2298 } 2299 2300 static int set_pages(struct ffa_address_range *regions, 2301 unsigned int num_regions, unsigned int num_pages, 2302 struct mobj_ffa *mf) 2303 { 2304 unsigned int n = 0; 2305 unsigned int idx = 0; 2306 2307 for (n = 0; n < num_regions; n++) { 2308 unsigned int page_count = READ_ONCE(regions[n].page_count); 2309 uint64_t addr = READ_ONCE(regions[n].address); 2310 2311 if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) 2312 return FFA_INVALID_PARAMETERS; 2313 } 2314 2315 if (idx != num_pages) 2316 return FFA_INVALID_PARAMETERS; 2317 2318 return 0; 2319 } 2320 2321 struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) 2322 { 2323 struct mobj_ffa *ret = NULL; 2324 struct ffa_mem_transaction_x retrieve_desc = { }; 2325 struct ffa_mem_access *descr_array = NULL; 2326 struct ffa_mem_region *descr = NULL; 2327 struct mobj_ffa *mf = NULL; 2328 unsigned int num_pages = 0; 2329 unsigned int offs = 0; 2330 void *buf = NULL; 2331 struct thread_smc_args ffa_rx_release_args = { 2332 .a0 = FFA_RX_RELEASE 2333 }; 2334 2335 /* 2336 * OP-TEE is only supporting a single mem_region while the 2337 * specification allows for more than one. 2338 */ 2339 buf = spmc_retrieve_req(cookie, &retrieve_desc); 2340 if (!buf) { 2341 EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, 2342 cookie); 2343 return NULL; 2344 } 2345 2346 descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); 2347 offs = READ_ONCE(descr_array->region_offs); 2348 descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); 2349 2350 num_pages = READ_ONCE(descr->total_page_count); 2351 mf = mobj_ffa_spmc_new(cookie, num_pages); 2352 if (!mf) 2353 goto out; 2354 2355 if (set_pages(descr->address_range_array, 2356 READ_ONCE(descr->address_range_count), num_pages, mf)) { 2357 mobj_ffa_spmc_delete(mf); 2358 goto out; 2359 } 2360 2361 ret = mf; 2362 2363 out: 2364 /* Release RX buffer after the mem retrieve request. */ 2365 thread_smccc(&ffa_rx_release_args); 2366 2367 return ret; 2368 } 2369 2370 static TEE_Result spmc_init(void) 2371 { 2372 unsigned int major = 0; 2373 unsigned int minor __maybe_unused = 0; 2374 uint32_t my_vers = 0; 2375 uint32_t vers = 0; 2376 2377 my_vers = MAKE_FFA_VERSION(FFA_VERSION_MAJOR, FFA_VERSION_MINOR); 2378 vers = get_ffa_version(my_vers); 2379 major = (vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK; 2380 minor = (vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK; 2381 DMSG("SPMC reported version %u.%u", major, minor); 2382 if (major != FFA_VERSION_MAJOR) { 2383 EMSG("Incompatible major version %u, expected %u", 2384 major, FFA_VERSION_MAJOR); 2385 panic(); 2386 } 2387 if (vers < my_vers) 2388 my_vers = vers; 2389 DMSG("Using version %u.%u", 2390 (my_vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK, 2391 (my_vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK); 2392 my_rxtx.ffa_vers = my_vers; 2393 2394 spmc_rxtx_map(&my_rxtx); 2395 2396 spmc_id = ffa_spm_id_get(); 2397 DMSG("SPMC ID %#"PRIx16, spmc_id); 2398 2399 optee_endpoint_id = ffa_id_get(); 2400 DMSG("OP-TEE endpoint ID %#"PRIx16, optee_endpoint_id); 2401 2402 if (!ffa_features(FFA_NOTIFICATION_SET)) { 2403 spmc_notif_is_ready = true; 2404 DMSG("Asynchronous notifications are ready"); 2405 } 2406 2407 return TEE_SUCCESS; 2408 } 2409 #endif /* !defined(CFG_CORE_SEL1_SPMC) */ 2410 2411 /* 2412 * boot_final() is always done before exiting at end of boot 2413 * initialization. In case of virtualization the init-calls are done only 2414 * once a OP-TEE partition has been created. So with virtualization we have 2415 * to initialize via boot_final() to make sure we have a value assigned 2416 * before it's used the first time. 2417 */ 2418 #ifdef CFG_NS_VIRTUALIZATION 2419 boot_final(spmc_init); 2420 #else 2421 service_init(spmc_init); 2422 #endif 2423