1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2020-2021, Linaro Limited. 4 * Copyright (c) 2019-2021, Arm Limited. All rights reserved. 5 */ 6 7 #include <assert.h> 8 #include <ffa.h> 9 #include <initcall.h> 10 #include <io.h> 11 #include <kernel/interrupt.h> 12 #include <kernel/panic.h> 13 #include <kernel/secure_partition.h> 14 #include <kernel/spinlock.h> 15 #include <kernel/spmc_sp_handler.h> 16 #include <kernel/tee_misc.h> 17 #include <kernel/thread.h> 18 #include <kernel/thread_private.h> 19 #include <kernel/thread_spmc.h> 20 #include <kernel/virtualization.h> 21 #include <mm/core_mmu.h> 22 #include <mm/mobj.h> 23 #include <optee_ffa.h> 24 #include <optee_msg.h> 25 #include <optee_rpc_cmd.h> 26 #include <string.h> 27 #include <sys/queue.h> 28 #include <tee/entry_std.h> 29 #include <tee/uuid.h> 30 #include <util.h> 31 32 #if defined(CFG_CORE_SEL1_SPMC) 33 struct mem_share_state { 34 struct mobj_ffa *mf; 35 unsigned int page_count; 36 unsigned int region_count; 37 unsigned int current_page_idx; 38 }; 39 40 struct mem_frag_state { 41 struct mem_share_state share; 42 tee_mm_entry_t *mm; 43 unsigned int frag_offset; 44 SLIST_ENTRY(mem_frag_state) link; 45 }; 46 #endif 47 48 /* Initialized in spmc_init() below */ 49 static uint16_t my_endpoint_id __nex_bss; 50 #ifdef CFG_CORE_SEL1_SPMC 51 static const uint32_t my_part_props = FFA_PART_PROP_DIRECT_REQ_RECV | 52 FFA_PART_PROP_DIRECT_REQ_SEND | 53 #ifdef CFG_NS_VIRTUALIZATION 54 FFA_PART_PROP_NOTIF_CREATED | 55 FFA_PART_PROP_NOTIF_DESTROYED | 56 #endif 57 #ifdef ARM64 58 FFA_PART_PROP_AARCH64_STATE | 59 #endif 60 FFA_PART_PROP_IS_PE_ID; 61 62 static uint32_t my_uuid_words[] = { 63 /* 64 * - if the SPMC is in S-EL2 this UUID describes OP-TEE as a S-EL1 65 * SP, or 66 * - if the SPMC is in S-EL1 then this UUID is for OP-TEE as a 67 * logical partition, residing in the same exception level as the 68 * SPMC 69 * UUID 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 70 */ 71 0xe0786148, 0xe311f8e7, 0x02005ebc, 0x1bc5d5a5, 72 }; 73 74 /* 75 * If struct ffa_rxtx::size is 0 RX/TX buffers are not mapped or initialized. 76 * 77 * struct ffa_rxtx::spin_lock protects the variables below from concurrent 78 * access this includes the use of content of struct ffa_rxtx::rx and 79 * @frag_state_head. 80 * 81 * struct ffa_rxtx::tx_buf_is_mine is true when we may write to struct 82 * ffa_rxtx::tx and false when it is owned by normal world. 83 * 84 * Note that we can't prevent normal world from updating the content of 85 * these buffers so we must always be careful when reading. while we hold 86 * the lock. 87 */ 88 89 static struct ffa_rxtx my_rxtx __nex_bss; 90 91 static bool is_nw_buf(struct ffa_rxtx *rxtx) 92 { 93 return rxtx == &my_rxtx; 94 } 95 96 static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = 97 SLIST_HEAD_INITIALIZER(&frag_state_head); 98 #else 99 static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 100 static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); 101 static struct ffa_rxtx my_rxtx = { 102 .rx = __rx_buf, 103 .tx = __tx_buf, 104 .size = sizeof(__rx_buf), 105 }; 106 #endif 107 108 static uint32_t swap_src_dst(uint32_t src_dst) 109 { 110 return (src_dst >> 16) | (src_dst << 16); 111 } 112 113 static uint16_t get_sender_id(uint32_t src_dst) 114 { 115 return src_dst >> 16; 116 } 117 118 void spmc_set_args(struct thread_smc_args *args, uint32_t fid, uint32_t src_dst, 119 uint32_t w2, uint32_t w3, uint32_t w4, uint32_t w5) 120 { 121 *args = (struct thread_smc_args){ .a0 = fid, 122 .a1 = src_dst, 123 .a2 = w2, 124 .a3 = w3, 125 .a4 = w4, 126 .a5 = w5, }; 127 } 128 129 uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) 130 { 131 /* 132 * No locking, if the caller does concurrent calls to this it's 133 * only making a mess for itself. We must be able to renegotiate 134 * the FF-A version in order to support differing versions between 135 * the loader and the driver. 136 */ 137 if (vers < FFA_VERSION_1_1) 138 rxtx->ffa_vers = FFA_VERSION_1_0; 139 else 140 rxtx->ffa_vers = FFA_VERSION_1_1; 141 142 return rxtx->ffa_vers; 143 } 144 145 #if defined(CFG_CORE_SEL1_SPMC) 146 static void handle_features(struct thread_smc_args *args) 147 { 148 uint32_t ret_fid = 0; 149 uint32_t ret_w2 = FFA_PARAM_MBZ; 150 151 switch (args->a1) { 152 #ifdef ARM64 153 case FFA_RXTX_MAP_64: 154 #endif 155 case FFA_RXTX_MAP_32: 156 ret_fid = FFA_SUCCESS_32; 157 ret_w2 = 0; /* 4kB Minimum buffer size and alignment boundary */ 158 break; 159 #ifdef ARM64 160 case FFA_MEM_SHARE_64: 161 #endif 162 case FFA_MEM_SHARE_32: 163 ret_fid = FFA_SUCCESS_32; 164 /* 165 * Partition manager supports transmission of a memory 166 * transaction descriptor in a buffer dynamically allocated 167 * by the endpoint. 168 */ 169 ret_w2 = BIT(0); 170 break; 171 172 case FFA_ERROR: 173 case FFA_VERSION: 174 case FFA_SUCCESS_32: 175 #ifdef ARM64 176 case FFA_SUCCESS_64: 177 #endif 178 case FFA_FEATURES: 179 case FFA_SPM_ID_GET: 180 case FFA_MEM_FRAG_TX: 181 case FFA_MEM_RECLAIM: 182 case FFA_MSG_SEND_DIRECT_REQ_32: 183 case FFA_INTERRUPT: 184 case FFA_PARTITION_INFO_GET: 185 case FFA_RXTX_UNMAP: 186 case FFA_RX_RELEASE: 187 case FFA_FEATURE_MANAGED_EXIT_INTR: 188 ret_fid = FFA_SUCCESS_32; 189 break; 190 default: 191 ret_fid = FFA_ERROR; 192 ret_w2 = FFA_NOT_SUPPORTED; 193 break; 194 } 195 196 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, ret_w2, FFA_PARAM_MBZ, 197 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 198 } 199 200 static int map_buf(paddr_t pa, unsigned int sz, void **va_ret) 201 { 202 tee_mm_entry_t *mm = NULL; 203 204 if (!core_pbuf_is(CORE_MEM_NON_SEC, pa, sz)) 205 return FFA_INVALID_PARAMETERS; 206 207 mm = tee_mm_alloc(&tee_mm_shm, sz); 208 if (!mm) 209 return FFA_NO_MEMORY; 210 211 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pa, 212 sz / SMALL_PAGE_SIZE, 213 MEM_AREA_NSEC_SHM)) { 214 tee_mm_free(mm); 215 return FFA_INVALID_PARAMETERS; 216 } 217 218 *va_ret = (void *)tee_mm_get_smem(mm); 219 return 0; 220 } 221 222 static void handle_spm_id_get(struct thread_smc_args *args) 223 { 224 spmc_set_args(args, FFA_SUCCESS_32, FFA_PARAM_MBZ, my_endpoint_id, 225 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 226 } 227 228 static void unmap_buf(void *va, size_t sz) 229 { 230 tee_mm_entry_t *mm = tee_mm_find(&tee_mm_shm, (vaddr_t)va); 231 232 assert(mm); 233 core_mmu_unmap_pages(tee_mm_get_smem(mm), sz / SMALL_PAGE_SIZE); 234 tee_mm_free(mm); 235 } 236 237 void spmc_handle_rxtx_map(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 238 { 239 int rc = 0; 240 uint32_t ret_fid = FFA_ERROR; 241 unsigned int sz = 0; 242 paddr_t rx_pa = 0; 243 paddr_t tx_pa = 0; 244 void *rx = NULL; 245 void *tx = NULL; 246 247 cpu_spin_lock(&rxtx->spinlock); 248 249 if (args->a3 & GENMASK_64(63, 6)) { 250 rc = FFA_INVALID_PARAMETERS; 251 goto out; 252 } 253 254 sz = args->a3 * SMALL_PAGE_SIZE; 255 if (!sz) { 256 rc = FFA_INVALID_PARAMETERS; 257 goto out; 258 } 259 /* TX/RX are swapped compared to the caller */ 260 tx_pa = args->a2; 261 rx_pa = args->a1; 262 263 if (rxtx->size) { 264 rc = FFA_DENIED; 265 goto out; 266 } 267 268 /* 269 * If the buffer comes from a SP the address is virtual and already 270 * mapped. 271 */ 272 if (is_nw_buf(rxtx)) { 273 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 274 enum teecore_memtypes mt = MEM_AREA_NEX_NSEC_SHM; 275 bool tx_alloced = false; 276 277 /* 278 * With virtualization we establish this mapping in 279 * the nexus mapping which then is replicated to 280 * each partition. 281 * 282 * This means that this mapping must be done before 283 * any partition is created and then must not be 284 * changed. 285 */ 286 287 /* 288 * core_mmu_add_mapping() may reuse previous 289 * mappings. First check if there's any mappings to 290 * reuse so we know how to clean up in case of 291 * failure. 292 */ 293 tx = phys_to_virt(tx_pa, mt, sz); 294 rx = phys_to_virt(rx_pa, mt, sz); 295 if (!tx) { 296 tx = core_mmu_add_mapping(mt, tx_pa, sz); 297 if (!tx) { 298 rc = FFA_NO_MEMORY; 299 goto out; 300 } 301 tx_alloced = true; 302 } 303 if (!rx) 304 rx = core_mmu_add_mapping(mt, rx_pa, sz); 305 306 if (!rx) { 307 if (tx_alloced && tx) 308 core_mmu_remove_mapping(mt, tx, sz); 309 rc = FFA_NO_MEMORY; 310 goto out; 311 } 312 } else { 313 rc = map_buf(tx_pa, sz, &tx); 314 if (rc) 315 goto out; 316 rc = map_buf(rx_pa, sz, &rx); 317 if (rc) { 318 unmap_buf(tx, sz); 319 goto out; 320 } 321 } 322 rxtx->tx = tx; 323 rxtx->rx = rx; 324 } else { 325 if ((tx_pa & SMALL_PAGE_MASK) || (rx_pa & SMALL_PAGE_MASK)) { 326 rc = FFA_INVALID_PARAMETERS; 327 goto out; 328 } 329 330 if (!virt_to_phys((void *)tx_pa) || 331 !virt_to_phys((void *)rx_pa)) { 332 rc = FFA_INVALID_PARAMETERS; 333 goto out; 334 } 335 336 rxtx->tx = (void *)tx_pa; 337 rxtx->rx = (void *)rx_pa; 338 } 339 340 rxtx->size = sz; 341 rxtx->tx_is_mine = true; 342 ret_fid = FFA_SUCCESS_32; 343 DMSG("Mapped tx %#"PRIxPA" size %#x @ %p", tx_pa, sz, tx); 344 DMSG("Mapped rx %#"PRIxPA" size %#x @ %p", rx_pa, sz, rx); 345 out: 346 cpu_spin_unlock(&rxtx->spinlock); 347 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, 348 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 349 } 350 351 void spmc_handle_rxtx_unmap(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 352 { 353 uint32_t ret_fid = FFA_ERROR; 354 int rc = FFA_INVALID_PARAMETERS; 355 356 cpu_spin_lock(&rxtx->spinlock); 357 358 if (!rxtx->size) 359 goto out; 360 361 /* We don't unmap the SP memory as the SP might still use it */ 362 if (is_nw_buf(rxtx)) { 363 unmap_buf(rxtx->rx, rxtx->size); 364 unmap_buf(rxtx->tx, rxtx->size); 365 } 366 rxtx->size = 0; 367 rxtx->rx = NULL; 368 rxtx->tx = NULL; 369 ret_fid = FFA_SUCCESS_32; 370 rc = 0; 371 out: 372 cpu_spin_unlock(&rxtx->spinlock); 373 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, 374 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 375 } 376 377 void spmc_handle_rx_release(struct thread_smc_args *args, struct ffa_rxtx *rxtx) 378 { 379 uint32_t ret_fid = 0; 380 int rc = 0; 381 382 cpu_spin_lock(&rxtx->spinlock); 383 /* The senders RX is our TX */ 384 if (!rxtx->size || rxtx->tx_is_mine) { 385 ret_fid = FFA_ERROR; 386 rc = FFA_DENIED; 387 } else { 388 ret_fid = FFA_SUCCESS_32; 389 rc = 0; 390 rxtx->tx_is_mine = true; 391 } 392 cpu_spin_unlock(&rxtx->spinlock); 393 394 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, 395 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 396 } 397 398 static bool is_nil_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 399 { 400 return !w0 && !w1 && !w2 && !w3; 401 } 402 403 static bool is_my_uuid(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3) 404 { 405 /* 406 * This depends on which UUID we have been assigned. 407 * TODO add a generic mechanism to obtain our UUID. 408 * 409 * The test below is for the hard coded UUID 410 * 486178e0-e7f8-11e3-bc5e-0002a5d5c51b 411 */ 412 return w0 == my_uuid_words[0] && w1 == my_uuid_words[1] && 413 w2 == my_uuid_words[2] && w3 == my_uuid_words[3]; 414 } 415 416 TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, 417 size_t idx, uint16_t endpoint_id, 418 uint16_t execution_context, 419 uint32_t part_props, 420 const uint32_t uuid_words[4]) 421 { 422 struct ffa_partition_info_x *fpi = NULL; 423 size_t fpi_size = sizeof(*fpi); 424 425 if (ffa_vers >= FFA_VERSION_1_1) 426 fpi_size += FFA_UUID_SIZE; 427 428 if ((idx + 1) * fpi_size > blen) 429 return TEE_ERROR_OUT_OF_MEMORY; 430 431 fpi = (void *)((vaddr_t)buf + idx * fpi_size); 432 fpi->id = endpoint_id; 433 /* Number of execution contexts implemented by this partition */ 434 fpi->execution_context = execution_context; 435 436 fpi->partition_properties = part_props; 437 438 if (ffa_vers >= FFA_VERSION_1_1) { 439 if (uuid_words) 440 memcpy(fpi->uuid, uuid_words, FFA_UUID_SIZE); 441 else 442 memset(fpi->uuid, 0, FFA_UUID_SIZE); 443 } 444 445 return TEE_SUCCESS; 446 } 447 448 static int handle_partition_info_get_all(size_t *elem_count, 449 struct ffa_rxtx *rxtx, bool count_only) 450 { 451 if (!count_only) { 452 /* Add OP-TEE SP */ 453 if (spmc_fill_partition_entry(rxtx->ffa_vers, rxtx->tx, 454 rxtx->size, 0, my_endpoint_id, 455 CFG_TEE_CORE_NB_CORE, 456 my_part_props, my_uuid_words)) 457 return FFA_NO_MEMORY; 458 } 459 *elem_count = 1; 460 461 if (IS_ENABLED(CFG_SECURE_PARTITION)) { 462 if (sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, rxtx->size, 463 NULL, elem_count, count_only)) 464 return FFA_NO_MEMORY; 465 } 466 467 return FFA_OK; 468 } 469 470 void spmc_handle_partition_info_get(struct thread_smc_args *args, 471 struct ffa_rxtx *rxtx) 472 { 473 TEE_Result res = TEE_SUCCESS; 474 uint32_t ret_fid = FFA_ERROR; 475 uint32_t rc = 0; 476 bool count_only = args->a5 & FFA_PARTITION_INFO_GET_COUNT_FLAG; 477 478 if (!count_only) { 479 cpu_spin_lock(&rxtx->spinlock); 480 481 if (!rxtx->size || !rxtx->tx_is_mine) { 482 rc = FFA_BUSY; 483 goto out; 484 } 485 } 486 487 if (is_nil_uuid(args->a1, args->a2, args->a3, args->a4)) { 488 size_t elem_count = 0; 489 490 ret_fid = handle_partition_info_get_all(&elem_count, rxtx, 491 count_only); 492 493 if (ret_fid) { 494 rc = ret_fid; 495 ret_fid = FFA_ERROR; 496 } else { 497 ret_fid = FFA_SUCCESS_32; 498 rc = elem_count; 499 } 500 501 goto out; 502 } 503 504 if (is_my_uuid(args->a1, args->a2, args->a3, args->a4)) { 505 if (!count_only) { 506 res = spmc_fill_partition_entry(rxtx->ffa_vers, 507 rxtx->tx, rxtx->size, 0, 508 my_endpoint_id, 509 CFG_TEE_CORE_NB_CORE, 510 my_part_props, 511 my_uuid_words); 512 if (res) { 513 ret_fid = FFA_ERROR; 514 rc = FFA_INVALID_PARAMETERS; 515 goto out; 516 } 517 } 518 rc = 1; 519 } else if (IS_ENABLED(CFG_SECURE_PARTITION)) { 520 uint32_t uuid_array[4] = { 0 }; 521 TEE_UUID uuid = { }; 522 size_t count = 0; 523 524 uuid_array[0] = args->a1; 525 uuid_array[1] = args->a2; 526 uuid_array[2] = args->a3; 527 uuid_array[3] = args->a4; 528 tee_uuid_from_octets(&uuid, (uint8_t *)uuid_array); 529 530 res = sp_partition_info_get(rxtx->ffa_vers, rxtx->tx, 531 rxtx->size, &uuid, &count, 532 count_only); 533 if (res != TEE_SUCCESS) { 534 ret_fid = FFA_ERROR; 535 rc = FFA_INVALID_PARAMETERS; 536 goto out; 537 } 538 rc = count; 539 } else { 540 ret_fid = FFA_ERROR; 541 rc = FFA_INVALID_PARAMETERS; 542 goto out; 543 } 544 545 ret_fid = FFA_SUCCESS_32; 546 547 out: 548 spmc_set_args(args, ret_fid, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, 549 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 550 if (!count_only) { 551 rxtx->tx_is_mine = false; 552 cpu_spin_unlock(&rxtx->spinlock); 553 } 554 } 555 556 static void spmc_handle_run(struct thread_smc_args *args) 557 { 558 uint16_t endpoint = FFA_TARGET_INFO_GET_SP_ID(args->a1); 559 uint16_t thread_id = FFA_TARGET_INFO_GET_VCPU_ID(args->a1); 560 uint32_t rc = FFA_OK; 561 562 if (endpoint != my_endpoint_id) { 563 /* 564 * The endpoint should be an SP, try to resume the SP from 565 * preempted into busy state. 566 */ 567 rc = spmc_sp_resume_from_preempted(endpoint); 568 if (rc) 569 goto out; 570 } 571 572 thread_resume_from_rpc(thread_id, 0, 0, 0, 0); 573 574 /* thread_resume_from_rpc return only of the thread_id is invalid */ 575 rc = FFA_INVALID_PARAMETERS; 576 577 out: 578 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, rc, FFA_PARAM_MBZ, 579 FFA_PARAM_MBZ, FFA_PARAM_MBZ); 580 } 581 #endif /*CFG_CORE_SEL1_SPMC*/ 582 583 static void handle_yielding_call(struct thread_smc_args *args) 584 { 585 TEE_Result res = 0; 586 587 thread_check_canaries(); 588 589 if (args->a3 == OPTEE_FFA_YIELDING_CALL_RESUME) { 590 /* Note connection to struct thread_rpc_arg::ret */ 591 thread_resume_from_rpc(args->a7, args->a4, args->a5, args->a6, 592 0); 593 res = TEE_ERROR_BAD_PARAMETERS; 594 } else { 595 thread_alloc_and_run(args->a1, args->a3, args->a4, args->a5, 596 args->a6, args->a7); 597 res = TEE_ERROR_BUSY; 598 } 599 spmc_set_args(args, FFA_MSG_SEND_DIRECT_RESP_32, 600 swap_src_dst(args->a1), 0, res, 0, 0); 601 } 602 603 static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) 604 { 605 uint64_t cookie = reg_pair_to_64(a5, a4); 606 uint32_t res = 0; 607 608 res = mobj_ffa_unregister_by_cookie(cookie); 609 switch (res) { 610 case TEE_SUCCESS: 611 case TEE_ERROR_ITEM_NOT_FOUND: 612 return 0; 613 case TEE_ERROR_BUSY: 614 EMSG("res %#"PRIx32, res); 615 return FFA_BUSY; 616 default: 617 EMSG("res %#"PRIx32, res); 618 return FFA_INVALID_PARAMETERS; 619 } 620 } 621 622 static void handle_blocking_call(struct thread_smc_args *args) 623 { 624 switch (args->a3) { 625 case OPTEE_FFA_GET_API_VERSION: 626 spmc_set_args(args, FFA_MSG_SEND_DIRECT_RESP_32, 627 swap_src_dst(args->a1), 0, 628 OPTEE_FFA_VERSION_MAJOR, OPTEE_FFA_VERSION_MINOR, 629 0); 630 break; 631 case OPTEE_FFA_GET_OS_VERSION: 632 spmc_set_args(args, FFA_MSG_SEND_DIRECT_RESP_32, 633 swap_src_dst(args->a1), 0, 634 CFG_OPTEE_REVISION_MAJOR, 635 CFG_OPTEE_REVISION_MINOR, TEE_IMPL_GIT_SHA1); 636 break; 637 case OPTEE_FFA_EXCHANGE_CAPABILITIES: 638 spmc_set_args(args, FFA_MSG_SEND_DIRECT_RESP_32, 639 swap_src_dst(args->a1), 0, 0, 640 THREAD_RPC_MAX_NUM_PARAMS, 641 OPTEE_FFA_SEC_CAP_ARG_OFFSET); 642 break; 643 case OPTEE_FFA_UNREGISTER_SHM: 644 spmc_set_args(args, FFA_MSG_SEND_DIRECT_RESP_32, 645 swap_src_dst(args->a1), 0, 646 handle_unregister_shm(args->a4, args->a5), 0, 0); 647 break; 648 default: 649 EMSG("Unhandled blocking service ID %#"PRIx32, 650 (uint32_t)args->a3); 651 panic(); 652 } 653 } 654 655 static void handle_framework_direct_request(struct thread_smc_args *args, 656 struct ffa_rxtx *rxtx) 657 { 658 uint32_t w0 = FFA_ERROR; 659 uint32_t w1 = FFA_PARAM_MBZ; 660 uint32_t w2 = FFA_NOT_SUPPORTED; 661 uint32_t w3 = FFA_PARAM_MBZ; 662 663 switch (args->a2 & FFA_MSG_TYPE_MASK) { 664 case FFA_MSG_SEND_VM_CREATED: 665 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 666 uint16_t guest_id = args->a5; 667 TEE_Result res = virt_guest_created(guest_id); 668 669 w0 = FFA_MSG_SEND_DIRECT_RESP_32; 670 w1 = swap_src_dst(args->a1); 671 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_RESP_VM_CREATED; 672 if (res == TEE_SUCCESS) 673 w3 = FFA_OK; 674 else if (res == TEE_ERROR_OUT_OF_MEMORY) 675 w3 = FFA_DENIED; 676 else 677 w3 = FFA_INVALID_PARAMETERS; 678 } 679 break; 680 case FFA_MSG_VERSION_REQ: 681 w0 = FFA_MSG_SEND_DIRECT_RESP_32; 682 w1 = swap_src_dst(args->a1); 683 w2 = FFA_MSG_FLAG_FRAMEWORK | FFA_MSG_VERSION_RESP; 684 w3 = spmc_exchange_version(args->a3, rxtx); 685 break; 686 default: 687 break; 688 } 689 spmc_set_args(args, w0, w1, w2, w3, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 690 } 691 692 static void handle_direct_request(struct thread_smc_args *args, 693 struct ffa_rxtx *rxtx) 694 { 695 if (IS_ENABLED(CFG_SECURE_PARTITION) && 696 FFA_DST(args->a1) != my_endpoint_id) { 697 spmc_sp_start_thread(args); 698 return; 699 } 700 701 if (args->a2 & FFA_MSG_FLAG_FRAMEWORK) { 702 handle_framework_direct_request(args, rxtx); 703 return; 704 } 705 706 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 707 virt_set_guest(get_sender_id(args->a1))) { 708 spmc_set_args(args, FFA_MSG_SEND_DIRECT_RESP_32, 709 swap_src_dst(args->a1), 0, 710 TEE_ERROR_ITEM_NOT_FOUND, 0, 0); 711 return; 712 } 713 714 if (args->a3 & BIT32(OPTEE_FFA_YIELDING_CALL_BIT)) 715 handle_yielding_call(args); 716 else 717 handle_blocking_call(args); 718 719 /* 720 * Note that handle_yielding_call() typically only returns if a 721 * thread cannot be allocated or found. virt_unset_guest() is also 722 * called from thread_state_suspend() and thread_state_free(). 723 */ 724 virt_unset_guest(); 725 } 726 727 int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, 728 struct ffa_mem_transaction_x *trans) 729 { 730 uint16_t mem_reg_attr = 0; 731 uint32_t flags = 0; 732 uint32_t count = 0; 733 uint32_t offs = 0; 734 uint32_t size = 0; 735 size_t n = 0; 736 737 if (!IS_ALIGNED_WITH_TYPE(buf, uint64_t)) 738 return FFA_INVALID_PARAMETERS; 739 740 if (ffa_vers >= FFA_VERSION_1_1) { 741 struct ffa_mem_transaction_1_1 *descr = NULL; 742 743 if (blen < sizeof(*descr)) 744 return FFA_INVALID_PARAMETERS; 745 746 descr = buf; 747 trans->sender_id = READ_ONCE(descr->sender_id); 748 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 749 flags = READ_ONCE(descr->flags); 750 trans->global_handle = READ_ONCE(descr->global_handle); 751 trans->tag = READ_ONCE(descr->tag); 752 753 count = READ_ONCE(descr->mem_access_count); 754 size = READ_ONCE(descr->mem_access_size); 755 offs = READ_ONCE(descr->mem_access_offs); 756 } else { 757 struct ffa_mem_transaction_1_0 *descr = NULL; 758 759 if (blen < sizeof(*descr)) 760 return FFA_INVALID_PARAMETERS; 761 762 descr = buf; 763 trans->sender_id = READ_ONCE(descr->sender_id); 764 mem_reg_attr = READ_ONCE(descr->mem_reg_attr); 765 flags = READ_ONCE(descr->flags); 766 trans->global_handle = READ_ONCE(descr->global_handle); 767 trans->tag = READ_ONCE(descr->tag); 768 769 count = READ_ONCE(descr->mem_access_count); 770 size = sizeof(struct ffa_mem_access); 771 offs = offsetof(struct ffa_mem_transaction_1_0, 772 mem_access_array); 773 } 774 775 if (mem_reg_attr > UINT8_MAX || flags > UINT8_MAX || 776 size > UINT8_MAX || count > UINT8_MAX || offs > UINT16_MAX) 777 return FFA_INVALID_PARAMETERS; 778 779 /* Check that the endpoint memory access descriptor array fits */ 780 if (MUL_OVERFLOW(size, count, &n) || ADD_OVERFLOW(offs, n, &n) || 781 n > blen) 782 return FFA_INVALID_PARAMETERS; 783 784 trans->mem_reg_attr = mem_reg_attr; 785 trans->flags = flags; 786 trans->mem_access_size = size; 787 trans->mem_access_count = count; 788 trans->mem_access_offs = offs; 789 return 0; 790 } 791 792 #if defined(CFG_CORE_SEL1_SPMC) 793 static int get_acc_perms(vaddr_t mem_acc_base, unsigned int mem_access_size, 794 unsigned int mem_access_count, uint8_t *acc_perms, 795 unsigned int *region_offs) 796 { 797 struct ffa_mem_access_perm *descr = NULL; 798 struct ffa_mem_access *mem_acc = NULL; 799 unsigned int n = 0; 800 801 for (n = 0; n < mem_access_count; n++) { 802 mem_acc = (void *)(mem_acc_base + mem_access_size * n); 803 descr = &mem_acc->access_perm; 804 if (READ_ONCE(descr->endpoint_id) == my_endpoint_id) { 805 *acc_perms = READ_ONCE(descr->perm); 806 *region_offs = READ_ONCE(mem_acc[n].region_offs); 807 return 0; 808 } 809 } 810 811 return FFA_INVALID_PARAMETERS; 812 } 813 814 static int mem_share_init(struct ffa_mem_transaction_x *mem_trans, void *buf, 815 size_t blen, unsigned int *page_count, 816 unsigned int *region_count, size_t *addr_range_offs) 817 { 818 const uint16_t exp_mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 819 const uint8_t exp_mem_acc_perm = FFA_MEM_ACC_RW; 820 struct ffa_mem_region *region_descr = NULL; 821 unsigned int region_descr_offs = 0; 822 uint8_t mem_acc_perm = 0; 823 size_t n = 0; 824 825 if (mem_trans->mem_reg_attr != exp_mem_reg_attr) 826 return FFA_INVALID_PARAMETERS; 827 828 /* Check that the access permissions matches what's expected */ 829 if (get_acc_perms((vaddr_t)buf + mem_trans->mem_access_offs, 830 mem_trans->mem_access_size, 831 mem_trans->mem_access_count, 832 &mem_acc_perm, ®ion_descr_offs) || 833 mem_acc_perm != exp_mem_acc_perm) 834 return FFA_INVALID_PARAMETERS; 835 836 /* Check that the Composite memory region descriptor fits */ 837 if (ADD_OVERFLOW(region_descr_offs, sizeof(*region_descr), &n) || 838 n > blen) 839 return FFA_INVALID_PARAMETERS; 840 841 if (!IS_ALIGNED_WITH_TYPE((vaddr_t)buf + region_descr_offs, 842 struct ffa_mem_region)) 843 return FFA_INVALID_PARAMETERS; 844 845 region_descr = (struct ffa_mem_region *)((vaddr_t)buf + 846 region_descr_offs); 847 *page_count = READ_ONCE(region_descr->total_page_count); 848 *region_count = READ_ONCE(region_descr->address_range_count); 849 *addr_range_offs = n; 850 return 0; 851 } 852 853 static int add_mem_share_helper(struct mem_share_state *s, void *buf, 854 size_t flen) 855 { 856 unsigned int region_count = flen / sizeof(struct ffa_address_range); 857 struct ffa_address_range *arange = NULL; 858 unsigned int n = 0; 859 860 if (region_count > s->region_count) 861 region_count = s->region_count; 862 863 if (!IS_ALIGNED_WITH_TYPE(buf, struct ffa_address_range)) 864 return FFA_INVALID_PARAMETERS; 865 arange = buf; 866 867 for (n = 0; n < region_count; n++) { 868 unsigned int page_count = READ_ONCE(arange[n].page_count); 869 uint64_t addr = READ_ONCE(arange[n].address); 870 871 if (mobj_ffa_add_pages_at(s->mf, &s->current_page_idx, 872 addr, page_count)) 873 return FFA_INVALID_PARAMETERS; 874 } 875 876 s->region_count -= region_count; 877 if (s->region_count) 878 return region_count * sizeof(*arange); 879 880 if (s->current_page_idx != s->page_count) 881 return FFA_INVALID_PARAMETERS; 882 883 return 0; 884 } 885 886 static int add_mem_share_frag(struct mem_frag_state *s, void *buf, size_t flen) 887 { 888 int rc = 0; 889 890 rc = add_mem_share_helper(&s->share, buf, flen); 891 if (rc >= 0) { 892 if (!ADD_OVERFLOW(s->frag_offset, rc, &s->frag_offset)) { 893 /* We're not at the end of the descriptor yet */ 894 if (s->share.region_count) 895 return s->frag_offset; 896 897 /* We're done */ 898 rc = 0; 899 } else { 900 rc = FFA_INVALID_PARAMETERS; 901 } 902 } 903 904 SLIST_REMOVE(&frag_state_head, s, mem_frag_state, link); 905 if (rc < 0) 906 mobj_ffa_sel1_spmc_delete(s->share.mf); 907 else 908 mobj_ffa_push_to_inactive(s->share.mf); 909 free(s); 910 911 return rc; 912 } 913 914 static bool is_sp_share(struct ffa_mem_transaction_x *mem_trans, 915 void *buf) 916 { 917 struct ffa_mem_access_perm *perm = NULL; 918 struct ffa_mem_access *mem_acc = NULL; 919 920 if (!IS_ENABLED(CFG_SECURE_PARTITION)) 921 return false; 922 923 if (mem_trans->mem_access_count < 1) 924 return false; 925 926 mem_acc = (void *)((vaddr_t)buf + mem_trans->mem_access_offs); 927 perm = &mem_acc->access_perm; 928 929 /* 930 * perm->endpoint_id is read here only to check if the endpoint is 931 * OP-TEE. We do read it later on again, but there are some additional 932 * checks there to make sure that the data is correct. 933 */ 934 return READ_ONCE(perm->endpoint_id) != my_endpoint_id; 935 } 936 937 static int add_mem_share(struct ffa_mem_transaction_x *mem_trans, 938 tee_mm_entry_t *mm, void *buf, size_t blen, 939 size_t flen, uint64_t *global_handle) 940 { 941 int rc = 0; 942 struct mem_share_state share = { }; 943 size_t addr_range_offs = 0; 944 uint64_t cookie = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 945 size_t n = 0; 946 947 rc = mem_share_init(mem_trans, buf, flen, &share.page_count, 948 &share.region_count, &addr_range_offs); 949 if (rc) 950 return rc; 951 952 if (MUL_OVERFLOW(share.region_count, 953 sizeof(struct ffa_address_range), &n) || 954 ADD_OVERFLOW(n, addr_range_offs, &n) || n > blen) 955 return FFA_INVALID_PARAMETERS; 956 957 if (mem_trans->global_handle) 958 cookie = mem_trans->global_handle; 959 share.mf = mobj_ffa_sel1_spmc_new(cookie, share.page_count); 960 if (!share.mf) 961 return FFA_NO_MEMORY; 962 963 if (flen != blen) { 964 struct mem_frag_state *s = calloc(sizeof(*s), 1); 965 966 if (!s) { 967 rc = FFA_NO_MEMORY; 968 goto err; 969 } 970 s->share = share; 971 s->mm = mm; 972 s->frag_offset = addr_range_offs; 973 974 SLIST_INSERT_HEAD(&frag_state_head, s, link); 975 rc = add_mem_share_frag(s, (char *)buf + addr_range_offs, 976 flen - addr_range_offs); 977 978 if (rc >= 0) 979 *global_handle = mobj_ffa_get_cookie(share.mf); 980 981 return rc; 982 } 983 984 rc = add_mem_share_helper(&share, (char *)buf + addr_range_offs, 985 flen - addr_range_offs); 986 if (rc) { 987 /* 988 * Number of consumed bytes may be returned instead of 0 for 989 * done. 990 */ 991 rc = FFA_INVALID_PARAMETERS; 992 goto err; 993 } 994 995 *global_handle = mobj_ffa_push_to_inactive(share.mf); 996 997 return 0; 998 err: 999 mobj_ffa_sel1_spmc_delete(share.mf); 1000 return rc; 1001 } 1002 1003 static int handle_mem_share_tmem(paddr_t pbuf, size_t blen, size_t flen, 1004 unsigned int page_count, 1005 uint64_t *global_handle, struct ffa_rxtx *rxtx) 1006 { 1007 struct ffa_mem_transaction_x mem_trans = { }; 1008 int rc = 0; 1009 size_t len = 0; 1010 void *buf = NULL; 1011 tee_mm_entry_t *mm = NULL; 1012 vaddr_t offs = pbuf & SMALL_PAGE_MASK; 1013 1014 if (MUL_OVERFLOW(page_count, SMALL_PAGE_SIZE, &len)) 1015 return FFA_INVALID_PARAMETERS; 1016 if (!core_pbuf_is(CORE_MEM_NON_SEC, pbuf, len)) 1017 return FFA_INVALID_PARAMETERS; 1018 1019 /* 1020 * Check that the length reported in flen is covered by len even 1021 * if the offset is taken into account. 1022 */ 1023 if (len < flen || len - offs < flen) 1024 return FFA_INVALID_PARAMETERS; 1025 1026 mm = tee_mm_alloc(&tee_mm_shm, len); 1027 if (!mm) 1028 return FFA_NO_MEMORY; 1029 1030 if (core_mmu_map_contiguous_pages(tee_mm_get_smem(mm), pbuf, 1031 page_count, MEM_AREA_NSEC_SHM)) { 1032 rc = FFA_INVALID_PARAMETERS; 1033 goto out; 1034 } 1035 buf = (void *)(tee_mm_get_smem(mm) + offs); 1036 1037 cpu_spin_lock(&rxtx->spinlock); 1038 rc = spmc_read_mem_transaction(rxtx->ffa_vers, buf, flen, &mem_trans); 1039 if (!rc && IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1040 virt_set_guest(mem_trans.sender_id)) 1041 rc = FFA_DENIED; 1042 if (!rc) 1043 rc = add_mem_share(&mem_trans, mm, buf, blen, flen, 1044 global_handle); 1045 virt_unset_guest(); 1046 cpu_spin_unlock(&rxtx->spinlock); 1047 if (rc > 0) 1048 return rc; 1049 1050 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1051 out: 1052 tee_mm_free(mm); 1053 return rc; 1054 } 1055 1056 static int handle_mem_share_rxbuf(size_t blen, size_t flen, 1057 uint64_t *global_handle, 1058 struct ffa_rxtx *rxtx) 1059 { 1060 struct ffa_mem_transaction_x mem_trans = { }; 1061 int rc = FFA_DENIED; 1062 1063 cpu_spin_lock(&rxtx->spinlock); 1064 1065 if (!rxtx->rx || flen > rxtx->size) 1066 goto out; 1067 1068 rc = spmc_read_mem_transaction(rxtx->ffa_vers, rxtx->rx, flen, 1069 &mem_trans); 1070 if (rc) 1071 goto out; 1072 if (is_sp_share(&mem_trans, rxtx->rx)) { 1073 rc = spmc_sp_add_share(&mem_trans, rxtx, blen, 1074 global_handle, NULL); 1075 goto out; 1076 } 1077 1078 if (IS_ENABLED(CFG_NS_VIRTUALIZATION) && 1079 virt_set_guest(mem_trans.sender_id)) 1080 goto out; 1081 1082 rc = add_mem_share(&mem_trans, NULL, rxtx->rx, blen, flen, 1083 global_handle); 1084 1085 virt_unset_guest(); 1086 1087 out: 1088 cpu_spin_unlock(&rxtx->spinlock); 1089 1090 return rc; 1091 } 1092 1093 static void handle_mem_share(struct thread_smc_args *args, 1094 struct ffa_rxtx *rxtx) 1095 { 1096 uint32_t tot_len = args->a1; 1097 uint32_t frag_len = args->a2; 1098 uint64_t addr = args->a3; 1099 uint32_t page_count = args->a4; 1100 uint32_t ret_w1 = 0; 1101 uint32_t ret_w2 = FFA_INVALID_PARAMETERS; 1102 uint32_t ret_w3 = 0; 1103 uint32_t ret_fid = FFA_ERROR; 1104 uint64_t global_handle = 0; 1105 int rc = 0; 1106 1107 /* Check that the MBZs are indeed 0 */ 1108 if (args->a5 || args->a6 || args->a7) 1109 goto out; 1110 1111 /* Check that fragment length doesn't exceed total length */ 1112 if (frag_len > tot_len) 1113 goto out; 1114 1115 /* Check for 32-bit calling convention */ 1116 if (args->a0 == FFA_MEM_SHARE_32) 1117 addr &= UINT32_MAX; 1118 1119 if (!addr) { 1120 /* 1121 * The memory transaction descriptor is passed via our rx 1122 * buffer. 1123 */ 1124 if (page_count) 1125 goto out; 1126 rc = handle_mem_share_rxbuf(tot_len, frag_len, &global_handle, 1127 rxtx); 1128 } else { 1129 rc = handle_mem_share_tmem(addr, tot_len, frag_len, page_count, 1130 &global_handle, rxtx); 1131 } 1132 if (rc < 0) { 1133 ret_w2 = rc; 1134 } else if (rc > 0) { 1135 ret_fid = FFA_MEM_FRAG_RX; 1136 ret_w3 = rc; 1137 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1138 } else { 1139 ret_fid = FFA_SUCCESS_32; 1140 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1141 } 1142 out: 1143 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1144 } 1145 1146 static struct mem_frag_state *get_frag_state(uint64_t global_handle) 1147 { 1148 struct mem_frag_state *s = NULL; 1149 1150 SLIST_FOREACH(s, &frag_state_head, link) 1151 if (mobj_ffa_get_cookie(s->share.mf) == global_handle) 1152 return s; 1153 1154 return NULL; 1155 } 1156 1157 static void handle_mem_frag_tx(struct thread_smc_args *args, 1158 struct ffa_rxtx *rxtx) 1159 { 1160 uint64_t global_handle = reg_pair_to_64(args->a2, args->a1); 1161 size_t flen = args->a3; 1162 uint32_t endpoint_id = args->a4; 1163 struct mem_frag_state *s = NULL; 1164 tee_mm_entry_t *mm = NULL; 1165 unsigned int page_count = 0; 1166 void *buf = NULL; 1167 uint32_t ret_w1 = 0; 1168 uint32_t ret_w2 = 0; 1169 uint32_t ret_w3 = 0; 1170 uint32_t ret_fid = 0; 1171 int rc = 0; 1172 1173 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1174 uint16_t guest_id = endpoint_id >> 16; 1175 1176 if (!guest_id || virt_set_guest(guest_id)) { 1177 rc = FFA_INVALID_PARAMETERS; 1178 goto out_set_rc; 1179 } 1180 } 1181 1182 /* 1183 * Currently we're only doing this for fragmented FFA_MEM_SHARE_* 1184 * requests. 1185 */ 1186 1187 cpu_spin_lock(&rxtx->spinlock); 1188 1189 s = get_frag_state(global_handle); 1190 if (!s) { 1191 rc = FFA_INVALID_PARAMETERS; 1192 goto out; 1193 } 1194 1195 mm = s->mm; 1196 if (mm) { 1197 if (flen > tee_mm_get_bytes(mm)) { 1198 rc = FFA_INVALID_PARAMETERS; 1199 goto out; 1200 } 1201 page_count = s->share.page_count; 1202 buf = (void *)tee_mm_get_smem(mm); 1203 } else { 1204 if (flen > rxtx->size) { 1205 rc = FFA_INVALID_PARAMETERS; 1206 goto out; 1207 } 1208 buf = rxtx->rx; 1209 } 1210 1211 rc = add_mem_share_frag(s, buf, flen); 1212 out: 1213 virt_unset_guest(); 1214 cpu_spin_unlock(&rxtx->spinlock); 1215 1216 if (rc <= 0 && mm) { 1217 core_mmu_unmap_pages(tee_mm_get_smem(mm), page_count); 1218 tee_mm_free(mm); 1219 } 1220 1221 out_set_rc: 1222 if (rc < 0) { 1223 ret_fid = FFA_ERROR; 1224 ret_w2 = rc; 1225 } else if (rc > 0) { 1226 ret_fid = FFA_MEM_FRAG_RX; 1227 ret_w3 = rc; 1228 reg_pair_from_64(global_handle, &ret_w2, &ret_w1); 1229 } else { 1230 ret_fid = FFA_SUCCESS_32; 1231 reg_pair_from_64(global_handle, &ret_w3, &ret_w2); 1232 } 1233 1234 spmc_set_args(args, ret_fid, ret_w1, ret_w2, ret_w3, 0, 0); 1235 } 1236 1237 static void handle_mem_reclaim(struct thread_smc_args *args) 1238 { 1239 uint32_t ret_val = FFA_INVALID_PARAMETERS; 1240 uint32_t ret_fid = FFA_ERROR; 1241 uint64_t cookie = 0; 1242 1243 if (args->a3 || args->a4 || args->a5 || args->a6 || args->a7) 1244 goto out; 1245 1246 cookie = reg_pair_to_64(args->a2, args->a1); 1247 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1248 uint16_t guest_id = 0; 1249 1250 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 1251 guest_id = virt_find_guest_by_cookie(cookie); 1252 } else { 1253 guest_id = (cookie >> FFA_MEMORY_HANDLE_PRTN_SHIFT) & 1254 FFA_MEMORY_HANDLE_PRTN_MASK; 1255 } 1256 if (!guest_id || virt_set_guest(guest_id)) 1257 goto out; 1258 } 1259 1260 switch (mobj_ffa_sel1_spmc_reclaim(cookie)) { 1261 case TEE_SUCCESS: 1262 ret_fid = FFA_SUCCESS_32; 1263 ret_val = 0; 1264 break; 1265 case TEE_ERROR_ITEM_NOT_FOUND: 1266 DMSG("cookie %#"PRIx64" not found", cookie); 1267 ret_val = FFA_INVALID_PARAMETERS; 1268 break; 1269 default: 1270 DMSG("cookie %#"PRIx64" busy", cookie); 1271 ret_val = FFA_DENIED; 1272 break; 1273 } 1274 1275 virt_unset_guest(); 1276 1277 out: 1278 spmc_set_args(args, ret_fid, ret_val, 0, 0, 0, 0); 1279 } 1280 #endif 1281 1282 /* Only called from assembly */ 1283 void thread_spmc_msg_recv(struct thread_smc_args *args); 1284 void thread_spmc_msg_recv(struct thread_smc_args *args) 1285 { 1286 assert((thread_get_exceptions() & THREAD_EXCP_ALL) == THREAD_EXCP_ALL); 1287 switch (args->a0) { 1288 #if defined(CFG_CORE_SEL1_SPMC) 1289 case FFA_FEATURES: 1290 handle_features(args); 1291 break; 1292 case FFA_SPM_ID_GET: 1293 handle_spm_id_get(args); 1294 break; 1295 #ifdef ARM64 1296 case FFA_RXTX_MAP_64: 1297 #endif 1298 case FFA_RXTX_MAP_32: 1299 spmc_handle_rxtx_map(args, &my_rxtx); 1300 break; 1301 case FFA_RXTX_UNMAP: 1302 spmc_handle_rxtx_unmap(args, &my_rxtx); 1303 break; 1304 case FFA_RX_RELEASE: 1305 spmc_handle_rx_release(args, &my_rxtx); 1306 break; 1307 case FFA_PARTITION_INFO_GET: 1308 spmc_handle_partition_info_get(args, &my_rxtx); 1309 break; 1310 case FFA_RUN: 1311 spmc_handle_run(args); 1312 break; 1313 #endif /*CFG_CORE_SEL1_SPMC*/ 1314 case FFA_INTERRUPT: 1315 interrupt_main_handler(); 1316 spmc_set_args(args, FFA_MSG_WAIT, 0, 0, 0, 0, 0); 1317 break; 1318 #ifdef ARM64 1319 case FFA_MSG_SEND_DIRECT_REQ_64: 1320 #endif 1321 case FFA_MSG_SEND_DIRECT_REQ_32: 1322 handle_direct_request(args, &my_rxtx); 1323 break; 1324 #if defined(CFG_CORE_SEL1_SPMC) 1325 #ifdef ARM64 1326 case FFA_MEM_SHARE_64: 1327 #endif 1328 case FFA_MEM_SHARE_32: 1329 handle_mem_share(args, &my_rxtx); 1330 break; 1331 case FFA_MEM_RECLAIM: 1332 if (!IS_ENABLED(CFG_SECURE_PARTITION) || 1333 !ffa_mem_reclaim(args, NULL)) 1334 handle_mem_reclaim(args); 1335 break; 1336 case FFA_MEM_FRAG_TX: 1337 handle_mem_frag_tx(args, &my_rxtx); 1338 break; 1339 #endif /*CFG_CORE_SEL1_SPMC*/ 1340 default: 1341 EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); 1342 spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, 1343 FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); 1344 } 1345 } 1346 1347 static TEE_Result yielding_call_with_arg(uint64_t cookie, uint32_t offset) 1348 { 1349 size_t sz_rpc = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1350 struct thread_ctx *thr = threads + thread_get_id(); 1351 TEE_Result res = TEE_ERROR_BAD_PARAMETERS; 1352 struct optee_msg_arg *arg = NULL; 1353 struct mobj *mobj = NULL; 1354 uint32_t num_params = 0; 1355 size_t sz = 0; 1356 1357 mobj = mobj_ffa_get_by_cookie(cookie, 0); 1358 if (!mobj) { 1359 EMSG("Can't find cookie %#"PRIx64, cookie); 1360 return TEE_ERROR_BAD_PARAMETERS; 1361 } 1362 1363 res = mobj_inc_map(mobj); 1364 if (res) 1365 goto out_put_mobj; 1366 1367 res = TEE_ERROR_BAD_PARAMETERS; 1368 arg = mobj_get_va(mobj, offset, sizeof(*arg)); 1369 if (!arg) 1370 goto out_dec_map; 1371 1372 num_params = READ_ONCE(arg->num_params); 1373 if (num_params > OPTEE_MSG_MAX_NUM_PARAMS) 1374 goto out_dec_map; 1375 1376 sz = OPTEE_MSG_GET_ARG_SIZE(num_params); 1377 1378 thr->rpc_arg = mobj_get_va(mobj, offset + sz, sz_rpc); 1379 if (!thr->rpc_arg) 1380 goto out_dec_map; 1381 1382 virt_on_stdcall(); 1383 res = tee_entry_std(arg, num_params); 1384 1385 thread_rpc_shm_cache_clear(&thr->shm_cache); 1386 thr->rpc_arg = NULL; 1387 1388 out_dec_map: 1389 mobj_dec_map(mobj); 1390 out_put_mobj: 1391 mobj_put(mobj); 1392 return res; 1393 } 1394 1395 /* 1396 * Helper routine for the assembly function thread_std_smc_entry() 1397 * 1398 * Note: this function is weak just to make link_dummies_paged.c happy. 1399 */ 1400 uint32_t __weak __thread_std_smc_entry(uint32_t a0, uint32_t a1, 1401 uint32_t a2, uint32_t a3, 1402 uint32_t a4, uint32_t a5 __unused) 1403 { 1404 /* 1405 * Arguments are supplied from handle_yielding_call() as: 1406 * a0 <- w1 1407 * a1 <- w3 1408 * a2 <- w4 1409 * a3 <- w5 1410 * a4 <- w6 1411 * a5 <- w7 1412 */ 1413 thread_get_tsd()->rpc_target_info = swap_src_dst(a0); 1414 if (a1 == OPTEE_FFA_YIELDING_CALL_WITH_ARG) 1415 return yielding_call_with_arg(reg_pair_to_64(a3, a2), a4); 1416 return FFA_DENIED; 1417 } 1418 1419 static bool set_fmem(struct optee_msg_param *param, struct thread_param *tpm) 1420 { 1421 uint64_t offs = tpm->u.memref.offs; 1422 1423 param->attr = tpm->attr - THREAD_PARAM_ATTR_MEMREF_IN + 1424 OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; 1425 1426 param->u.fmem.offs_low = offs; 1427 param->u.fmem.offs_high = offs >> 32; 1428 if (param->u.fmem.offs_high != offs >> 32) 1429 return false; 1430 1431 param->u.fmem.size = tpm->u.memref.size; 1432 if (tpm->u.memref.mobj) { 1433 uint64_t cookie = mobj_get_cookie(tpm->u.memref.mobj); 1434 1435 /* If a mobj is passed it better be one with a valid cookie. */ 1436 if (cookie == OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) 1437 return false; 1438 param->u.fmem.global_id = cookie; 1439 } else { 1440 param->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; 1441 } 1442 1443 return true; 1444 } 1445 1446 static uint32_t get_rpc_arg(uint32_t cmd, size_t num_params, 1447 struct thread_param *params, 1448 struct optee_msg_arg **arg_ret) 1449 { 1450 size_t sz = OPTEE_MSG_GET_ARG_SIZE(THREAD_RPC_MAX_NUM_PARAMS); 1451 struct thread_ctx *thr = threads + thread_get_id(); 1452 struct optee_msg_arg *arg = thr->rpc_arg; 1453 1454 if (num_params > THREAD_RPC_MAX_NUM_PARAMS) 1455 return TEE_ERROR_BAD_PARAMETERS; 1456 1457 if (!arg) { 1458 EMSG("rpc_arg not set"); 1459 return TEE_ERROR_GENERIC; 1460 } 1461 1462 memset(arg, 0, sz); 1463 arg->cmd = cmd; 1464 arg->num_params = num_params; 1465 arg->ret = TEE_ERROR_GENERIC; /* in case value isn't updated */ 1466 1467 for (size_t n = 0; n < num_params; n++) { 1468 switch (params[n].attr) { 1469 case THREAD_PARAM_ATTR_NONE: 1470 arg->params[n].attr = OPTEE_MSG_ATTR_TYPE_NONE; 1471 break; 1472 case THREAD_PARAM_ATTR_VALUE_IN: 1473 case THREAD_PARAM_ATTR_VALUE_OUT: 1474 case THREAD_PARAM_ATTR_VALUE_INOUT: 1475 arg->params[n].attr = params[n].attr - 1476 THREAD_PARAM_ATTR_VALUE_IN + 1477 OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 1478 arg->params[n].u.value.a = params[n].u.value.a; 1479 arg->params[n].u.value.b = params[n].u.value.b; 1480 arg->params[n].u.value.c = params[n].u.value.c; 1481 break; 1482 case THREAD_PARAM_ATTR_MEMREF_IN: 1483 case THREAD_PARAM_ATTR_MEMREF_OUT: 1484 case THREAD_PARAM_ATTR_MEMREF_INOUT: 1485 if (!set_fmem(arg->params + n, params + n)) 1486 return TEE_ERROR_BAD_PARAMETERS; 1487 break; 1488 default: 1489 return TEE_ERROR_BAD_PARAMETERS; 1490 } 1491 } 1492 1493 if (arg_ret) 1494 *arg_ret = arg; 1495 1496 return TEE_SUCCESS; 1497 } 1498 1499 static uint32_t get_rpc_arg_res(struct optee_msg_arg *arg, size_t num_params, 1500 struct thread_param *params) 1501 { 1502 for (size_t n = 0; n < num_params; n++) { 1503 switch (params[n].attr) { 1504 case THREAD_PARAM_ATTR_VALUE_OUT: 1505 case THREAD_PARAM_ATTR_VALUE_INOUT: 1506 params[n].u.value.a = arg->params[n].u.value.a; 1507 params[n].u.value.b = arg->params[n].u.value.b; 1508 params[n].u.value.c = arg->params[n].u.value.c; 1509 break; 1510 case THREAD_PARAM_ATTR_MEMREF_OUT: 1511 case THREAD_PARAM_ATTR_MEMREF_INOUT: 1512 params[n].u.memref.size = arg->params[n].u.fmem.size; 1513 break; 1514 default: 1515 break; 1516 } 1517 } 1518 1519 return arg->ret; 1520 } 1521 1522 uint32_t thread_rpc_cmd(uint32_t cmd, size_t num_params, 1523 struct thread_param *params) 1524 { 1525 struct thread_rpc_arg rpc_arg = { .call = { 1526 .w1 = thread_get_tsd()->rpc_target_info, 1527 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1528 }, 1529 }; 1530 struct optee_msg_arg *arg = NULL; 1531 uint32_t ret = 0; 1532 1533 ret = get_rpc_arg(cmd, num_params, params, &arg); 1534 if (ret) 1535 return ret; 1536 1537 thread_rpc(&rpc_arg); 1538 1539 return get_rpc_arg_res(arg, num_params, params); 1540 } 1541 1542 static void thread_rpc_free(unsigned int bt, uint64_t cookie, struct mobj *mobj) 1543 { 1544 struct thread_rpc_arg rpc_arg = { .call = { 1545 .w1 = thread_get_tsd()->rpc_target_info, 1546 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1547 }, 1548 }; 1549 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, cookie, 0); 1550 uint32_t res2 = 0; 1551 uint32_t res = 0; 1552 1553 DMSG("freeing cookie %#"PRIx64, cookie); 1554 1555 res = get_rpc_arg(OPTEE_RPC_CMD_SHM_FREE, 1, ¶m, NULL); 1556 1557 mobj_put(mobj); 1558 res2 = mobj_ffa_unregister_by_cookie(cookie); 1559 if (res2) 1560 DMSG("mobj_ffa_unregister_by_cookie(%#"PRIx64"): %#"PRIx32, 1561 cookie, res2); 1562 if (!res) 1563 thread_rpc(&rpc_arg); 1564 } 1565 1566 static struct mobj *thread_rpc_alloc(size_t size, size_t align, unsigned int bt) 1567 { 1568 struct thread_rpc_arg rpc_arg = { .call = { 1569 .w1 = thread_get_tsd()->rpc_target_info, 1570 .w4 = OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD, 1571 }, 1572 }; 1573 struct thread_param param = THREAD_PARAM_VALUE(IN, bt, size, align); 1574 struct optee_msg_arg *arg = NULL; 1575 unsigned int internal_offset = 0; 1576 struct mobj *mobj = NULL; 1577 uint64_t cookie = 0; 1578 1579 if (get_rpc_arg(OPTEE_RPC_CMD_SHM_ALLOC, 1, ¶m, &arg)) 1580 return NULL; 1581 1582 thread_rpc(&rpc_arg); 1583 1584 if (arg->num_params != 1 || 1585 arg->params->attr != OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT) 1586 return NULL; 1587 1588 internal_offset = READ_ONCE(arg->params->u.fmem.internal_offs); 1589 cookie = READ_ONCE(arg->params->u.fmem.global_id); 1590 mobj = mobj_ffa_get_by_cookie(cookie, internal_offset); 1591 if (!mobj) { 1592 DMSG("mobj_ffa_get_by_cookie(%#"PRIx64", %#x): failed", 1593 cookie, internal_offset); 1594 return NULL; 1595 } 1596 1597 assert(mobj_is_nonsec(mobj)); 1598 1599 if (mobj->size < size) { 1600 DMSG("Mobj %#"PRIx64": wrong size", cookie); 1601 mobj_put(mobj); 1602 return NULL; 1603 } 1604 1605 if (mobj_inc_map(mobj)) { 1606 DMSG("mobj_inc_map(%#"PRIx64"): failed", cookie); 1607 mobj_put(mobj); 1608 return NULL; 1609 } 1610 1611 return mobj; 1612 } 1613 1614 struct mobj *thread_rpc_alloc_payload(size_t size) 1615 { 1616 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_APPL); 1617 } 1618 1619 struct mobj *thread_rpc_alloc_kernel_payload(size_t size) 1620 { 1621 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_KERNEL); 1622 } 1623 1624 void thread_rpc_free_kernel_payload(struct mobj *mobj) 1625 { 1626 thread_rpc_free(OPTEE_RPC_SHM_TYPE_KERNEL, mobj_get_cookie(mobj), mobj); 1627 } 1628 1629 void thread_rpc_free_payload(struct mobj *mobj) 1630 { 1631 thread_rpc_free(OPTEE_RPC_SHM_TYPE_APPL, mobj_get_cookie(mobj), 1632 mobj); 1633 } 1634 1635 struct mobj *thread_rpc_alloc_global_payload(size_t size) 1636 { 1637 return thread_rpc_alloc(size, 8, OPTEE_RPC_SHM_TYPE_GLOBAL); 1638 } 1639 1640 void thread_rpc_free_global_payload(struct mobj *mobj) 1641 { 1642 thread_rpc_free(OPTEE_RPC_SHM_TYPE_GLOBAL, mobj_get_cookie(mobj), 1643 mobj); 1644 } 1645 1646 void thread_spmc_register_secondary_ep(vaddr_t ep) 1647 { 1648 unsigned long ret = 0; 1649 1650 /* Let the SPM know the entry point for secondary CPUs */ 1651 ret = thread_smc(FFA_SECONDARY_EP_REGISTER_64, ep, 0, 0); 1652 1653 if (ret != FFA_SUCCESS_32 && ret != FFA_SUCCESS_64) 1654 EMSG("FFA_SECONDARY_EP_REGISTER_64 ret %#lx", ret); 1655 } 1656 1657 #if defined(CFG_CORE_SEL1_SPMC) 1658 static TEE_Result spmc_init(void) 1659 { 1660 my_endpoint_id = SPMC_ENDPOINT_ID; 1661 DMSG("My endpoint ID %#x", my_endpoint_id); 1662 1663 /* 1664 * If SPMD think we are version 1.0 it will report version 1.0 to 1665 * normal world regardless of what version we query the SPM with. 1666 * However, if SPMD think we are version 1.1 it will forward 1667 * queries from normal world to let us negotiate version. So by 1668 * setting version 1.0 here we should be compatible. 1669 * 1670 * Note that disagreement on negotiated version means that we'll 1671 * have communication problems with normal world. 1672 */ 1673 my_rxtx.ffa_vers = FFA_VERSION_1_0; 1674 1675 return TEE_SUCCESS; 1676 } 1677 #else /* !defined(CFG_CORE_SEL1_SPMC) */ 1678 static bool is_ffa_success(uint32_t fid) 1679 { 1680 #ifdef ARM64 1681 if (fid == FFA_SUCCESS_64) 1682 return true; 1683 #endif 1684 return fid == FFA_SUCCESS_32; 1685 } 1686 1687 static void spmc_rxtx_map(struct ffa_rxtx *rxtx) 1688 { 1689 struct thread_smc_args args = { 1690 #ifdef ARM64 1691 .a0 = FFA_RXTX_MAP_64, 1692 #else 1693 .a0 = FFA_RXTX_MAP_32, 1694 #endif 1695 .a1 = virt_to_phys(rxtx->tx), 1696 .a2 = virt_to_phys(rxtx->rx), 1697 .a3 = 1, 1698 }; 1699 1700 thread_smccc(&args); 1701 if (!is_ffa_success(args.a0)) { 1702 if (args.a0 == FFA_ERROR) 1703 EMSG("rxtx map failed with error %ld", args.a2); 1704 else 1705 EMSG("rxtx map failed"); 1706 panic(); 1707 } 1708 } 1709 1710 static uint16_t get_my_id(void) 1711 { 1712 struct thread_smc_args args = { 1713 .a0 = FFA_ID_GET, 1714 }; 1715 1716 thread_smccc(&args); 1717 if (!is_ffa_success(args.a0)) { 1718 if (args.a0 == FFA_ERROR) 1719 EMSG("Get id failed with error %ld", args.a2); 1720 else 1721 EMSG("Get id failed"); 1722 panic(); 1723 } 1724 1725 return args.a2; 1726 } 1727 1728 static uint32_t get_ffa_version(uint32_t my_version) 1729 { 1730 struct thread_smc_args args = { 1731 .a0 = FFA_VERSION, 1732 .a1 = my_version, 1733 }; 1734 1735 thread_smccc(&args); 1736 if (args.a0 & BIT(31)) { 1737 EMSG("FF-A version failed with error %ld", args.a0); 1738 panic(); 1739 } 1740 1741 return args.a0; 1742 } 1743 1744 static void *spmc_retrieve_req(uint64_t cookie, 1745 struct ffa_mem_transaction_x *trans) 1746 { 1747 struct ffa_mem_access *acc_descr_array = NULL; 1748 struct ffa_mem_access_perm *perm_descr = NULL; 1749 struct thread_smc_args args = { 1750 .a0 = FFA_MEM_RETRIEVE_REQ_32, 1751 .a3 = 0, /* Address, Using TX -> MBZ */ 1752 .a4 = 0, /* Using TX -> MBZ */ 1753 }; 1754 size_t size = 0; 1755 int rc = 0; 1756 1757 if (my_rxtx.ffa_vers == FFA_VERSION_1_0) { 1758 struct ffa_mem_transaction_1_0 *trans_descr = my_rxtx.tx; 1759 1760 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 1761 memset(trans_descr, 0, size); 1762 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 1763 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 1764 trans_descr->global_handle = cookie; 1765 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 1766 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 1767 trans_descr->mem_access_count = 1; 1768 acc_descr_array = trans_descr->mem_access_array; 1769 } else { 1770 struct ffa_mem_transaction_1_1 *trans_descr = my_rxtx.tx; 1771 1772 size = sizeof(*trans_descr) + 1 * sizeof(struct ffa_mem_access); 1773 memset(trans_descr, 0, size); 1774 trans_descr->sender_id = thread_get_tsd()->rpc_target_info; 1775 trans_descr->mem_reg_attr = FFA_NORMAL_MEM_REG_ATTR; 1776 trans_descr->global_handle = cookie; 1777 trans_descr->flags = FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE | 1778 FFA_MEMORY_REGION_FLAG_ANY_ALIGNMENT; 1779 trans_descr->mem_access_count = 1; 1780 trans_descr->mem_access_offs = sizeof(*trans_descr); 1781 trans_descr->mem_access_size = sizeof(struct ffa_mem_access); 1782 acc_descr_array = (void *)((vaddr_t)my_rxtx.tx + 1783 sizeof(*trans_descr)); 1784 } 1785 acc_descr_array->region_offs = 0; 1786 acc_descr_array->reserved = 0; 1787 perm_descr = &acc_descr_array->access_perm; 1788 perm_descr->endpoint_id = my_endpoint_id; 1789 perm_descr->perm = FFA_MEM_ACC_RW; 1790 perm_descr->flags = 0; 1791 1792 args.a1 = size; /* Total Length */ 1793 args.a2 = size; /* Frag Length == Total length */ 1794 thread_smccc(&args); 1795 if (args.a0 != FFA_MEM_RETRIEVE_RESP) { 1796 if (args.a0 == FFA_ERROR) 1797 EMSG("Failed to fetch cookie %#"PRIx64" error code %d", 1798 cookie, (int)args.a2); 1799 else 1800 EMSG("Failed to fetch cookie %#"PRIx64" a0 %#"PRIx64, 1801 cookie, args.a0); 1802 return NULL; 1803 } 1804 rc = spmc_read_mem_transaction(my_rxtx.ffa_vers, my_rxtx.tx, 1805 my_rxtx.size, trans); 1806 if (rc) { 1807 EMSG("Memory transaction failure for cookie %#"PRIx64" rc %d", 1808 cookie, rc); 1809 return NULL; 1810 } 1811 1812 return my_rxtx.rx; 1813 } 1814 1815 void thread_spmc_relinquish(uint64_t cookie) 1816 { 1817 struct ffa_mem_relinquish *relinquish_desc = my_rxtx.tx; 1818 struct thread_smc_args args = { 1819 .a0 = FFA_MEM_RELINQUISH, 1820 }; 1821 1822 memset(relinquish_desc, 0, sizeof(*relinquish_desc)); 1823 relinquish_desc->handle = cookie; 1824 relinquish_desc->flags = 0; 1825 relinquish_desc->endpoint_count = 1; 1826 relinquish_desc->endpoint_id_array[0] = my_endpoint_id; 1827 thread_smccc(&args); 1828 if (!is_ffa_success(args.a0)) 1829 EMSG("Failed to relinquish cookie %#"PRIx64, cookie); 1830 } 1831 1832 static int set_pages(struct ffa_address_range *regions, 1833 unsigned int num_regions, unsigned int num_pages, 1834 struct mobj_ffa *mf) 1835 { 1836 unsigned int n = 0; 1837 unsigned int idx = 0; 1838 1839 for (n = 0; n < num_regions; n++) { 1840 unsigned int page_count = READ_ONCE(regions[n].page_count); 1841 uint64_t addr = READ_ONCE(regions[n].address); 1842 1843 if (mobj_ffa_add_pages_at(mf, &idx, addr, page_count)) 1844 return FFA_INVALID_PARAMETERS; 1845 } 1846 1847 if (idx != num_pages) 1848 return FFA_INVALID_PARAMETERS; 1849 1850 return 0; 1851 } 1852 1853 struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie) 1854 { 1855 struct mobj_ffa *ret = NULL; 1856 struct ffa_mem_transaction_x retrieve_desc = { }; 1857 struct ffa_mem_access *descr_array = NULL; 1858 struct ffa_mem_region *descr = NULL; 1859 struct mobj_ffa *mf = NULL; 1860 unsigned int num_pages = 0; 1861 unsigned int offs = 0; 1862 void *buf = NULL; 1863 struct thread_smc_args ffa_rx_release_args = { 1864 .a0 = FFA_RX_RELEASE 1865 }; 1866 1867 /* 1868 * OP-TEE is only supporting a single mem_region while the 1869 * specification allows for more than one. 1870 */ 1871 buf = spmc_retrieve_req(cookie, &retrieve_desc); 1872 if (!buf) { 1873 EMSG("Failed to retrieve cookie from rx buffer %#"PRIx64, 1874 cookie); 1875 return NULL; 1876 } 1877 1878 descr_array = (void *)((vaddr_t)buf + retrieve_desc.mem_access_offs); 1879 offs = READ_ONCE(descr_array->region_offs); 1880 descr = (struct ffa_mem_region *)((vaddr_t)buf + offs); 1881 1882 num_pages = READ_ONCE(descr->total_page_count); 1883 mf = mobj_ffa_spmc_new(cookie, num_pages); 1884 if (!mf) 1885 goto out; 1886 1887 if (set_pages(descr->address_range_array, 1888 READ_ONCE(descr->address_range_count), num_pages, mf)) { 1889 mobj_ffa_spmc_delete(mf); 1890 goto out; 1891 } 1892 1893 ret = mf; 1894 1895 out: 1896 /* Release RX buffer after the mem retrieve request. */ 1897 thread_smccc(&ffa_rx_release_args); 1898 1899 return ret; 1900 } 1901 1902 static TEE_Result spmc_init(void) 1903 { 1904 unsigned int major = 0; 1905 unsigned int minor __maybe_unused = 0; 1906 uint32_t my_vers = 0; 1907 uint32_t vers = 0; 1908 1909 my_vers = MAKE_FFA_VERSION(FFA_VERSION_MAJOR, FFA_VERSION_MINOR); 1910 vers = get_ffa_version(my_vers); 1911 major = (vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK; 1912 minor = (vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK; 1913 DMSG("SPMC reported version %u.%u", major, minor); 1914 if (major != FFA_VERSION_MAJOR) { 1915 EMSG("Incompatible major version %u, expected %u", 1916 major, FFA_VERSION_MAJOR); 1917 panic(); 1918 } 1919 if (vers < my_vers) 1920 my_vers = vers; 1921 DMSG("Using version %u.%u", 1922 (my_vers >> FFA_VERSION_MAJOR_SHIFT) & FFA_VERSION_MAJOR_MASK, 1923 (my_vers >> FFA_VERSION_MINOR_SHIFT) & FFA_VERSION_MINOR_MASK); 1924 my_rxtx.ffa_vers = my_vers; 1925 1926 spmc_rxtx_map(&my_rxtx); 1927 my_endpoint_id = get_my_id(); 1928 DMSG("My endpoint ID %#x", my_endpoint_id); 1929 1930 return TEE_SUCCESS; 1931 } 1932 #endif /* !defined(CFG_CORE_SEL1_SPMC) */ 1933 1934 /* 1935 * boot_final() is always done before exiting at end of boot 1936 * initialization. In case of virtualization the init-calls are done only 1937 * once a OP-TEE partition has been created. So with virtualization we have 1938 * to initialize via boot_final() to make sure we have a value assigned 1939 * before it's used the first time. 1940 */ 1941 #ifdef CFG_NS_VIRTUALIZATION 1942 boot_final(spmc_init); 1943 #else 1944 service_init(spmc_init); 1945 #endif 1946