1 /* 2 * Copyright (c) 2022, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 #include <assert.h> 7 #include <errno.h> 8 9 #include <common/debug.h> 10 #include <common/runtime_svc.h> 11 #include <lib/object_pool.h> 12 #include <lib/spinlock.h> 13 #include <lib/xlat_tables/xlat_tables_v2.h> 14 #include <services/ffa_svc.h> 15 #include "spmc.h" 16 #include "spmc_shared_mem.h" 17 18 #include <platform_def.h> 19 20 /** 21 * struct spmc_shmem_obj - Shared memory object. 22 * @desc_size: Size of @desc. 23 * @desc_filled: Size of @desc already received. 24 * @in_use: Number of clients that have called ffa_mem_retrieve_req 25 * without a matching ffa_mem_relinquish call. 26 * @desc: FF-A memory region descriptor passed in ffa_mem_share. 27 */ 28 struct spmc_shmem_obj { 29 size_t desc_size; 30 size_t desc_filled; 31 size_t in_use; 32 struct ffa_mtd desc; 33 }; 34 35 /* 36 * Declare our data structure to store the metadata of memory share requests. 37 * The main datastore is allocated on a per platform basis to ensure enough 38 * storage can be made available. 39 * The address of the data store will be populated by the SPMC during its 40 * initialization. 41 */ 42 43 struct spmc_shmem_obj_state spmc_shmem_obj_state = { 44 /* Set start value for handle so top 32 bits are needed quickly. */ 45 .next_handle = 0xffffffc0U, 46 }; 47 48 /** 49 * spmc_shmem_obj_size - Convert from descriptor size to object size. 50 * @desc_size: Size of struct ffa_memory_region_descriptor object. 51 * 52 * Return: Size of struct spmc_shmem_obj object. 53 */ 54 static size_t spmc_shmem_obj_size(size_t desc_size) 55 { 56 return desc_size + offsetof(struct spmc_shmem_obj, desc); 57 } 58 59 /** 60 * spmc_shmem_obj_alloc - Allocate struct spmc_shmem_obj. 61 * @state: Global state. 62 * @desc_size: Size of struct ffa_memory_region_descriptor object that 63 * allocated object will hold. 64 * 65 * Return: Pointer to newly allocated object, or %NULL if there not enough space 66 * left. The returned pointer is only valid while @state is locked, to 67 * used it again after unlocking @state, spmc_shmem_obj_lookup must be 68 * called. 69 */ 70 static struct spmc_shmem_obj * 71 spmc_shmem_obj_alloc(struct spmc_shmem_obj_state *state, size_t desc_size) 72 { 73 struct spmc_shmem_obj *obj; 74 size_t free = state->data_size - state->allocated; 75 76 if (state->data == NULL) { 77 ERROR("Missing shmem datastore!\n"); 78 return NULL; 79 } 80 81 if (spmc_shmem_obj_size(desc_size) > free) { 82 WARN("%s(0x%zx) failed, free 0x%zx\n", 83 __func__, desc_size, free); 84 return NULL; 85 } 86 obj = (struct spmc_shmem_obj *)(state->data + state->allocated); 87 obj->desc = (struct ffa_mtd) {0}; 88 obj->desc_size = desc_size; 89 obj->desc_filled = 0; 90 obj->in_use = 0; 91 state->allocated += spmc_shmem_obj_size(desc_size); 92 return obj; 93 } 94 95 /** 96 * spmc_shmem_obj_free - Free struct spmc_shmem_obj. 97 * @state: Global state. 98 * @obj: Object to free. 99 * 100 * Release memory used by @obj. Other objects may move, so on return all 101 * pointers to struct spmc_shmem_obj object should be considered invalid, not 102 * just @obj. 103 * 104 * The current implementation always compacts the remaining objects to simplify 105 * the allocator and to avoid fragmentation. 106 */ 107 108 static void spmc_shmem_obj_free(struct spmc_shmem_obj_state *state, 109 struct spmc_shmem_obj *obj) 110 { 111 size_t free_size = spmc_shmem_obj_size(obj->desc_size); 112 uint8_t *shift_dest = (uint8_t *)obj; 113 uint8_t *shift_src = shift_dest + free_size; 114 size_t shift_size = state->allocated - (shift_src - state->data); 115 116 if (shift_size != 0U) { 117 memmove(shift_dest, shift_src, shift_size); 118 } 119 state->allocated -= free_size; 120 } 121 122 /** 123 * spmc_shmem_obj_lookup - Lookup struct spmc_shmem_obj by handle. 124 * @state: Global state. 125 * @handle: Unique handle of object to return. 126 * 127 * Return: struct spmc_shmem_obj_state object with handle matching @handle. 128 * %NULL, if not object in @state->data has a matching handle. 129 */ 130 static struct spmc_shmem_obj * 131 spmc_shmem_obj_lookup(struct spmc_shmem_obj_state *state, uint64_t handle) 132 { 133 uint8_t *curr = state->data; 134 135 while (curr - state->data < state->allocated) { 136 struct spmc_shmem_obj *obj = (struct spmc_shmem_obj *)curr; 137 138 if (obj->desc.handle == handle) { 139 return obj; 140 } 141 curr += spmc_shmem_obj_size(obj->desc_size); 142 } 143 return NULL; 144 } 145 146 /** 147 * spmc_shmem_obj_get_next - Get the next memory object from an offset. 148 * @offset: Offset used to track which objects have previously been 149 * returned. 150 * 151 * Return: the next struct spmc_shmem_obj_state object from the provided 152 * offset. 153 * %NULL, if there are no more objects. 154 */ 155 static struct spmc_shmem_obj * 156 spmc_shmem_obj_get_next(struct spmc_shmem_obj_state *state, size_t *offset) 157 { 158 uint8_t *curr = state->data + *offset; 159 160 if (curr - state->data < state->allocated) { 161 struct spmc_shmem_obj *obj = (struct spmc_shmem_obj *)curr; 162 163 *offset += spmc_shmem_obj_size(obj->desc_size); 164 165 return obj; 166 } 167 return NULL; 168 } 169 170 /******************************************************************************* 171 * FF-A memory descriptor helper functions. 172 ******************************************************************************/ 173 /** 174 * spmc_shmem_obj_get_emad - Get the emad from a given index depending on the 175 * clients FF-A version. 176 * @desc: The memory transaction descriptor. 177 * @index: The index of the emad element to be accessed. 178 * @ffa_version: FF-A version of the provided structure. 179 * @emad_size: Will be populated with the size of the returned emad 180 * descriptor. 181 * Return: A pointer to the requested emad structure. 182 */ 183 static void * 184 spmc_shmem_obj_get_emad(const struct ffa_mtd *desc, uint32_t index, 185 uint32_t ffa_version, size_t *emad_size) 186 { 187 uint8_t *emad; 188 /* 189 * If the caller is using FF-A v1.0 interpret the descriptor as a v1.0 190 * format, otherwise assume it is a v1.1 format. 191 */ 192 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 193 /* Cast our descriptor to the v1.0 format. */ 194 struct ffa_mtd_v1_0 *mtd_v1_0 = 195 (struct ffa_mtd_v1_0 *) desc; 196 emad = (uint8_t *) &(mtd_v1_0->emad); 197 *emad_size = sizeof(struct ffa_emad_v1_0); 198 } else { 199 if (!is_aligned(desc->emad_offset, 16)) { 200 WARN("Emad offset is not aligned.\n"); 201 return NULL; 202 } 203 emad = ((uint8_t *) desc + desc->emad_offset); 204 *emad_size = desc->emad_size; 205 } 206 return (emad + (*emad_size * index)); 207 } 208 209 /** 210 * spmc_shmem_obj_get_comp_mrd - Get comp_mrd from a mtd struct based on the 211 * FF-A version of the descriptor. 212 * @obj: Object containing ffa_memory_region_descriptor. 213 * 214 * Return: struct ffa_comp_mrd object corresponding to the composite memory 215 * region descriptor. 216 */ 217 static struct ffa_comp_mrd * 218 spmc_shmem_obj_get_comp_mrd(struct spmc_shmem_obj *obj, uint32_t ffa_version) 219 { 220 size_t emad_size; 221 /* 222 * The comp_mrd_offset field of the emad descriptor remains consistent 223 * between FF-A versions therefore we can use the v1.0 descriptor here 224 * in all cases. 225 */ 226 struct ffa_emad_v1_0 *emad = spmc_shmem_obj_get_emad(&obj->desc, 0, 227 ffa_version, 228 &emad_size); 229 /* Ensure the emad array was found. */ 230 if (emad == NULL) { 231 return NULL; 232 } 233 234 /* Ensure the composite descriptor offset is aligned. */ 235 if (!is_aligned(emad->comp_mrd_offset, 8)) { 236 WARN("Unaligned composite memory region descriptor offset.\n"); 237 return NULL; 238 } 239 240 return (struct ffa_comp_mrd *) 241 ((uint8_t *)(&obj->desc) + emad->comp_mrd_offset); 242 } 243 244 /** 245 * spmc_shmem_obj_ffa_constituent_size - Calculate variable size part of obj. 246 * @obj: Object containing ffa_memory_region_descriptor. 247 * 248 * Return: Size of ffa_constituent_memory_region_descriptors in @obj. 249 */ 250 static size_t 251 spmc_shmem_obj_ffa_constituent_size(struct spmc_shmem_obj *obj, 252 uint32_t ffa_version) 253 { 254 struct ffa_comp_mrd *comp_mrd; 255 256 comp_mrd = spmc_shmem_obj_get_comp_mrd(obj, ffa_version); 257 if (comp_mrd == NULL) { 258 return 0; 259 } 260 return comp_mrd->address_range_count * sizeof(struct ffa_cons_mrd); 261 } 262 263 /* 264 * Compare two memory regions to determine if any range overlaps with another 265 * ongoing memory transaction. 266 */ 267 static bool 268 overlapping_memory_regions(struct ffa_comp_mrd *region1, 269 struct ffa_comp_mrd *region2) 270 { 271 uint64_t region1_start; 272 uint64_t region1_size; 273 uint64_t region1_end; 274 uint64_t region2_start; 275 uint64_t region2_size; 276 uint64_t region2_end; 277 278 assert(region1 != NULL); 279 assert(region2 != NULL); 280 281 if (region1 == region2) { 282 return true; 283 } 284 285 /* 286 * Check each memory region in the request against existing 287 * transactions. 288 */ 289 for (size_t i = 0; i < region1->address_range_count; i++) { 290 291 region1_start = region1->address_range_array[i].address; 292 region1_size = 293 region1->address_range_array[i].page_count * 294 PAGE_SIZE_4KB; 295 region1_end = region1_start + region1_size; 296 297 for (size_t j = 0; j < region2->address_range_count; j++) { 298 299 region2_start = region2->address_range_array[j].address; 300 region2_size = 301 region2->address_range_array[j].page_count * 302 PAGE_SIZE_4KB; 303 region2_end = region2_start + region2_size; 304 305 if ((region1_start >= region2_start && 306 region1_start < region2_end) || 307 (region1_end > region2_start 308 && region1_end < region2_end)) { 309 WARN("Overlapping mem regions 0x%lx-0x%lx & 0x%lx-0x%lx\n", 310 region1_start, region1_end, 311 region2_start, region2_end); 312 return true; 313 } 314 } 315 } 316 return false; 317 } 318 319 /******************************************************************************* 320 * FF-A v1.0 Memory Descriptor Conversion Helpers. 321 ******************************************************************************/ 322 /** 323 * spmc_shm_get_v1_1_descriptor_size - Calculate the required size for a v1.1 324 * converted descriptor. 325 * @orig: The original v1.0 memory transaction descriptor. 326 * @desc_size: The size of the original v1.0 memory transaction descriptor. 327 * 328 * Return: the size required to store the descriptor store in the v1.1 format. 329 */ 330 static size_t 331 spmc_shm_get_v1_1_descriptor_size(struct ffa_mtd_v1_0 *orig, size_t desc_size) 332 { 333 size_t size = 0; 334 struct ffa_comp_mrd *mrd; 335 struct ffa_emad_v1_0 *emad_array = orig->emad; 336 337 /* Get the size of the v1.1 descriptor. */ 338 size += sizeof(struct ffa_mtd); 339 340 /* Add the size of the emad descriptors. */ 341 size += orig->emad_count * sizeof(struct ffa_emad_v1_0); 342 343 /* Add the size of the composite mrds. */ 344 size += sizeof(struct ffa_comp_mrd); 345 346 /* Add the size of the constituent mrds. */ 347 mrd = (struct ffa_comp_mrd *) ((uint8_t *) orig + 348 emad_array[0].comp_mrd_offset); 349 350 /* Check the calculated address is within the memory descriptor. */ 351 if ((uintptr_t) mrd >= (uintptr_t)((uint8_t *) orig + desc_size)) { 352 return 0; 353 } 354 size += mrd->address_range_count * sizeof(struct ffa_cons_mrd); 355 356 return size; 357 } 358 359 /** 360 * spmc_shm_get_v1_0_descriptor_size - Calculate the required size for a v1.0 361 * converted descriptor. 362 * @orig: The original v1.1 memory transaction descriptor. 363 * @desc_size: The size of the original v1.1 memory transaction descriptor. 364 * 365 * Return: the size required to store the descriptor store in the v1.0 format. 366 */ 367 static size_t 368 spmc_shm_get_v1_0_descriptor_size(struct ffa_mtd *orig, size_t desc_size) 369 { 370 size_t size = 0; 371 struct ffa_comp_mrd *mrd; 372 struct ffa_emad_v1_0 *emad_array = (struct ffa_emad_v1_0 *) 373 ((uint8_t *) orig + 374 orig->emad_offset); 375 376 /* Get the size of the v1.0 descriptor. */ 377 size += sizeof(struct ffa_mtd_v1_0); 378 379 /* Add the size of the v1.0 emad descriptors. */ 380 size += orig->emad_count * sizeof(struct ffa_emad_v1_0); 381 382 /* Add the size of the composite mrds. */ 383 size += sizeof(struct ffa_comp_mrd); 384 385 /* Add the size of the constituent mrds. */ 386 mrd = (struct ffa_comp_mrd *) ((uint8_t *) orig + 387 emad_array[0].comp_mrd_offset); 388 389 /* Check the calculated address is within the memory descriptor. */ 390 if ((uintptr_t) mrd >= (uintptr_t)((uint8_t *) orig + desc_size)) { 391 return 0; 392 } 393 size += mrd->address_range_count * sizeof(struct ffa_cons_mrd); 394 395 return size; 396 } 397 398 /** 399 * spmc_shm_convert_shmem_obj_from_v1_0 - Converts a given v1.0 memory object. 400 * @out_obj: The shared memory object to populate the converted descriptor. 401 * @orig: The shared memory object containing the v1.0 descriptor. 402 * 403 * Return: true if the conversion is successful else false. 404 */ 405 static bool 406 spmc_shm_convert_shmem_obj_from_v1_0(struct spmc_shmem_obj *out_obj, 407 struct spmc_shmem_obj *orig) 408 { 409 struct ffa_mtd_v1_0 *mtd_orig = (struct ffa_mtd_v1_0 *) &orig->desc; 410 struct ffa_mtd *out = &out_obj->desc; 411 struct ffa_emad_v1_0 *emad_array_in; 412 struct ffa_emad_v1_0 *emad_array_out; 413 struct ffa_comp_mrd *mrd_in; 414 struct ffa_comp_mrd *mrd_out; 415 416 size_t mrd_in_offset; 417 size_t mrd_out_offset; 418 size_t mrd_size = 0; 419 420 /* Populate the new descriptor format from the v1.0 struct. */ 421 out->sender_id = mtd_orig->sender_id; 422 out->memory_region_attributes = mtd_orig->memory_region_attributes; 423 out->flags = mtd_orig->flags; 424 out->handle = mtd_orig->handle; 425 out->tag = mtd_orig->tag; 426 out->emad_count = mtd_orig->emad_count; 427 out->emad_size = sizeof(struct ffa_emad_v1_0); 428 429 /* 430 * We will locate the emad descriptors directly after the ffa_mtd 431 * struct. This will be 8-byte aligned. 432 */ 433 out->emad_offset = sizeof(struct ffa_mtd); 434 435 emad_array_in = mtd_orig->emad; 436 emad_array_out = (struct ffa_emad_v1_0 *) 437 ((uint8_t *) out + out->emad_offset); 438 439 /* Copy across the emad structs. */ 440 for (unsigned int i = 0U; i < out->emad_count; i++) { 441 memcpy(&emad_array_out[i], &emad_array_in[i], 442 sizeof(struct ffa_emad_v1_0)); 443 } 444 445 /* Place the mrd descriptors after the end of the emad descriptors.*/ 446 mrd_in_offset = emad_array_in->comp_mrd_offset; 447 mrd_out_offset = out->emad_offset + (out->emad_size * out->emad_count); 448 mrd_out = (struct ffa_comp_mrd *) ((uint8_t *) out + mrd_out_offset); 449 450 /* Add the size of the composite memory region descriptor. */ 451 mrd_size += sizeof(struct ffa_comp_mrd); 452 453 /* Find the mrd descriptor. */ 454 mrd_in = (struct ffa_comp_mrd *) ((uint8_t *) mtd_orig + mrd_in_offset); 455 456 /* Add the size of the constituent memory region descriptors. */ 457 mrd_size += mrd_in->address_range_count * sizeof(struct ffa_cons_mrd); 458 459 /* 460 * Update the offset in the emads by the delta between the input and 461 * output addresses. 462 */ 463 for (unsigned int i = 0U; i < out->emad_count; i++) { 464 emad_array_out[i].comp_mrd_offset = 465 emad_array_in[i].comp_mrd_offset + 466 (mrd_out_offset - mrd_in_offset); 467 } 468 469 /* Verify that we stay within bound of the memory descriptors. */ 470 if ((uintptr_t)((uint8_t *) mrd_in + mrd_size) > 471 (uintptr_t)((uint8_t *) mtd_orig + orig->desc_size) || 472 ((uintptr_t)((uint8_t *) mrd_out + mrd_size) > 473 (uintptr_t)((uint8_t *) out + out_obj->desc_size))) { 474 ERROR("%s: Invalid mrd structure.\n", __func__); 475 return false; 476 } 477 478 /* Copy the mrd descriptors directly. */ 479 memcpy(mrd_out, mrd_in, mrd_size); 480 481 return true; 482 } 483 484 /** 485 * spmc_shm_convert_mtd_to_v1_0 - Converts a given v1.1 memory object to 486 * v1.0 memory object. 487 * @out_obj: The shared memory object to populate the v1.0 descriptor. 488 * @orig: The shared memory object containing the v1.1 descriptor. 489 * 490 * Return: true if the conversion is successful else false. 491 */ 492 static bool 493 spmc_shm_convert_mtd_to_v1_0(struct spmc_shmem_obj *out_obj, 494 struct spmc_shmem_obj *orig) 495 { 496 struct ffa_mtd *mtd_orig = &orig->desc; 497 struct ffa_mtd_v1_0 *out = (struct ffa_mtd_v1_0 *) &out_obj->desc; 498 struct ffa_emad_v1_0 *emad_in; 499 struct ffa_emad_v1_0 *emad_array_in; 500 struct ffa_emad_v1_0 *emad_array_out; 501 struct ffa_comp_mrd *mrd_in; 502 struct ffa_comp_mrd *mrd_out; 503 504 size_t mrd_in_offset; 505 size_t mrd_out_offset; 506 size_t emad_out_array_size; 507 size_t mrd_size = 0; 508 509 /* Populate the v1.0 descriptor format from the v1.1 struct. */ 510 out->sender_id = mtd_orig->sender_id; 511 out->memory_region_attributes = mtd_orig->memory_region_attributes; 512 out->flags = mtd_orig->flags; 513 out->handle = mtd_orig->handle; 514 out->tag = mtd_orig->tag; 515 out->emad_count = mtd_orig->emad_count; 516 517 /* Determine the location of the emad array in both descriptors. */ 518 emad_array_in = (struct ffa_emad_v1_0 *) 519 ((uint8_t *) mtd_orig + mtd_orig->emad_offset); 520 emad_array_out = out->emad; 521 522 /* Copy across the emad structs. */ 523 emad_in = emad_array_in; 524 for (unsigned int i = 0U; i < out->emad_count; i++) { 525 memcpy(&emad_array_out[i], emad_in, 526 sizeof(struct ffa_emad_v1_0)); 527 528 emad_in += mtd_orig->emad_size; 529 } 530 531 /* Place the mrd descriptors after the end of the emad descriptors. */ 532 emad_out_array_size = sizeof(struct ffa_emad_v1_0) * out->emad_count; 533 534 mrd_out_offset = (uint8_t *) out->emad - (uint8_t *) out + 535 emad_out_array_size; 536 537 mrd_out = (struct ffa_comp_mrd *) ((uint8_t *) out + mrd_out_offset); 538 539 mrd_in_offset = mtd_orig->emad_offset + 540 (mtd_orig->emad_size * mtd_orig->emad_count); 541 542 /* Add the size of the composite memory region descriptor. */ 543 mrd_size += sizeof(struct ffa_comp_mrd); 544 545 /* Find the mrd descriptor. */ 546 mrd_in = (struct ffa_comp_mrd *) ((uint8_t *) mtd_orig + mrd_in_offset); 547 548 /* Add the size of the constituent memory region descriptors. */ 549 mrd_size += mrd_in->address_range_count * sizeof(struct ffa_cons_mrd); 550 551 /* 552 * Update the offset in the emads by the delta between the input and 553 * output addresses. 554 */ 555 emad_in = emad_array_in; 556 557 for (unsigned int i = 0U; i < out->emad_count; i++) { 558 emad_array_out[i].comp_mrd_offset = emad_in->comp_mrd_offset + 559 (mrd_out_offset - 560 mrd_in_offset); 561 emad_in += mtd_orig->emad_size; 562 } 563 564 /* Verify that we stay within bound of the memory descriptors. */ 565 if ((uintptr_t)((uint8_t *) mrd_in + mrd_size) > 566 (uintptr_t)((uint8_t *) mtd_orig + orig->desc_size) || 567 ((uintptr_t)((uint8_t *) mrd_out + mrd_size) > 568 (uintptr_t)((uint8_t *) out + out_obj->desc_size))) { 569 ERROR("%s: Invalid mrd structure.\n", __func__); 570 return false; 571 } 572 573 /* Copy the mrd descriptors directly. */ 574 memcpy(mrd_out, mrd_in, mrd_size); 575 576 return true; 577 } 578 579 /** 580 * spmc_populate_ffa_v1_0_descriptor - Converts a given v1.1 memory object to 581 * the v1.0 format and populates the 582 * provided buffer. 583 * @dst: Buffer to populate v1.0 ffa_memory_region_descriptor. 584 * @orig_obj: Object containing v1.1 ffa_memory_region_descriptor. 585 * @buf_size: Size of the buffer to populate. 586 * @offset: The offset of the converted descriptor to copy. 587 * @copy_size: Will be populated with the number of bytes copied. 588 * @out_desc_size: Will be populated with the total size of the v1.0 589 * descriptor. 590 * 591 * Return: 0 if conversion and population succeeded. 592 * Note: This function invalidates the reference to @orig therefore 593 * `spmc_shmem_obj_lookup` must be called if further usage is required. 594 */ 595 static uint32_t 596 spmc_populate_ffa_v1_0_descriptor(void *dst, struct spmc_shmem_obj *orig_obj, 597 size_t buf_size, size_t offset, 598 size_t *copy_size, size_t *v1_0_desc_size) 599 { 600 struct spmc_shmem_obj *v1_0_obj; 601 602 /* Calculate the size that the v1.0 descriptor will require. */ 603 *v1_0_desc_size = spmc_shm_get_v1_0_descriptor_size( 604 &orig_obj->desc, orig_obj->desc_size); 605 606 if (*v1_0_desc_size == 0) { 607 ERROR("%s: cannot determine size of descriptor.\n", 608 __func__); 609 return FFA_ERROR_INVALID_PARAMETER; 610 } 611 612 /* Get a new obj to store the v1.0 descriptor. */ 613 v1_0_obj = spmc_shmem_obj_alloc(&spmc_shmem_obj_state, 614 *v1_0_desc_size); 615 616 if (!v1_0_obj) { 617 return FFA_ERROR_NO_MEMORY; 618 } 619 620 /* Perform the conversion from v1.1 to v1.0. */ 621 if (!spmc_shm_convert_mtd_to_v1_0(v1_0_obj, orig_obj)) { 622 spmc_shmem_obj_free(&spmc_shmem_obj_state, v1_0_obj); 623 return FFA_ERROR_INVALID_PARAMETER; 624 } 625 626 *copy_size = MIN(v1_0_obj->desc_size - offset, buf_size); 627 memcpy(dst, (uint8_t *) &v1_0_obj->desc + offset, *copy_size); 628 629 /* 630 * We're finished with the v1.0 descriptor for now so free it. 631 * Note that this will invalidate any references to the v1.1 632 * descriptor. 633 */ 634 spmc_shmem_obj_free(&spmc_shmem_obj_state, v1_0_obj); 635 636 return 0; 637 } 638 639 /** 640 * spmc_shmem_check_obj - Check that counts in descriptor match overall size. 641 * @obj: Object containing ffa_memory_region_descriptor. 642 * @ffa_version: FF-A version of the provided descriptor. 643 * 644 * Return: 0 if object is valid, -EINVAL if constituent_memory_region_descriptor 645 * offset or count is invalid. 646 */ 647 static int spmc_shmem_check_obj(struct spmc_shmem_obj *obj, 648 uint32_t ffa_version) 649 { 650 uint32_t comp_mrd_offset = 0; 651 652 if (obj->desc.emad_count == 0U) { 653 WARN("%s: unsupported attribute desc count %u.\n", 654 __func__, obj->desc.emad_count); 655 return -EINVAL; 656 } 657 658 for (size_t emad_num = 0; emad_num < obj->desc.emad_count; emad_num++) { 659 size_t size; 660 size_t count; 661 size_t expected_size; 662 size_t total_page_count; 663 size_t emad_size; 664 size_t desc_size; 665 size_t header_emad_size; 666 uint32_t offset; 667 struct ffa_comp_mrd *comp; 668 struct ffa_emad_v1_0 *emad; 669 670 emad = spmc_shmem_obj_get_emad(&obj->desc, emad_num, 671 ffa_version, &emad_size); 672 if (emad == NULL) { 673 WARN("%s: invalid emad structure.\n", __func__); 674 return -EINVAL; 675 } 676 677 /* 678 * Validate the calculated emad address resides within the 679 * descriptor. 680 */ 681 if ((uintptr_t) emad >= 682 (uintptr_t)((uint8_t *) &obj->desc + obj->desc_size)) { 683 WARN("Invalid emad access.\n"); 684 return -EINVAL; 685 } 686 687 offset = emad->comp_mrd_offset; 688 689 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 690 desc_size = sizeof(struct ffa_mtd_v1_0); 691 } else { 692 desc_size = sizeof(struct ffa_mtd); 693 } 694 695 header_emad_size = desc_size + 696 (obj->desc.emad_count * emad_size); 697 698 if (offset < header_emad_size) { 699 WARN("%s: invalid object, offset %u < header + emad %zu\n", 700 __func__, offset, header_emad_size); 701 return -EINVAL; 702 } 703 704 size = obj->desc_size; 705 706 if (offset > size) { 707 WARN("%s: invalid object, offset %u > total size %zu\n", 708 __func__, offset, obj->desc_size); 709 return -EINVAL; 710 } 711 size -= offset; 712 713 if (size < sizeof(struct ffa_comp_mrd)) { 714 WARN("%s: invalid object, offset %u, total size %zu, no header space.\n", 715 __func__, offset, obj->desc_size); 716 return -EINVAL; 717 } 718 size -= sizeof(struct ffa_comp_mrd); 719 720 count = size / sizeof(struct ffa_cons_mrd); 721 722 comp = spmc_shmem_obj_get_comp_mrd(obj, ffa_version); 723 724 if (comp == NULL) { 725 WARN("%s: invalid comp_mrd offset\n", __func__); 726 return -EINVAL; 727 } 728 729 if (comp->address_range_count != count) { 730 WARN("%s: invalid object, desc count %u != %zu\n", 731 __func__, comp->address_range_count, count); 732 return -EINVAL; 733 } 734 735 expected_size = offset + sizeof(*comp) + 736 spmc_shmem_obj_ffa_constituent_size(obj, 737 ffa_version); 738 739 if (expected_size != obj->desc_size) { 740 WARN("%s: invalid object, computed size %zu != size %zu\n", 741 __func__, expected_size, obj->desc_size); 742 return -EINVAL; 743 } 744 745 if (obj->desc_filled < obj->desc_size) { 746 /* 747 * The whole descriptor has not yet been received. 748 * Skip final checks. 749 */ 750 return 0; 751 } 752 753 /* 754 * The offset provided to the composite memory region descriptor 755 * should be consistent across endpoint descriptors. Store the 756 * first entry and compare against subsequent entries. 757 */ 758 if (comp_mrd_offset == 0) { 759 comp_mrd_offset = offset; 760 } else { 761 if (comp_mrd_offset != offset) { 762 ERROR("%s: mismatching offsets provided, %u != %u\n", 763 __func__, offset, comp_mrd_offset); 764 return -EINVAL; 765 } 766 } 767 768 total_page_count = 0; 769 770 for (size_t i = 0; i < count; i++) { 771 total_page_count += 772 comp->address_range_array[i].page_count; 773 } 774 if (comp->total_page_count != total_page_count) { 775 WARN("%s: invalid object, desc total_page_count %u != %zu\n", 776 __func__, comp->total_page_count, 777 total_page_count); 778 return -EINVAL; 779 } 780 } 781 return 0; 782 } 783 784 /** 785 * spmc_shmem_check_state_obj - Check if the descriptor describes memory 786 * regions that are currently involved with an 787 * existing memory transactions. This implies that 788 * the memory is not in a valid state for lending. 789 * @obj: Object containing ffa_memory_region_descriptor. 790 * 791 * Return: 0 if object is valid, -EINVAL if invalid memory state. 792 */ 793 static int spmc_shmem_check_state_obj(struct spmc_shmem_obj *obj, 794 uint32_t ffa_version) 795 { 796 size_t obj_offset = 0; 797 struct spmc_shmem_obj *inflight_obj; 798 799 struct ffa_comp_mrd *other_mrd; 800 struct ffa_comp_mrd *requested_mrd = spmc_shmem_obj_get_comp_mrd(obj, 801 ffa_version); 802 803 if (requested_mrd == NULL) { 804 return -EINVAL; 805 } 806 807 inflight_obj = spmc_shmem_obj_get_next(&spmc_shmem_obj_state, 808 &obj_offset); 809 810 while (inflight_obj != NULL) { 811 /* 812 * Don't compare the transaction to itself or to partially 813 * transmitted descriptors. 814 */ 815 if ((obj->desc.handle != inflight_obj->desc.handle) && 816 (obj->desc_size == obj->desc_filled)) { 817 other_mrd = spmc_shmem_obj_get_comp_mrd(inflight_obj, 818 FFA_VERSION_COMPILED); 819 if (other_mrd == NULL) { 820 return -EINVAL; 821 } 822 if (overlapping_memory_regions(requested_mrd, 823 other_mrd)) { 824 return -EINVAL; 825 } 826 } 827 828 inflight_obj = spmc_shmem_obj_get_next(&spmc_shmem_obj_state, 829 &obj_offset); 830 } 831 return 0; 832 } 833 834 static long spmc_ffa_fill_desc(struct mailbox *mbox, 835 struct spmc_shmem_obj *obj, 836 uint32_t fragment_length, 837 ffa_mtd_flag32_t mtd_flag, 838 uint32_t ffa_version, 839 void *smc_handle) 840 { 841 int ret; 842 size_t emad_size; 843 uint32_t handle_low; 844 uint32_t handle_high; 845 struct ffa_emad_v1_0 *emad; 846 struct ffa_emad_v1_0 *other_emad; 847 848 if (mbox->rxtx_page_count == 0U) { 849 WARN("%s: buffer pair not registered.\n", __func__); 850 ret = FFA_ERROR_INVALID_PARAMETER; 851 goto err_arg; 852 } 853 854 if (fragment_length > mbox->rxtx_page_count * PAGE_SIZE_4KB) { 855 WARN("%s: bad fragment size %u > %u buffer size\n", __func__, 856 fragment_length, mbox->rxtx_page_count * PAGE_SIZE_4KB); 857 ret = FFA_ERROR_INVALID_PARAMETER; 858 goto err_arg; 859 } 860 861 memcpy((uint8_t *)&obj->desc + obj->desc_filled, 862 (uint8_t *) mbox->tx_buffer, fragment_length); 863 864 if (fragment_length > obj->desc_size - obj->desc_filled) { 865 WARN("%s: bad fragment size %u > %zu remaining\n", __func__, 866 fragment_length, obj->desc_size - obj->desc_filled); 867 ret = FFA_ERROR_INVALID_PARAMETER; 868 goto err_arg; 869 } 870 871 /* Ensure that the sender ID resides in the normal world. */ 872 if (ffa_is_secure_world_id(obj->desc.sender_id)) { 873 WARN("%s: Invalid sender ID 0x%x.\n", 874 __func__, obj->desc.sender_id); 875 ret = FFA_ERROR_DENIED; 876 goto err_arg; 877 } 878 879 /* Ensure the NS bit is set to 0. */ 880 if ((obj->desc.memory_region_attributes & FFA_MEM_ATTR_NS_BIT) != 0U) { 881 WARN("%s: NS mem attributes flags MBZ.\n", __func__); 882 ret = FFA_ERROR_INVALID_PARAMETER; 883 goto err_arg; 884 } 885 886 /* 887 * We don't currently support any optional flags so ensure none are 888 * requested. 889 */ 890 if (obj->desc.flags != 0U && mtd_flag != 0U && 891 (obj->desc.flags != mtd_flag)) { 892 WARN("%s: invalid memory transaction flags %u != %u\n", 893 __func__, obj->desc.flags, mtd_flag); 894 ret = FFA_ERROR_INVALID_PARAMETER; 895 goto err_arg; 896 } 897 898 if (obj->desc_filled == 0U) { 899 /* First fragment, descriptor header has been copied */ 900 obj->desc.handle = spmc_shmem_obj_state.next_handle++; 901 obj->desc.flags |= mtd_flag; 902 } 903 904 obj->desc_filled += fragment_length; 905 ret = spmc_shmem_check_obj(obj, ffa_version); 906 if (ret != 0) { 907 ret = FFA_ERROR_INVALID_PARAMETER; 908 goto err_bad_desc; 909 } 910 911 handle_low = (uint32_t)obj->desc.handle; 912 handle_high = obj->desc.handle >> 32; 913 914 if (obj->desc_filled != obj->desc_size) { 915 SMC_RET8(smc_handle, FFA_MEM_FRAG_RX, handle_low, 916 handle_high, obj->desc_filled, 917 (uint32_t)obj->desc.sender_id << 16, 0, 0, 0); 918 } 919 920 /* The full descriptor has been received, perform any final checks. */ 921 922 /* 923 * If a partition ID resides in the secure world validate that the 924 * partition ID is for a known partition. Ignore any partition ID 925 * belonging to the normal world as it is assumed the Hypervisor will 926 * have validated these. 927 */ 928 for (size_t i = 0; i < obj->desc.emad_count; i++) { 929 emad = spmc_shmem_obj_get_emad(&obj->desc, i, ffa_version, 930 &emad_size); 931 if (emad == NULL) { 932 ret = FFA_ERROR_INVALID_PARAMETER; 933 goto err_bad_desc; 934 } 935 936 ffa_endpoint_id16_t ep_id = emad->mapd.endpoint_id; 937 938 if (ffa_is_secure_world_id(ep_id)) { 939 if (spmc_get_sp_ctx(ep_id) == NULL) { 940 WARN("%s: Invalid receiver id 0x%x\n", 941 __func__, ep_id); 942 ret = FFA_ERROR_INVALID_PARAMETER; 943 goto err_bad_desc; 944 } 945 } 946 } 947 948 /* Ensure partition IDs are not duplicated. */ 949 for (size_t i = 0; i < obj->desc.emad_count; i++) { 950 emad = spmc_shmem_obj_get_emad(&obj->desc, i, ffa_version, 951 &emad_size); 952 if (emad == NULL) { 953 ret = FFA_ERROR_INVALID_PARAMETER; 954 goto err_bad_desc; 955 } 956 for (size_t j = i + 1; j < obj->desc.emad_count; j++) { 957 other_emad = spmc_shmem_obj_get_emad(&obj->desc, j, 958 ffa_version, 959 &emad_size); 960 if (other_emad == NULL) { 961 ret = FFA_ERROR_INVALID_PARAMETER; 962 goto err_bad_desc; 963 } 964 965 if (emad->mapd.endpoint_id == 966 other_emad->mapd.endpoint_id) { 967 WARN("%s: Duplicated endpoint id 0x%x\n", 968 __func__, emad->mapd.endpoint_id); 969 ret = FFA_ERROR_INVALID_PARAMETER; 970 goto err_bad_desc; 971 } 972 } 973 } 974 975 ret = spmc_shmem_check_state_obj(obj, ffa_version); 976 if (ret) { 977 ERROR("%s: invalid memory region descriptor.\n", __func__); 978 ret = FFA_ERROR_INVALID_PARAMETER; 979 goto err_bad_desc; 980 } 981 982 /* 983 * Everything checks out, if the sender was using FF-A v1.0, convert 984 * the descriptor format to use the v1.1 structures. 985 */ 986 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 987 struct spmc_shmem_obj *v1_1_obj; 988 uint64_t mem_handle; 989 990 /* Calculate the size that the v1.1 descriptor will required. */ 991 size_t v1_1_desc_size = 992 spmc_shm_get_v1_1_descriptor_size((void *) &obj->desc, 993 fragment_length); 994 995 if (v1_1_desc_size == 0U) { 996 ERROR("%s: cannot determine size of descriptor.\n", 997 __func__); 998 goto err_arg; 999 } 1000 1001 /* Get a new obj to store the v1.1 descriptor. */ 1002 v1_1_obj = 1003 spmc_shmem_obj_alloc(&spmc_shmem_obj_state, v1_1_desc_size); 1004 1005 if (!obj) { 1006 ret = FFA_ERROR_NO_MEMORY; 1007 goto err_arg; 1008 } 1009 1010 /* Perform the conversion from v1.0 to v1.1. */ 1011 v1_1_obj->desc_size = v1_1_desc_size; 1012 v1_1_obj->desc_filled = v1_1_desc_size; 1013 if (!spmc_shm_convert_shmem_obj_from_v1_0(v1_1_obj, obj)) { 1014 ERROR("%s: Could not convert mtd!\n", __func__); 1015 spmc_shmem_obj_free(&spmc_shmem_obj_state, v1_1_obj); 1016 goto err_arg; 1017 } 1018 1019 /* 1020 * We're finished with the v1.0 descriptor so free it 1021 * and continue our checks with the new v1.1 descriptor. 1022 */ 1023 mem_handle = obj->desc.handle; 1024 spmc_shmem_obj_free(&spmc_shmem_obj_state, obj); 1025 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, mem_handle); 1026 if (obj == NULL) { 1027 ERROR("%s: Failed to find converted descriptor.\n", 1028 __func__); 1029 ret = FFA_ERROR_INVALID_PARAMETER; 1030 return spmc_ffa_error_return(smc_handle, ret); 1031 } 1032 } 1033 1034 /* Allow for platform specific operations to be performed. */ 1035 ret = plat_spmc_shmem_begin(&obj->desc); 1036 if (ret != 0) { 1037 goto err_arg; 1038 } 1039 1040 SMC_RET8(smc_handle, FFA_SUCCESS_SMC32, 0, handle_low, handle_high, 0, 1041 0, 0, 0); 1042 1043 err_bad_desc: 1044 err_arg: 1045 spmc_shmem_obj_free(&spmc_shmem_obj_state, obj); 1046 return spmc_ffa_error_return(smc_handle, ret); 1047 } 1048 1049 /** 1050 * spmc_ffa_mem_send - FFA_MEM_SHARE/LEND implementation. 1051 * @client: Client state. 1052 * @total_length: Total length of shared memory descriptor. 1053 * @fragment_length: Length of fragment of shared memory descriptor passed in 1054 * this call. 1055 * @address: Not supported, must be 0. 1056 * @page_count: Not supported, must be 0. 1057 * @smc_handle: Handle passed to smc call. Used to return 1058 * FFA_MEM_FRAG_RX or SMC_FC_FFA_SUCCESS. 1059 * 1060 * Implements a subset of the FF-A FFA_MEM_SHARE and FFA_MEM_LEND calls needed 1061 * to share or lend memory from non-secure os to secure os (with no stream 1062 * endpoints). 1063 * 1064 * Return: 0 on success, error code on failure. 1065 */ 1066 long spmc_ffa_mem_send(uint32_t smc_fid, 1067 bool secure_origin, 1068 uint64_t total_length, 1069 uint32_t fragment_length, 1070 uint64_t address, 1071 uint32_t page_count, 1072 void *cookie, 1073 void *handle, 1074 uint64_t flags) 1075 1076 { 1077 long ret; 1078 struct spmc_shmem_obj *obj; 1079 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1080 ffa_mtd_flag32_t mtd_flag; 1081 uint32_t ffa_version = get_partition_ffa_version(secure_origin); 1082 1083 if (address != 0U || page_count != 0U) { 1084 WARN("%s: custom memory region for message not supported.\n", 1085 __func__); 1086 return spmc_ffa_error_return(handle, 1087 FFA_ERROR_INVALID_PARAMETER); 1088 } 1089 1090 if (secure_origin) { 1091 WARN("%s: unsupported share direction.\n", __func__); 1092 return spmc_ffa_error_return(handle, 1093 FFA_ERROR_INVALID_PARAMETER); 1094 } 1095 1096 /* 1097 * Check if the descriptor is smaller than the v1.0 descriptor. The 1098 * descriptor cannot be smaller than this structure. 1099 */ 1100 if (fragment_length < sizeof(struct ffa_mtd_v1_0)) { 1101 WARN("%s: bad first fragment size %u < %zu\n", 1102 __func__, fragment_length, sizeof(struct ffa_mtd_v1_0)); 1103 return spmc_ffa_error_return(handle, 1104 FFA_ERROR_INVALID_PARAMETER); 1105 } 1106 1107 if ((smc_fid & FUNCID_NUM_MASK) == FFA_FNUM_MEM_SHARE) { 1108 mtd_flag = FFA_MTD_FLAG_TYPE_SHARE_MEMORY; 1109 } else if ((smc_fid & FUNCID_NUM_MASK) == FFA_FNUM_MEM_LEND) { 1110 mtd_flag = FFA_MTD_FLAG_TYPE_LEND_MEMORY; 1111 } else { 1112 WARN("%s: invalid memory management operation.\n", __func__); 1113 return spmc_ffa_error_return(handle, 1114 FFA_ERROR_INVALID_PARAMETER); 1115 } 1116 1117 spin_lock(&spmc_shmem_obj_state.lock); 1118 obj = spmc_shmem_obj_alloc(&spmc_shmem_obj_state, total_length); 1119 if (obj == NULL) { 1120 ret = FFA_ERROR_NO_MEMORY; 1121 goto err_unlock; 1122 } 1123 1124 spin_lock(&mbox->lock); 1125 ret = spmc_ffa_fill_desc(mbox, obj, fragment_length, mtd_flag, 1126 ffa_version, handle); 1127 spin_unlock(&mbox->lock); 1128 1129 spin_unlock(&spmc_shmem_obj_state.lock); 1130 return ret; 1131 1132 err_unlock: 1133 spin_unlock(&spmc_shmem_obj_state.lock); 1134 return spmc_ffa_error_return(handle, ret); 1135 } 1136 1137 /** 1138 * spmc_ffa_mem_frag_tx - FFA_MEM_FRAG_TX implementation. 1139 * @client: Client state. 1140 * @handle_low: Handle_low value returned from FFA_MEM_FRAG_RX. 1141 * @handle_high: Handle_high value returned from FFA_MEM_FRAG_RX. 1142 * @fragment_length: Length of fragments transmitted. 1143 * @sender_id: Vmid of sender in bits [31:16] 1144 * @smc_handle: Handle passed to smc call. Used to return 1145 * FFA_MEM_FRAG_RX or SMC_FC_FFA_SUCCESS. 1146 * 1147 * Return: @smc_handle on success, error code on failure. 1148 */ 1149 long spmc_ffa_mem_frag_tx(uint32_t smc_fid, 1150 bool secure_origin, 1151 uint64_t handle_low, 1152 uint64_t handle_high, 1153 uint32_t fragment_length, 1154 uint32_t sender_id, 1155 void *cookie, 1156 void *handle, 1157 uint64_t flags) 1158 { 1159 long ret; 1160 uint32_t desc_sender_id; 1161 uint32_t ffa_version = get_partition_ffa_version(secure_origin); 1162 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1163 1164 struct spmc_shmem_obj *obj; 1165 uint64_t mem_handle = handle_low | (((uint64_t)handle_high) << 32); 1166 1167 spin_lock(&spmc_shmem_obj_state.lock); 1168 1169 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, mem_handle); 1170 if (obj == NULL) { 1171 WARN("%s: invalid handle, 0x%lx, not a valid handle.\n", 1172 __func__, mem_handle); 1173 ret = FFA_ERROR_INVALID_PARAMETER; 1174 goto err_unlock; 1175 } 1176 1177 desc_sender_id = (uint32_t)obj->desc.sender_id << 16; 1178 if (sender_id != desc_sender_id) { 1179 WARN("%s: invalid sender_id 0x%x != 0x%x\n", __func__, 1180 sender_id, desc_sender_id); 1181 ret = FFA_ERROR_INVALID_PARAMETER; 1182 goto err_unlock; 1183 } 1184 1185 if (obj->desc_filled == obj->desc_size) { 1186 WARN("%s: object desc already filled, %zu\n", __func__, 1187 obj->desc_filled); 1188 ret = FFA_ERROR_INVALID_PARAMETER; 1189 goto err_unlock; 1190 } 1191 1192 spin_lock(&mbox->lock); 1193 ret = spmc_ffa_fill_desc(mbox, obj, fragment_length, 0, ffa_version, 1194 handle); 1195 spin_unlock(&mbox->lock); 1196 1197 spin_unlock(&spmc_shmem_obj_state.lock); 1198 return ret; 1199 1200 err_unlock: 1201 spin_unlock(&spmc_shmem_obj_state.lock); 1202 return spmc_ffa_error_return(handle, ret); 1203 } 1204 1205 /** 1206 * spmc_ffa_mem_retrieve_set_ns_bit - Set the NS bit in the response descriptor 1207 * if the caller implements a version greater 1208 * than FF-A 1.0 or if they have requested 1209 * the functionality. 1210 * TODO: We are assuming that the caller is 1211 * an SP. To support retrieval from the 1212 * normal world this function will need to be 1213 * expanded accordingly. 1214 * @resp: Descriptor populated in callers RX buffer. 1215 * @sp_ctx: Context of the calling SP. 1216 */ 1217 void spmc_ffa_mem_retrieve_set_ns_bit(struct ffa_mtd *resp, 1218 struct secure_partition_desc *sp_ctx) 1219 { 1220 if (sp_ctx->ffa_version > MAKE_FFA_VERSION(1, 0) || 1221 sp_ctx->ns_bit_requested) { 1222 /* 1223 * Currently memory senders must reside in the normal 1224 * world, and we do not have the functionlaity to change 1225 * the state of memory dynamically. Therefore we can always set 1226 * the NS bit to 1. 1227 */ 1228 resp->memory_region_attributes |= FFA_MEM_ATTR_NS_BIT; 1229 } 1230 } 1231 1232 /** 1233 * spmc_ffa_mem_retrieve_req - FFA_MEM_RETRIEVE_REQ implementation. 1234 * @smc_fid: FID of SMC 1235 * @total_length: Total length of retrieve request descriptor if this is 1236 * the first call. Otherwise (unsupported) must be 0. 1237 * @fragment_length: Length of fragment of retrieve request descriptor passed 1238 * in this call. Only @fragment_length == @length is 1239 * supported by this implementation. 1240 * @address: Not supported, must be 0. 1241 * @page_count: Not supported, must be 0. 1242 * @smc_handle: Handle passed to smc call. Used to return 1243 * FFA_MEM_RETRIEVE_RESP. 1244 * 1245 * Implements a subset of the FF-A FFA_MEM_RETRIEVE_REQ call. 1246 * Used by secure os to retrieve memory already shared by non-secure os. 1247 * If the data does not fit in a single FFA_MEM_RETRIEVE_RESP message, 1248 * the client must call FFA_MEM_FRAG_RX until the full response has been 1249 * received. 1250 * 1251 * Return: @handle on success, error code on failure. 1252 */ 1253 long 1254 spmc_ffa_mem_retrieve_req(uint32_t smc_fid, 1255 bool secure_origin, 1256 uint32_t total_length, 1257 uint32_t fragment_length, 1258 uint64_t address, 1259 uint32_t page_count, 1260 void *cookie, 1261 void *handle, 1262 uint64_t flags) 1263 { 1264 int ret; 1265 size_t buf_size; 1266 size_t copy_size = 0; 1267 size_t min_desc_size; 1268 size_t out_desc_size = 0; 1269 1270 /* 1271 * Currently we are only accessing fields that are the same in both the 1272 * v1.0 and v1.1 mtd struct therefore we can use a v1.1 struct directly 1273 * here. We only need validate against the appropriate struct size. 1274 */ 1275 struct ffa_mtd *resp; 1276 const struct ffa_mtd *req; 1277 struct spmc_shmem_obj *obj = NULL; 1278 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1279 uint32_t ffa_version = get_partition_ffa_version(secure_origin); 1280 struct secure_partition_desc *sp_ctx = spmc_get_current_sp_ctx(); 1281 1282 if (!secure_origin) { 1283 WARN("%s: unsupported retrieve req direction.\n", __func__); 1284 return spmc_ffa_error_return(handle, 1285 FFA_ERROR_INVALID_PARAMETER); 1286 } 1287 1288 if (address != 0U || page_count != 0U) { 1289 WARN("%s: custom memory region not supported.\n", __func__); 1290 return spmc_ffa_error_return(handle, 1291 FFA_ERROR_INVALID_PARAMETER); 1292 } 1293 1294 spin_lock(&mbox->lock); 1295 1296 req = mbox->tx_buffer; 1297 resp = mbox->rx_buffer; 1298 buf_size = mbox->rxtx_page_count * FFA_PAGE_SIZE; 1299 1300 if (mbox->rxtx_page_count == 0U) { 1301 WARN("%s: buffer pair not registered.\n", __func__); 1302 ret = FFA_ERROR_INVALID_PARAMETER; 1303 goto err_unlock_mailbox; 1304 } 1305 1306 if (mbox->state != MAILBOX_STATE_EMPTY) { 1307 WARN("%s: RX Buffer is full! %d\n", __func__, mbox->state); 1308 ret = FFA_ERROR_DENIED; 1309 goto err_unlock_mailbox; 1310 } 1311 1312 if (fragment_length != total_length) { 1313 WARN("%s: fragmented retrieve request not supported.\n", 1314 __func__); 1315 ret = FFA_ERROR_INVALID_PARAMETER; 1316 goto err_unlock_mailbox; 1317 } 1318 1319 if (req->emad_count == 0U) { 1320 WARN("%s: unsupported attribute desc count %u.\n", 1321 __func__, obj->desc.emad_count); 1322 return -EINVAL; 1323 } 1324 1325 /* Determine the appropriate minimum descriptor size. */ 1326 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 1327 min_desc_size = sizeof(struct ffa_mtd_v1_0); 1328 } else { 1329 min_desc_size = sizeof(struct ffa_mtd); 1330 } 1331 if (total_length < min_desc_size) { 1332 WARN("%s: invalid length %u < %zu\n", __func__, total_length, 1333 min_desc_size); 1334 ret = FFA_ERROR_INVALID_PARAMETER; 1335 goto err_unlock_mailbox; 1336 } 1337 1338 spin_lock(&spmc_shmem_obj_state.lock); 1339 1340 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, req->handle); 1341 if (obj == NULL) { 1342 ret = FFA_ERROR_INVALID_PARAMETER; 1343 goto err_unlock_all; 1344 } 1345 1346 if (obj->desc_filled != obj->desc_size) { 1347 WARN("%s: incomplete object desc filled %zu < size %zu\n", 1348 __func__, obj->desc_filled, obj->desc_size); 1349 ret = FFA_ERROR_INVALID_PARAMETER; 1350 goto err_unlock_all; 1351 } 1352 1353 if (req->emad_count != 0U && req->sender_id != obj->desc.sender_id) { 1354 WARN("%s: wrong sender id 0x%x != 0x%x\n", 1355 __func__, req->sender_id, obj->desc.sender_id); 1356 ret = FFA_ERROR_INVALID_PARAMETER; 1357 goto err_unlock_all; 1358 } 1359 1360 if (req->emad_count != 0U && req->tag != obj->desc.tag) { 1361 WARN("%s: wrong tag 0x%lx != 0x%lx\n", 1362 __func__, req->tag, obj->desc.tag); 1363 ret = FFA_ERROR_INVALID_PARAMETER; 1364 goto err_unlock_all; 1365 } 1366 1367 if (req->emad_count != 0U && req->emad_count != obj->desc.emad_count) { 1368 WARN("%s: mistmatch of endpoint counts %u != %u\n", 1369 __func__, req->emad_count, obj->desc.emad_count); 1370 ret = FFA_ERROR_INVALID_PARAMETER; 1371 goto err_unlock_all; 1372 } 1373 1374 /* Ensure the NS bit is set to 0 in the request. */ 1375 if ((req->memory_region_attributes & FFA_MEM_ATTR_NS_BIT) != 0U) { 1376 WARN("%s: NS mem attributes flags MBZ.\n", __func__); 1377 ret = FFA_ERROR_INVALID_PARAMETER; 1378 goto err_unlock_all; 1379 } 1380 1381 if (req->flags != 0U) { 1382 if ((req->flags & FFA_MTD_FLAG_TYPE_MASK) != 1383 (obj->desc.flags & FFA_MTD_FLAG_TYPE_MASK)) { 1384 /* 1385 * If the retrieve request specifies the memory 1386 * transaction ensure it matches what we expect. 1387 */ 1388 WARN("%s: wrong mem transaction flags %x != %x\n", 1389 __func__, req->flags, obj->desc.flags); 1390 ret = FFA_ERROR_INVALID_PARAMETER; 1391 goto err_unlock_all; 1392 } 1393 1394 if (req->flags != FFA_MTD_FLAG_TYPE_SHARE_MEMORY && 1395 req->flags != FFA_MTD_FLAG_TYPE_LEND_MEMORY) { 1396 /* 1397 * Current implementation does not support donate and 1398 * it supports no other flags. 1399 */ 1400 WARN("%s: invalid flags 0x%x\n", __func__, req->flags); 1401 ret = FFA_ERROR_INVALID_PARAMETER; 1402 goto err_unlock_all; 1403 } 1404 } 1405 1406 /* Validate that the provided emad offset and structure is valid.*/ 1407 for (size_t i = 0; i < req->emad_count; i++) { 1408 size_t emad_size; 1409 struct ffa_emad_v1_0 *emad; 1410 1411 emad = spmc_shmem_obj_get_emad(req, i, ffa_version, 1412 &emad_size); 1413 if (emad == NULL) { 1414 WARN("%s: invalid emad structure.\n", __func__); 1415 ret = FFA_ERROR_INVALID_PARAMETER; 1416 goto err_unlock_all; 1417 } 1418 1419 if ((uintptr_t) emad >= (uintptr_t) 1420 ((uint8_t *) req + total_length)) { 1421 WARN("Invalid emad access.\n"); 1422 ret = FFA_ERROR_INVALID_PARAMETER; 1423 goto err_unlock_all; 1424 } 1425 } 1426 1427 /* 1428 * Validate all the endpoints match in the case of multiple 1429 * borrowers. We don't mandate that the order of the borrowers 1430 * must match in the descriptors therefore check to see if the 1431 * endpoints match in any order. 1432 */ 1433 for (size_t i = 0; i < req->emad_count; i++) { 1434 bool found = false; 1435 size_t emad_size; 1436 struct ffa_emad_v1_0 *emad; 1437 struct ffa_emad_v1_0 *other_emad; 1438 1439 emad = spmc_shmem_obj_get_emad(req, i, ffa_version, 1440 &emad_size); 1441 if (emad == NULL) { 1442 ret = FFA_ERROR_INVALID_PARAMETER; 1443 goto err_unlock_all; 1444 } 1445 1446 for (size_t j = 0; j < obj->desc.emad_count; j++) { 1447 other_emad = spmc_shmem_obj_get_emad( 1448 &obj->desc, j, MAKE_FFA_VERSION(1, 1), 1449 &emad_size); 1450 1451 if (other_emad == NULL) { 1452 ret = FFA_ERROR_INVALID_PARAMETER; 1453 goto err_unlock_all; 1454 } 1455 1456 if (req->emad_count && 1457 emad->mapd.endpoint_id == 1458 other_emad->mapd.endpoint_id) { 1459 found = true; 1460 break; 1461 } 1462 } 1463 1464 if (!found) { 1465 WARN("%s: invalid receiver id (0x%x).\n", 1466 __func__, emad->mapd.endpoint_id); 1467 ret = FFA_ERROR_INVALID_PARAMETER; 1468 goto err_unlock_all; 1469 } 1470 } 1471 1472 mbox->state = MAILBOX_STATE_FULL; 1473 1474 if (req->emad_count != 0U) { 1475 obj->in_use++; 1476 } 1477 1478 /* 1479 * If the caller is v1.0 convert the descriptor, otherwise copy 1480 * directly. 1481 */ 1482 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 1483 ret = spmc_populate_ffa_v1_0_descriptor(resp, obj, buf_size, 0, 1484 ©_size, 1485 &out_desc_size); 1486 if (ret != 0U) { 1487 ERROR("%s: Failed to process descriptor.\n", __func__); 1488 goto err_unlock_all; 1489 } 1490 } else { 1491 copy_size = MIN(obj->desc_size, buf_size); 1492 out_desc_size = obj->desc_size; 1493 1494 memcpy(resp, &obj->desc, copy_size); 1495 } 1496 1497 /* Set the NS bit in the response if applicable. */ 1498 spmc_ffa_mem_retrieve_set_ns_bit(resp, sp_ctx); 1499 1500 spin_unlock(&spmc_shmem_obj_state.lock); 1501 spin_unlock(&mbox->lock); 1502 1503 SMC_RET8(handle, FFA_MEM_RETRIEVE_RESP, out_desc_size, 1504 copy_size, 0, 0, 0, 0, 0); 1505 1506 err_unlock_all: 1507 spin_unlock(&spmc_shmem_obj_state.lock); 1508 err_unlock_mailbox: 1509 spin_unlock(&mbox->lock); 1510 return spmc_ffa_error_return(handle, ret); 1511 } 1512 1513 /** 1514 * spmc_ffa_mem_frag_rx - FFA_MEM_FRAG_RX implementation. 1515 * @client: Client state. 1516 * @handle_low: Handle passed to &FFA_MEM_RETRIEVE_REQ. Bit[31:0]. 1517 * @handle_high: Handle passed to &FFA_MEM_RETRIEVE_REQ. Bit[63:32]. 1518 * @fragment_offset: Byte offset in descriptor to resume at. 1519 * @sender_id: Bit[31:16]: Endpoint id of sender if client is a 1520 * hypervisor. 0 otherwise. 1521 * @smc_handle: Handle passed to smc call. Used to return 1522 * FFA_MEM_FRAG_TX. 1523 * 1524 * Return: @smc_handle on success, error code on failure. 1525 */ 1526 long spmc_ffa_mem_frag_rx(uint32_t smc_fid, 1527 bool secure_origin, 1528 uint32_t handle_low, 1529 uint32_t handle_high, 1530 uint32_t fragment_offset, 1531 uint32_t sender_id, 1532 void *cookie, 1533 void *handle, 1534 uint64_t flags) 1535 { 1536 int ret; 1537 void *src; 1538 size_t buf_size; 1539 size_t copy_size; 1540 size_t full_copy_size; 1541 uint32_t desc_sender_id; 1542 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1543 uint64_t mem_handle = handle_low | (((uint64_t)handle_high) << 32); 1544 struct spmc_shmem_obj *obj; 1545 uint32_t ffa_version = get_partition_ffa_version(secure_origin); 1546 1547 if (!secure_origin) { 1548 WARN("%s: can only be called from swld.\n", 1549 __func__); 1550 return spmc_ffa_error_return(handle, 1551 FFA_ERROR_INVALID_PARAMETER); 1552 } 1553 1554 spin_lock(&spmc_shmem_obj_state.lock); 1555 1556 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, mem_handle); 1557 if (obj == NULL) { 1558 WARN("%s: invalid handle, 0x%lx, not a valid handle.\n", 1559 __func__, mem_handle); 1560 ret = FFA_ERROR_INVALID_PARAMETER; 1561 goto err_unlock_shmem; 1562 } 1563 1564 desc_sender_id = (uint32_t)obj->desc.sender_id << 16; 1565 if (sender_id != 0U && sender_id != desc_sender_id) { 1566 WARN("%s: invalid sender_id 0x%x != 0x%x\n", __func__, 1567 sender_id, desc_sender_id); 1568 ret = FFA_ERROR_INVALID_PARAMETER; 1569 goto err_unlock_shmem; 1570 } 1571 1572 if (fragment_offset >= obj->desc_size) { 1573 WARN("%s: invalid fragment_offset 0x%x >= 0x%zx\n", 1574 __func__, fragment_offset, obj->desc_size); 1575 ret = FFA_ERROR_INVALID_PARAMETER; 1576 goto err_unlock_shmem; 1577 } 1578 1579 spin_lock(&mbox->lock); 1580 1581 if (mbox->rxtx_page_count == 0U) { 1582 WARN("%s: buffer pair not registered.\n", __func__); 1583 ret = FFA_ERROR_INVALID_PARAMETER; 1584 goto err_unlock_all; 1585 } 1586 1587 if (mbox->state != MAILBOX_STATE_EMPTY) { 1588 WARN("%s: RX Buffer is full!\n", __func__); 1589 ret = FFA_ERROR_DENIED; 1590 goto err_unlock_all; 1591 } 1592 1593 buf_size = mbox->rxtx_page_count * FFA_PAGE_SIZE; 1594 1595 mbox->state = MAILBOX_STATE_FULL; 1596 1597 /* 1598 * If the caller is v1.0 convert the descriptor, otherwise copy 1599 * directly. 1600 */ 1601 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 1602 size_t out_desc_size; 1603 1604 ret = spmc_populate_ffa_v1_0_descriptor(mbox->rx_buffer, obj, 1605 buf_size, 1606 fragment_offset, 1607 ©_size, 1608 &out_desc_size); 1609 if (ret != 0U) { 1610 ERROR("%s: Failed to process descriptor.\n", __func__); 1611 goto err_unlock_all; 1612 } 1613 } else { 1614 full_copy_size = obj->desc_size - fragment_offset; 1615 copy_size = MIN(full_copy_size, buf_size); 1616 1617 src = &obj->desc; 1618 1619 memcpy(mbox->rx_buffer, src + fragment_offset, copy_size); 1620 } 1621 1622 spin_unlock(&mbox->lock); 1623 spin_unlock(&spmc_shmem_obj_state.lock); 1624 1625 SMC_RET8(handle, FFA_MEM_FRAG_TX, handle_low, handle_high, 1626 copy_size, sender_id, 0, 0, 0); 1627 1628 err_unlock_all: 1629 spin_unlock(&mbox->lock); 1630 err_unlock_shmem: 1631 spin_unlock(&spmc_shmem_obj_state.lock); 1632 return spmc_ffa_error_return(handle, ret); 1633 } 1634 1635 /** 1636 * spmc_ffa_mem_relinquish - FFA_MEM_RELINQUISH implementation. 1637 * @client: Client state. 1638 * 1639 * Implements a subset of the FF-A FFA_MEM_RELINQUISH call. 1640 * Used by secure os release previously shared memory to non-secure os. 1641 * 1642 * The handle to release must be in the client's (secure os's) transmit buffer. 1643 * 1644 * Return: 0 on success, error code on failure. 1645 */ 1646 int spmc_ffa_mem_relinquish(uint32_t smc_fid, 1647 bool secure_origin, 1648 uint32_t handle_low, 1649 uint32_t handle_high, 1650 uint32_t fragment_offset, 1651 uint32_t sender_id, 1652 void *cookie, 1653 void *handle, 1654 uint64_t flags) 1655 { 1656 int ret; 1657 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1658 struct spmc_shmem_obj *obj; 1659 const struct ffa_mem_relinquish_descriptor *req; 1660 1661 if (!secure_origin) { 1662 WARN("%s: unsupported relinquish direction.\n", __func__); 1663 return spmc_ffa_error_return(handle, 1664 FFA_ERROR_INVALID_PARAMETER); 1665 } 1666 1667 spin_lock(&mbox->lock); 1668 1669 if (mbox->rxtx_page_count == 0U) { 1670 WARN("%s: buffer pair not registered.\n", __func__); 1671 ret = FFA_ERROR_INVALID_PARAMETER; 1672 goto err_unlock_mailbox; 1673 } 1674 1675 req = mbox->tx_buffer; 1676 1677 if (req->flags != 0U) { 1678 WARN("%s: unsupported flags 0x%x\n", __func__, req->flags); 1679 ret = FFA_ERROR_INVALID_PARAMETER; 1680 goto err_unlock_mailbox; 1681 } 1682 1683 if (req->endpoint_count == 0) { 1684 WARN("%s: endpoint count cannot be 0.\n", __func__); 1685 ret = FFA_ERROR_INVALID_PARAMETER; 1686 goto err_unlock_mailbox; 1687 } 1688 1689 spin_lock(&spmc_shmem_obj_state.lock); 1690 1691 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, req->handle); 1692 if (obj == NULL) { 1693 ret = FFA_ERROR_INVALID_PARAMETER; 1694 goto err_unlock_all; 1695 } 1696 1697 if (obj->desc.emad_count != req->endpoint_count) { 1698 WARN("%s: mismatch of endpoint count %u != %u\n", __func__, 1699 obj->desc.emad_count, req->endpoint_count); 1700 ret = FFA_ERROR_INVALID_PARAMETER; 1701 goto err_unlock_all; 1702 } 1703 1704 /* Validate requested endpoint IDs match descriptor. */ 1705 for (size_t i = 0; i < req->endpoint_count; i++) { 1706 bool found = false; 1707 size_t emad_size; 1708 struct ffa_emad_v1_0 *emad; 1709 1710 for (unsigned int j = 0; j < obj->desc.emad_count; j++) { 1711 emad = spmc_shmem_obj_get_emad(&obj->desc, j, 1712 MAKE_FFA_VERSION(1, 1), 1713 &emad_size); 1714 if (req->endpoint_array[i] == 1715 emad->mapd.endpoint_id) { 1716 found = true; 1717 break; 1718 } 1719 } 1720 1721 if (!found) { 1722 WARN("%s: Invalid endpoint ID (0x%x).\n", 1723 __func__, req->endpoint_array[i]); 1724 ret = FFA_ERROR_INVALID_PARAMETER; 1725 goto err_unlock_all; 1726 } 1727 } 1728 1729 if (obj->in_use == 0U) { 1730 ret = FFA_ERROR_INVALID_PARAMETER; 1731 goto err_unlock_all; 1732 } 1733 obj->in_use--; 1734 1735 spin_unlock(&spmc_shmem_obj_state.lock); 1736 spin_unlock(&mbox->lock); 1737 1738 SMC_RET1(handle, FFA_SUCCESS_SMC32); 1739 1740 err_unlock_all: 1741 spin_unlock(&spmc_shmem_obj_state.lock); 1742 err_unlock_mailbox: 1743 spin_unlock(&mbox->lock); 1744 return spmc_ffa_error_return(handle, ret); 1745 } 1746 1747 /** 1748 * spmc_ffa_mem_reclaim - FFA_MEM_RECLAIM implementation. 1749 * @client: Client state. 1750 * @handle_low: Unique handle of shared memory object to reclaim. Bit[31:0]. 1751 * @handle_high: Unique handle of shared memory object to reclaim. 1752 * Bit[63:32]. 1753 * @flags: Unsupported, ignored. 1754 * 1755 * Implements a subset of the FF-A FFA_MEM_RECLAIM call. 1756 * Used by non-secure os reclaim memory previously shared with secure os. 1757 * 1758 * Return: 0 on success, error code on failure. 1759 */ 1760 int spmc_ffa_mem_reclaim(uint32_t smc_fid, 1761 bool secure_origin, 1762 uint32_t handle_low, 1763 uint32_t handle_high, 1764 uint32_t mem_flags, 1765 uint64_t x4, 1766 void *cookie, 1767 void *handle, 1768 uint64_t flags) 1769 { 1770 int ret; 1771 struct spmc_shmem_obj *obj; 1772 uint64_t mem_handle = handle_low | (((uint64_t)handle_high) << 32); 1773 1774 if (secure_origin) { 1775 WARN("%s: unsupported reclaim direction.\n", __func__); 1776 return spmc_ffa_error_return(handle, 1777 FFA_ERROR_INVALID_PARAMETER); 1778 } 1779 1780 if (mem_flags != 0U) { 1781 WARN("%s: unsupported flags 0x%x\n", __func__, mem_flags); 1782 return spmc_ffa_error_return(handle, 1783 FFA_ERROR_INVALID_PARAMETER); 1784 } 1785 1786 spin_lock(&spmc_shmem_obj_state.lock); 1787 1788 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, mem_handle); 1789 if (obj == NULL) { 1790 ret = FFA_ERROR_INVALID_PARAMETER; 1791 goto err_unlock; 1792 } 1793 if (obj->in_use != 0U) { 1794 ret = FFA_ERROR_DENIED; 1795 goto err_unlock; 1796 } 1797 1798 /* Allow for platform specific operations to be performed. */ 1799 ret = plat_spmc_shmem_reclaim(&obj->desc); 1800 if (ret != 0) { 1801 goto err_unlock; 1802 } 1803 1804 spmc_shmem_obj_free(&spmc_shmem_obj_state, obj); 1805 spin_unlock(&spmc_shmem_obj_state.lock); 1806 1807 SMC_RET1(handle, FFA_SUCCESS_SMC32); 1808 1809 err_unlock: 1810 spin_unlock(&spmc_shmem_obj_state.lock); 1811 return spmc_ffa_error_return(handle, ret); 1812 } 1813