1 /* 2 * Copyright (c) 2022, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 #include <assert.h> 7 #include <errno.h> 8 9 #include <common/debug.h> 10 #include <common/runtime_svc.h> 11 #include <lib/object_pool.h> 12 #include <lib/spinlock.h> 13 #include <lib/xlat_tables/xlat_tables_v2.h> 14 #include <services/ffa_svc.h> 15 #include "spmc.h" 16 #include "spmc_shared_mem.h" 17 18 #include <platform_def.h> 19 20 /** 21 * struct spmc_shmem_obj - Shared memory object. 22 * @desc_size: Size of @desc. 23 * @desc_filled: Size of @desc already received. 24 * @in_use: Number of clients that have called ffa_mem_retrieve_req 25 * without a matching ffa_mem_relinquish call. 26 * @desc: FF-A memory region descriptor passed in ffa_mem_share. 27 */ 28 struct spmc_shmem_obj { 29 size_t desc_size; 30 size_t desc_filled; 31 size_t in_use; 32 struct ffa_mtd desc; 33 }; 34 35 /* 36 * Declare our data structure to store the metadata of memory share requests. 37 * The main datastore is allocated on a per platform basis to ensure enough 38 * storage can be made available. 39 * The address of the data store will be populated by the SPMC during its 40 * initialization. 41 */ 42 43 struct spmc_shmem_obj_state spmc_shmem_obj_state = { 44 /* Set start value for handle so top 32 bits are needed quickly. */ 45 .next_handle = 0xffffffc0U, 46 }; 47 48 /** 49 * spmc_shmem_obj_size - Convert from descriptor size to object size. 50 * @desc_size: Size of struct ffa_memory_region_descriptor object. 51 * 52 * Return: Size of struct spmc_shmem_obj object. 53 */ 54 static size_t spmc_shmem_obj_size(size_t desc_size) 55 { 56 return desc_size + offsetof(struct spmc_shmem_obj, desc); 57 } 58 59 /** 60 * spmc_shmem_obj_alloc - Allocate struct spmc_shmem_obj. 61 * @state: Global state. 62 * @desc_size: Size of struct ffa_memory_region_descriptor object that 63 * allocated object will hold. 64 * 65 * Return: Pointer to newly allocated object, or %NULL if there not enough space 66 * left. The returned pointer is only valid while @state is locked, to 67 * used it again after unlocking @state, spmc_shmem_obj_lookup must be 68 * called. 69 */ 70 static struct spmc_shmem_obj * 71 spmc_shmem_obj_alloc(struct spmc_shmem_obj_state *state, size_t desc_size) 72 { 73 struct spmc_shmem_obj *obj; 74 size_t free = state->data_size - state->allocated; 75 size_t obj_size; 76 77 if (state->data == NULL) { 78 ERROR("Missing shmem datastore!\n"); 79 return NULL; 80 } 81 82 obj_size = spmc_shmem_obj_size(desc_size); 83 84 /* Ensure the obj size has not overflowed. */ 85 if (obj_size < desc_size) { 86 WARN("%s(0x%zx) desc_size overflow\n", 87 __func__, desc_size); 88 return NULL; 89 } 90 91 if (obj_size > free) { 92 WARN("%s(0x%zx) failed, free 0x%zx\n", 93 __func__, desc_size, free); 94 return NULL; 95 } 96 obj = (struct spmc_shmem_obj *)(state->data + state->allocated); 97 obj->desc = (struct ffa_mtd) {0}; 98 obj->desc_size = desc_size; 99 obj->desc_filled = 0; 100 obj->in_use = 0; 101 state->allocated += obj_size; 102 return obj; 103 } 104 105 /** 106 * spmc_shmem_obj_free - Free struct spmc_shmem_obj. 107 * @state: Global state. 108 * @obj: Object to free. 109 * 110 * Release memory used by @obj. Other objects may move, so on return all 111 * pointers to struct spmc_shmem_obj object should be considered invalid, not 112 * just @obj. 113 * 114 * The current implementation always compacts the remaining objects to simplify 115 * the allocator and to avoid fragmentation. 116 */ 117 118 static void spmc_shmem_obj_free(struct spmc_shmem_obj_state *state, 119 struct spmc_shmem_obj *obj) 120 { 121 size_t free_size = spmc_shmem_obj_size(obj->desc_size); 122 uint8_t *shift_dest = (uint8_t *)obj; 123 uint8_t *shift_src = shift_dest + free_size; 124 size_t shift_size = state->allocated - (shift_src - state->data); 125 126 if (shift_size != 0U) { 127 memmove(shift_dest, shift_src, shift_size); 128 } 129 state->allocated -= free_size; 130 } 131 132 /** 133 * spmc_shmem_obj_lookup - Lookup struct spmc_shmem_obj by handle. 134 * @state: Global state. 135 * @handle: Unique handle of object to return. 136 * 137 * Return: struct spmc_shmem_obj_state object with handle matching @handle. 138 * %NULL, if not object in @state->data has a matching handle. 139 */ 140 static struct spmc_shmem_obj * 141 spmc_shmem_obj_lookup(struct spmc_shmem_obj_state *state, uint64_t handle) 142 { 143 uint8_t *curr = state->data; 144 145 while (curr - state->data < state->allocated) { 146 struct spmc_shmem_obj *obj = (struct spmc_shmem_obj *)curr; 147 148 if (obj->desc.handle == handle) { 149 return obj; 150 } 151 curr += spmc_shmem_obj_size(obj->desc_size); 152 } 153 return NULL; 154 } 155 156 /** 157 * spmc_shmem_obj_get_next - Get the next memory object from an offset. 158 * @offset: Offset used to track which objects have previously been 159 * returned. 160 * 161 * Return: the next struct spmc_shmem_obj_state object from the provided 162 * offset. 163 * %NULL, if there are no more objects. 164 */ 165 static struct spmc_shmem_obj * 166 spmc_shmem_obj_get_next(struct spmc_shmem_obj_state *state, size_t *offset) 167 { 168 uint8_t *curr = state->data + *offset; 169 170 if (curr - state->data < state->allocated) { 171 struct spmc_shmem_obj *obj = (struct spmc_shmem_obj *)curr; 172 173 *offset += spmc_shmem_obj_size(obj->desc_size); 174 175 return obj; 176 } 177 return NULL; 178 } 179 180 /******************************************************************************* 181 * FF-A memory descriptor helper functions. 182 ******************************************************************************/ 183 /** 184 * spmc_shmem_obj_get_emad - Get the emad from a given index depending on the 185 * clients FF-A version. 186 * @desc: The memory transaction descriptor. 187 * @index: The index of the emad element to be accessed. 188 * @ffa_version: FF-A version of the provided structure. 189 * @emad_size: Will be populated with the size of the returned emad 190 * descriptor. 191 * Return: A pointer to the requested emad structure. 192 */ 193 static void * 194 spmc_shmem_obj_get_emad(const struct ffa_mtd *desc, uint32_t index, 195 uint32_t ffa_version, size_t *emad_size) 196 { 197 uint8_t *emad; 198 /* 199 * If the caller is using FF-A v1.0 interpret the descriptor as a v1.0 200 * format, otherwise assume it is a v1.1 format. 201 */ 202 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 203 /* Cast our descriptor to the v1.0 format. */ 204 struct ffa_mtd_v1_0 *mtd_v1_0 = 205 (struct ffa_mtd_v1_0 *) desc; 206 emad = (uint8_t *) &(mtd_v1_0->emad); 207 *emad_size = sizeof(struct ffa_emad_v1_0); 208 } else { 209 if (!is_aligned(desc->emad_offset, 16)) { 210 WARN("Emad offset is not aligned.\n"); 211 return NULL; 212 } 213 emad = ((uint8_t *) desc + desc->emad_offset); 214 *emad_size = desc->emad_size; 215 } 216 return (emad + (*emad_size * index)); 217 } 218 219 /** 220 * spmc_shmem_obj_get_comp_mrd - Get comp_mrd from a mtd struct based on the 221 * FF-A version of the descriptor. 222 * @obj: Object containing ffa_memory_region_descriptor. 223 * 224 * Return: struct ffa_comp_mrd object corresponding to the composite memory 225 * region descriptor. 226 */ 227 static struct ffa_comp_mrd * 228 spmc_shmem_obj_get_comp_mrd(struct spmc_shmem_obj *obj, uint32_t ffa_version) 229 { 230 size_t emad_size; 231 /* 232 * The comp_mrd_offset field of the emad descriptor remains consistent 233 * between FF-A versions therefore we can use the v1.0 descriptor here 234 * in all cases. 235 */ 236 struct ffa_emad_v1_0 *emad = spmc_shmem_obj_get_emad(&obj->desc, 0, 237 ffa_version, 238 &emad_size); 239 /* Ensure the emad array was found. */ 240 if (emad == NULL) { 241 return NULL; 242 } 243 244 /* Ensure the composite descriptor offset is aligned. */ 245 if (!is_aligned(emad->comp_mrd_offset, 8)) { 246 WARN("Unaligned composite memory region descriptor offset.\n"); 247 return NULL; 248 } 249 250 return (struct ffa_comp_mrd *) 251 ((uint8_t *)(&obj->desc) + emad->comp_mrd_offset); 252 } 253 254 /** 255 * spmc_shmem_obj_ffa_constituent_size - Calculate variable size part of obj. 256 * @obj: Object containing ffa_memory_region_descriptor. 257 * 258 * Return: Size of ffa_constituent_memory_region_descriptors in @obj. 259 */ 260 static size_t 261 spmc_shmem_obj_ffa_constituent_size(struct spmc_shmem_obj *obj, 262 uint32_t ffa_version) 263 { 264 struct ffa_comp_mrd *comp_mrd; 265 266 comp_mrd = spmc_shmem_obj_get_comp_mrd(obj, ffa_version); 267 if (comp_mrd == NULL) { 268 return 0; 269 } 270 return comp_mrd->address_range_count * sizeof(struct ffa_cons_mrd); 271 } 272 273 /** 274 * spmc_shmem_obj_validate_id - Validate a partition ID is participating in 275 * a given memory transaction. 276 * @sp_id: Partition ID to validate. 277 * @desc: Descriptor of the memory transaction. 278 * 279 * Return: true if ID is valid, else false. 280 */ 281 bool spmc_shmem_obj_validate_id(const struct ffa_mtd *desc, uint16_t sp_id) 282 { 283 bool found = false; 284 285 /* Validate the partition is a valid participant. */ 286 for (unsigned int i = 0U; i < desc->emad_count; i++) { 287 size_t emad_size; 288 struct ffa_emad_v1_0 *emad; 289 290 emad = spmc_shmem_obj_get_emad(desc, i, 291 MAKE_FFA_VERSION(1, 1), 292 &emad_size); 293 if (sp_id == emad->mapd.endpoint_id) { 294 found = true; 295 break; 296 } 297 } 298 return found; 299 } 300 301 /* 302 * Compare two memory regions to determine if any range overlaps with another 303 * ongoing memory transaction. 304 */ 305 static bool 306 overlapping_memory_regions(struct ffa_comp_mrd *region1, 307 struct ffa_comp_mrd *region2) 308 { 309 uint64_t region1_start; 310 uint64_t region1_size; 311 uint64_t region1_end; 312 uint64_t region2_start; 313 uint64_t region2_size; 314 uint64_t region2_end; 315 316 assert(region1 != NULL); 317 assert(region2 != NULL); 318 319 if (region1 == region2) { 320 return true; 321 } 322 323 /* 324 * Check each memory region in the request against existing 325 * transactions. 326 */ 327 for (size_t i = 0; i < region1->address_range_count; i++) { 328 329 region1_start = region1->address_range_array[i].address; 330 region1_size = 331 region1->address_range_array[i].page_count * 332 PAGE_SIZE_4KB; 333 region1_end = region1_start + region1_size; 334 335 for (size_t j = 0; j < region2->address_range_count; j++) { 336 337 region2_start = region2->address_range_array[j].address; 338 region2_size = 339 region2->address_range_array[j].page_count * 340 PAGE_SIZE_4KB; 341 region2_end = region2_start + region2_size; 342 343 /* Check if regions are not overlapping. */ 344 if (!((region2_end <= region1_start) || 345 (region1_end <= region2_start))) { 346 WARN("Overlapping mem regions 0x%lx-0x%lx & 0x%lx-0x%lx\n", 347 region1_start, region1_end, 348 region2_start, region2_end); 349 return true; 350 } 351 } 352 } 353 return false; 354 } 355 356 /******************************************************************************* 357 * FF-A v1.0 Memory Descriptor Conversion Helpers. 358 ******************************************************************************/ 359 /** 360 * spmc_shm_get_v1_1_descriptor_size - Calculate the required size for a v1.1 361 * converted descriptor. 362 * @orig: The original v1.0 memory transaction descriptor. 363 * @desc_size: The size of the original v1.0 memory transaction descriptor. 364 * 365 * Return: the size required to store the descriptor store in the v1.1 format. 366 */ 367 static size_t 368 spmc_shm_get_v1_1_descriptor_size(struct ffa_mtd_v1_0 *orig, size_t desc_size) 369 { 370 size_t size = 0; 371 struct ffa_comp_mrd *mrd; 372 struct ffa_emad_v1_0 *emad_array = orig->emad; 373 374 /* Get the size of the v1.1 descriptor. */ 375 size += sizeof(struct ffa_mtd); 376 377 /* Add the size of the emad descriptors. */ 378 size += orig->emad_count * sizeof(struct ffa_emad_v1_0); 379 380 /* Add the size of the composite mrds. */ 381 size += sizeof(struct ffa_comp_mrd); 382 383 /* Add the size of the constituent mrds. */ 384 mrd = (struct ffa_comp_mrd *) ((uint8_t *) orig + 385 emad_array[0].comp_mrd_offset); 386 387 /* Check the calculated address is within the memory descriptor. */ 388 if (((uintptr_t) mrd + sizeof(struct ffa_comp_mrd)) > 389 (uintptr_t)((uint8_t *) orig + desc_size)) { 390 return 0; 391 } 392 size += mrd->address_range_count * sizeof(struct ffa_cons_mrd); 393 394 return size; 395 } 396 397 /** 398 * spmc_shm_get_v1_0_descriptor_size - Calculate the required size for a v1.0 399 * converted descriptor. 400 * @orig: The original v1.1 memory transaction descriptor. 401 * @desc_size: The size of the original v1.1 memory transaction descriptor. 402 * 403 * Return: the size required to store the descriptor store in the v1.0 format. 404 */ 405 static size_t 406 spmc_shm_get_v1_0_descriptor_size(struct ffa_mtd *orig, size_t desc_size) 407 { 408 size_t size = 0; 409 struct ffa_comp_mrd *mrd; 410 struct ffa_emad_v1_0 *emad_array = (struct ffa_emad_v1_0 *) 411 ((uint8_t *) orig + 412 orig->emad_offset); 413 414 /* Get the size of the v1.0 descriptor. */ 415 size += sizeof(struct ffa_mtd_v1_0); 416 417 /* Add the size of the v1.0 emad descriptors. */ 418 size += orig->emad_count * sizeof(struct ffa_emad_v1_0); 419 420 /* Add the size of the composite mrds. */ 421 size += sizeof(struct ffa_comp_mrd); 422 423 /* Add the size of the constituent mrds. */ 424 mrd = (struct ffa_comp_mrd *) ((uint8_t *) orig + 425 emad_array[0].comp_mrd_offset); 426 427 /* Check the calculated address is within the memory descriptor. */ 428 if (((uintptr_t) mrd + sizeof(struct ffa_comp_mrd)) > 429 (uintptr_t)((uint8_t *) orig + desc_size)) { 430 return 0; 431 } 432 size += mrd->address_range_count * sizeof(struct ffa_cons_mrd); 433 434 return size; 435 } 436 437 /** 438 * spmc_shm_convert_shmem_obj_from_v1_0 - Converts a given v1.0 memory object. 439 * @out_obj: The shared memory object to populate the converted descriptor. 440 * @orig: The shared memory object containing the v1.0 descriptor. 441 * 442 * Return: true if the conversion is successful else false. 443 */ 444 static bool 445 spmc_shm_convert_shmem_obj_from_v1_0(struct spmc_shmem_obj *out_obj, 446 struct spmc_shmem_obj *orig) 447 { 448 struct ffa_mtd_v1_0 *mtd_orig = (struct ffa_mtd_v1_0 *) &orig->desc; 449 struct ffa_mtd *out = &out_obj->desc; 450 struct ffa_emad_v1_0 *emad_array_in; 451 struct ffa_emad_v1_0 *emad_array_out; 452 struct ffa_comp_mrd *mrd_in; 453 struct ffa_comp_mrd *mrd_out; 454 455 size_t mrd_in_offset; 456 size_t mrd_out_offset; 457 size_t mrd_size = 0; 458 459 /* Populate the new descriptor format from the v1.0 struct. */ 460 out->sender_id = mtd_orig->sender_id; 461 out->memory_region_attributes = mtd_orig->memory_region_attributes; 462 out->flags = mtd_orig->flags; 463 out->handle = mtd_orig->handle; 464 out->tag = mtd_orig->tag; 465 out->emad_count = mtd_orig->emad_count; 466 out->emad_size = sizeof(struct ffa_emad_v1_0); 467 468 /* 469 * We will locate the emad descriptors directly after the ffa_mtd 470 * struct. This will be 8-byte aligned. 471 */ 472 out->emad_offset = sizeof(struct ffa_mtd); 473 474 emad_array_in = mtd_orig->emad; 475 emad_array_out = (struct ffa_emad_v1_0 *) 476 ((uint8_t *) out + out->emad_offset); 477 478 /* Copy across the emad structs. */ 479 for (unsigned int i = 0U; i < out->emad_count; i++) { 480 memcpy(&emad_array_out[i], &emad_array_in[i], 481 sizeof(struct ffa_emad_v1_0)); 482 } 483 484 /* Place the mrd descriptors after the end of the emad descriptors.*/ 485 mrd_in_offset = emad_array_in->comp_mrd_offset; 486 mrd_out_offset = out->emad_offset + (out->emad_size * out->emad_count); 487 mrd_out = (struct ffa_comp_mrd *) ((uint8_t *) out + mrd_out_offset); 488 489 /* Add the size of the composite memory region descriptor. */ 490 mrd_size += sizeof(struct ffa_comp_mrd); 491 492 /* Find the mrd descriptor. */ 493 mrd_in = (struct ffa_comp_mrd *) ((uint8_t *) mtd_orig + mrd_in_offset); 494 495 /* Add the size of the constituent memory region descriptors. */ 496 mrd_size += mrd_in->address_range_count * sizeof(struct ffa_cons_mrd); 497 498 /* 499 * Update the offset in the emads by the delta between the input and 500 * output addresses. 501 */ 502 for (unsigned int i = 0U; i < out->emad_count; i++) { 503 emad_array_out[i].comp_mrd_offset = 504 emad_array_in[i].comp_mrd_offset + 505 (mrd_out_offset - mrd_in_offset); 506 } 507 508 /* Verify that we stay within bound of the memory descriptors. */ 509 if ((uintptr_t)((uint8_t *) mrd_in + mrd_size) > 510 (uintptr_t)((uint8_t *) mtd_orig + orig->desc_size) || 511 ((uintptr_t)((uint8_t *) mrd_out + mrd_size) > 512 (uintptr_t)((uint8_t *) out + out_obj->desc_size))) { 513 ERROR("%s: Invalid mrd structure.\n", __func__); 514 return false; 515 } 516 517 /* Copy the mrd descriptors directly. */ 518 memcpy(mrd_out, mrd_in, mrd_size); 519 520 return true; 521 } 522 523 /** 524 * spmc_shm_convert_mtd_to_v1_0 - Converts a given v1.1 memory object to 525 * v1.0 memory object. 526 * @out_obj: The shared memory object to populate the v1.0 descriptor. 527 * @orig: The shared memory object containing the v1.1 descriptor. 528 * 529 * Return: true if the conversion is successful else false. 530 */ 531 static bool 532 spmc_shm_convert_mtd_to_v1_0(struct spmc_shmem_obj *out_obj, 533 struct spmc_shmem_obj *orig) 534 { 535 struct ffa_mtd *mtd_orig = &orig->desc; 536 struct ffa_mtd_v1_0 *out = (struct ffa_mtd_v1_0 *) &out_obj->desc; 537 struct ffa_emad_v1_0 *emad_in; 538 struct ffa_emad_v1_0 *emad_array_in; 539 struct ffa_emad_v1_0 *emad_array_out; 540 struct ffa_comp_mrd *mrd_in; 541 struct ffa_comp_mrd *mrd_out; 542 543 size_t mrd_in_offset; 544 size_t mrd_out_offset; 545 size_t emad_out_array_size; 546 size_t mrd_size = 0; 547 548 /* Populate the v1.0 descriptor format from the v1.1 struct. */ 549 out->sender_id = mtd_orig->sender_id; 550 out->memory_region_attributes = mtd_orig->memory_region_attributes; 551 out->flags = mtd_orig->flags; 552 out->handle = mtd_orig->handle; 553 out->tag = mtd_orig->tag; 554 out->emad_count = mtd_orig->emad_count; 555 556 /* Determine the location of the emad array in both descriptors. */ 557 emad_array_in = (struct ffa_emad_v1_0 *) 558 ((uint8_t *) mtd_orig + mtd_orig->emad_offset); 559 emad_array_out = out->emad; 560 561 /* Copy across the emad structs. */ 562 emad_in = emad_array_in; 563 for (unsigned int i = 0U; i < out->emad_count; i++) { 564 memcpy(&emad_array_out[i], emad_in, 565 sizeof(struct ffa_emad_v1_0)); 566 567 emad_in += mtd_orig->emad_size; 568 } 569 570 /* Place the mrd descriptors after the end of the emad descriptors. */ 571 emad_out_array_size = sizeof(struct ffa_emad_v1_0) * out->emad_count; 572 573 mrd_out_offset = (uint8_t *) out->emad - (uint8_t *) out + 574 emad_out_array_size; 575 576 mrd_out = (struct ffa_comp_mrd *) ((uint8_t *) out + mrd_out_offset); 577 578 mrd_in_offset = mtd_orig->emad_offset + 579 (mtd_orig->emad_size * mtd_orig->emad_count); 580 581 /* Add the size of the composite memory region descriptor. */ 582 mrd_size += sizeof(struct ffa_comp_mrd); 583 584 /* Find the mrd descriptor. */ 585 mrd_in = (struct ffa_comp_mrd *) ((uint8_t *) mtd_orig + mrd_in_offset); 586 587 /* Add the size of the constituent memory region descriptors. */ 588 mrd_size += mrd_in->address_range_count * sizeof(struct ffa_cons_mrd); 589 590 /* 591 * Update the offset in the emads by the delta between the input and 592 * output addresses. 593 */ 594 emad_in = emad_array_in; 595 596 for (unsigned int i = 0U; i < out->emad_count; i++) { 597 emad_array_out[i].comp_mrd_offset = emad_in->comp_mrd_offset + 598 (mrd_out_offset - 599 mrd_in_offset); 600 emad_in += mtd_orig->emad_size; 601 } 602 603 /* Verify that we stay within bound of the memory descriptors. */ 604 if ((uintptr_t)((uint8_t *) mrd_in + mrd_size) > 605 (uintptr_t)((uint8_t *) mtd_orig + orig->desc_size) || 606 ((uintptr_t)((uint8_t *) mrd_out + mrd_size) > 607 (uintptr_t)((uint8_t *) out + out_obj->desc_size))) { 608 ERROR("%s: Invalid mrd structure.\n", __func__); 609 return false; 610 } 611 612 /* Copy the mrd descriptors directly. */ 613 memcpy(mrd_out, mrd_in, mrd_size); 614 615 return true; 616 } 617 618 /** 619 * spmc_populate_ffa_v1_0_descriptor - Converts a given v1.1 memory object to 620 * the v1.0 format and populates the 621 * provided buffer. 622 * @dst: Buffer to populate v1.0 ffa_memory_region_descriptor. 623 * @orig_obj: Object containing v1.1 ffa_memory_region_descriptor. 624 * @buf_size: Size of the buffer to populate. 625 * @offset: The offset of the converted descriptor to copy. 626 * @copy_size: Will be populated with the number of bytes copied. 627 * @out_desc_size: Will be populated with the total size of the v1.0 628 * descriptor. 629 * 630 * Return: 0 if conversion and population succeeded. 631 * Note: This function invalidates the reference to @orig therefore 632 * `spmc_shmem_obj_lookup` must be called if further usage is required. 633 */ 634 static uint32_t 635 spmc_populate_ffa_v1_0_descriptor(void *dst, struct spmc_shmem_obj *orig_obj, 636 size_t buf_size, size_t offset, 637 size_t *copy_size, size_t *v1_0_desc_size) 638 { 639 struct spmc_shmem_obj *v1_0_obj; 640 641 /* Calculate the size that the v1.0 descriptor will require. */ 642 *v1_0_desc_size = spmc_shm_get_v1_0_descriptor_size( 643 &orig_obj->desc, orig_obj->desc_size); 644 645 if (*v1_0_desc_size == 0) { 646 ERROR("%s: cannot determine size of descriptor.\n", 647 __func__); 648 return FFA_ERROR_INVALID_PARAMETER; 649 } 650 651 /* Get a new obj to store the v1.0 descriptor. */ 652 v1_0_obj = spmc_shmem_obj_alloc(&spmc_shmem_obj_state, 653 *v1_0_desc_size); 654 655 if (!v1_0_obj) { 656 return FFA_ERROR_NO_MEMORY; 657 } 658 659 /* Perform the conversion from v1.1 to v1.0. */ 660 if (!spmc_shm_convert_mtd_to_v1_0(v1_0_obj, orig_obj)) { 661 spmc_shmem_obj_free(&spmc_shmem_obj_state, v1_0_obj); 662 return FFA_ERROR_INVALID_PARAMETER; 663 } 664 665 *copy_size = MIN(v1_0_obj->desc_size - offset, buf_size); 666 memcpy(dst, (uint8_t *) &v1_0_obj->desc + offset, *copy_size); 667 668 /* 669 * We're finished with the v1.0 descriptor for now so free it. 670 * Note that this will invalidate any references to the v1.1 671 * descriptor. 672 */ 673 spmc_shmem_obj_free(&spmc_shmem_obj_state, v1_0_obj); 674 675 return 0; 676 } 677 678 /** 679 * spmc_shmem_check_obj - Check that counts in descriptor match overall size. 680 * @obj: Object containing ffa_memory_region_descriptor. 681 * @ffa_version: FF-A version of the provided descriptor. 682 * 683 * Return: 0 if object is valid, -EINVAL if constituent_memory_region_descriptor 684 * offset or count is invalid. 685 */ 686 static int spmc_shmem_check_obj(struct spmc_shmem_obj *obj, 687 uint32_t ffa_version) 688 { 689 uint32_t comp_mrd_offset = 0; 690 691 if (obj->desc.emad_count == 0U) { 692 WARN("%s: unsupported attribute desc count %u.\n", 693 __func__, obj->desc.emad_count); 694 return -EINVAL; 695 } 696 697 for (size_t emad_num = 0; emad_num < obj->desc.emad_count; emad_num++) { 698 size_t size; 699 size_t count; 700 size_t expected_size; 701 size_t total_page_count; 702 size_t emad_size; 703 size_t desc_size; 704 size_t header_emad_size; 705 uint32_t offset; 706 struct ffa_comp_mrd *comp; 707 struct ffa_emad_v1_0 *emad; 708 709 emad = spmc_shmem_obj_get_emad(&obj->desc, emad_num, 710 ffa_version, &emad_size); 711 if (emad == NULL) { 712 WARN("%s: invalid emad structure.\n", __func__); 713 return -EINVAL; 714 } 715 716 /* 717 * Validate the calculated emad address resides within the 718 * descriptor. 719 */ 720 if ((uintptr_t) emad >= 721 (uintptr_t)((uint8_t *) &obj->desc + obj->desc_size)) { 722 WARN("Invalid emad access.\n"); 723 return -EINVAL; 724 } 725 726 offset = emad->comp_mrd_offset; 727 728 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 729 desc_size = sizeof(struct ffa_mtd_v1_0); 730 } else { 731 desc_size = sizeof(struct ffa_mtd); 732 } 733 734 header_emad_size = desc_size + 735 (obj->desc.emad_count * emad_size); 736 737 if (offset < header_emad_size) { 738 WARN("%s: invalid object, offset %u < header + emad %zu\n", 739 __func__, offset, header_emad_size); 740 return -EINVAL; 741 } 742 743 size = obj->desc_size; 744 745 if (offset > size) { 746 WARN("%s: invalid object, offset %u > total size %zu\n", 747 __func__, offset, obj->desc_size); 748 return -EINVAL; 749 } 750 size -= offset; 751 752 if (size < sizeof(struct ffa_comp_mrd)) { 753 WARN("%s: invalid object, offset %u, total size %zu, no header space.\n", 754 __func__, offset, obj->desc_size); 755 return -EINVAL; 756 } 757 size -= sizeof(struct ffa_comp_mrd); 758 759 count = size / sizeof(struct ffa_cons_mrd); 760 761 comp = spmc_shmem_obj_get_comp_mrd(obj, ffa_version); 762 763 if (comp == NULL) { 764 WARN("%s: invalid comp_mrd offset\n", __func__); 765 return -EINVAL; 766 } 767 768 if (comp->address_range_count != count) { 769 WARN("%s: invalid object, desc count %u != %zu\n", 770 __func__, comp->address_range_count, count); 771 return -EINVAL; 772 } 773 774 expected_size = offset + sizeof(*comp) + 775 spmc_shmem_obj_ffa_constituent_size(obj, 776 ffa_version); 777 778 if (expected_size != obj->desc_size) { 779 WARN("%s: invalid object, computed size %zu != size %zu\n", 780 __func__, expected_size, obj->desc_size); 781 return -EINVAL; 782 } 783 784 if (obj->desc_filled < obj->desc_size) { 785 /* 786 * The whole descriptor has not yet been received. 787 * Skip final checks. 788 */ 789 return 0; 790 } 791 792 /* 793 * The offset provided to the composite memory region descriptor 794 * should be consistent across endpoint descriptors. Store the 795 * first entry and compare against subsequent entries. 796 */ 797 if (comp_mrd_offset == 0) { 798 comp_mrd_offset = offset; 799 } else { 800 if (comp_mrd_offset != offset) { 801 ERROR("%s: mismatching offsets provided, %u != %u\n", 802 __func__, offset, comp_mrd_offset); 803 return -EINVAL; 804 } 805 } 806 807 total_page_count = 0; 808 809 for (size_t i = 0; i < count; i++) { 810 total_page_count += 811 comp->address_range_array[i].page_count; 812 } 813 if (comp->total_page_count != total_page_count) { 814 WARN("%s: invalid object, desc total_page_count %u != %zu\n", 815 __func__, comp->total_page_count, 816 total_page_count); 817 return -EINVAL; 818 } 819 } 820 return 0; 821 } 822 823 /** 824 * spmc_shmem_check_state_obj - Check if the descriptor describes memory 825 * regions that are currently involved with an 826 * existing memory transactions. This implies that 827 * the memory is not in a valid state for lending. 828 * @obj: Object containing ffa_memory_region_descriptor. 829 * 830 * Return: 0 if object is valid, -EINVAL if invalid memory state. 831 */ 832 static int spmc_shmem_check_state_obj(struct spmc_shmem_obj *obj, 833 uint32_t ffa_version) 834 { 835 size_t obj_offset = 0; 836 struct spmc_shmem_obj *inflight_obj; 837 838 struct ffa_comp_mrd *other_mrd; 839 struct ffa_comp_mrd *requested_mrd = spmc_shmem_obj_get_comp_mrd(obj, 840 ffa_version); 841 842 if (requested_mrd == NULL) { 843 return -EINVAL; 844 } 845 846 inflight_obj = spmc_shmem_obj_get_next(&spmc_shmem_obj_state, 847 &obj_offset); 848 849 while (inflight_obj != NULL) { 850 /* 851 * Don't compare the transaction to itself or to partially 852 * transmitted descriptors. 853 */ 854 if ((obj->desc.handle != inflight_obj->desc.handle) && 855 (obj->desc_size == obj->desc_filled)) { 856 other_mrd = spmc_shmem_obj_get_comp_mrd(inflight_obj, 857 FFA_VERSION_COMPILED); 858 if (other_mrd == NULL) { 859 return -EINVAL; 860 } 861 if (overlapping_memory_regions(requested_mrd, 862 other_mrd)) { 863 return -EINVAL; 864 } 865 } 866 867 inflight_obj = spmc_shmem_obj_get_next(&spmc_shmem_obj_state, 868 &obj_offset); 869 } 870 return 0; 871 } 872 873 static long spmc_ffa_fill_desc(struct mailbox *mbox, 874 struct spmc_shmem_obj *obj, 875 uint32_t fragment_length, 876 ffa_mtd_flag32_t mtd_flag, 877 uint32_t ffa_version, 878 void *smc_handle) 879 { 880 int ret; 881 size_t emad_size; 882 uint32_t handle_low; 883 uint32_t handle_high; 884 struct ffa_emad_v1_0 *emad; 885 struct ffa_emad_v1_0 *other_emad; 886 887 if (mbox->rxtx_page_count == 0U) { 888 WARN("%s: buffer pair not registered.\n", __func__); 889 ret = FFA_ERROR_INVALID_PARAMETER; 890 goto err_arg; 891 } 892 893 if (fragment_length > mbox->rxtx_page_count * PAGE_SIZE_4KB) { 894 WARN("%s: bad fragment size %u > %u buffer size\n", __func__, 895 fragment_length, mbox->rxtx_page_count * PAGE_SIZE_4KB); 896 ret = FFA_ERROR_INVALID_PARAMETER; 897 goto err_arg; 898 } 899 900 if (fragment_length > obj->desc_size - obj->desc_filled) { 901 WARN("%s: bad fragment size %u > %zu remaining\n", __func__, 902 fragment_length, obj->desc_size - obj->desc_filled); 903 ret = FFA_ERROR_INVALID_PARAMETER; 904 goto err_arg; 905 } 906 907 memcpy((uint8_t *)&obj->desc + obj->desc_filled, 908 (uint8_t *) mbox->tx_buffer, fragment_length); 909 910 /* Ensure that the sender ID resides in the normal world. */ 911 if (ffa_is_secure_world_id(obj->desc.sender_id)) { 912 WARN("%s: Invalid sender ID 0x%x.\n", 913 __func__, obj->desc.sender_id); 914 ret = FFA_ERROR_DENIED; 915 goto err_arg; 916 } 917 918 /* Ensure the NS bit is set to 0. */ 919 if ((obj->desc.memory_region_attributes & FFA_MEM_ATTR_NS_BIT) != 0U) { 920 WARN("%s: NS mem attributes flags MBZ.\n", __func__); 921 ret = FFA_ERROR_INVALID_PARAMETER; 922 goto err_arg; 923 } 924 925 /* 926 * We don't currently support any optional flags so ensure none are 927 * requested. 928 */ 929 if (obj->desc.flags != 0U && mtd_flag != 0U && 930 (obj->desc.flags != mtd_flag)) { 931 WARN("%s: invalid memory transaction flags %u != %u\n", 932 __func__, obj->desc.flags, mtd_flag); 933 ret = FFA_ERROR_INVALID_PARAMETER; 934 goto err_arg; 935 } 936 937 if (obj->desc_filled == 0U) { 938 /* First fragment, descriptor header has been copied */ 939 obj->desc.handle = spmc_shmem_obj_state.next_handle++; 940 obj->desc.flags |= mtd_flag; 941 } 942 943 obj->desc_filled += fragment_length; 944 ret = spmc_shmem_check_obj(obj, ffa_version); 945 if (ret != 0) { 946 ret = FFA_ERROR_INVALID_PARAMETER; 947 goto err_bad_desc; 948 } 949 950 handle_low = (uint32_t)obj->desc.handle; 951 handle_high = obj->desc.handle >> 32; 952 953 if (obj->desc_filled != obj->desc_size) { 954 SMC_RET8(smc_handle, FFA_MEM_FRAG_RX, handle_low, 955 handle_high, obj->desc_filled, 956 (uint32_t)obj->desc.sender_id << 16, 0, 0, 0); 957 } 958 959 /* The full descriptor has been received, perform any final checks. */ 960 961 /* 962 * If a partition ID resides in the secure world validate that the 963 * partition ID is for a known partition. Ignore any partition ID 964 * belonging to the normal world as it is assumed the Hypervisor will 965 * have validated these. 966 */ 967 for (size_t i = 0; i < obj->desc.emad_count; i++) { 968 emad = spmc_shmem_obj_get_emad(&obj->desc, i, ffa_version, 969 &emad_size); 970 if (emad == NULL) { 971 ret = FFA_ERROR_INVALID_PARAMETER; 972 goto err_bad_desc; 973 } 974 975 ffa_endpoint_id16_t ep_id = emad->mapd.endpoint_id; 976 977 if (ffa_is_secure_world_id(ep_id)) { 978 if (spmc_get_sp_ctx(ep_id) == NULL) { 979 WARN("%s: Invalid receiver id 0x%x\n", 980 __func__, ep_id); 981 ret = FFA_ERROR_INVALID_PARAMETER; 982 goto err_bad_desc; 983 } 984 } 985 } 986 987 /* Ensure partition IDs are not duplicated. */ 988 for (size_t i = 0; i < obj->desc.emad_count; i++) { 989 emad = spmc_shmem_obj_get_emad(&obj->desc, i, ffa_version, 990 &emad_size); 991 if (emad == NULL) { 992 ret = FFA_ERROR_INVALID_PARAMETER; 993 goto err_bad_desc; 994 } 995 for (size_t j = i + 1; j < obj->desc.emad_count; j++) { 996 other_emad = spmc_shmem_obj_get_emad(&obj->desc, j, 997 ffa_version, 998 &emad_size); 999 if (other_emad == NULL) { 1000 ret = FFA_ERROR_INVALID_PARAMETER; 1001 goto err_bad_desc; 1002 } 1003 1004 if (emad->mapd.endpoint_id == 1005 other_emad->mapd.endpoint_id) { 1006 WARN("%s: Duplicated endpoint id 0x%x\n", 1007 __func__, emad->mapd.endpoint_id); 1008 ret = FFA_ERROR_INVALID_PARAMETER; 1009 goto err_bad_desc; 1010 } 1011 } 1012 } 1013 1014 ret = spmc_shmem_check_state_obj(obj, ffa_version); 1015 if (ret) { 1016 ERROR("%s: invalid memory region descriptor.\n", __func__); 1017 ret = FFA_ERROR_INVALID_PARAMETER; 1018 goto err_bad_desc; 1019 } 1020 1021 /* 1022 * Everything checks out, if the sender was using FF-A v1.0, convert 1023 * the descriptor format to use the v1.1 structures. 1024 */ 1025 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 1026 struct spmc_shmem_obj *v1_1_obj; 1027 uint64_t mem_handle; 1028 1029 /* Calculate the size that the v1.1 descriptor will required. */ 1030 size_t v1_1_desc_size = 1031 spmc_shm_get_v1_1_descriptor_size((void *) &obj->desc, 1032 obj->desc_size); 1033 1034 if (v1_1_desc_size == 0U) { 1035 ERROR("%s: cannot determine size of descriptor.\n", 1036 __func__); 1037 goto err_arg; 1038 } 1039 1040 /* Get a new obj to store the v1.1 descriptor. */ 1041 v1_1_obj = 1042 spmc_shmem_obj_alloc(&spmc_shmem_obj_state, v1_1_desc_size); 1043 1044 if (!v1_1_obj) { 1045 ret = FFA_ERROR_NO_MEMORY; 1046 goto err_arg; 1047 } 1048 1049 /* Perform the conversion from v1.0 to v1.1. */ 1050 v1_1_obj->desc_size = v1_1_desc_size; 1051 v1_1_obj->desc_filled = v1_1_desc_size; 1052 if (!spmc_shm_convert_shmem_obj_from_v1_0(v1_1_obj, obj)) { 1053 ERROR("%s: Could not convert mtd!\n", __func__); 1054 spmc_shmem_obj_free(&spmc_shmem_obj_state, v1_1_obj); 1055 goto err_arg; 1056 } 1057 1058 /* 1059 * We're finished with the v1.0 descriptor so free it 1060 * and continue our checks with the new v1.1 descriptor. 1061 */ 1062 mem_handle = obj->desc.handle; 1063 spmc_shmem_obj_free(&spmc_shmem_obj_state, obj); 1064 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, mem_handle); 1065 if (obj == NULL) { 1066 ERROR("%s: Failed to find converted descriptor.\n", 1067 __func__); 1068 ret = FFA_ERROR_INVALID_PARAMETER; 1069 return spmc_ffa_error_return(smc_handle, ret); 1070 } 1071 } 1072 1073 /* Allow for platform specific operations to be performed. */ 1074 ret = plat_spmc_shmem_begin(&obj->desc); 1075 if (ret != 0) { 1076 goto err_arg; 1077 } 1078 1079 SMC_RET8(smc_handle, FFA_SUCCESS_SMC32, 0, handle_low, handle_high, 0, 1080 0, 0, 0); 1081 1082 err_bad_desc: 1083 err_arg: 1084 spmc_shmem_obj_free(&spmc_shmem_obj_state, obj); 1085 return spmc_ffa_error_return(smc_handle, ret); 1086 } 1087 1088 /** 1089 * spmc_ffa_mem_send - FFA_MEM_SHARE/LEND implementation. 1090 * @client: Client state. 1091 * @total_length: Total length of shared memory descriptor. 1092 * @fragment_length: Length of fragment of shared memory descriptor passed in 1093 * this call. 1094 * @address: Not supported, must be 0. 1095 * @page_count: Not supported, must be 0. 1096 * @smc_handle: Handle passed to smc call. Used to return 1097 * FFA_MEM_FRAG_RX or SMC_FC_FFA_SUCCESS. 1098 * 1099 * Implements a subset of the FF-A FFA_MEM_SHARE and FFA_MEM_LEND calls needed 1100 * to share or lend memory from non-secure os to secure os (with no stream 1101 * endpoints). 1102 * 1103 * Return: 0 on success, error code on failure. 1104 */ 1105 long spmc_ffa_mem_send(uint32_t smc_fid, 1106 bool secure_origin, 1107 uint64_t total_length, 1108 uint32_t fragment_length, 1109 uint64_t address, 1110 uint32_t page_count, 1111 void *cookie, 1112 void *handle, 1113 uint64_t flags) 1114 1115 { 1116 long ret; 1117 struct spmc_shmem_obj *obj; 1118 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1119 ffa_mtd_flag32_t mtd_flag; 1120 uint32_t ffa_version = get_partition_ffa_version(secure_origin); 1121 1122 if (address != 0U || page_count != 0U) { 1123 WARN("%s: custom memory region for message not supported.\n", 1124 __func__); 1125 return spmc_ffa_error_return(handle, 1126 FFA_ERROR_INVALID_PARAMETER); 1127 } 1128 1129 if (secure_origin) { 1130 WARN("%s: unsupported share direction.\n", __func__); 1131 return spmc_ffa_error_return(handle, 1132 FFA_ERROR_INVALID_PARAMETER); 1133 } 1134 1135 /* 1136 * Check if the descriptor is smaller than the v1.0 descriptor. The 1137 * descriptor cannot be smaller than this structure. 1138 */ 1139 if (fragment_length < sizeof(struct ffa_mtd_v1_0)) { 1140 WARN("%s: bad first fragment size %u < %zu\n", 1141 __func__, fragment_length, sizeof(struct ffa_mtd_v1_0)); 1142 return spmc_ffa_error_return(handle, 1143 FFA_ERROR_INVALID_PARAMETER); 1144 } 1145 1146 if ((smc_fid & FUNCID_NUM_MASK) == FFA_FNUM_MEM_SHARE) { 1147 mtd_flag = FFA_MTD_FLAG_TYPE_SHARE_MEMORY; 1148 } else if ((smc_fid & FUNCID_NUM_MASK) == FFA_FNUM_MEM_LEND) { 1149 mtd_flag = FFA_MTD_FLAG_TYPE_LEND_MEMORY; 1150 } else { 1151 WARN("%s: invalid memory management operation.\n", __func__); 1152 return spmc_ffa_error_return(handle, 1153 FFA_ERROR_INVALID_PARAMETER); 1154 } 1155 1156 spin_lock(&spmc_shmem_obj_state.lock); 1157 obj = spmc_shmem_obj_alloc(&spmc_shmem_obj_state, total_length); 1158 if (obj == NULL) { 1159 ret = FFA_ERROR_NO_MEMORY; 1160 goto err_unlock; 1161 } 1162 1163 spin_lock(&mbox->lock); 1164 ret = spmc_ffa_fill_desc(mbox, obj, fragment_length, mtd_flag, 1165 ffa_version, handle); 1166 spin_unlock(&mbox->lock); 1167 1168 spin_unlock(&spmc_shmem_obj_state.lock); 1169 return ret; 1170 1171 err_unlock: 1172 spin_unlock(&spmc_shmem_obj_state.lock); 1173 return spmc_ffa_error_return(handle, ret); 1174 } 1175 1176 /** 1177 * spmc_ffa_mem_frag_tx - FFA_MEM_FRAG_TX implementation. 1178 * @client: Client state. 1179 * @handle_low: Handle_low value returned from FFA_MEM_FRAG_RX. 1180 * @handle_high: Handle_high value returned from FFA_MEM_FRAG_RX. 1181 * @fragment_length: Length of fragments transmitted. 1182 * @sender_id: Vmid of sender in bits [31:16] 1183 * @smc_handle: Handle passed to smc call. Used to return 1184 * FFA_MEM_FRAG_RX or SMC_FC_FFA_SUCCESS. 1185 * 1186 * Return: @smc_handle on success, error code on failure. 1187 */ 1188 long spmc_ffa_mem_frag_tx(uint32_t smc_fid, 1189 bool secure_origin, 1190 uint64_t handle_low, 1191 uint64_t handle_high, 1192 uint32_t fragment_length, 1193 uint32_t sender_id, 1194 void *cookie, 1195 void *handle, 1196 uint64_t flags) 1197 { 1198 long ret; 1199 uint32_t desc_sender_id; 1200 uint32_t ffa_version = get_partition_ffa_version(secure_origin); 1201 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1202 1203 struct spmc_shmem_obj *obj; 1204 uint64_t mem_handle = handle_low | (((uint64_t)handle_high) << 32); 1205 1206 spin_lock(&spmc_shmem_obj_state.lock); 1207 1208 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, mem_handle); 1209 if (obj == NULL) { 1210 WARN("%s: invalid handle, 0x%lx, not a valid handle.\n", 1211 __func__, mem_handle); 1212 ret = FFA_ERROR_INVALID_PARAMETER; 1213 goto err_unlock; 1214 } 1215 1216 desc_sender_id = (uint32_t)obj->desc.sender_id << 16; 1217 if (sender_id != desc_sender_id) { 1218 WARN("%s: invalid sender_id 0x%x != 0x%x\n", __func__, 1219 sender_id, desc_sender_id); 1220 ret = FFA_ERROR_INVALID_PARAMETER; 1221 goto err_unlock; 1222 } 1223 1224 if (obj->desc_filled == obj->desc_size) { 1225 WARN("%s: object desc already filled, %zu\n", __func__, 1226 obj->desc_filled); 1227 ret = FFA_ERROR_INVALID_PARAMETER; 1228 goto err_unlock; 1229 } 1230 1231 spin_lock(&mbox->lock); 1232 ret = spmc_ffa_fill_desc(mbox, obj, fragment_length, 0, ffa_version, 1233 handle); 1234 spin_unlock(&mbox->lock); 1235 1236 spin_unlock(&spmc_shmem_obj_state.lock); 1237 return ret; 1238 1239 err_unlock: 1240 spin_unlock(&spmc_shmem_obj_state.lock); 1241 return spmc_ffa_error_return(handle, ret); 1242 } 1243 1244 /** 1245 * spmc_ffa_mem_retrieve_set_ns_bit - Set the NS bit in the response descriptor 1246 * if the caller implements a version greater 1247 * than FF-A 1.0 or if they have requested 1248 * the functionality. 1249 * TODO: We are assuming that the caller is 1250 * an SP. To support retrieval from the 1251 * normal world this function will need to be 1252 * expanded accordingly. 1253 * @resp: Descriptor populated in callers RX buffer. 1254 * @sp_ctx: Context of the calling SP. 1255 */ 1256 void spmc_ffa_mem_retrieve_set_ns_bit(struct ffa_mtd *resp, 1257 struct secure_partition_desc *sp_ctx) 1258 { 1259 if (sp_ctx->ffa_version > MAKE_FFA_VERSION(1, 0) || 1260 sp_ctx->ns_bit_requested) { 1261 /* 1262 * Currently memory senders must reside in the normal 1263 * world, and we do not have the functionlaity to change 1264 * the state of memory dynamically. Therefore we can always set 1265 * the NS bit to 1. 1266 */ 1267 resp->memory_region_attributes |= FFA_MEM_ATTR_NS_BIT; 1268 } 1269 } 1270 1271 /** 1272 * spmc_ffa_mem_retrieve_req - FFA_MEM_RETRIEVE_REQ implementation. 1273 * @smc_fid: FID of SMC 1274 * @total_length: Total length of retrieve request descriptor if this is 1275 * the first call. Otherwise (unsupported) must be 0. 1276 * @fragment_length: Length of fragment of retrieve request descriptor passed 1277 * in this call. Only @fragment_length == @length is 1278 * supported by this implementation. 1279 * @address: Not supported, must be 0. 1280 * @page_count: Not supported, must be 0. 1281 * @smc_handle: Handle passed to smc call. Used to return 1282 * FFA_MEM_RETRIEVE_RESP. 1283 * 1284 * Implements a subset of the FF-A FFA_MEM_RETRIEVE_REQ call. 1285 * Used by secure os to retrieve memory already shared by non-secure os. 1286 * If the data does not fit in a single FFA_MEM_RETRIEVE_RESP message, 1287 * the client must call FFA_MEM_FRAG_RX until the full response has been 1288 * received. 1289 * 1290 * Return: @handle on success, error code on failure. 1291 */ 1292 long 1293 spmc_ffa_mem_retrieve_req(uint32_t smc_fid, 1294 bool secure_origin, 1295 uint32_t total_length, 1296 uint32_t fragment_length, 1297 uint64_t address, 1298 uint32_t page_count, 1299 void *cookie, 1300 void *handle, 1301 uint64_t flags) 1302 { 1303 int ret; 1304 size_t buf_size; 1305 size_t copy_size = 0; 1306 size_t min_desc_size; 1307 size_t out_desc_size = 0; 1308 1309 /* 1310 * Currently we are only accessing fields that are the same in both the 1311 * v1.0 and v1.1 mtd struct therefore we can use a v1.1 struct directly 1312 * here. We only need validate against the appropriate struct size. 1313 */ 1314 struct ffa_mtd *resp; 1315 const struct ffa_mtd *req; 1316 struct spmc_shmem_obj *obj = NULL; 1317 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1318 uint32_t ffa_version = get_partition_ffa_version(secure_origin); 1319 struct secure_partition_desc *sp_ctx = spmc_get_current_sp_ctx(); 1320 1321 if (!secure_origin) { 1322 WARN("%s: unsupported retrieve req direction.\n", __func__); 1323 return spmc_ffa_error_return(handle, 1324 FFA_ERROR_INVALID_PARAMETER); 1325 } 1326 1327 if (address != 0U || page_count != 0U) { 1328 WARN("%s: custom memory region not supported.\n", __func__); 1329 return spmc_ffa_error_return(handle, 1330 FFA_ERROR_INVALID_PARAMETER); 1331 } 1332 1333 spin_lock(&mbox->lock); 1334 1335 req = mbox->tx_buffer; 1336 resp = mbox->rx_buffer; 1337 buf_size = mbox->rxtx_page_count * FFA_PAGE_SIZE; 1338 1339 if (mbox->rxtx_page_count == 0U) { 1340 WARN("%s: buffer pair not registered.\n", __func__); 1341 ret = FFA_ERROR_INVALID_PARAMETER; 1342 goto err_unlock_mailbox; 1343 } 1344 1345 if (mbox->state != MAILBOX_STATE_EMPTY) { 1346 WARN("%s: RX Buffer is full! %d\n", __func__, mbox->state); 1347 ret = FFA_ERROR_DENIED; 1348 goto err_unlock_mailbox; 1349 } 1350 1351 if (fragment_length != total_length) { 1352 WARN("%s: fragmented retrieve request not supported.\n", 1353 __func__); 1354 ret = FFA_ERROR_INVALID_PARAMETER; 1355 goto err_unlock_mailbox; 1356 } 1357 1358 if (req->emad_count == 0U) { 1359 WARN("%s: unsupported attribute desc count %u.\n", 1360 __func__, obj->desc.emad_count); 1361 ret = FFA_ERROR_INVALID_PARAMETER; 1362 goto err_unlock_mailbox; 1363 } 1364 1365 /* Determine the appropriate minimum descriptor size. */ 1366 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 1367 min_desc_size = sizeof(struct ffa_mtd_v1_0); 1368 } else { 1369 min_desc_size = sizeof(struct ffa_mtd); 1370 } 1371 if (total_length < min_desc_size) { 1372 WARN("%s: invalid length %u < %zu\n", __func__, total_length, 1373 min_desc_size); 1374 ret = FFA_ERROR_INVALID_PARAMETER; 1375 goto err_unlock_mailbox; 1376 } 1377 1378 spin_lock(&spmc_shmem_obj_state.lock); 1379 1380 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, req->handle); 1381 if (obj == NULL) { 1382 ret = FFA_ERROR_INVALID_PARAMETER; 1383 goto err_unlock_all; 1384 } 1385 1386 if (obj->desc_filled != obj->desc_size) { 1387 WARN("%s: incomplete object desc filled %zu < size %zu\n", 1388 __func__, obj->desc_filled, obj->desc_size); 1389 ret = FFA_ERROR_INVALID_PARAMETER; 1390 goto err_unlock_all; 1391 } 1392 1393 if (req->emad_count != 0U && req->sender_id != obj->desc.sender_id) { 1394 WARN("%s: wrong sender id 0x%x != 0x%x\n", 1395 __func__, req->sender_id, obj->desc.sender_id); 1396 ret = FFA_ERROR_INVALID_PARAMETER; 1397 goto err_unlock_all; 1398 } 1399 1400 if (req->emad_count != 0U && req->tag != obj->desc.tag) { 1401 WARN("%s: wrong tag 0x%lx != 0x%lx\n", 1402 __func__, req->tag, obj->desc.tag); 1403 ret = FFA_ERROR_INVALID_PARAMETER; 1404 goto err_unlock_all; 1405 } 1406 1407 if (req->emad_count != 0U && req->emad_count != obj->desc.emad_count) { 1408 WARN("%s: mistmatch of endpoint counts %u != %u\n", 1409 __func__, req->emad_count, obj->desc.emad_count); 1410 ret = FFA_ERROR_INVALID_PARAMETER; 1411 goto err_unlock_all; 1412 } 1413 1414 /* Ensure the NS bit is set to 0 in the request. */ 1415 if ((req->memory_region_attributes & FFA_MEM_ATTR_NS_BIT) != 0U) { 1416 WARN("%s: NS mem attributes flags MBZ.\n", __func__); 1417 ret = FFA_ERROR_INVALID_PARAMETER; 1418 goto err_unlock_all; 1419 } 1420 1421 if (req->flags != 0U) { 1422 if ((req->flags & FFA_MTD_FLAG_TYPE_MASK) != 1423 (obj->desc.flags & FFA_MTD_FLAG_TYPE_MASK)) { 1424 /* 1425 * If the retrieve request specifies the memory 1426 * transaction ensure it matches what we expect. 1427 */ 1428 WARN("%s: wrong mem transaction flags %x != %x\n", 1429 __func__, req->flags, obj->desc.flags); 1430 ret = FFA_ERROR_INVALID_PARAMETER; 1431 goto err_unlock_all; 1432 } 1433 1434 if (req->flags != FFA_MTD_FLAG_TYPE_SHARE_MEMORY && 1435 req->flags != FFA_MTD_FLAG_TYPE_LEND_MEMORY) { 1436 /* 1437 * Current implementation does not support donate and 1438 * it supports no other flags. 1439 */ 1440 WARN("%s: invalid flags 0x%x\n", __func__, req->flags); 1441 ret = FFA_ERROR_INVALID_PARAMETER; 1442 goto err_unlock_all; 1443 } 1444 } 1445 1446 /* Validate the caller is a valid participant. */ 1447 if (!spmc_shmem_obj_validate_id(&obj->desc, sp_ctx->sp_id)) { 1448 WARN("%s: Invalid endpoint ID (0x%x).\n", 1449 __func__, sp_ctx->sp_id); 1450 ret = FFA_ERROR_INVALID_PARAMETER; 1451 goto err_unlock_all; 1452 } 1453 1454 /* Validate that the provided emad offset and structure is valid.*/ 1455 for (size_t i = 0; i < req->emad_count; i++) { 1456 size_t emad_size; 1457 struct ffa_emad_v1_0 *emad; 1458 1459 emad = spmc_shmem_obj_get_emad(req, i, ffa_version, 1460 &emad_size); 1461 if (emad == NULL) { 1462 WARN("%s: invalid emad structure.\n", __func__); 1463 ret = FFA_ERROR_INVALID_PARAMETER; 1464 goto err_unlock_all; 1465 } 1466 1467 if ((uintptr_t) emad >= (uintptr_t) 1468 ((uint8_t *) req + total_length)) { 1469 WARN("Invalid emad access.\n"); 1470 ret = FFA_ERROR_INVALID_PARAMETER; 1471 goto err_unlock_all; 1472 } 1473 } 1474 1475 /* 1476 * Validate all the endpoints match in the case of multiple 1477 * borrowers. We don't mandate that the order of the borrowers 1478 * must match in the descriptors therefore check to see if the 1479 * endpoints match in any order. 1480 */ 1481 for (size_t i = 0; i < req->emad_count; i++) { 1482 bool found = false; 1483 size_t emad_size; 1484 struct ffa_emad_v1_0 *emad; 1485 struct ffa_emad_v1_0 *other_emad; 1486 1487 emad = spmc_shmem_obj_get_emad(req, i, ffa_version, 1488 &emad_size); 1489 if (emad == NULL) { 1490 ret = FFA_ERROR_INVALID_PARAMETER; 1491 goto err_unlock_all; 1492 } 1493 1494 for (size_t j = 0; j < obj->desc.emad_count; j++) { 1495 other_emad = spmc_shmem_obj_get_emad( 1496 &obj->desc, j, MAKE_FFA_VERSION(1, 1), 1497 &emad_size); 1498 1499 if (other_emad == NULL) { 1500 ret = FFA_ERROR_INVALID_PARAMETER; 1501 goto err_unlock_all; 1502 } 1503 1504 if (req->emad_count && 1505 emad->mapd.endpoint_id == 1506 other_emad->mapd.endpoint_id) { 1507 found = true; 1508 break; 1509 } 1510 } 1511 1512 if (!found) { 1513 WARN("%s: invalid receiver id (0x%x).\n", 1514 __func__, emad->mapd.endpoint_id); 1515 ret = FFA_ERROR_INVALID_PARAMETER; 1516 goto err_unlock_all; 1517 } 1518 } 1519 1520 mbox->state = MAILBOX_STATE_FULL; 1521 1522 if (req->emad_count != 0U) { 1523 obj->in_use++; 1524 } 1525 1526 /* 1527 * If the caller is v1.0 convert the descriptor, otherwise copy 1528 * directly. 1529 */ 1530 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 1531 ret = spmc_populate_ffa_v1_0_descriptor(resp, obj, buf_size, 0, 1532 ©_size, 1533 &out_desc_size); 1534 if (ret != 0U) { 1535 ERROR("%s: Failed to process descriptor.\n", __func__); 1536 goto err_unlock_all; 1537 } 1538 } else { 1539 copy_size = MIN(obj->desc_size, buf_size); 1540 out_desc_size = obj->desc_size; 1541 1542 memcpy(resp, &obj->desc, copy_size); 1543 } 1544 1545 /* Set the NS bit in the response if applicable. */ 1546 spmc_ffa_mem_retrieve_set_ns_bit(resp, sp_ctx); 1547 1548 spin_unlock(&spmc_shmem_obj_state.lock); 1549 spin_unlock(&mbox->lock); 1550 1551 SMC_RET8(handle, FFA_MEM_RETRIEVE_RESP, out_desc_size, 1552 copy_size, 0, 0, 0, 0, 0); 1553 1554 err_unlock_all: 1555 spin_unlock(&spmc_shmem_obj_state.lock); 1556 err_unlock_mailbox: 1557 spin_unlock(&mbox->lock); 1558 return spmc_ffa_error_return(handle, ret); 1559 } 1560 1561 /** 1562 * spmc_ffa_mem_frag_rx - FFA_MEM_FRAG_RX implementation. 1563 * @client: Client state. 1564 * @handle_low: Handle passed to &FFA_MEM_RETRIEVE_REQ. Bit[31:0]. 1565 * @handle_high: Handle passed to &FFA_MEM_RETRIEVE_REQ. Bit[63:32]. 1566 * @fragment_offset: Byte offset in descriptor to resume at. 1567 * @sender_id: Bit[31:16]: Endpoint id of sender if client is a 1568 * hypervisor. 0 otherwise. 1569 * @smc_handle: Handle passed to smc call. Used to return 1570 * FFA_MEM_FRAG_TX. 1571 * 1572 * Return: @smc_handle on success, error code on failure. 1573 */ 1574 long spmc_ffa_mem_frag_rx(uint32_t smc_fid, 1575 bool secure_origin, 1576 uint32_t handle_low, 1577 uint32_t handle_high, 1578 uint32_t fragment_offset, 1579 uint32_t sender_id, 1580 void *cookie, 1581 void *handle, 1582 uint64_t flags) 1583 { 1584 int ret; 1585 void *src; 1586 size_t buf_size; 1587 size_t copy_size; 1588 size_t full_copy_size; 1589 uint32_t desc_sender_id; 1590 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1591 uint64_t mem_handle = handle_low | (((uint64_t)handle_high) << 32); 1592 struct spmc_shmem_obj *obj; 1593 uint32_t ffa_version = get_partition_ffa_version(secure_origin); 1594 1595 if (!secure_origin) { 1596 WARN("%s: can only be called from swld.\n", 1597 __func__); 1598 return spmc_ffa_error_return(handle, 1599 FFA_ERROR_INVALID_PARAMETER); 1600 } 1601 1602 spin_lock(&spmc_shmem_obj_state.lock); 1603 1604 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, mem_handle); 1605 if (obj == NULL) { 1606 WARN("%s: invalid handle, 0x%lx, not a valid handle.\n", 1607 __func__, mem_handle); 1608 ret = FFA_ERROR_INVALID_PARAMETER; 1609 goto err_unlock_shmem; 1610 } 1611 1612 desc_sender_id = (uint32_t)obj->desc.sender_id << 16; 1613 if (sender_id != 0U && sender_id != desc_sender_id) { 1614 WARN("%s: invalid sender_id 0x%x != 0x%x\n", __func__, 1615 sender_id, desc_sender_id); 1616 ret = FFA_ERROR_INVALID_PARAMETER; 1617 goto err_unlock_shmem; 1618 } 1619 1620 if (fragment_offset >= obj->desc_size) { 1621 WARN("%s: invalid fragment_offset 0x%x >= 0x%zx\n", 1622 __func__, fragment_offset, obj->desc_size); 1623 ret = FFA_ERROR_INVALID_PARAMETER; 1624 goto err_unlock_shmem; 1625 } 1626 1627 spin_lock(&mbox->lock); 1628 1629 if (mbox->rxtx_page_count == 0U) { 1630 WARN("%s: buffer pair not registered.\n", __func__); 1631 ret = FFA_ERROR_INVALID_PARAMETER; 1632 goto err_unlock_all; 1633 } 1634 1635 if (mbox->state != MAILBOX_STATE_EMPTY) { 1636 WARN("%s: RX Buffer is full!\n", __func__); 1637 ret = FFA_ERROR_DENIED; 1638 goto err_unlock_all; 1639 } 1640 1641 buf_size = mbox->rxtx_page_count * FFA_PAGE_SIZE; 1642 1643 mbox->state = MAILBOX_STATE_FULL; 1644 1645 /* 1646 * If the caller is v1.0 convert the descriptor, otherwise copy 1647 * directly. 1648 */ 1649 if (ffa_version == MAKE_FFA_VERSION(1, 0)) { 1650 size_t out_desc_size; 1651 1652 ret = spmc_populate_ffa_v1_0_descriptor(mbox->rx_buffer, obj, 1653 buf_size, 1654 fragment_offset, 1655 ©_size, 1656 &out_desc_size); 1657 if (ret != 0U) { 1658 ERROR("%s: Failed to process descriptor.\n", __func__); 1659 goto err_unlock_all; 1660 } 1661 } else { 1662 full_copy_size = obj->desc_size - fragment_offset; 1663 copy_size = MIN(full_copy_size, buf_size); 1664 1665 src = &obj->desc; 1666 1667 memcpy(mbox->rx_buffer, src + fragment_offset, copy_size); 1668 } 1669 1670 spin_unlock(&mbox->lock); 1671 spin_unlock(&spmc_shmem_obj_state.lock); 1672 1673 SMC_RET8(handle, FFA_MEM_FRAG_TX, handle_low, handle_high, 1674 copy_size, sender_id, 0, 0, 0); 1675 1676 err_unlock_all: 1677 spin_unlock(&mbox->lock); 1678 err_unlock_shmem: 1679 spin_unlock(&spmc_shmem_obj_state.lock); 1680 return spmc_ffa_error_return(handle, ret); 1681 } 1682 1683 /** 1684 * spmc_ffa_mem_relinquish - FFA_MEM_RELINQUISH implementation. 1685 * @client: Client state. 1686 * 1687 * Implements a subset of the FF-A FFA_MEM_RELINQUISH call. 1688 * Used by secure os release previously shared memory to non-secure os. 1689 * 1690 * The handle to release must be in the client's (secure os's) transmit buffer. 1691 * 1692 * Return: 0 on success, error code on failure. 1693 */ 1694 int spmc_ffa_mem_relinquish(uint32_t smc_fid, 1695 bool secure_origin, 1696 uint32_t handle_low, 1697 uint32_t handle_high, 1698 uint32_t fragment_offset, 1699 uint32_t sender_id, 1700 void *cookie, 1701 void *handle, 1702 uint64_t flags) 1703 { 1704 int ret; 1705 struct mailbox *mbox = spmc_get_mbox_desc(secure_origin); 1706 struct spmc_shmem_obj *obj; 1707 const struct ffa_mem_relinquish_descriptor *req; 1708 struct secure_partition_desc *sp_ctx = spmc_get_current_sp_ctx(); 1709 1710 if (!secure_origin) { 1711 WARN("%s: unsupported relinquish direction.\n", __func__); 1712 return spmc_ffa_error_return(handle, 1713 FFA_ERROR_INVALID_PARAMETER); 1714 } 1715 1716 spin_lock(&mbox->lock); 1717 1718 if (mbox->rxtx_page_count == 0U) { 1719 WARN("%s: buffer pair not registered.\n", __func__); 1720 ret = FFA_ERROR_INVALID_PARAMETER; 1721 goto err_unlock_mailbox; 1722 } 1723 1724 req = mbox->tx_buffer; 1725 1726 if (req->flags != 0U) { 1727 WARN("%s: unsupported flags 0x%x\n", __func__, req->flags); 1728 ret = FFA_ERROR_INVALID_PARAMETER; 1729 goto err_unlock_mailbox; 1730 } 1731 1732 if (req->endpoint_count == 0) { 1733 WARN("%s: endpoint count cannot be 0.\n", __func__); 1734 ret = FFA_ERROR_INVALID_PARAMETER; 1735 goto err_unlock_mailbox; 1736 } 1737 1738 spin_lock(&spmc_shmem_obj_state.lock); 1739 1740 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, req->handle); 1741 if (obj == NULL) { 1742 ret = FFA_ERROR_INVALID_PARAMETER; 1743 goto err_unlock_all; 1744 } 1745 1746 /* 1747 * Validate the endpoint ID was populated correctly. We don't currently 1748 * support proxy endpoints so the endpoint count should always be 1. 1749 */ 1750 if (req->endpoint_count != 1U) { 1751 WARN("%s: unsupported endpoint count %u != 1\n", __func__, 1752 req->endpoint_count); 1753 ret = FFA_ERROR_INVALID_PARAMETER; 1754 goto err_unlock_all; 1755 } 1756 1757 /* Validate provided endpoint ID matches the partition ID. */ 1758 if (req->endpoint_array[0] != sp_ctx->sp_id) { 1759 WARN("%s: invalid endpoint ID %u != %u\n", __func__, 1760 req->endpoint_array[0], sp_ctx->sp_id); 1761 ret = FFA_ERROR_INVALID_PARAMETER; 1762 goto err_unlock_all; 1763 } 1764 1765 /* Validate the caller is a valid participant. */ 1766 if (!spmc_shmem_obj_validate_id(&obj->desc, sp_ctx->sp_id)) { 1767 WARN("%s: Invalid endpoint ID (0x%x).\n", 1768 __func__, req->endpoint_array[0]); 1769 ret = FFA_ERROR_INVALID_PARAMETER; 1770 goto err_unlock_all; 1771 } 1772 1773 if (obj->in_use == 0U) { 1774 ret = FFA_ERROR_INVALID_PARAMETER; 1775 goto err_unlock_all; 1776 } 1777 obj->in_use--; 1778 1779 spin_unlock(&spmc_shmem_obj_state.lock); 1780 spin_unlock(&mbox->lock); 1781 1782 SMC_RET1(handle, FFA_SUCCESS_SMC32); 1783 1784 err_unlock_all: 1785 spin_unlock(&spmc_shmem_obj_state.lock); 1786 err_unlock_mailbox: 1787 spin_unlock(&mbox->lock); 1788 return spmc_ffa_error_return(handle, ret); 1789 } 1790 1791 /** 1792 * spmc_ffa_mem_reclaim - FFA_MEM_RECLAIM implementation. 1793 * @client: Client state. 1794 * @handle_low: Unique handle of shared memory object to reclaim. Bit[31:0]. 1795 * @handle_high: Unique handle of shared memory object to reclaim. 1796 * Bit[63:32]. 1797 * @flags: Unsupported, ignored. 1798 * 1799 * Implements a subset of the FF-A FFA_MEM_RECLAIM call. 1800 * Used by non-secure os reclaim memory previously shared with secure os. 1801 * 1802 * Return: 0 on success, error code on failure. 1803 */ 1804 int spmc_ffa_mem_reclaim(uint32_t smc_fid, 1805 bool secure_origin, 1806 uint32_t handle_low, 1807 uint32_t handle_high, 1808 uint32_t mem_flags, 1809 uint64_t x4, 1810 void *cookie, 1811 void *handle, 1812 uint64_t flags) 1813 { 1814 int ret; 1815 struct spmc_shmem_obj *obj; 1816 uint64_t mem_handle = handle_low | (((uint64_t)handle_high) << 32); 1817 1818 if (secure_origin) { 1819 WARN("%s: unsupported reclaim direction.\n", __func__); 1820 return spmc_ffa_error_return(handle, 1821 FFA_ERROR_INVALID_PARAMETER); 1822 } 1823 1824 if (mem_flags != 0U) { 1825 WARN("%s: unsupported flags 0x%x\n", __func__, mem_flags); 1826 return spmc_ffa_error_return(handle, 1827 FFA_ERROR_INVALID_PARAMETER); 1828 } 1829 1830 spin_lock(&spmc_shmem_obj_state.lock); 1831 1832 obj = spmc_shmem_obj_lookup(&spmc_shmem_obj_state, mem_handle); 1833 if (obj == NULL) { 1834 ret = FFA_ERROR_INVALID_PARAMETER; 1835 goto err_unlock; 1836 } 1837 if (obj->in_use != 0U) { 1838 ret = FFA_ERROR_DENIED; 1839 goto err_unlock; 1840 } 1841 1842 if (obj->desc_filled != obj->desc_size) { 1843 WARN("%s: incomplete object desc filled %zu < size %zu\n", 1844 __func__, obj->desc_filled, obj->desc_size); 1845 ret = FFA_ERROR_INVALID_PARAMETER; 1846 goto err_unlock; 1847 } 1848 1849 /* Allow for platform specific operations to be performed. */ 1850 ret = plat_spmc_shmem_reclaim(&obj->desc); 1851 if (ret != 0) { 1852 goto err_unlock; 1853 } 1854 1855 spmc_shmem_obj_free(&spmc_shmem_obj_state, obj); 1856 spin_unlock(&spmc_shmem_obj_state.lock); 1857 1858 SMC_RET1(handle, FFA_SUCCESS_SMC32); 1859 1860 err_unlock: 1861 spin_unlock(&spmc_shmem_obj_state.lock); 1862 return spmc_ffa_error_return(handle, ret); 1863 } 1864