1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2018, EPAM Systems. All rights reserved. 4 * Copyright (c) 2023, Linaro Limited 5 */ 6 7 #include <bitstring.h> 8 #include <compiler.h> 9 #include <kernel/boot.h> 10 #include <kernel/linker.h> 11 #include <kernel/misc.h> 12 #include <kernel/mutex.h> 13 #include <kernel/panic.h> 14 #include <kernel/refcount.h> 15 #include <kernel/spinlock.h> 16 #include <kernel/thread_spmc.h> 17 #include <kernel/virtualization.h> 18 #include <mm/core_memprot.h> 19 #include <mm/core_mmu.h> 20 #include <mm/tee_mm.h> 21 #include <platform_config.h> 22 #include <sm/optee_smc.h> 23 #include <string.h> 24 #include <util.h> 25 26 LIST_HEAD(prtn_list_head, guest_partition); 27 28 static unsigned int prtn_list_lock __nex_data = SPINLOCK_UNLOCK; 29 30 static struct prtn_list_head prtn_list __nex_data = 31 LIST_HEAD_INITIALIZER(prtn_list); 32 static struct prtn_list_head prtn_destroy_list __nex_data = 33 LIST_HEAD_INITIALIZER(prtn_destroy_list); 34 35 /* Free pages used for guest partitions */ 36 tee_mm_pool_t virt_mapper_pool __nex_bss; 37 38 /* Memory used by OP-TEE core */ 39 struct tee_mmap_region *kmemory_map __nex_bss; 40 41 struct guest_partition { 42 LIST_ENTRY(guest_partition) link; 43 struct mmu_partition *mmu_prtn; 44 struct tee_mmap_region *memory_map; 45 struct mutex mutex; 46 void *tables_va; 47 tee_mm_entry_t *tee_ram; 48 tee_mm_entry_t *ta_ram; 49 tee_mm_entry_t *tables; 50 bool runtime_initialized; 51 uint16_t id; 52 struct refcount refc; 53 #ifdef CFG_CORE_SEL1_SPMC 54 uint64_t cookies[SPMC_CORE_SEL1_MAX_SHM_COUNT]; 55 uint8_t cookie_count; 56 bitstr_t bit_decl(shm_bits, SPMC_CORE_SEL1_MAX_SHM_COUNT); 57 #endif 58 }; 59 60 struct guest_partition *current_partition[CFG_TEE_CORE_NB_CORE] __nex_bss; 61 62 static struct guest_partition *get_current_prtn(void) 63 { 64 struct guest_partition *ret; 65 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 66 67 ret = current_partition[get_core_pos()]; 68 69 thread_unmask_exceptions(exceptions); 70 71 return ret; 72 } 73 74 uint16_t virt_get_current_guest_id(void) 75 { 76 struct guest_partition *prtn = get_current_prtn(); 77 78 if (!prtn) 79 return 0; 80 return prtn->id; 81 } 82 83 static void set_current_prtn(struct guest_partition *prtn) 84 { 85 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); 86 87 current_partition[get_core_pos()] = prtn; 88 89 thread_unmask_exceptions(exceptions); 90 } 91 92 static size_t get_ta_ram_size(void) 93 { 94 size_t ta_size = 0; 95 96 core_mmu_get_ta_range(NULL, &ta_size); 97 return ROUNDDOWN(ta_size / CFG_VIRT_GUEST_COUNT - VCORE_UNPG_RW_SZ - 98 core_mmu_get_total_pages_size(), SMALL_PAGE_SIZE); 99 } 100 101 static struct tee_mmap_region *prepare_memory_map(paddr_t tee_data, 102 paddr_t ta_ram) 103 { 104 int i, entries; 105 vaddr_t max_va = 0; 106 struct tee_mmap_region *map; 107 /* 108 * This function assumes that at time of operation, 109 * kmemory_map (aka static_memory_map from core_mmu.c) 110 * will not be altered. This is true, because all 111 * changes to static_memory_map are done during 112 * OP-TEE initialization, while this function will 113 * called when hypervisor creates a guest. 114 */ 115 116 /* Count number of entries in nexus memory map */ 117 for (map = kmemory_map, entries = 1; map->type != MEM_AREA_END; 118 map++, entries++) 119 ; 120 121 /* Allocate entries for virtual guest map */ 122 map = nex_calloc(entries + 1, sizeof(struct tee_mmap_region)); 123 if (!map) 124 return NULL; 125 126 memcpy(map, kmemory_map, sizeof(*map) * entries); 127 128 /* Map TEE .data and .bss sections */ 129 for (i = 0; i < entries; i++) { 130 if (map[i].va == (vaddr_t)(VCORE_UNPG_RW_PA)) { 131 map[i].type = MEM_AREA_TEE_RAM_RW; 132 map[i].attr = core_mmu_type_to_attr(map[i].type); 133 map[i].pa = tee_data; 134 } 135 if (map[i].va + map[i].size > max_va) 136 max_va = map[i].va + map[i].size; 137 } 138 139 /* Map TA_RAM */ 140 assert(map[entries - 1].type == MEM_AREA_END); 141 map[entries] = map[entries - 1]; 142 map[entries - 1].region_size = SMALL_PAGE_SIZE; 143 map[entries - 1].va = ROUNDUP(max_va, map[entries - 1].region_size); 144 map[entries - 1].va += 145 (ta_ram - map[entries - 1].va) & CORE_MMU_PGDIR_MASK; 146 map[entries - 1].pa = ta_ram; 147 map[entries - 1].size = get_ta_ram_size(); 148 map[entries - 1].type = MEM_AREA_TA_RAM; 149 map[entries - 1].attr = core_mmu_type_to_attr(map[entries - 1].type); 150 151 DMSG("New map (%08lx):", (vaddr_t)(VCORE_UNPG_RW_PA)); 152 153 for (i = 0; i < entries; i++) 154 DMSG("T: %-16s rsz: %08x, pa: %08lx, va: %08lx, sz: %08lx attr: %x", 155 teecore_memtype_name(map[i].type), 156 map[i].region_size, map[i].pa, map[i].va, 157 map[i].size, map[i].attr); 158 return map; 159 } 160 161 void virt_init_memory(struct tee_mmap_region *memory_map, paddr_t secmem0_base, 162 paddr_size_t secmem0_size, paddr_t secmem1_base, 163 paddr_size_t secmem1_size) 164 { 165 struct tee_mmap_region *map = NULL; 166 paddr_size_t size = secmem0_size; 167 paddr_t base = secmem0_base; 168 169 if (secmem1_size) { 170 assert(secmem0_base + secmem0_size <= secmem1_base); 171 size = secmem1_base + secmem1_size - base; 172 } 173 174 /* Init page pool that covers all secure RAM */ 175 if (!tee_mm_init(&virt_mapper_pool, base, size, 176 SMALL_PAGE_SHIFT, TEE_MM_POOL_NEX_MALLOC)) 177 panic("Can't create pool with free pages"); 178 DMSG("Created virtual mapper pool from %"PRIxPA" to %"PRIxPA, 179 base, base + size); 180 181 if (secmem1_size) { 182 /* Carve out an eventual gap between secmem0 and secmem1 */ 183 base = secmem0_base + secmem0_size; 184 size = secmem1_base - base; 185 if (size) { 186 DMSG("Carving out gap between secmem0 and secmem1 (0x%"PRIxPA":0x%"PRIxPASZ")", 187 base, size); 188 if (!tee_mm_alloc2(&virt_mapper_pool, base, size)) 189 panic("Can't carve out secmem gap"); 190 } 191 } 192 193 194 /* Carve out areas that are used by OP-TEE core */ 195 for (map = memory_map; map->type != MEM_AREA_END; map++) { 196 switch (map->type) { 197 case MEM_AREA_TEE_RAM_RX: 198 case MEM_AREA_TEE_RAM_RO: 199 case MEM_AREA_NEX_RAM_RO: 200 case MEM_AREA_NEX_RAM_RW: 201 DMSG("Carving out area of type %d (0x%08lx-0x%08lx)", 202 map->type, map->pa, map->pa + map->size); 203 if (!tee_mm_alloc2(&virt_mapper_pool, map->pa, 204 map->size)) 205 panic("Can't carve out used area"); 206 break; 207 default: 208 continue; 209 } 210 } 211 212 kmemory_map = memory_map; 213 } 214 215 216 static TEE_Result configure_guest_prtn_mem(struct guest_partition *prtn) 217 { 218 TEE_Result res = TEE_SUCCESS; 219 paddr_t original_data_pa = 0; 220 221 prtn->tee_ram = tee_mm_alloc(&virt_mapper_pool, VCORE_UNPG_RW_SZ); 222 if (!prtn->tee_ram) { 223 EMSG("Can't allocate memory for TEE runtime context"); 224 res = TEE_ERROR_OUT_OF_MEMORY; 225 goto err; 226 } 227 DMSG("TEE RAM: %08" PRIxPA, tee_mm_get_smem(prtn->tee_ram)); 228 229 prtn->ta_ram = tee_mm_alloc(&virt_mapper_pool, get_ta_ram_size()); 230 if (!prtn->ta_ram) { 231 EMSG("Can't allocate memory for TA data"); 232 res = TEE_ERROR_OUT_OF_MEMORY; 233 goto err; 234 } 235 DMSG("TA RAM: %08" PRIxPA, tee_mm_get_smem(prtn->ta_ram)); 236 237 prtn->tables = tee_mm_alloc(&virt_mapper_pool, 238 core_mmu_get_total_pages_size()); 239 if (!prtn->tables) { 240 EMSG("Can't allocate memory for page tables"); 241 res = TEE_ERROR_OUT_OF_MEMORY; 242 goto err; 243 } 244 245 prtn->tables_va = phys_to_virt(tee_mm_get_smem(prtn->tables), 246 MEM_AREA_SEC_RAM_OVERALL, 247 core_mmu_get_total_pages_size()); 248 assert(prtn->tables_va); 249 250 prtn->mmu_prtn = core_alloc_mmu_prtn(prtn->tables_va); 251 if (!prtn->mmu_prtn) { 252 res = TEE_ERROR_OUT_OF_MEMORY; 253 goto err; 254 } 255 256 prtn->memory_map = prepare_memory_map(tee_mm_get_smem(prtn->tee_ram), 257 tee_mm_get_smem(prtn->ta_ram)); 258 if (!prtn->memory_map) { 259 res = TEE_ERROR_OUT_OF_MEMORY; 260 goto err; 261 } 262 263 core_init_mmu_prtn(prtn->mmu_prtn, prtn->memory_map); 264 265 original_data_pa = virt_to_phys(__data_start); 266 /* Switch to guest's mappings */ 267 core_mmu_set_prtn(prtn->mmu_prtn); 268 269 /* clear .bss */ 270 memset((void *)(VCORE_UNPG_RW_PA), 0, VCORE_UNPG_RW_SZ); 271 272 /* copy .data section from R/O original */ 273 memcpy(__data_start, 274 phys_to_virt(original_data_pa, MEM_AREA_SEC_RAM_OVERALL, 275 __data_end - __data_start), 276 __data_end - __data_start); 277 278 return TEE_SUCCESS; 279 280 err: 281 if (prtn->tee_ram) 282 tee_mm_free(prtn->tee_ram); 283 if (prtn->ta_ram) 284 tee_mm_free(prtn->ta_ram); 285 if (prtn->tables) 286 tee_mm_free(prtn->tables); 287 nex_free(prtn->mmu_prtn); 288 nex_free(prtn->memory_map); 289 290 return res; 291 } 292 293 TEE_Result virt_guest_created(uint16_t guest_id) 294 { 295 struct guest_partition *prtn = NULL; 296 TEE_Result res = TEE_SUCCESS; 297 uint32_t exceptions = 0; 298 299 prtn = nex_calloc(1, sizeof(*prtn)); 300 if (!prtn) 301 return TEE_ERROR_OUT_OF_MEMORY; 302 303 prtn->id = guest_id; 304 mutex_init(&prtn->mutex); 305 refcount_set(&prtn->refc, 1); 306 res = configure_guest_prtn_mem(prtn); 307 if (res) { 308 nex_free(prtn); 309 return res; 310 } 311 312 set_current_prtn(prtn); 313 314 /* Initialize threads */ 315 thread_init_threads(); 316 /* Do the preinitcalls */ 317 call_preinitcalls(); 318 319 exceptions = cpu_spin_lock_xsave(&prtn_list_lock); 320 LIST_INSERT_HEAD(&prtn_list, prtn, link); 321 cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); 322 323 IMSG("Added guest %d", guest_id); 324 325 set_current_prtn(NULL); 326 core_mmu_set_default_prtn(); 327 328 return TEE_SUCCESS; 329 } 330 331 static bool 332 prtn_have_remaining_resources(struct guest_partition *prtn __maybe_unused) 333 { 334 #ifdef CFG_CORE_SEL1_SPMC 335 int i = 0; 336 337 if (prtn->cookie_count) 338 return true; 339 bit_ffs(prtn->shm_bits, SPMC_CORE_SEL1_MAX_SHM_COUNT, &i); 340 return i >= 0; 341 #else 342 return false; 343 #endif 344 } 345 346 TEE_Result virt_guest_destroyed(uint16_t guest_id) 347 { 348 struct guest_partition *prtn = NULL; 349 uint32_t exceptions = 0; 350 bool do_free = true; 351 352 IMSG("Removing guest %d", guest_id); 353 354 exceptions = cpu_spin_lock_xsave(&prtn_list_lock); 355 356 LIST_FOREACH(prtn, &prtn_list, link) { 357 if (prtn->id == guest_id) { 358 if (!refcount_dec(&prtn->refc)) { 359 EMSG("Guest thread(s) is still running. refc = %d", 360 refcount_val(&prtn->refc)); 361 panic(); 362 } 363 LIST_REMOVE(prtn, link); 364 if (prtn_have_remaining_resources(prtn)) { 365 LIST_INSERT_HEAD(&prtn_destroy_list, prtn, 366 link); 367 /* 368 * Delay the nex_free() until 369 * virt_reclaim_cookie_from_destroyed_guest() 370 * is done with this partition. 371 */ 372 do_free = false; 373 } 374 break; 375 } 376 } 377 cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); 378 379 if (prtn) { 380 tee_mm_free(prtn->tee_ram); 381 prtn->tee_ram = NULL; 382 tee_mm_free(prtn->ta_ram); 383 prtn->ta_ram = NULL; 384 tee_mm_free(prtn->tables); 385 prtn->tables = NULL; 386 core_free_mmu_prtn(prtn->mmu_prtn); 387 prtn->mmu_prtn = NULL; 388 nex_free(prtn->memory_map); 389 prtn->memory_map = NULL; 390 if (do_free) 391 nex_free(prtn); 392 } else 393 EMSG("Client with id %d is not found", guest_id); 394 395 return TEE_SUCCESS; 396 } 397 398 TEE_Result virt_set_guest(uint16_t guest_id) 399 { 400 struct guest_partition *prtn; 401 uint32_t exceptions; 402 403 prtn = get_current_prtn(); 404 405 /* This can be true only if we return from IRQ RPC */ 406 if (prtn && prtn->id == guest_id) 407 return TEE_SUCCESS; 408 409 if (prtn) 410 panic("Virtual guest partition is already set"); 411 412 exceptions = cpu_spin_lock_xsave(&prtn_list_lock); 413 LIST_FOREACH(prtn, &prtn_list, link) { 414 if (prtn->id == guest_id) { 415 set_current_prtn(prtn); 416 core_mmu_set_prtn(prtn->mmu_prtn); 417 refcount_inc(&prtn->refc); 418 cpu_spin_unlock_xrestore(&prtn_list_lock, 419 exceptions); 420 return TEE_SUCCESS; 421 } 422 } 423 cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); 424 425 return TEE_ERROR_ITEM_NOT_FOUND; 426 } 427 428 void virt_unset_guest(void) 429 { 430 struct guest_partition *prtn = get_current_prtn(); 431 432 if (!prtn) 433 return; 434 435 set_current_prtn(NULL); 436 core_mmu_set_default_prtn(); 437 if (refcount_dec(&prtn->refc)) 438 panic(); 439 } 440 441 void virt_on_stdcall(void) 442 { 443 struct guest_partition *prtn = get_current_prtn(); 444 445 /* Initialize runtime on first std call */ 446 if (!prtn->runtime_initialized) { 447 mutex_lock(&prtn->mutex); 448 if (!prtn->runtime_initialized) { 449 init_tee_runtime(); 450 prtn->runtime_initialized = true; 451 } 452 mutex_unlock(&prtn->mutex); 453 } 454 } 455 456 struct tee_mmap_region *virt_get_memory_map(void) 457 { 458 struct guest_partition *prtn; 459 460 prtn = get_current_prtn(); 461 462 if (!prtn) 463 return NULL; 464 465 return prtn->memory_map; 466 } 467 468 void virt_get_ta_ram(vaddr_t *start, vaddr_t *end) 469 { 470 struct guest_partition *prtn = get_current_prtn(); 471 472 *start = (vaddr_t)phys_to_virt(tee_mm_get_smem(prtn->ta_ram), 473 MEM_AREA_TA_RAM, 474 tee_mm_get_bytes(prtn->ta_ram)); 475 *end = *start + tee_mm_get_bytes(prtn->ta_ram); 476 } 477 478 #ifdef CFG_CORE_SEL1_SPMC 479 static int find_cookie(struct guest_partition *prtn, uint64_t cookie) 480 { 481 int i = 0; 482 483 for (i = 0; i < prtn->cookie_count; i++) 484 if (prtn->cookies[i] == cookie) 485 return i; 486 return -1; 487 } 488 489 static struct guest_partition *find_prtn_cookie(uint64_t cookie, int *idx) 490 { 491 struct guest_partition *prtn = NULL; 492 int i = 0; 493 494 LIST_FOREACH(prtn, &prtn_list, link) { 495 i = find_cookie(prtn, cookie); 496 if (i >= 0) { 497 if (idx) 498 *idx = i; 499 return prtn; 500 } 501 } 502 503 return NULL; 504 } 505 506 TEE_Result virt_add_cookie_to_current_guest(uint64_t cookie) 507 { 508 TEE_Result res = TEE_ERROR_ACCESS_DENIED; 509 struct guest_partition *prtn = NULL; 510 uint32_t exceptions = 0; 511 512 exceptions = cpu_spin_lock_xsave(&prtn_list_lock); 513 if (find_prtn_cookie(cookie, NULL)) 514 goto out; 515 516 prtn = current_partition[get_core_pos()]; 517 if (prtn->cookie_count < ARRAY_SIZE(prtn->cookies)) { 518 prtn->cookies[prtn->cookie_count] = cookie; 519 prtn->cookie_count++; 520 res = TEE_SUCCESS; 521 } 522 out: 523 cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); 524 525 return res; 526 } 527 528 void virt_remove_cookie(uint64_t cookie) 529 { 530 struct guest_partition *prtn = NULL; 531 uint32_t exceptions = 0; 532 int i = 0; 533 534 exceptions = cpu_spin_lock_xsave(&prtn_list_lock); 535 prtn = find_prtn_cookie(cookie, &i); 536 if (prtn) { 537 memmove(prtn->cookies + i, prtn->cookies + i + 1, 538 sizeof(uint64_t) * (prtn->cookie_count - i - 1)); 539 prtn->cookie_count--; 540 } 541 cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); 542 } 543 544 uint16_t virt_find_guest_by_cookie(uint64_t cookie) 545 { 546 struct guest_partition *prtn = NULL; 547 uint32_t exceptions = 0; 548 uint16_t ret = 0; 549 550 exceptions = cpu_spin_lock_xsave(&prtn_list_lock); 551 prtn = find_prtn_cookie(cookie, NULL); 552 if (prtn) 553 ret = prtn->id; 554 555 cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); 556 557 return ret; 558 } 559 560 bitstr_t *virt_get_shm_bits(void) 561 { 562 return get_current_prtn()->shm_bits; 563 } 564 565 static TEE_Result reclaim_cookie(struct guest_partition *prtn, uint64_t cookie) 566 { 567 if (cookie & FFA_MEMORY_HANDLE_HYPERVISOR_BIT) { 568 size_t n = 0; 569 570 for (n = 0; n < prtn->cookie_count; n++) { 571 if (prtn->cookies[n] == cookie) { 572 memmove(prtn->cookies + n, 573 prtn->cookies + n + 1, 574 sizeof(uint64_t) * 575 (prtn->cookie_count - n - 1)); 576 prtn->cookie_count--; 577 return TEE_SUCCESS; 578 } 579 } 580 } else { 581 uint64_t mask = FFA_MEMORY_HANDLE_NON_SECURE_BIT | 582 SHIFT_U64(FFA_MEMORY_HANDLE_PRTN_MASK, 583 FFA_MEMORY_HANDLE_PRTN_SHIFT); 584 int64_t i = cookie & ~mask; 585 586 if (i >= 0 && i < SPMC_CORE_SEL1_MAX_SHM_COUNT && 587 bit_test(prtn->shm_bits, i)) { 588 bit_clear(prtn->shm_bits, i); 589 return TEE_SUCCESS; 590 } 591 } 592 593 return TEE_ERROR_ITEM_NOT_FOUND; 594 } 595 596 TEE_Result virt_reclaim_cookie_from_destroyed_guest(uint16_t guest_id, 597 uint64_t cookie) 598 599 { 600 struct guest_partition *prtn = NULL; 601 TEE_Result res = TEE_ERROR_ITEM_NOT_FOUND; 602 uint32_t exceptions = 0; 603 604 exceptions = cpu_spin_lock_xsave(&prtn_list_lock); 605 LIST_FOREACH(prtn, &prtn_destroy_list, link) { 606 if (prtn->id == guest_id) { 607 res = reclaim_cookie(prtn, cookie); 608 if (prtn_have_remaining_resources(prtn)) 609 prtn = NULL; 610 else 611 LIST_REMOVE(prtn, link); 612 break; 613 } 614 } 615 cpu_spin_unlock_xrestore(&prtn_list_lock, exceptions); 616 617 nex_free(prtn); 618 619 return res; 620 } 621 #endif /*CFG_CORE_SEL1_SPMC*/ 622