1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016, 2022 Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * Copyright (c) 2022, Arm Limited and Contributors. All rights reserved. 6 */ 7 8 #include <assert.h> 9 #include <config.h> 10 #include <kernel/boot.h> 11 #include <kernel/dt.h> 12 #include <kernel/linker.h> 13 #include <kernel/panic.h> 14 #include <kernel/spinlock.h> 15 #include <kernel/tee_l2cc_mutex.h> 16 #include <kernel/tee_misc.h> 17 #include <kernel/tlb_helpers.h> 18 #include <kernel/user_mode_ctx.h> 19 #include <kernel/virtualization.h> 20 #include <libfdt.h> 21 #include <memtag.h> 22 #include <mm/core_memprot.h> 23 #include <mm/core_mmu.h> 24 #include <mm/mobj.h> 25 #include <mm/pgt_cache.h> 26 #include <mm/phys_mem.h> 27 #include <mm/tee_pager.h> 28 #include <mm/vm.h> 29 #include <platform_config.h> 30 #include <stdalign.h> 31 #include <string.h> 32 #include <trace.h> 33 #include <util.h> 34 35 #ifndef DEBUG_XLAT_TABLE 36 #define DEBUG_XLAT_TABLE 0 37 #endif 38 39 #define SHM_VASPACE_SIZE (1024 * 1024 * 32) 40 41 /* Virtual memory pool for core mappings */ 42 tee_mm_pool_t core_virt_mem_pool; 43 44 /* Virtual memory pool for shared memory mappings */ 45 tee_mm_pool_t core_virt_shm_pool; 46 47 #ifdef CFG_CORE_PHYS_RELOCATABLE 48 unsigned long core_mmu_tee_load_pa __nex_bss; 49 #else 50 const unsigned long core_mmu_tee_load_pa = TEE_LOAD_ADDR; 51 #endif 52 53 /* 54 * These variables are initialized before .bss is cleared. To avoid 55 * resetting them when .bss is cleared we're storing them in .data instead, 56 * even if they initially are zero. 57 */ 58 59 #ifdef CFG_CORE_RESERVED_SHM 60 /* Default NSec shared memory allocated from NSec world */ 61 unsigned long default_nsec_shm_size __nex_bss; 62 unsigned long default_nsec_shm_paddr __nex_bss; 63 #endif 64 65 #ifdef CFG_BOOT_MEM 66 static struct memory_map static_memory_map __nex_bss; 67 #else 68 static struct tee_mmap_region static_mmap_regions[CFG_MMAP_REGIONS 69 #if defined(CFG_CORE_ASLR) || defined(CFG_CORE_PHYS_RELOCATABLE) 70 + 1 71 #endif 72 + 4] __nex_bss; 73 static struct memory_map static_memory_map __nex_data = { 74 .map = static_mmap_regions, 75 .alloc_count = ARRAY_SIZE(static_mmap_regions), 76 }; 77 #endif 78 void (*memory_map_realloc_func)(struct memory_map *mem_map) __nex_bss; 79 80 /* Offset of the first TEE RAM mapping from start of secure RAM */ 81 static size_t tee_ram_initial_offs __nex_bss; 82 83 /* Define the platform's memory layout. */ 84 struct memaccess_area { 85 paddr_t paddr; 86 size_t size; 87 }; 88 89 #define MEMACCESS_AREA(a, s) { .paddr = a, .size = s } 90 91 static struct memaccess_area secure_only[] __nex_data = { 92 #ifdef CFG_CORE_PHYS_RELOCATABLE 93 MEMACCESS_AREA(0, 0), 94 #else 95 #ifdef TRUSTED_SRAM_BASE 96 MEMACCESS_AREA(TRUSTED_SRAM_BASE, TRUSTED_SRAM_SIZE), 97 #endif 98 MEMACCESS_AREA(TRUSTED_DRAM_BASE, TRUSTED_DRAM_SIZE), 99 #endif 100 }; 101 102 static struct memaccess_area nsec_shared[] __nex_data = { 103 #ifdef CFG_CORE_RESERVED_SHM 104 MEMACCESS_AREA(TEE_SHMEM_START, TEE_SHMEM_SIZE), 105 #endif 106 }; 107 108 #if defined(CFG_SECURE_DATA_PATH) 109 static const char *tz_sdp_match = "linaro,secure-heap"; 110 static struct memaccess_area sec_sdp; 111 #ifdef CFG_TEE_SDP_MEM_BASE 112 register_sdp_mem(CFG_TEE_SDP_MEM_BASE, CFG_TEE_SDP_MEM_SIZE); 113 #endif 114 #ifdef TEE_SDP_TEST_MEM_BASE 115 register_sdp_mem(TEE_SDP_TEST_MEM_BASE, TEE_SDP_TEST_MEM_SIZE); 116 #endif 117 #endif 118 119 #ifdef CFG_CORE_RESERVED_SHM 120 register_phys_mem(MEM_AREA_NSEC_SHM, TEE_SHMEM_START, TEE_SHMEM_SIZE); 121 #endif 122 static unsigned int mmu_spinlock; 123 124 static uint32_t mmu_lock(void) 125 { 126 return cpu_spin_lock_xsave(&mmu_spinlock); 127 } 128 129 static void mmu_unlock(uint32_t exceptions) 130 { 131 cpu_spin_unlock_xrestore(&mmu_spinlock, exceptions); 132 } 133 134 static void heap_realloc_memory_map(struct memory_map *mem_map) 135 { 136 struct tee_mmap_region *m = NULL; 137 struct tee_mmap_region *old = mem_map->map; 138 size_t old_sz = sizeof(*old) * mem_map->alloc_count; 139 size_t sz = old_sz + sizeof(*m); 140 141 assert(nex_malloc_buffer_is_within_alloced(old, old_sz)); 142 m = nex_realloc(old, sz); 143 if (!m) 144 panic(); 145 mem_map->map = m; 146 mem_map->alloc_count++; 147 } 148 149 static void boot_mem_realloc_memory_map(struct memory_map *mem_map) 150 { 151 struct tee_mmap_region *m = NULL; 152 struct tee_mmap_region *old = mem_map->map; 153 size_t old_sz = sizeof(*old) * mem_map->alloc_count; 154 size_t sz = old_sz * 2; 155 156 m = boot_mem_alloc_tmp(sz, alignof(*m)); 157 memcpy(m, old, old_sz); 158 mem_map->map = m; 159 mem_map->alloc_count *= 2; 160 } 161 162 static void grow_mem_map(struct memory_map *mem_map) 163 { 164 if (mem_map->count == mem_map->alloc_count) { 165 if (!memory_map_realloc_func) { 166 EMSG("Out of entries (%zu) in mem_map", 167 mem_map->alloc_count); 168 panic(); 169 } 170 memory_map_realloc_func(mem_map); 171 } 172 mem_map->count++; 173 } 174 175 void core_mmu_get_secure_memory(paddr_t *base, paddr_size_t *size) 176 { 177 /* 178 * The first range is always used to cover OP-TEE core memory, but 179 * depending on configuration it may cover more than that. 180 */ 181 *base = secure_only[0].paddr; 182 *size = secure_only[0].size; 183 } 184 185 void core_mmu_set_secure_memory(paddr_t base, size_t size) 186 { 187 #ifdef CFG_CORE_PHYS_RELOCATABLE 188 static_assert(ARRAY_SIZE(secure_only) == 1); 189 #endif 190 runtime_assert(IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE)); 191 assert(!secure_only[0].size); 192 assert(base && size); 193 194 DMSG("Physical secure memory base %#"PRIxPA" size %#zx", base, size); 195 secure_only[0].paddr = base; 196 secure_only[0].size = size; 197 } 198 199 static struct memory_map *get_memory_map(void) 200 { 201 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 202 struct memory_map *map = virt_get_memory_map(); 203 204 if (map) 205 return map; 206 } 207 208 return &static_memory_map; 209 } 210 211 static bool _pbuf_intersects(struct memaccess_area *a, size_t alen, 212 paddr_t pa, size_t size) 213 { 214 size_t n; 215 216 for (n = 0; n < alen; n++) 217 if (core_is_buffer_intersect(pa, size, a[n].paddr, a[n].size)) 218 return true; 219 return false; 220 } 221 222 #define pbuf_intersects(a, pa, size) \ 223 _pbuf_intersects((a), ARRAY_SIZE(a), (pa), (size)) 224 225 static bool _pbuf_is_inside(struct memaccess_area *a, size_t alen, 226 paddr_t pa, size_t size) 227 { 228 size_t n; 229 230 for (n = 0; n < alen; n++) 231 if (core_is_buffer_inside(pa, size, a[n].paddr, a[n].size)) 232 return true; 233 return false; 234 } 235 236 #define pbuf_is_inside(a, pa, size) \ 237 _pbuf_is_inside((a), ARRAY_SIZE(a), (pa), (size)) 238 239 static bool pa_is_in_map(struct tee_mmap_region *map, paddr_t pa, size_t len) 240 { 241 paddr_t end_pa = 0; 242 243 if (!map) 244 return false; 245 246 if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) 247 return false; 248 249 return (pa >= map->pa && end_pa <= map->pa + map->size - 1); 250 } 251 252 static bool va_is_in_map(struct tee_mmap_region *map, vaddr_t va) 253 { 254 if (!map) 255 return false; 256 return (va >= map->va && va <= (map->va + map->size - 1)); 257 } 258 259 /* check if target buffer fits in a core default map area */ 260 static bool pbuf_inside_map_area(unsigned long p, size_t l, 261 struct tee_mmap_region *map) 262 { 263 return core_is_buffer_inside(p, l, map->pa, map->size); 264 } 265 266 TEE_Result core_mmu_for_each_map(void *ptr, 267 TEE_Result (*fn)(struct tee_mmap_region *map, 268 void *ptr)) 269 { 270 struct memory_map *mem_map = get_memory_map(); 271 TEE_Result res = TEE_SUCCESS; 272 size_t n = 0; 273 274 for (n = 0; n < mem_map->count; n++) { 275 res = fn(mem_map->map + n, ptr); 276 if (res) 277 return res; 278 } 279 280 return TEE_SUCCESS; 281 } 282 283 static struct tee_mmap_region *find_map_by_type(enum teecore_memtypes type) 284 { 285 struct memory_map *mem_map = get_memory_map(); 286 size_t n = 0; 287 288 for (n = 0; n < mem_map->count; n++) { 289 if (mem_map->map[n].type == type) 290 return mem_map->map + n; 291 } 292 return NULL; 293 } 294 295 static struct tee_mmap_region * 296 find_map_by_type_and_pa(enum teecore_memtypes type, paddr_t pa, size_t len) 297 { 298 struct memory_map *mem_map = get_memory_map(); 299 size_t n = 0; 300 301 for (n = 0; n < mem_map->count; n++) { 302 if (mem_map->map[n].type != type) 303 continue; 304 if (pa_is_in_map(mem_map->map + n, pa, len)) 305 return mem_map->map + n; 306 } 307 return NULL; 308 } 309 310 static struct tee_mmap_region *find_map_by_va(void *va) 311 { 312 struct memory_map *mem_map = get_memory_map(); 313 vaddr_t a = (vaddr_t)va; 314 size_t n = 0; 315 316 for (n = 0; n < mem_map->count; n++) { 317 if (a >= mem_map->map[n].va && 318 a <= (mem_map->map[n].va - 1 + mem_map->map[n].size)) 319 return mem_map->map + n; 320 } 321 322 return NULL; 323 } 324 325 static struct tee_mmap_region *find_map_by_pa(unsigned long pa) 326 { 327 struct memory_map *mem_map = get_memory_map(); 328 size_t n = 0; 329 330 for (n = 0; n < mem_map->count; n++) { 331 /* Skip unmapped regions */ 332 if ((mem_map->map[n].attr & TEE_MATTR_VALID_BLOCK) && 333 pa >= mem_map->map[n].pa && 334 pa <= (mem_map->map[n].pa - 1 + mem_map->map[n].size)) 335 return mem_map->map + n; 336 } 337 338 return NULL; 339 } 340 341 #if defined(CFG_SECURE_DATA_PATH) 342 static bool dtb_get_sdp_region(void) 343 { 344 void *fdt = NULL; 345 int node = 0; 346 int tmp_node = 0; 347 paddr_t tmp_addr = 0; 348 size_t tmp_size = 0; 349 350 if (!IS_ENABLED(CFG_EMBED_DTB)) 351 return false; 352 353 fdt = get_embedded_dt(); 354 if (!fdt) 355 panic("No DTB found"); 356 357 node = fdt_node_offset_by_compatible(fdt, -1, tz_sdp_match); 358 if (node < 0) { 359 DMSG("No %s compatible node found", tz_sdp_match); 360 return false; 361 } 362 tmp_node = node; 363 while (tmp_node >= 0) { 364 tmp_node = fdt_node_offset_by_compatible(fdt, tmp_node, 365 tz_sdp_match); 366 if (tmp_node >= 0) 367 DMSG("Ignore SDP pool node %s, supports only 1 node", 368 fdt_get_name(fdt, tmp_node, NULL)); 369 } 370 371 if (fdt_reg_info(fdt, node, &tmp_addr, &tmp_size)) { 372 EMSG("%s: Unable to get base addr or size from DT", 373 tz_sdp_match); 374 return false; 375 } 376 377 sec_sdp.paddr = tmp_addr; 378 sec_sdp.size = tmp_size; 379 380 return true; 381 } 382 #endif 383 384 #if defined(CFG_CORE_DYN_SHM) || defined(CFG_SECURE_DATA_PATH) 385 static bool pbuf_is_special_mem(paddr_t pbuf, size_t len, 386 const struct core_mmu_phys_mem *start, 387 const struct core_mmu_phys_mem *end) 388 { 389 const struct core_mmu_phys_mem *mem; 390 391 for (mem = start; mem < end; mem++) { 392 if (core_is_buffer_inside(pbuf, len, mem->addr, mem->size)) 393 return true; 394 } 395 396 return false; 397 } 398 #endif 399 400 #ifdef CFG_CORE_DYN_SHM 401 static void carve_out_phys_mem(struct core_mmu_phys_mem **mem, size_t *nelems, 402 paddr_t pa, size_t size) 403 { 404 struct core_mmu_phys_mem *m = *mem; 405 size_t n = 0; 406 407 while (true) { 408 if (n >= *nelems) { 409 DMSG("No need to carve out %#" PRIxPA " size %#zx", 410 pa, size); 411 return; 412 } 413 if (core_is_buffer_inside(pa, size, m[n].addr, m[n].size)) 414 break; 415 if (!core_is_buffer_outside(pa, size, m[n].addr, m[n].size)) 416 panic(); 417 n++; 418 } 419 420 if (pa == m[n].addr && size == m[n].size) { 421 /* Remove this entry */ 422 (*nelems)--; 423 memmove(m + n, m + n + 1, sizeof(*m) * (*nelems - n)); 424 m = nex_realloc(m, sizeof(*m) * *nelems); 425 if (!m) 426 panic(); 427 *mem = m; 428 } else if (pa == m[n].addr) { 429 m[n].addr += size; 430 m[n].size -= size; 431 } else if ((pa + size) == (m[n].addr + m[n].size)) { 432 m[n].size -= size; 433 } else { 434 /* Need to split the memory entry */ 435 m = nex_realloc(m, sizeof(*m) * (*nelems + 1)); 436 if (!m) 437 panic(); 438 *mem = m; 439 memmove(m + n + 1, m + n, sizeof(*m) * (*nelems - n)); 440 (*nelems)++; 441 m[n].size = pa - m[n].addr; 442 m[n + 1].size -= size + m[n].size; 443 m[n + 1].addr = pa + size; 444 } 445 } 446 447 static void check_phys_mem_is_outside(struct core_mmu_phys_mem *start, 448 size_t nelems, 449 struct tee_mmap_region *map) 450 { 451 size_t n; 452 453 for (n = 0; n < nelems; n++) { 454 if (!core_is_buffer_outside(start[n].addr, start[n].size, 455 map->pa, map->size)) { 456 EMSG("Non-sec mem (%#" PRIxPA ":%#" PRIxPASZ 457 ") overlaps map (type %d %#" PRIxPA ":%#zx)", 458 start[n].addr, start[n].size, 459 map->type, map->pa, map->size); 460 panic(); 461 } 462 } 463 } 464 465 static const struct core_mmu_phys_mem *discovered_nsec_ddr_start __nex_bss; 466 static size_t discovered_nsec_ddr_nelems __nex_bss; 467 468 static int cmp_pmem_by_addr(const void *a, const void *b) 469 { 470 const struct core_mmu_phys_mem *pmem_a = a; 471 const struct core_mmu_phys_mem *pmem_b = b; 472 473 return CMP_TRILEAN(pmem_a->addr, pmem_b->addr); 474 } 475 476 void core_mmu_set_discovered_nsec_ddr(struct core_mmu_phys_mem *start, 477 size_t nelems) 478 { 479 struct core_mmu_phys_mem *m = start; 480 size_t num_elems = nelems; 481 struct memory_map *mem_map = &static_memory_map; 482 const struct core_mmu_phys_mem __maybe_unused *pmem; 483 size_t n = 0; 484 485 assert(!discovered_nsec_ddr_start); 486 assert(m && num_elems); 487 488 qsort(m, num_elems, sizeof(*m), cmp_pmem_by_addr); 489 490 /* 491 * Non-secure shared memory and also secure data 492 * path memory are supposed to reside inside 493 * non-secure memory. Since NSEC_SHM and SDP_MEM 494 * are used for a specific purpose make holes for 495 * those memory in the normal non-secure memory. 496 * 497 * This has to be done since for instance QEMU 498 * isn't aware of which memory range in the 499 * non-secure memory is used for NSEC_SHM. 500 */ 501 502 #ifdef CFG_SECURE_DATA_PATH 503 if (dtb_get_sdp_region()) 504 carve_out_phys_mem(&m, &num_elems, sec_sdp.paddr, sec_sdp.size); 505 506 for (pmem = phys_sdp_mem_begin; pmem < phys_sdp_mem_end; pmem++) 507 carve_out_phys_mem(&m, &num_elems, pmem->addr, pmem->size); 508 #endif 509 510 for (n = 0; n < ARRAY_SIZE(secure_only); n++) 511 carve_out_phys_mem(&m, &num_elems, secure_only[n].paddr, 512 secure_only[n].size); 513 514 for (n = 0; n < mem_map->count; n++) { 515 switch (mem_map->map[n].type) { 516 case MEM_AREA_NSEC_SHM: 517 carve_out_phys_mem(&m, &num_elems, mem_map->map[n].pa, 518 mem_map->map[n].size); 519 break; 520 case MEM_AREA_EXT_DT: 521 case MEM_AREA_MANIFEST_DT: 522 case MEM_AREA_RAM_NSEC: 523 case MEM_AREA_RES_VASPACE: 524 case MEM_AREA_SHM_VASPACE: 525 case MEM_AREA_TS_VASPACE: 526 case MEM_AREA_PAGER_VASPACE: 527 break; 528 default: 529 check_phys_mem_is_outside(m, num_elems, 530 mem_map->map + n); 531 } 532 } 533 534 discovered_nsec_ddr_start = m; 535 discovered_nsec_ddr_nelems = num_elems; 536 537 if (!core_mmu_check_end_pa(m[num_elems - 1].addr, 538 m[num_elems - 1].size)) 539 panic(); 540 } 541 542 static bool get_discovered_nsec_ddr(const struct core_mmu_phys_mem **start, 543 const struct core_mmu_phys_mem **end) 544 { 545 if (!discovered_nsec_ddr_start) 546 return false; 547 548 *start = discovered_nsec_ddr_start; 549 *end = discovered_nsec_ddr_start + discovered_nsec_ddr_nelems; 550 551 return true; 552 } 553 554 static bool pbuf_is_nsec_ddr(paddr_t pbuf, size_t len) 555 { 556 const struct core_mmu_phys_mem *start; 557 const struct core_mmu_phys_mem *end; 558 559 if (!get_discovered_nsec_ddr(&start, &end)) 560 return false; 561 562 return pbuf_is_special_mem(pbuf, len, start, end); 563 } 564 565 bool core_mmu_nsec_ddr_is_defined(void) 566 { 567 const struct core_mmu_phys_mem *start; 568 const struct core_mmu_phys_mem *end; 569 570 if (!get_discovered_nsec_ddr(&start, &end)) 571 return false; 572 573 return start != end; 574 } 575 #else 576 static bool pbuf_is_nsec_ddr(paddr_t pbuf __unused, size_t len __unused) 577 { 578 return false; 579 } 580 #endif /*CFG_CORE_DYN_SHM*/ 581 582 #define MSG_MEM_INSTERSECT(pa1, sz1, pa2, sz2) \ 583 EMSG("[%" PRIxPA " %" PRIx64 "] intersects [%" PRIxPA " %" PRIx64 "]", \ 584 pa1, (uint64_t)pa1 + (sz1), pa2, (uint64_t)pa2 + (sz2)) 585 586 #ifdef CFG_SECURE_DATA_PATH 587 static bool pbuf_is_sdp_mem(paddr_t pbuf, size_t len) 588 { 589 bool is_sdp_mem = false; 590 591 if (sec_sdp.size) 592 is_sdp_mem = core_is_buffer_inside(pbuf, len, sec_sdp.paddr, 593 sec_sdp.size); 594 595 if (!is_sdp_mem) 596 is_sdp_mem = pbuf_is_special_mem(pbuf, len, phys_sdp_mem_begin, 597 phys_sdp_mem_end); 598 599 return is_sdp_mem; 600 } 601 602 static struct mobj *core_sdp_mem_alloc_mobj(paddr_t pa, size_t size) 603 { 604 struct mobj *mobj = mobj_phys_alloc(pa, size, TEE_MATTR_MEM_TYPE_CACHED, 605 CORE_MEM_SDP_MEM); 606 607 if (!mobj) 608 panic("can't create SDP physical memory object"); 609 610 return mobj; 611 } 612 613 struct mobj **core_sdp_mem_create_mobjs(void) 614 { 615 const struct core_mmu_phys_mem *mem = NULL; 616 struct mobj **mobj_base = NULL; 617 struct mobj **mobj = NULL; 618 int cnt = phys_sdp_mem_end - phys_sdp_mem_begin; 619 620 if (sec_sdp.size) 621 cnt++; 622 623 /* SDP mobjs table must end with a NULL entry */ 624 mobj_base = calloc(cnt + 1, sizeof(struct mobj *)); 625 if (!mobj_base) 626 panic("Out of memory"); 627 628 mobj = mobj_base; 629 630 for (mem = phys_sdp_mem_begin; mem < phys_sdp_mem_end; mem++, mobj++) 631 *mobj = core_sdp_mem_alloc_mobj(mem->addr, mem->size); 632 633 if (sec_sdp.size) 634 *mobj = core_sdp_mem_alloc_mobj(sec_sdp.paddr, sec_sdp.size); 635 636 return mobj_base; 637 } 638 639 #else /* CFG_SECURE_DATA_PATH */ 640 static bool pbuf_is_sdp_mem(paddr_t pbuf __unused, size_t len __unused) 641 { 642 return false; 643 } 644 645 #endif /* CFG_SECURE_DATA_PATH */ 646 647 /* Check special memories comply with registered memories */ 648 static void verify_special_mem_areas(struct memory_map *mem_map, 649 const struct core_mmu_phys_mem *start, 650 const struct core_mmu_phys_mem *end, 651 const char *area_name __maybe_unused) 652 { 653 const struct core_mmu_phys_mem *mem = NULL; 654 const struct core_mmu_phys_mem *mem2 = NULL; 655 size_t n = 0; 656 657 if (start == end) { 658 DMSG("No %s memory area defined", area_name); 659 return; 660 } 661 662 for (mem = start; mem < end; mem++) 663 DMSG("%s memory [%" PRIxPA " %" PRIx64 "]", 664 area_name, mem->addr, (uint64_t)mem->addr + mem->size); 665 666 /* Check memories do not intersect each other */ 667 for (mem = start; mem + 1 < end; mem++) { 668 for (mem2 = mem + 1; mem2 < end; mem2++) { 669 if (core_is_buffer_intersect(mem2->addr, mem2->size, 670 mem->addr, mem->size)) { 671 MSG_MEM_INSTERSECT(mem2->addr, mem2->size, 672 mem->addr, mem->size); 673 panic("Special memory intersection"); 674 } 675 } 676 } 677 678 /* 679 * Check memories do not intersect any mapped memory. 680 * This is called before reserved VA space is loaded in mem_map. 681 */ 682 for (mem = start; mem < end; mem++) { 683 for (n = 0; n < mem_map->count; n++) { 684 if (core_is_buffer_intersect(mem->addr, mem->size, 685 mem_map->map[n].pa, 686 mem_map->map[n].size)) { 687 MSG_MEM_INSTERSECT(mem->addr, mem->size, 688 mem_map->map[n].pa, 689 mem_map->map[n].size); 690 panic("Special memory intersection"); 691 } 692 } 693 } 694 } 695 696 static void merge_mmaps(struct tee_mmap_region *dst, 697 const struct tee_mmap_region *src) 698 { 699 paddr_t end_pa = MAX(dst->pa + dst->size - 1, src->pa + src->size - 1); 700 paddr_t pa = MIN(dst->pa, src->pa); 701 702 DMSG("Merging %#"PRIxPA"..%#"PRIxPA" and %#"PRIxPA"..%#"PRIxPA, 703 dst->pa, dst->pa + dst->size - 1, src->pa, 704 src->pa + src->size - 1); 705 dst->pa = pa; 706 dst->size = end_pa - pa + 1; 707 } 708 709 static bool mmaps_are_mergeable(const struct tee_mmap_region *r1, 710 const struct tee_mmap_region *r2) 711 { 712 if (r1->type != r2->type) 713 return false; 714 715 if (r1->pa == r2->pa) 716 return true; 717 718 if (r1->pa < r2->pa) 719 return r1->pa + r1->size >= r2->pa; 720 else 721 return r2->pa + r2->size >= r1->pa; 722 } 723 724 static void add_phys_mem(struct memory_map *mem_map, 725 const char *mem_name __maybe_unused, 726 enum teecore_memtypes mem_type, 727 paddr_t mem_addr, paddr_size_t mem_size) 728 { 729 size_t n = 0; 730 const struct tee_mmap_region m0 = { 731 .type = mem_type, 732 .pa = mem_addr, 733 .size = mem_size, 734 }; 735 736 if (!mem_size) /* Discard null size entries */ 737 return; 738 739 /* 740 * If some ranges of memory of the same type do overlap 741 * each others they are coalesced into one entry. To help this 742 * added entries are sorted by increasing physical. 743 * 744 * Note that it's valid to have the same physical memory as several 745 * different memory types, for instance the same device memory 746 * mapped as both secure and non-secure. This will probably not 747 * happen often in practice. 748 */ 749 DMSG("%s type %s 0x%08" PRIxPA " size 0x%08" PRIxPASZ, 750 mem_name, teecore_memtype_name(mem_type), mem_addr, mem_size); 751 for (n = 0; n < mem_map->count; n++) { 752 if (mmaps_are_mergeable(mem_map->map + n, &m0)) { 753 merge_mmaps(mem_map->map + n, &m0); 754 /* 755 * The merged result might be mergeable with the 756 * next or previous entry. 757 */ 758 if (n + 1 < mem_map->count && 759 mmaps_are_mergeable(mem_map->map + n, 760 mem_map->map + n + 1)) { 761 merge_mmaps(mem_map->map + n, 762 mem_map->map + n + 1); 763 rem_array_elem(mem_map->map, mem_map->count, 764 sizeof(*mem_map->map), n + 1); 765 mem_map->count--; 766 } 767 if (n > 0 && mmaps_are_mergeable(mem_map->map + n - 1, 768 mem_map->map + n)) { 769 merge_mmaps(mem_map->map + n - 1, 770 mem_map->map + n); 771 rem_array_elem(mem_map->map, mem_map->count, 772 sizeof(*mem_map->map), n); 773 mem_map->count--; 774 } 775 return; 776 } 777 if (mem_type < mem_map->map[n].type || 778 (mem_type == mem_map->map[n].type && 779 mem_addr < mem_map->map[n].pa)) 780 break; /* found the spot where to insert this memory */ 781 } 782 783 grow_mem_map(mem_map); 784 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 785 n, &m0); 786 } 787 788 static void add_va_space(struct memory_map *mem_map, 789 enum teecore_memtypes type, size_t size) 790 { 791 size_t n = 0; 792 793 DMSG("type %s size 0x%08zx", teecore_memtype_name(type), size); 794 for (n = 0; n < mem_map->count; n++) { 795 if (type < mem_map->map[n].type) 796 break; 797 } 798 799 grow_mem_map(mem_map); 800 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 801 n, NULL); 802 mem_map->map[n] = (struct tee_mmap_region){ 803 .type = type, 804 .size = size, 805 }; 806 } 807 808 uint32_t core_mmu_type_to_attr(enum teecore_memtypes t) 809 { 810 const uint32_t attr = TEE_MATTR_VALID_BLOCK; 811 const uint32_t tagged = TEE_MATTR_MEM_TYPE_TAGGED << 812 TEE_MATTR_MEM_TYPE_SHIFT; 813 const uint32_t cached = TEE_MATTR_MEM_TYPE_CACHED << 814 TEE_MATTR_MEM_TYPE_SHIFT; 815 const uint32_t noncache = TEE_MATTR_MEM_TYPE_DEV << 816 TEE_MATTR_MEM_TYPE_SHIFT; 817 818 switch (t) { 819 case MEM_AREA_TEE_RAM: 820 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | tagged; 821 case MEM_AREA_TEE_RAM_RX: 822 case MEM_AREA_INIT_RAM_RX: 823 case MEM_AREA_IDENTITY_MAP_RX: 824 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRX | tagged; 825 case MEM_AREA_TEE_RAM_RO: 826 case MEM_AREA_INIT_RAM_RO: 827 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | tagged; 828 case MEM_AREA_TEE_RAM_RW: 829 case MEM_AREA_NEX_RAM_RO: /* This has to be r/w during init runtime */ 830 case MEM_AREA_NEX_RAM_RW: 831 case MEM_AREA_TEE_ASAN: 832 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | tagged; 833 case MEM_AREA_TEE_COHERENT: 834 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | noncache; 835 case MEM_AREA_NSEC_SHM: 836 case MEM_AREA_NEX_NSEC_SHM: 837 return attr | TEE_MATTR_PRW | cached; 838 case MEM_AREA_MANIFEST_DT: 839 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | cached; 840 case MEM_AREA_TRANSFER_LIST: 841 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 842 case MEM_AREA_EXT_DT: 843 /* 844 * If CFG_MAP_EXT_DT_SECURE is enabled map the external device 845 * tree as secure non-cached memory, otherwise, fall back to 846 * non-secure mapping. 847 */ 848 if (IS_ENABLED(CFG_MAP_EXT_DT_SECURE)) 849 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | 850 noncache; 851 fallthrough; 852 case MEM_AREA_IO_NSEC: 853 return attr | TEE_MATTR_PRW | noncache; 854 case MEM_AREA_IO_SEC: 855 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | noncache; 856 case MEM_AREA_RAM_NSEC: 857 return attr | TEE_MATTR_PRW | cached; 858 case MEM_AREA_RAM_SEC: 859 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 860 case MEM_AREA_SEC_RAM_OVERALL: 861 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | tagged; 862 case MEM_AREA_ROM_SEC: 863 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | cached; 864 case MEM_AREA_RES_VASPACE: 865 case MEM_AREA_SHM_VASPACE: 866 return 0; 867 case MEM_AREA_PAGER_VASPACE: 868 return TEE_MATTR_SECURE; 869 default: 870 panic("invalid type"); 871 } 872 } 873 874 static bool __maybe_unused map_is_tee_ram(const struct tee_mmap_region *mm) 875 { 876 switch (mm->type) { 877 case MEM_AREA_TEE_RAM: 878 case MEM_AREA_TEE_RAM_RX: 879 case MEM_AREA_TEE_RAM_RO: 880 case MEM_AREA_TEE_RAM_RW: 881 case MEM_AREA_INIT_RAM_RX: 882 case MEM_AREA_INIT_RAM_RO: 883 case MEM_AREA_NEX_RAM_RW: 884 case MEM_AREA_NEX_RAM_RO: 885 case MEM_AREA_TEE_ASAN: 886 return true; 887 default: 888 return false; 889 } 890 } 891 892 static bool __maybe_unused map_is_secure(const struct tee_mmap_region *mm) 893 { 894 return !!(core_mmu_type_to_attr(mm->type) & TEE_MATTR_SECURE); 895 } 896 897 static bool __maybe_unused map_is_pgdir(const struct tee_mmap_region *mm) 898 { 899 return mm->region_size == CORE_MMU_PGDIR_SIZE; 900 } 901 902 static int cmp_mmap_by_lower_va(const void *a, const void *b) 903 { 904 const struct tee_mmap_region *mm_a = a; 905 const struct tee_mmap_region *mm_b = b; 906 907 return CMP_TRILEAN(mm_a->va, mm_b->va); 908 } 909 910 static void dump_mmap_table(struct memory_map *mem_map) 911 { 912 size_t n = 0; 913 914 for (n = 0; n < mem_map->count; n++) { 915 struct tee_mmap_region *map __maybe_unused = mem_map->map + n; 916 917 DMSG("type %-12s va 0x%08" PRIxVA "..0x%08" PRIxVA 918 " pa 0x%08" PRIxPA "..0x%08" PRIxPA " size 0x%08zx (%s)", 919 teecore_memtype_name(map->type), map->va, 920 map->va + map->size - 1, map->pa, 921 (paddr_t)(map->pa + map->size - 1), map->size, 922 map->region_size == SMALL_PAGE_SIZE ? "smallpg" : "pgdir"); 923 } 924 } 925 926 #if DEBUG_XLAT_TABLE 927 928 static void dump_xlat_table(vaddr_t va, unsigned int level) 929 { 930 struct core_mmu_table_info tbl_info; 931 unsigned int idx = 0; 932 paddr_t pa; 933 uint32_t attr; 934 935 core_mmu_find_table(NULL, va, level, &tbl_info); 936 va = tbl_info.va_base; 937 for (idx = 0; idx < tbl_info.num_entries; idx++) { 938 core_mmu_get_entry(&tbl_info, idx, &pa, &attr); 939 if (attr || level > CORE_MMU_BASE_TABLE_LEVEL) { 940 const char *security_bit = ""; 941 942 if (core_mmu_entry_have_security_bit(attr)) { 943 if (attr & TEE_MATTR_SECURE) 944 security_bit = "S"; 945 else 946 security_bit = "NS"; 947 } 948 949 if (attr & TEE_MATTR_TABLE) { 950 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 951 " TBL:0x%010" PRIxPA " %s", 952 level * 2, "", level, va, pa, 953 security_bit); 954 dump_xlat_table(va, level + 1); 955 } else if (attr) { 956 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 957 " PA:0x%010" PRIxPA " %s-%s-%s-%s", 958 level * 2, "", level, va, pa, 959 mattr_is_cached(attr) ? "MEM" : 960 "DEV", 961 attr & TEE_MATTR_PW ? "RW" : "RO", 962 attr & TEE_MATTR_PX ? "X " : "XN", 963 security_bit); 964 } else { 965 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 966 " INVALID\n", 967 level * 2, "", level, va); 968 } 969 } 970 va += BIT64(tbl_info.shift); 971 } 972 } 973 974 #else 975 976 static void dump_xlat_table(vaddr_t va __unused, int level __unused) 977 { 978 } 979 980 #endif 981 982 /* 983 * Reserves virtual memory space for pager usage. 984 * 985 * From the start of the first memory used by the link script + 986 * TEE_RAM_VA_SIZE should be covered, either with a direct mapping or empty 987 * mapping for pager usage. This adds translation tables as needed for the 988 * pager to operate. 989 */ 990 static void add_pager_vaspace(struct memory_map *mem_map) 991 { 992 paddr_t begin = 0; 993 paddr_t end = 0; 994 size_t size = 0; 995 size_t pos = 0; 996 size_t n = 0; 997 998 999 for (n = 0; n < mem_map->count; n++) { 1000 if (map_is_tee_ram(mem_map->map + n)) { 1001 if (!begin) 1002 begin = mem_map->map[n].pa; 1003 pos = n + 1; 1004 } 1005 } 1006 1007 end = mem_map->map[pos - 1].pa + mem_map->map[pos - 1].size; 1008 assert(end - begin < TEE_RAM_VA_SIZE); 1009 size = TEE_RAM_VA_SIZE - (end - begin); 1010 1011 grow_mem_map(mem_map); 1012 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 1013 n, NULL); 1014 mem_map->map[n] = (struct tee_mmap_region){ 1015 .type = MEM_AREA_PAGER_VASPACE, 1016 .size = size, 1017 .region_size = SMALL_PAGE_SIZE, 1018 .attr = core_mmu_type_to_attr(MEM_AREA_PAGER_VASPACE), 1019 }; 1020 } 1021 1022 static void check_sec_nsec_mem_config(void) 1023 { 1024 size_t n = 0; 1025 1026 for (n = 0; n < ARRAY_SIZE(secure_only); n++) { 1027 if (pbuf_intersects(nsec_shared, secure_only[n].paddr, 1028 secure_only[n].size)) 1029 panic("Invalid memory access config: sec/nsec"); 1030 } 1031 } 1032 1033 static void collect_device_mem_ranges(struct memory_map *mem_map) 1034 { 1035 const char *compatible = "arm,ffa-manifest-device-regions"; 1036 void *fdt = get_manifest_dt(); 1037 const char *name = NULL; 1038 uint64_t page_count = 0; 1039 uint64_t base = 0; 1040 int subnode = 0; 1041 int node = 0; 1042 1043 assert(fdt); 1044 1045 node = fdt_node_offset_by_compatible(fdt, 0, compatible); 1046 if (node < 0) 1047 return; 1048 1049 fdt_for_each_subnode(subnode, fdt, node) { 1050 name = fdt_get_name(fdt, subnode, NULL); 1051 if (!name) 1052 continue; 1053 1054 if (dt_getprop_as_number(fdt, subnode, "base-address", 1055 &base)) { 1056 EMSG("Mandatory field is missing: base-address"); 1057 continue; 1058 } 1059 1060 if (base & SMALL_PAGE_MASK) { 1061 EMSG("base-address is not page aligned"); 1062 continue; 1063 } 1064 1065 if (dt_getprop_as_number(fdt, subnode, "pages-count", 1066 &page_count)) { 1067 EMSG("Mandatory field is missing: pages-count"); 1068 continue; 1069 } 1070 1071 add_phys_mem(mem_map, name, MEM_AREA_IO_SEC, 1072 base, base + page_count * SMALL_PAGE_SIZE); 1073 } 1074 } 1075 1076 static void collect_mem_ranges(struct memory_map *mem_map) 1077 { 1078 const struct core_mmu_phys_mem *mem = NULL; 1079 vaddr_t ram_start = secure_only[0].paddr; 1080 size_t n = 0; 1081 1082 #define ADD_PHYS_MEM(_type, _addr, _size) \ 1083 add_phys_mem(mem_map, #_addr, (_type), (_addr), (_size)) 1084 1085 if (IS_ENABLED(CFG_CORE_RWDATA_NOEXEC)) { 1086 paddr_t next_pa = 0; 1087 1088 /* 1089 * Read-only and read-execute physical memory areas must 1090 * not be mapped by MEM_AREA_SEC_RAM_OVERALL, but all the 1091 * read/write should. 1092 */ 1093 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, ram_start, 1094 VCORE_UNPG_RX_PA - ram_start); 1095 assert(VCORE_UNPG_RX_PA >= ram_start); 1096 tee_ram_initial_offs = VCORE_UNPG_RX_PA - ram_start; 1097 DMSG("tee_ram_initial_offs %#zx", tee_ram_initial_offs); 1098 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RX, VCORE_UNPG_RX_PA, 1099 VCORE_UNPG_RX_SZ); 1100 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, VCORE_UNPG_RO_PA, 1101 VCORE_UNPG_RO_SZ); 1102 1103 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1104 ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RO, VCORE_UNPG_RW_PA, 1105 VCORE_UNPG_RW_SZ); 1106 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_UNPG_RW_PA, 1107 VCORE_UNPG_RW_SZ); 1108 1109 ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RW, VCORE_NEX_RW_PA, 1110 VCORE_NEX_RW_SZ); 1111 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_NEX_RW_PA, 1112 VCORE_NEX_RW_SZ); 1113 1114 ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RW, VCORE_FREE_PA, 1115 VCORE_FREE_SZ); 1116 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_FREE_PA, 1117 VCORE_FREE_SZ); 1118 next_pa = VCORE_FREE_PA + VCORE_FREE_SZ; 1119 } else { 1120 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RW, VCORE_UNPG_RW_PA, 1121 VCORE_UNPG_RW_SZ); 1122 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_UNPG_RW_PA, 1123 VCORE_UNPG_RW_SZ); 1124 1125 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RW, VCORE_FREE_PA, 1126 VCORE_FREE_SZ); 1127 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_FREE_PA, 1128 VCORE_FREE_SZ); 1129 next_pa = VCORE_FREE_PA + VCORE_FREE_SZ; 1130 } 1131 1132 if (IS_ENABLED(CFG_WITH_PAGER)) { 1133 paddr_t pa = 0; 1134 size_t sz = 0; 1135 1136 ADD_PHYS_MEM(MEM_AREA_INIT_RAM_RX, VCORE_INIT_RX_PA, 1137 VCORE_INIT_RX_SZ); 1138 ADD_PHYS_MEM(MEM_AREA_INIT_RAM_RO, VCORE_INIT_RO_PA, 1139 VCORE_INIT_RO_SZ); 1140 /* 1141 * Core init mapping shall cover up to end of the 1142 * physical RAM. This is required since the hash 1143 * table is appended to the binary data after the 1144 * firmware build sequence. 1145 */ 1146 pa = VCORE_INIT_RO_PA + VCORE_INIT_RO_SZ; 1147 sz = TEE_RAM_START + TEE_RAM_PH_SIZE - pa; 1148 ADD_PHYS_MEM(MEM_AREA_TEE_RAM, pa, sz); 1149 } else { 1150 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, next_pa, 1151 secure_only[0].paddr + 1152 secure_only[0].size - next_pa); 1153 } 1154 } else { 1155 ADD_PHYS_MEM(MEM_AREA_TEE_RAM, TEE_RAM_START, TEE_RAM_PH_SIZE); 1156 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, secure_only[n].paddr, 1157 secure_only[0].size); 1158 } 1159 1160 for (n = 1; n < ARRAY_SIZE(secure_only); n++) 1161 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, secure_only[n].paddr, 1162 secure_only[n].size); 1163 1164 if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS) && 1165 IS_ENABLED(CFG_WITH_PAGER)) { 1166 /* 1167 * Asan ram is part of MEM_AREA_TEE_RAM_RW when pager is 1168 * disabled. 1169 */ 1170 ADD_PHYS_MEM(MEM_AREA_TEE_ASAN, ASAN_MAP_PA, ASAN_MAP_SZ); 1171 } 1172 1173 #undef ADD_PHYS_MEM 1174 1175 /* Collect device memory info from SP manifest */ 1176 if (IS_ENABLED(CFG_CORE_SEL2_SPMC)) 1177 collect_device_mem_ranges(mem_map); 1178 1179 for (mem = phys_mem_map_begin; mem < phys_mem_map_end; mem++) { 1180 /* Only unmapped virtual range may have a null phys addr */ 1181 assert(mem->addr || !core_mmu_type_to_attr(mem->type)); 1182 1183 add_phys_mem(mem_map, mem->name, mem->type, 1184 mem->addr, mem->size); 1185 } 1186 1187 if (IS_ENABLED(CFG_SECURE_DATA_PATH)) 1188 verify_special_mem_areas(mem_map, phys_sdp_mem_begin, 1189 phys_sdp_mem_end, "SDP"); 1190 1191 add_va_space(mem_map, MEM_AREA_RES_VASPACE, CFG_RESERVED_VASPACE_SIZE); 1192 add_va_space(mem_map, MEM_AREA_SHM_VASPACE, SHM_VASPACE_SIZE); 1193 } 1194 1195 static void assign_mem_granularity(struct memory_map *mem_map) 1196 { 1197 size_t n = 0; 1198 1199 /* 1200 * Assign region sizes, note that MEM_AREA_TEE_RAM always uses 1201 * SMALL_PAGE_SIZE. 1202 */ 1203 for (n = 0; n < mem_map->count; n++) { 1204 paddr_t mask = mem_map->map[n].pa | mem_map->map[n].size; 1205 1206 if (mask & SMALL_PAGE_MASK) 1207 panic("Impossible memory alignment"); 1208 1209 if (map_is_tee_ram(mem_map->map + n)) 1210 mem_map->map[n].region_size = SMALL_PAGE_SIZE; 1211 else 1212 mem_map->map[n].region_size = CORE_MMU_PGDIR_SIZE; 1213 } 1214 } 1215 1216 static bool place_tee_ram_at_top(paddr_t paddr) 1217 { 1218 return paddr > BIT64(core_mmu_get_va_width()) / 2; 1219 } 1220 1221 /* 1222 * MMU arch driver shall override this function if it helps 1223 * optimizing the memory footprint of the address translation tables. 1224 */ 1225 bool __weak core_mmu_prefer_tee_ram_at_top(paddr_t paddr) 1226 { 1227 return place_tee_ram_at_top(paddr); 1228 } 1229 1230 static bool assign_mem_va_dir(vaddr_t tee_ram_va, struct memory_map *mem_map, 1231 bool tee_ram_at_top) 1232 { 1233 struct tee_mmap_region *map = NULL; 1234 vaddr_t va = 0; 1235 bool va_is_secure = true; 1236 size_t n = 0; 1237 1238 /* 1239 * tee_ram_va might equals 0 when CFG_CORE_ASLR=y. 1240 * 0 is by design an invalid va, so return false directly. 1241 */ 1242 if (!tee_ram_va) 1243 return false; 1244 1245 /* Clear eventual previous assignments */ 1246 for (n = 0; n < mem_map->count; n++) 1247 mem_map->map[n].va = 0; 1248 1249 /* 1250 * TEE RAM regions are always aligned with region_size. 1251 * 1252 * Note that MEM_AREA_PAGER_VASPACE also counts as TEE RAM here 1253 * since it handles virtual memory which covers the part of the ELF 1254 * that cannot fit directly into memory. 1255 */ 1256 va = tee_ram_va + tee_ram_initial_offs; 1257 for (n = 0; n < mem_map->count; n++) { 1258 map = mem_map->map + n; 1259 if (map_is_tee_ram(map) || 1260 map->type == MEM_AREA_PAGER_VASPACE) { 1261 assert(!(va & (map->region_size - 1))); 1262 assert(!(map->size & (map->region_size - 1))); 1263 map->va = va; 1264 if (ADD_OVERFLOW(va, map->size, &va)) 1265 return false; 1266 if (va >= BIT64(core_mmu_get_va_width())) 1267 return false; 1268 } 1269 } 1270 1271 if (tee_ram_at_top) { 1272 /* 1273 * Map non-tee ram regions at addresses lower than the tee 1274 * ram region. 1275 */ 1276 va = tee_ram_va; 1277 for (n = 0; n < mem_map->count; n++) { 1278 map = mem_map->map + n; 1279 map->attr = core_mmu_type_to_attr(map->type); 1280 if (map->va) 1281 continue; 1282 1283 if (!IS_ENABLED(CFG_WITH_LPAE) && 1284 va_is_secure != map_is_secure(map)) { 1285 va_is_secure = !va_is_secure; 1286 va = ROUNDDOWN(va, CORE_MMU_PGDIR_SIZE); 1287 } 1288 1289 if (SUB_OVERFLOW(va, map->size, &va)) 1290 return false; 1291 va = ROUNDDOWN2(va, map->region_size); 1292 /* 1293 * Make sure that va is aligned with pa for 1294 * efficient pgdir mapping. Basically pa & 1295 * pgdir_mask should be == va & pgdir_mask 1296 */ 1297 if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { 1298 if (SUB_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, &va)) 1299 return false; 1300 va += (map->pa - va) & CORE_MMU_PGDIR_MASK; 1301 } 1302 map->va = va; 1303 } 1304 } else { 1305 /* 1306 * Map non-tee ram regions at addresses higher than the tee 1307 * ram region. 1308 */ 1309 for (n = 0; n < mem_map->count; n++) { 1310 map = mem_map->map + n; 1311 map->attr = core_mmu_type_to_attr(map->type); 1312 if (map->va) 1313 continue; 1314 1315 if (!IS_ENABLED(CFG_WITH_LPAE) && 1316 va_is_secure != map_is_secure(map)) { 1317 va_is_secure = !va_is_secure; 1318 if (ROUNDUP_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, 1319 &va)) 1320 return false; 1321 } 1322 1323 if (ROUNDUP2_OVERFLOW(va, map->region_size, &va)) 1324 return false; 1325 /* 1326 * Make sure that va is aligned with pa for 1327 * efficient pgdir mapping. Basically pa & 1328 * pgdir_mask should be == va & pgdir_mask 1329 */ 1330 if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { 1331 vaddr_t offs = (map->pa - va) & 1332 CORE_MMU_PGDIR_MASK; 1333 1334 if (ADD_OVERFLOW(va, offs, &va)) 1335 return false; 1336 } 1337 1338 map->va = va; 1339 if (ADD_OVERFLOW(va, map->size, &va)) 1340 return false; 1341 if (va >= BIT64(core_mmu_get_va_width())) 1342 return false; 1343 } 1344 } 1345 1346 return true; 1347 } 1348 1349 static bool assign_mem_va(vaddr_t tee_ram_va, struct memory_map *mem_map) 1350 { 1351 bool tee_ram_at_top = place_tee_ram_at_top(tee_ram_va); 1352 1353 /* 1354 * Check that we're not overlapping with the user VA range. 1355 */ 1356 if (IS_ENABLED(CFG_WITH_LPAE)) { 1357 /* 1358 * User VA range is supposed to be defined after these 1359 * mappings have been established. 1360 */ 1361 assert(!core_mmu_user_va_range_is_defined()); 1362 } else { 1363 vaddr_t user_va_base = 0; 1364 size_t user_va_size = 0; 1365 1366 assert(core_mmu_user_va_range_is_defined()); 1367 core_mmu_get_user_va_range(&user_va_base, &user_va_size); 1368 if (tee_ram_va < (user_va_base + user_va_size)) 1369 return false; 1370 } 1371 1372 if (IS_ENABLED(CFG_WITH_PAGER)) { 1373 bool prefered_dir = core_mmu_prefer_tee_ram_at_top(tee_ram_va); 1374 1375 /* Try whole mapping covered by a single base xlat entry */ 1376 if (prefered_dir != tee_ram_at_top && 1377 assign_mem_va_dir(tee_ram_va, mem_map, prefered_dir)) 1378 return true; 1379 } 1380 1381 return assign_mem_va_dir(tee_ram_va, mem_map, tee_ram_at_top); 1382 } 1383 1384 static int cmp_init_mem_map(const void *a, const void *b) 1385 { 1386 const struct tee_mmap_region *mm_a = a; 1387 const struct tee_mmap_region *mm_b = b; 1388 int rc = 0; 1389 1390 rc = CMP_TRILEAN(mm_a->region_size, mm_b->region_size); 1391 if (!rc) 1392 rc = CMP_TRILEAN(mm_a->pa, mm_b->pa); 1393 /* 1394 * 32bit MMU descriptors cannot mix secure and non-secure mapping in 1395 * the same level2 table. Hence sort secure mapping from non-secure 1396 * mapping. 1397 */ 1398 if (!rc && !IS_ENABLED(CFG_WITH_LPAE)) 1399 rc = CMP_TRILEAN(map_is_secure(mm_a), map_is_secure(mm_b)); 1400 1401 return rc; 1402 } 1403 1404 static bool mem_map_add_id_map(struct memory_map *mem_map, 1405 vaddr_t id_map_start, vaddr_t id_map_end) 1406 { 1407 vaddr_t start = ROUNDDOWN(id_map_start, SMALL_PAGE_SIZE); 1408 vaddr_t end = ROUNDUP(id_map_end, SMALL_PAGE_SIZE); 1409 size_t len = end - start; 1410 size_t n = 0; 1411 1412 1413 for (n = 0; n < mem_map->count; n++) 1414 if (core_is_buffer_intersect(mem_map->map[n].va, 1415 mem_map->map[n].size, start, len)) 1416 return false; 1417 1418 grow_mem_map(mem_map); 1419 mem_map->map[mem_map->count - 1] = (struct tee_mmap_region){ 1420 .type = MEM_AREA_IDENTITY_MAP_RX, 1421 /* 1422 * Could use CORE_MMU_PGDIR_SIZE to potentially save a 1423 * translation table, at the increased risk of clashes with 1424 * the rest of the memory map. 1425 */ 1426 .region_size = SMALL_PAGE_SIZE, 1427 .pa = start, 1428 .va = start, 1429 .size = len, 1430 .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), 1431 }; 1432 1433 return true; 1434 } 1435 1436 static struct memory_map *init_mem_map(struct memory_map *mem_map, 1437 unsigned long seed, 1438 unsigned long *ret_offs) 1439 { 1440 /* 1441 * @id_map_start and @id_map_end describes a physical memory range 1442 * that must be mapped Read-Only eXecutable at identical virtual 1443 * addresses. 1444 */ 1445 vaddr_t id_map_start = (vaddr_t)__identity_map_init_start; 1446 vaddr_t id_map_end = (vaddr_t)__identity_map_init_end; 1447 vaddr_t start_addr = secure_only[0].paddr; 1448 unsigned long offs = 0; 1449 1450 collect_mem_ranges(mem_map); 1451 assign_mem_granularity(mem_map); 1452 1453 /* 1454 * To ease mapping and lower use of xlat tables, sort mapping 1455 * description moving small-page regions after the pgdir regions. 1456 */ 1457 qsort(mem_map->map, mem_map->count, sizeof(struct tee_mmap_region), 1458 cmp_init_mem_map); 1459 1460 if (IS_ENABLED(CFG_WITH_PAGER)) 1461 add_pager_vaspace(mem_map); 1462 1463 if (IS_ENABLED(CFG_CORE_ASLR) && seed) { 1464 vaddr_t base_addr = start_addr + seed; 1465 const unsigned int va_width = core_mmu_get_va_width(); 1466 const vaddr_t va_mask = GENMASK_64(va_width - 1, 1467 SMALL_PAGE_SHIFT); 1468 vaddr_t ba = base_addr; 1469 size_t n = 0; 1470 1471 for (n = 0; n < 3; n++) { 1472 if (n) 1473 ba = base_addr ^ BIT64(va_width - n); 1474 ba &= va_mask; 1475 if (assign_mem_va(ba, mem_map) && 1476 mem_map_add_id_map(mem_map, id_map_start, 1477 id_map_end)) { 1478 offs = ba - start_addr; 1479 DMSG("Mapping core at %#"PRIxVA" offs %#lx", 1480 ba, offs); 1481 goto out; 1482 } else { 1483 DMSG("Failed to map core at %#"PRIxVA, ba); 1484 } 1485 } 1486 EMSG("Failed to map core with seed %#lx", seed); 1487 } 1488 1489 if (!assign_mem_va(start_addr, mem_map)) 1490 panic(); 1491 1492 out: 1493 qsort(mem_map->map, mem_map->count, sizeof(struct tee_mmap_region), 1494 cmp_mmap_by_lower_va); 1495 1496 dump_mmap_table(mem_map); 1497 1498 *ret_offs = offs; 1499 return mem_map; 1500 } 1501 1502 static void check_mem_map(struct memory_map *mem_map) 1503 { 1504 struct tee_mmap_region *m = NULL; 1505 size_t n = 0; 1506 1507 for (n = 0; n < mem_map->count; n++) { 1508 m = mem_map->map + n; 1509 switch (m->type) { 1510 case MEM_AREA_TEE_RAM: 1511 case MEM_AREA_TEE_RAM_RX: 1512 case MEM_AREA_TEE_RAM_RO: 1513 case MEM_AREA_TEE_RAM_RW: 1514 case MEM_AREA_INIT_RAM_RX: 1515 case MEM_AREA_INIT_RAM_RO: 1516 case MEM_AREA_NEX_RAM_RW: 1517 case MEM_AREA_NEX_RAM_RO: 1518 case MEM_AREA_IDENTITY_MAP_RX: 1519 if (!pbuf_is_inside(secure_only, m->pa, m->size)) 1520 panic("TEE_RAM can't fit in secure_only"); 1521 break; 1522 case MEM_AREA_SEC_RAM_OVERALL: 1523 if (!pbuf_is_inside(secure_only, m->pa, m->size)) 1524 panic("SEC_RAM_OVERALL can't fit in secure_only"); 1525 break; 1526 case MEM_AREA_NSEC_SHM: 1527 if (!pbuf_is_inside(nsec_shared, m->pa, m->size)) 1528 panic("NS_SHM can't fit in nsec_shared"); 1529 break; 1530 case MEM_AREA_TEE_COHERENT: 1531 case MEM_AREA_TEE_ASAN: 1532 case MEM_AREA_IO_SEC: 1533 case MEM_AREA_IO_NSEC: 1534 case MEM_AREA_EXT_DT: 1535 case MEM_AREA_MANIFEST_DT: 1536 case MEM_AREA_TRANSFER_LIST: 1537 case MEM_AREA_RAM_SEC: 1538 case MEM_AREA_RAM_NSEC: 1539 case MEM_AREA_ROM_SEC: 1540 case MEM_AREA_RES_VASPACE: 1541 case MEM_AREA_SHM_VASPACE: 1542 case MEM_AREA_PAGER_VASPACE: 1543 break; 1544 default: 1545 EMSG("Uhandled memtype %d", m->type); 1546 panic(); 1547 } 1548 } 1549 } 1550 1551 /* 1552 * core_init_mmu_map() - init tee core default memory mapping 1553 * 1554 * This routine sets the static default TEE core mapping. If @seed is > 0 1555 * and configured with CFG_CORE_ASLR it will map tee core at a location 1556 * based on the seed and return the offset from the link address. 1557 * 1558 * If an error happened: core_init_mmu_map is expected to panic. 1559 * 1560 * Note: this function is weak just to make it possible to exclude it from 1561 * the unpaged area. 1562 */ 1563 void __weak core_init_mmu_map(unsigned long seed, struct core_mmu_config *cfg) 1564 { 1565 #ifndef CFG_NS_VIRTUALIZATION 1566 vaddr_t start = ROUNDDOWN((vaddr_t)__nozi_start, SMALL_PAGE_SIZE); 1567 #else 1568 vaddr_t start = ROUNDDOWN((vaddr_t)__vcore_nex_rw_start, 1569 SMALL_PAGE_SIZE); 1570 #endif 1571 #ifdef CFG_DYN_CONFIG 1572 vaddr_t len = ROUNDUP(VCORE_FREE_END_PA, SMALL_PAGE_SIZE) - start; 1573 #else 1574 vaddr_t len = ROUNDUP((vaddr_t)__nozi_end, SMALL_PAGE_SIZE) - start; 1575 #endif 1576 struct tee_mmap_region tmp_mmap_region = { }; 1577 struct memory_map mem_map = { }; 1578 unsigned long offs = 0; 1579 1580 if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) && 1581 (core_mmu_tee_load_pa & SMALL_PAGE_MASK)) 1582 panic("OP-TEE load address is not page aligned"); 1583 1584 check_sec_nsec_mem_config(); 1585 1586 if (IS_ENABLED(CFG_BOOT_MEM)) { 1587 mem_map.alloc_count = CFG_MMAP_REGIONS; 1588 mem_map.map = boot_mem_alloc_tmp(mem_map.alloc_count * 1589 sizeof(*mem_map.map), 1590 alignof(*mem_map.map)); 1591 memory_map_realloc_func = boot_mem_realloc_memory_map; 1592 } else { 1593 mem_map = static_memory_map; 1594 } 1595 1596 static_memory_map = (struct memory_map){ 1597 .map = &tmp_mmap_region, 1598 .alloc_count = 1, 1599 .count = 1, 1600 }; 1601 /* 1602 * Add a entry covering the translation tables which will be 1603 * involved in some virt_to_phys() and phys_to_virt() conversions. 1604 */ 1605 static_memory_map.map[0] = (struct tee_mmap_region){ 1606 .type = MEM_AREA_TEE_RAM, 1607 .region_size = SMALL_PAGE_SIZE, 1608 .pa = start, 1609 .va = start, 1610 .size = len, 1611 .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), 1612 }; 1613 1614 init_mem_map(&mem_map, seed, &offs); 1615 1616 check_mem_map(&mem_map); 1617 core_init_mmu(&mem_map); 1618 dump_xlat_table(0x0, CORE_MMU_BASE_TABLE_LEVEL); 1619 core_init_mmu_regs(cfg); 1620 cfg->map_offset = offs; 1621 static_memory_map = mem_map; 1622 boot_mem_add_reloc(&static_memory_map.map); 1623 } 1624 1625 void core_mmu_save_mem_map(void) 1626 { 1627 if (IS_ENABLED(CFG_BOOT_MEM)) { 1628 size_t alloc_count = static_memory_map.count + 5; 1629 size_t elem_sz = sizeof(*static_memory_map.map); 1630 void *p = NULL; 1631 1632 p = nex_calloc(alloc_count, elem_sz); 1633 if (!p) 1634 panic(); 1635 memcpy(p, static_memory_map.map, 1636 static_memory_map.count * elem_sz); 1637 static_memory_map.map = p; 1638 static_memory_map.alloc_count = alloc_count; 1639 memory_map_realloc_func = heap_realloc_memory_map; 1640 } 1641 } 1642 1643 bool core_mmu_mattr_is_ok(uint32_t mattr) 1644 { 1645 /* 1646 * Keep in sync with core_mmu_lpae.c:mattr_to_desc and 1647 * core_mmu_v7.c:mattr_to_texcb 1648 */ 1649 1650 switch ((mattr >> TEE_MATTR_MEM_TYPE_SHIFT) & TEE_MATTR_MEM_TYPE_MASK) { 1651 case TEE_MATTR_MEM_TYPE_DEV: 1652 case TEE_MATTR_MEM_TYPE_STRONGLY_O: 1653 case TEE_MATTR_MEM_TYPE_CACHED: 1654 case TEE_MATTR_MEM_TYPE_TAGGED: 1655 return true; 1656 default: 1657 return false; 1658 } 1659 } 1660 1661 /* 1662 * test attributes of target physical buffer 1663 * 1664 * Flags: pbuf_is(SECURE, NOT_SECURE, RAM, IOMEM, KEYVAULT). 1665 * 1666 */ 1667 bool core_pbuf_is(uint32_t attr, paddr_t pbuf, size_t len) 1668 { 1669 struct tee_mmap_region *map; 1670 1671 /* Empty buffers complies with anything */ 1672 if (len == 0) 1673 return true; 1674 1675 switch (attr) { 1676 case CORE_MEM_SEC: 1677 return pbuf_is_inside(secure_only, pbuf, len); 1678 case CORE_MEM_NON_SEC: 1679 return pbuf_is_inside(nsec_shared, pbuf, len) || 1680 pbuf_is_nsec_ddr(pbuf, len); 1681 case CORE_MEM_TEE_RAM: 1682 return core_is_buffer_inside(pbuf, len, TEE_RAM_START, 1683 TEE_RAM_PH_SIZE); 1684 #ifdef CFG_CORE_RESERVED_SHM 1685 case CORE_MEM_NSEC_SHM: 1686 return core_is_buffer_inside(pbuf, len, TEE_SHMEM_START, 1687 TEE_SHMEM_SIZE); 1688 #endif 1689 case CORE_MEM_SDP_MEM: 1690 return pbuf_is_sdp_mem(pbuf, len); 1691 case CORE_MEM_CACHED: 1692 map = find_map_by_pa(pbuf); 1693 if (!map || !pbuf_inside_map_area(pbuf, len, map)) 1694 return false; 1695 return mattr_is_cached(map->attr); 1696 default: 1697 return false; 1698 } 1699 } 1700 1701 /* test attributes of target virtual buffer (in core mapping) */ 1702 bool core_vbuf_is(uint32_t attr, const void *vbuf, size_t len) 1703 { 1704 paddr_t p; 1705 1706 /* Empty buffers complies with anything */ 1707 if (len == 0) 1708 return true; 1709 1710 p = virt_to_phys((void *)vbuf); 1711 if (!p) 1712 return false; 1713 1714 return core_pbuf_is(attr, p, len); 1715 } 1716 1717 /* core_va2pa - teecore exported service */ 1718 static int __maybe_unused core_va2pa_helper(void *va, paddr_t *pa) 1719 { 1720 struct tee_mmap_region *map; 1721 1722 map = find_map_by_va(va); 1723 if (!va_is_in_map(map, (vaddr_t)va)) 1724 return -1; 1725 1726 /* 1727 * We can calculate PA for static map. Virtual address ranges 1728 * reserved to core dynamic mapping return a 'match' (return 0;) 1729 * together with an invalid null physical address. 1730 */ 1731 if (map->pa) 1732 *pa = map->pa + (vaddr_t)va - map->va; 1733 else 1734 *pa = 0; 1735 1736 return 0; 1737 } 1738 1739 static void *map_pa2va(struct tee_mmap_region *map, paddr_t pa, size_t len) 1740 { 1741 if (!pa_is_in_map(map, pa, len)) 1742 return NULL; 1743 1744 return (void *)(vaddr_t)(map->va + pa - map->pa); 1745 } 1746 1747 /* 1748 * teecore gets some memory area definitions 1749 */ 1750 void core_mmu_get_mem_by_type(enum teecore_memtypes type, vaddr_t *s, 1751 vaddr_t *e) 1752 { 1753 struct tee_mmap_region *map = find_map_by_type(type); 1754 1755 if (map) { 1756 *s = map->va; 1757 *e = map->va + map->size; 1758 } else { 1759 *s = 0; 1760 *e = 0; 1761 } 1762 } 1763 1764 enum teecore_memtypes core_mmu_get_type_by_pa(paddr_t pa) 1765 { 1766 struct tee_mmap_region *map = find_map_by_pa(pa); 1767 1768 if (!map) 1769 return MEM_AREA_MAXTYPE; 1770 return map->type; 1771 } 1772 1773 void core_mmu_set_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, 1774 paddr_t pa, uint32_t attr) 1775 { 1776 assert(idx < tbl_info->num_entries); 1777 core_mmu_set_entry_primitive(tbl_info->table, tbl_info->level, 1778 idx, pa, attr); 1779 } 1780 1781 void core_mmu_get_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, 1782 paddr_t *pa, uint32_t *attr) 1783 { 1784 assert(idx < tbl_info->num_entries); 1785 core_mmu_get_entry_primitive(tbl_info->table, tbl_info->level, 1786 idx, pa, attr); 1787 } 1788 1789 static void clear_region(struct core_mmu_table_info *tbl_info, 1790 struct tee_mmap_region *region) 1791 { 1792 unsigned int end = 0; 1793 unsigned int idx = 0; 1794 1795 /* va, len and pa should be block aligned */ 1796 assert(!core_mmu_get_block_offset(tbl_info, region->va)); 1797 assert(!core_mmu_get_block_offset(tbl_info, region->size)); 1798 assert(!core_mmu_get_block_offset(tbl_info, region->pa)); 1799 1800 idx = core_mmu_va2idx(tbl_info, region->va); 1801 end = core_mmu_va2idx(tbl_info, region->va + region->size); 1802 1803 while (idx < end) { 1804 core_mmu_set_entry(tbl_info, idx, 0, 0); 1805 idx++; 1806 } 1807 } 1808 1809 static void set_region(struct core_mmu_table_info *tbl_info, 1810 struct tee_mmap_region *region) 1811 { 1812 unsigned int end; 1813 unsigned int idx; 1814 paddr_t pa; 1815 1816 /* va, len and pa should be block aligned */ 1817 assert(!core_mmu_get_block_offset(tbl_info, region->va)); 1818 assert(!core_mmu_get_block_offset(tbl_info, region->size)); 1819 assert(!core_mmu_get_block_offset(tbl_info, region->pa)); 1820 1821 idx = core_mmu_va2idx(tbl_info, region->va); 1822 end = core_mmu_va2idx(tbl_info, region->va + region->size); 1823 pa = region->pa; 1824 1825 while (idx < end) { 1826 core_mmu_set_entry(tbl_info, idx, pa, region->attr); 1827 idx++; 1828 pa += BIT64(tbl_info->shift); 1829 } 1830 } 1831 1832 static void set_pg_region(struct core_mmu_table_info *dir_info, 1833 struct vm_region *region, struct pgt **pgt, 1834 struct core_mmu_table_info *pg_info) 1835 { 1836 struct tee_mmap_region r = { 1837 .va = region->va, 1838 .size = region->size, 1839 .attr = region->attr, 1840 }; 1841 vaddr_t end = r.va + r.size; 1842 uint32_t pgt_attr = (r.attr & TEE_MATTR_SECURE) | TEE_MATTR_TABLE; 1843 1844 while (r.va < end) { 1845 if (!pg_info->table || 1846 r.va >= (pg_info->va_base + CORE_MMU_PGDIR_SIZE)) { 1847 /* 1848 * We're assigning a new translation table. 1849 */ 1850 unsigned int idx; 1851 1852 /* Virtual addresses must grow */ 1853 assert(r.va > pg_info->va_base); 1854 1855 idx = core_mmu_va2idx(dir_info, r.va); 1856 pg_info->va_base = core_mmu_idx2va(dir_info, idx); 1857 1858 /* 1859 * Advance pgt to va_base, note that we may need to 1860 * skip multiple page tables if there are large 1861 * holes in the vm map. 1862 */ 1863 while ((*pgt)->vabase < pg_info->va_base) { 1864 *pgt = SLIST_NEXT(*pgt, link); 1865 /* We should have allocated enough */ 1866 assert(*pgt); 1867 } 1868 assert((*pgt)->vabase == pg_info->va_base); 1869 pg_info->table = (*pgt)->tbl; 1870 1871 core_mmu_set_entry(dir_info, idx, 1872 virt_to_phys(pg_info->table), 1873 pgt_attr); 1874 } 1875 1876 r.size = MIN(CORE_MMU_PGDIR_SIZE - (r.va - pg_info->va_base), 1877 end - r.va); 1878 1879 if (!(*pgt)->populated && !mobj_is_paged(region->mobj)) { 1880 size_t granule = BIT(pg_info->shift); 1881 size_t offset = r.va - region->va + region->offset; 1882 1883 r.size = MIN(r.size, 1884 mobj_get_phys_granule(region->mobj)); 1885 r.size = ROUNDUP(r.size, SMALL_PAGE_SIZE); 1886 1887 if (mobj_get_pa(region->mobj, offset, granule, 1888 &r.pa) != TEE_SUCCESS) 1889 panic("Failed to get PA of unpaged mobj"); 1890 set_region(pg_info, &r); 1891 } 1892 r.va += r.size; 1893 } 1894 } 1895 1896 static bool can_map_at_level(paddr_t paddr, vaddr_t vaddr, 1897 size_t size_left, paddr_t block_size, 1898 struct tee_mmap_region *mm) 1899 { 1900 /* VA and PA are aligned to block size at current level */ 1901 if ((vaddr | paddr) & (block_size - 1)) 1902 return false; 1903 1904 /* Remainder fits into block at current level */ 1905 if (size_left < block_size) 1906 return false; 1907 1908 /* 1909 * The required block size of the region is compatible with the 1910 * block size of the current level. 1911 */ 1912 if (mm->region_size < block_size) 1913 return false; 1914 1915 #ifdef CFG_WITH_PAGER 1916 /* 1917 * If pager is enabled, we need to map TEE RAM and the whole pager 1918 * regions with small pages only 1919 */ 1920 if ((map_is_tee_ram(mm) || mm->type == MEM_AREA_PAGER_VASPACE) && 1921 block_size != SMALL_PAGE_SIZE) 1922 return false; 1923 #endif 1924 1925 return true; 1926 } 1927 1928 void core_mmu_map_region(struct mmu_partition *prtn, struct tee_mmap_region *mm) 1929 { 1930 struct core_mmu_table_info tbl_info; 1931 unsigned int idx; 1932 vaddr_t vaddr = mm->va; 1933 paddr_t paddr = mm->pa; 1934 ssize_t size_left = mm->size; 1935 unsigned int level; 1936 bool table_found; 1937 uint32_t old_attr; 1938 1939 assert(!((vaddr | paddr) & SMALL_PAGE_MASK)); 1940 1941 while (size_left > 0) { 1942 level = CORE_MMU_BASE_TABLE_LEVEL; 1943 1944 while (true) { 1945 paddr_t block_size = 0; 1946 1947 assert(core_mmu_level_in_range(level)); 1948 1949 table_found = core_mmu_find_table(prtn, vaddr, level, 1950 &tbl_info); 1951 if (!table_found) 1952 panic("can't find table for mapping"); 1953 1954 block_size = BIT64(tbl_info.shift); 1955 1956 idx = core_mmu_va2idx(&tbl_info, vaddr); 1957 if (!can_map_at_level(paddr, vaddr, size_left, 1958 block_size, mm)) { 1959 bool secure = mm->attr & TEE_MATTR_SECURE; 1960 1961 /* 1962 * This part of the region can't be mapped at 1963 * this level. Need to go deeper. 1964 */ 1965 if (!core_mmu_entry_to_finer_grained(&tbl_info, 1966 idx, 1967 secure)) 1968 panic("Can't divide MMU entry"); 1969 level = tbl_info.next_level; 1970 continue; 1971 } 1972 1973 /* We can map part of the region at current level */ 1974 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 1975 if (old_attr) 1976 panic("Page is already mapped"); 1977 1978 core_mmu_set_entry(&tbl_info, idx, paddr, mm->attr); 1979 paddr += block_size; 1980 vaddr += block_size; 1981 size_left -= block_size; 1982 1983 break; 1984 } 1985 } 1986 } 1987 1988 TEE_Result core_mmu_map_pages(vaddr_t vstart, paddr_t *pages, size_t num_pages, 1989 enum teecore_memtypes memtype) 1990 { 1991 TEE_Result ret; 1992 struct core_mmu_table_info tbl_info; 1993 struct tee_mmap_region *mm; 1994 unsigned int idx; 1995 uint32_t old_attr; 1996 uint32_t exceptions; 1997 vaddr_t vaddr = vstart; 1998 size_t i; 1999 bool secure; 2000 2001 assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); 2002 2003 secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; 2004 2005 if (vaddr & SMALL_PAGE_MASK) 2006 return TEE_ERROR_BAD_PARAMETERS; 2007 2008 exceptions = mmu_lock(); 2009 2010 mm = find_map_by_va((void *)vaddr); 2011 if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) 2012 panic("VA does not belong to any known mm region"); 2013 2014 if (!core_mmu_is_dynamic_vaspace(mm)) 2015 panic("Trying to map into static region"); 2016 2017 for (i = 0; i < num_pages; i++) { 2018 if (pages[i] & SMALL_PAGE_MASK) { 2019 ret = TEE_ERROR_BAD_PARAMETERS; 2020 goto err; 2021 } 2022 2023 while (true) { 2024 if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, 2025 &tbl_info)) 2026 panic("Can't find pagetable for vaddr "); 2027 2028 idx = core_mmu_va2idx(&tbl_info, vaddr); 2029 if (tbl_info.shift == SMALL_PAGE_SHIFT) 2030 break; 2031 2032 /* This is supertable. Need to divide it. */ 2033 if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, 2034 secure)) 2035 panic("Failed to spread pgdir on small tables"); 2036 } 2037 2038 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 2039 if (old_attr) 2040 panic("Page is already mapped"); 2041 2042 core_mmu_set_entry(&tbl_info, idx, pages[i], 2043 core_mmu_type_to_attr(memtype)); 2044 vaddr += SMALL_PAGE_SIZE; 2045 } 2046 2047 /* 2048 * Make sure all the changes to translation tables are visible 2049 * before returning. TLB doesn't need to be invalidated as we are 2050 * guaranteed that there's no valid mapping in this range. 2051 */ 2052 core_mmu_table_write_barrier(); 2053 mmu_unlock(exceptions); 2054 2055 return TEE_SUCCESS; 2056 err: 2057 mmu_unlock(exceptions); 2058 2059 if (i) 2060 core_mmu_unmap_pages(vstart, i); 2061 2062 return ret; 2063 } 2064 2065 TEE_Result core_mmu_map_contiguous_pages(vaddr_t vstart, paddr_t pstart, 2066 size_t num_pages, 2067 enum teecore_memtypes memtype) 2068 { 2069 struct core_mmu_table_info tbl_info = { }; 2070 struct tee_mmap_region *mm = NULL; 2071 unsigned int idx = 0; 2072 uint32_t old_attr = 0; 2073 uint32_t exceptions = 0; 2074 vaddr_t vaddr = vstart; 2075 paddr_t paddr = pstart; 2076 size_t i = 0; 2077 bool secure = false; 2078 2079 assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); 2080 2081 secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; 2082 2083 if ((vaddr | paddr) & SMALL_PAGE_MASK) 2084 return TEE_ERROR_BAD_PARAMETERS; 2085 2086 exceptions = mmu_lock(); 2087 2088 mm = find_map_by_va((void *)vaddr); 2089 if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) 2090 panic("VA does not belong to any known mm region"); 2091 2092 if (!core_mmu_is_dynamic_vaspace(mm)) 2093 panic("Trying to map into static region"); 2094 2095 for (i = 0; i < num_pages; i++) { 2096 while (true) { 2097 if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, 2098 &tbl_info)) 2099 panic("Can't find pagetable for vaddr "); 2100 2101 idx = core_mmu_va2idx(&tbl_info, vaddr); 2102 if (tbl_info.shift == SMALL_PAGE_SHIFT) 2103 break; 2104 2105 /* This is supertable. Need to divide it. */ 2106 if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, 2107 secure)) 2108 panic("Failed to spread pgdir on small tables"); 2109 } 2110 2111 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 2112 if (old_attr) 2113 panic("Page is already mapped"); 2114 2115 core_mmu_set_entry(&tbl_info, idx, paddr, 2116 core_mmu_type_to_attr(memtype)); 2117 paddr += SMALL_PAGE_SIZE; 2118 vaddr += SMALL_PAGE_SIZE; 2119 } 2120 2121 /* 2122 * Make sure all the changes to translation tables are visible 2123 * before returning. TLB doesn't need to be invalidated as we are 2124 * guaranteed that there's no valid mapping in this range. 2125 */ 2126 core_mmu_table_write_barrier(); 2127 mmu_unlock(exceptions); 2128 2129 return TEE_SUCCESS; 2130 } 2131 2132 static bool mem_range_is_in_vcore_free(vaddr_t vstart, size_t num_pages) 2133 { 2134 return core_is_buffer_inside(vstart, num_pages * SMALL_PAGE_SIZE, 2135 VCORE_FREE_PA, VCORE_FREE_SZ); 2136 } 2137 2138 void core_mmu_unmap_pages(vaddr_t vstart, size_t num_pages) 2139 { 2140 struct core_mmu_table_info tbl_info; 2141 struct tee_mmap_region *mm; 2142 size_t i; 2143 unsigned int idx; 2144 uint32_t exceptions; 2145 2146 exceptions = mmu_lock(); 2147 2148 mm = find_map_by_va((void *)vstart); 2149 if (!mm || !va_is_in_map(mm, vstart + num_pages * SMALL_PAGE_SIZE - 1)) 2150 panic("VA does not belong to any known mm region"); 2151 2152 if (!core_mmu_is_dynamic_vaspace(mm) && 2153 !mem_range_is_in_vcore_free(vstart, num_pages)) 2154 panic("Trying to unmap static region"); 2155 2156 for (i = 0; i < num_pages; i++, vstart += SMALL_PAGE_SIZE) { 2157 if (!core_mmu_find_table(NULL, vstart, UINT_MAX, &tbl_info)) 2158 panic("Can't find pagetable"); 2159 2160 if (tbl_info.shift != SMALL_PAGE_SHIFT) 2161 panic("Invalid pagetable level"); 2162 2163 idx = core_mmu_va2idx(&tbl_info, vstart); 2164 core_mmu_set_entry(&tbl_info, idx, 0, 0); 2165 } 2166 tlbi_all(); 2167 2168 mmu_unlock(exceptions); 2169 } 2170 2171 void core_mmu_populate_user_map(struct core_mmu_table_info *dir_info, 2172 struct user_mode_ctx *uctx) 2173 { 2174 struct core_mmu_table_info pg_info = { }; 2175 struct pgt_cache *pgt_cache = &uctx->pgt_cache; 2176 struct pgt *pgt = NULL; 2177 struct pgt *p = NULL; 2178 struct vm_region *r = NULL; 2179 2180 if (TAILQ_EMPTY(&uctx->vm_info.regions)) 2181 return; /* Nothing to map */ 2182 2183 /* 2184 * Allocate all page tables in advance. 2185 */ 2186 pgt_get_all(uctx); 2187 pgt = SLIST_FIRST(pgt_cache); 2188 2189 core_mmu_set_info_table(&pg_info, dir_info->next_level, 0, NULL); 2190 2191 TAILQ_FOREACH(r, &uctx->vm_info.regions, link) 2192 set_pg_region(dir_info, r, &pgt, &pg_info); 2193 /* Record that the translation tables now are populated. */ 2194 SLIST_FOREACH(p, pgt_cache, link) { 2195 p->populated = true; 2196 if (p == pgt) 2197 break; 2198 } 2199 assert(p == pgt); 2200 } 2201 2202 TEE_Result core_mmu_remove_mapping(enum teecore_memtypes type, void *addr, 2203 size_t len) 2204 { 2205 struct core_mmu_table_info tbl_info = { }; 2206 struct tee_mmap_region *res_map = NULL; 2207 struct tee_mmap_region *map = NULL; 2208 paddr_t pa = virt_to_phys(addr); 2209 size_t granule = 0; 2210 ptrdiff_t i = 0; 2211 paddr_t p = 0; 2212 size_t l = 0; 2213 2214 map = find_map_by_type_and_pa(type, pa, len); 2215 if (!map) 2216 return TEE_ERROR_GENERIC; 2217 2218 res_map = find_map_by_type(MEM_AREA_RES_VASPACE); 2219 if (!res_map) 2220 return TEE_ERROR_GENERIC; 2221 if (!core_mmu_find_table(NULL, res_map->va, UINT_MAX, &tbl_info)) 2222 return TEE_ERROR_GENERIC; 2223 granule = BIT(tbl_info.shift); 2224 2225 if (map < static_memory_map.map || 2226 map >= static_memory_map.map + static_memory_map.count) 2227 return TEE_ERROR_GENERIC; 2228 i = map - static_memory_map.map; 2229 2230 /* Check that we have a full match */ 2231 p = ROUNDDOWN2(pa, granule); 2232 l = ROUNDUP2(len + pa - p, granule); 2233 if (map->pa != p || map->size != l) 2234 return TEE_ERROR_GENERIC; 2235 2236 clear_region(&tbl_info, map); 2237 tlbi_all(); 2238 2239 /* If possible remove the va range from res_map */ 2240 if (res_map->va - map->size == map->va) { 2241 res_map->va -= map->size; 2242 res_map->size += map->size; 2243 } 2244 2245 /* Remove the entry. */ 2246 rem_array_elem(static_memory_map.map, static_memory_map.count, 2247 sizeof(*static_memory_map.map), i); 2248 static_memory_map.count--; 2249 2250 return TEE_SUCCESS; 2251 } 2252 2253 struct tee_mmap_region * 2254 core_mmu_find_mapping_exclusive(enum teecore_memtypes type, size_t len) 2255 { 2256 struct memory_map *mem_map = get_memory_map(); 2257 struct tee_mmap_region *map_found = NULL; 2258 size_t n = 0; 2259 2260 if (!len) 2261 return NULL; 2262 2263 for (n = 0; n < mem_map->count; n++) { 2264 if (mem_map->map[n].type != type) 2265 continue; 2266 2267 if (map_found) 2268 return NULL; 2269 2270 map_found = mem_map->map + n; 2271 } 2272 2273 if (!map_found || map_found->size < len) 2274 return NULL; 2275 2276 return map_found; 2277 } 2278 2279 void *core_mmu_add_mapping(enum teecore_memtypes type, paddr_t addr, size_t len) 2280 { 2281 struct memory_map *mem_map = &static_memory_map; 2282 struct core_mmu_table_info tbl_info = { }; 2283 struct tee_mmap_region *map = NULL; 2284 size_t granule = 0; 2285 paddr_t p = 0; 2286 size_t l = 0; 2287 2288 if (!len) 2289 return NULL; 2290 2291 if (!core_mmu_check_end_pa(addr, len)) 2292 return NULL; 2293 2294 /* Check if the memory is already mapped */ 2295 map = find_map_by_type_and_pa(type, addr, len); 2296 if (map && pbuf_inside_map_area(addr, len, map)) 2297 return (void *)(vaddr_t)(map->va + addr - map->pa); 2298 2299 /* Find the reserved va space used for late mappings */ 2300 map = find_map_by_type(MEM_AREA_RES_VASPACE); 2301 if (!map) 2302 return NULL; 2303 2304 if (!core_mmu_find_table(NULL, map->va, UINT_MAX, &tbl_info)) 2305 return NULL; 2306 2307 granule = BIT64(tbl_info.shift); 2308 p = ROUNDDOWN2(addr, granule); 2309 l = ROUNDUP2(len + addr - p, granule); 2310 2311 /* Ban overflowing virtual addresses */ 2312 if (map->size < l) 2313 return NULL; 2314 2315 /* 2316 * Something is wrong, we can't fit the va range into the selected 2317 * table. The reserved va range is possibly missaligned with 2318 * granule. 2319 */ 2320 if (core_mmu_va2idx(&tbl_info, map->va + len) >= tbl_info.num_entries) 2321 return NULL; 2322 2323 if (static_memory_map.count >= static_memory_map.alloc_count) 2324 return NULL; 2325 2326 mem_map->map[mem_map->count] = (struct tee_mmap_region){ 2327 .va = map->va, 2328 .size = l, 2329 .type = type, 2330 .region_size = granule, 2331 .attr = core_mmu_type_to_attr(type), 2332 .pa = p, 2333 }; 2334 map->va += l; 2335 map->size -= l; 2336 map = mem_map->map + mem_map->count; 2337 mem_map->count++; 2338 2339 set_region(&tbl_info, map); 2340 2341 /* Make sure the new entry is visible before continuing. */ 2342 core_mmu_table_write_barrier(); 2343 2344 return (void *)(vaddr_t)(map->va + addr - map->pa); 2345 } 2346 2347 #ifdef CFG_WITH_PAGER 2348 static vaddr_t get_linear_map_end_va(void) 2349 { 2350 /* this is synced with the generic linker file kern.ld.S */ 2351 return (vaddr_t)__heap2_end; 2352 } 2353 2354 static paddr_t get_linear_map_end_pa(void) 2355 { 2356 return get_linear_map_end_va() - boot_mmu_config.map_offset; 2357 } 2358 #endif 2359 2360 #if defined(CFG_TEE_CORE_DEBUG) 2361 static void check_pa_matches_va(void *va, paddr_t pa) 2362 { 2363 TEE_Result res = TEE_ERROR_GENERIC; 2364 vaddr_t v = (vaddr_t)va; 2365 paddr_t p = 0; 2366 struct core_mmu_table_info ti __maybe_unused = { }; 2367 2368 if (core_mmu_user_va_range_is_defined()) { 2369 vaddr_t user_va_base = 0; 2370 size_t user_va_size = 0; 2371 2372 core_mmu_get_user_va_range(&user_va_base, &user_va_size); 2373 if (v >= user_va_base && 2374 v <= (user_va_base - 1 + user_va_size)) { 2375 if (!core_mmu_user_mapping_is_active()) { 2376 if (pa) 2377 panic("issue in linear address space"); 2378 return; 2379 } 2380 2381 res = vm_va2pa(to_user_mode_ctx(thread_get_tsd()->ctx), 2382 va, &p); 2383 if (res == TEE_ERROR_NOT_SUPPORTED) 2384 return; 2385 if (res == TEE_SUCCESS && pa != p) 2386 panic("bad pa"); 2387 if (res != TEE_SUCCESS && pa) 2388 panic("false pa"); 2389 return; 2390 } 2391 } 2392 #ifdef CFG_WITH_PAGER 2393 if (is_unpaged(va)) { 2394 if (v - boot_mmu_config.map_offset != pa) 2395 panic("issue in linear address space"); 2396 return; 2397 } 2398 2399 if (tee_pager_get_table_info(v, &ti)) { 2400 uint32_t a; 2401 2402 /* 2403 * Lookups in the page table managed by the pager is 2404 * dangerous for addresses in the paged area as those pages 2405 * changes all the time. But some ranges are safe, 2406 * rw-locked areas when the page is populated for instance. 2407 */ 2408 core_mmu_get_entry(&ti, core_mmu_va2idx(&ti, v), &p, &a); 2409 if (a & TEE_MATTR_VALID_BLOCK) { 2410 paddr_t mask = BIT64(ti.shift) - 1; 2411 2412 p |= v & mask; 2413 if (pa != p) 2414 panic(); 2415 } else { 2416 if (pa) 2417 panic(); 2418 } 2419 return; 2420 } 2421 #endif 2422 2423 if (!core_va2pa_helper(va, &p)) { 2424 /* Verfiy only the static mapping (case non null phys addr) */ 2425 if (p && pa != p) { 2426 DMSG("va %p maps 0x%" PRIxPA ", expect 0x%" PRIxPA, 2427 va, p, pa); 2428 panic(); 2429 } 2430 } else { 2431 if (pa) { 2432 DMSG("va %p unmapped, expect 0x%" PRIxPA, va, pa); 2433 panic(); 2434 } 2435 } 2436 } 2437 #else 2438 static void check_pa_matches_va(void *va __unused, paddr_t pa __unused) 2439 { 2440 } 2441 #endif 2442 2443 paddr_t virt_to_phys(void *va) 2444 { 2445 paddr_t pa = 0; 2446 2447 if (!arch_va2pa_helper(va, &pa)) 2448 pa = 0; 2449 check_pa_matches_va(memtag_strip_tag(va), pa); 2450 return pa; 2451 } 2452 2453 /* 2454 * Don't use check_va_matches_pa() for RISC-V, as its callee 2455 * arch_va2pa_helper() will call it eventually, this creates 2456 * indirect recursion and can lead to a stack overflow. 2457 * Moreover, if arch_va2pa_helper() returns true, it implies 2458 * the va2pa mapping is matched, no need to check it again. 2459 */ 2460 #if defined(CFG_TEE_CORE_DEBUG) && !defined(__riscv) 2461 static void check_va_matches_pa(paddr_t pa, void *va) 2462 { 2463 paddr_t p = 0; 2464 2465 if (!va) 2466 return; 2467 2468 p = virt_to_phys(va); 2469 if (p != pa) { 2470 DMSG("va %p maps 0x%" PRIxPA " expect 0x%" PRIxPA, va, p, pa); 2471 panic(); 2472 } 2473 } 2474 #else 2475 static void check_va_matches_pa(paddr_t pa __unused, void *va __unused) 2476 { 2477 } 2478 #endif 2479 2480 static void *phys_to_virt_ts_vaspace(paddr_t pa, size_t len) 2481 { 2482 if (!core_mmu_user_mapping_is_active()) 2483 return NULL; 2484 2485 return vm_pa2va(to_user_mode_ctx(thread_get_tsd()->ctx), pa, len); 2486 } 2487 2488 #ifdef CFG_WITH_PAGER 2489 static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) 2490 { 2491 paddr_t end_pa = 0; 2492 2493 if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) 2494 return NULL; 2495 2496 if (pa >= TEE_LOAD_ADDR && pa < get_linear_map_end_pa()) { 2497 if (end_pa > get_linear_map_end_pa()) 2498 return NULL; 2499 return (void *)(vaddr_t)(pa + boot_mmu_config.map_offset); 2500 } 2501 2502 return tee_pager_phys_to_virt(pa, len); 2503 } 2504 #else 2505 static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) 2506 { 2507 struct tee_mmap_region *mmap = NULL; 2508 void *va = NULL; 2509 2510 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM, pa, len); 2511 if (!mmap) 2512 mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RW, pa, len); 2513 if (!mmap) 2514 mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RO, pa, len); 2515 if (!mmap) 2516 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RW, pa, len); 2517 if (!mmap) 2518 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RO, pa, len); 2519 if (!mmap) 2520 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RX, pa, len); 2521 /* 2522 * Note that MEM_AREA_INIT_RAM_RO and MEM_AREA_INIT_RAM_RX are only 2523 * used with pager and not needed here. 2524 */ 2525 va = map_pa2va(mmap, pa, len); 2526 2527 if (va && mmap->type == MEM_AREA_TEE_RAM_RW) { 2528 /* 2529 * Parts of the "unused" memory area covered by 2530 * MEM_AREA_TEE_RAM_RW can be unmaped, but map_pa2va() 2531 * doesn't check for holes in the map. Now that we have a 2532 * possible virtual address, check that it's mapped. 2533 */ 2534 paddr_t p = 0; 2535 2536 if (!arch_va2pa_helper(va, &p)) 2537 va = NULL; 2538 } 2539 2540 return va; 2541 } 2542 #endif 2543 2544 void *phys_to_virt(paddr_t pa, enum teecore_memtypes m, size_t len) 2545 { 2546 void *va = NULL; 2547 2548 switch (m) { 2549 case MEM_AREA_TS_VASPACE: 2550 va = phys_to_virt_ts_vaspace(pa, len); 2551 break; 2552 case MEM_AREA_TEE_RAM: 2553 case MEM_AREA_TEE_RAM_RX: 2554 case MEM_AREA_TEE_RAM_RO: 2555 case MEM_AREA_TEE_RAM_RW: 2556 case MEM_AREA_NEX_RAM_RO: 2557 case MEM_AREA_NEX_RAM_RW: 2558 va = phys_to_virt_tee_ram(pa, len); 2559 break; 2560 case MEM_AREA_SHM_VASPACE: 2561 /* Find VA from PA in dynamic SHM is not yet supported */ 2562 va = NULL; 2563 break; 2564 default: 2565 va = map_pa2va(find_map_by_type_and_pa(m, pa, len), pa, len); 2566 } 2567 if (m != MEM_AREA_SEC_RAM_OVERALL) 2568 check_va_matches_pa(pa, va); 2569 return va; 2570 } 2571 2572 void *phys_to_virt_io(paddr_t pa, size_t len) 2573 { 2574 struct tee_mmap_region *map = NULL; 2575 void *va = NULL; 2576 2577 map = find_map_by_type_and_pa(MEM_AREA_IO_SEC, pa, len); 2578 if (!map) 2579 map = find_map_by_type_and_pa(MEM_AREA_IO_NSEC, pa, len); 2580 if (!map) 2581 return NULL; 2582 va = map_pa2va(map, pa, len); 2583 check_va_matches_pa(pa, va); 2584 return va; 2585 } 2586 2587 vaddr_t core_mmu_get_va(paddr_t pa, enum teecore_memtypes type, size_t len) 2588 { 2589 if (cpu_mmu_enabled()) 2590 return (vaddr_t)phys_to_virt(pa, type, len); 2591 2592 return (vaddr_t)pa; 2593 } 2594 2595 #ifdef CFG_WITH_PAGER 2596 bool is_unpaged(const void *va) 2597 { 2598 vaddr_t v = (vaddr_t)va; 2599 2600 return v >= VCORE_START_VA && v < get_linear_map_end_va(); 2601 } 2602 #endif 2603 2604 #ifdef CFG_NS_VIRTUALIZATION 2605 bool is_nexus(const void *va) 2606 { 2607 vaddr_t v = (vaddr_t)va; 2608 2609 return v >= VCORE_START_VA && v < VCORE_NEX_RW_PA + VCORE_NEX_RW_SZ; 2610 } 2611 #endif 2612 2613 vaddr_t io_pa_or_va(struct io_pa_va *p, size_t len) 2614 { 2615 assert(p->pa); 2616 if (cpu_mmu_enabled()) { 2617 if (!p->va) 2618 p->va = (vaddr_t)phys_to_virt_io(p->pa, len); 2619 assert(p->va); 2620 return p->va; 2621 } 2622 return p->pa; 2623 } 2624 2625 vaddr_t io_pa_or_va_secure(struct io_pa_va *p, size_t len) 2626 { 2627 assert(p->pa); 2628 if (cpu_mmu_enabled()) { 2629 if (!p->va) 2630 p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_SEC, 2631 len); 2632 assert(p->va); 2633 return p->va; 2634 } 2635 return p->pa; 2636 } 2637 2638 vaddr_t io_pa_or_va_nsec(struct io_pa_va *p, size_t len) 2639 { 2640 assert(p->pa); 2641 if (cpu_mmu_enabled()) { 2642 if (!p->va) 2643 p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_NSEC, 2644 len); 2645 assert(p->va); 2646 return p->va; 2647 } 2648 return p->pa; 2649 } 2650 2651 #ifdef CFG_CORE_RESERVED_SHM 2652 static TEE_Result teecore_init_pub_ram(void) 2653 { 2654 vaddr_t s = 0; 2655 vaddr_t e = 0; 2656 2657 /* get virtual addr/size of NSec shared mem allocated from teecore */ 2658 core_mmu_get_mem_by_type(MEM_AREA_NSEC_SHM, &s, &e); 2659 2660 if (s >= e || s & SMALL_PAGE_MASK || e & SMALL_PAGE_MASK) 2661 panic("invalid PUB RAM"); 2662 2663 /* extra check: we could rely on core_mmu_get_mem_by_type() */ 2664 if (!tee_vbuf_is_non_sec(s, e - s)) 2665 panic("PUB RAM is not non-secure"); 2666 2667 #ifdef CFG_PL310 2668 /* Allocate statically the l2cc mutex */ 2669 tee_l2cc_store_mutex_boot_pa(virt_to_phys((void *)s)); 2670 s += sizeof(uint32_t); /* size of a pl310 mutex */ 2671 s = ROUNDUP(s, SMALL_PAGE_SIZE); /* keep required alignment */ 2672 #endif 2673 2674 default_nsec_shm_paddr = virt_to_phys((void *)s); 2675 default_nsec_shm_size = e - s; 2676 2677 return TEE_SUCCESS; 2678 } 2679 early_init(teecore_init_pub_ram); 2680 #endif /*CFG_CORE_RESERVED_SHM*/ 2681 2682 static void __maybe_unused carve_out_core_mem(paddr_t pa, paddr_t end_pa) 2683 { 2684 tee_mm_entry_t *mm __maybe_unused = NULL; 2685 2686 DMSG("%#"PRIxPA" .. %#"PRIxPA, pa, end_pa); 2687 mm = phys_mem_alloc2(pa, end_pa - pa); 2688 assert(mm); 2689 } 2690 2691 void core_mmu_init_phys_mem(void) 2692 { 2693 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 2694 paddr_t b1 = 0; 2695 paddr_size_t s1 = 0; 2696 2697 static_assert(ARRAY_SIZE(secure_only) <= 2); 2698 2699 if (ARRAY_SIZE(secure_only) == 2) { 2700 b1 = secure_only[1].paddr; 2701 s1 = secure_only[1].size; 2702 } 2703 virt_init_memory(&static_memory_map, secure_only[0].paddr, 2704 secure_only[0].size, b1, s1); 2705 } else { 2706 #ifdef CFG_WITH_PAGER 2707 /* 2708 * The pager uses all core memory so there's no need to add 2709 * it to the pool. 2710 */ 2711 static_assert(ARRAY_SIZE(secure_only) == 2); 2712 phys_mem_init(0, 0, secure_only[1].paddr, secure_only[1].size); 2713 #else /*!CFG_WITH_PAGER*/ 2714 size_t align = BIT(CORE_MMU_USER_CODE_SHIFT); 2715 paddr_t end_pa = 0; 2716 size_t size = 0; 2717 paddr_t ps = 0; 2718 paddr_t pa = 0; 2719 2720 static_assert(ARRAY_SIZE(secure_only) <= 2); 2721 if (ARRAY_SIZE(secure_only) == 2) { 2722 ps = secure_only[1].paddr; 2723 size = secure_only[1].size; 2724 } 2725 phys_mem_init(secure_only[0].paddr, secure_only[0].size, 2726 ps, size); 2727 2728 /* 2729 * The VCORE macros are relocatable so we need to translate 2730 * the addresses now that the MMU is enabled. 2731 */ 2732 end_pa = vaddr_to_phys(ROUNDUP2(VCORE_FREE_END_PA, 2733 align) - 1) + 1; 2734 /* Carve out the part used by OP-TEE core */ 2735 carve_out_core_mem(vaddr_to_phys(VCORE_UNPG_RX_PA), end_pa); 2736 if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS)) { 2737 pa = vaddr_to_phys(ROUNDUP2(ASAN_MAP_PA, align)); 2738 carve_out_core_mem(pa, pa + ASAN_MAP_SZ); 2739 } 2740 2741 /* Carve out test SDP memory */ 2742 #ifdef TEE_SDP_TEST_MEM_BASE 2743 if (TEE_SDP_TEST_MEM_SIZE) { 2744 pa = vaddr_to_phys(TEE_SDP_TEST_MEM_BASE); 2745 carve_out_core_mem(pa, pa + TEE_SDP_TEST_MEM_SIZE); 2746 } 2747 #endif 2748 #endif /*!CFG_WITH_PAGER*/ 2749 } 2750 } 2751