1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016, 2022 Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * Copyright (c) 2022, Arm Limited and Contributors. All rights reserved. 6 */ 7 8 #include <assert.h> 9 #include <config.h> 10 #include <kernel/boot.h> 11 #include <kernel/dt.h> 12 #include <kernel/linker.h> 13 #include <kernel/panic.h> 14 #include <kernel/spinlock.h> 15 #include <kernel/tee_l2cc_mutex.h> 16 #include <kernel/tee_misc.h> 17 #include <kernel/tlb_helpers.h> 18 #include <kernel/user_mode_ctx.h> 19 #include <kernel/virtualization.h> 20 #include <libfdt.h> 21 #include <memtag.h> 22 #include <mm/core_memprot.h> 23 #include <mm/core_mmu.h> 24 #include <mm/mobj.h> 25 #include <mm/pgt_cache.h> 26 #include <mm/phys_mem.h> 27 #include <mm/tee_pager.h> 28 #include <mm/vm.h> 29 #include <platform_config.h> 30 #include <stdalign.h> 31 #include <string.h> 32 #include <trace.h> 33 #include <util.h> 34 35 #ifndef DEBUG_XLAT_TABLE 36 #define DEBUG_XLAT_TABLE 0 37 #endif 38 39 #define SHM_VASPACE_SIZE (1024 * 1024 * 32) 40 41 /* Virtual memory pool for core mappings */ 42 tee_mm_pool_t core_virt_mem_pool; 43 44 /* Virtual memory pool for shared memory mappings */ 45 tee_mm_pool_t core_virt_shm_pool; 46 47 #ifdef CFG_CORE_PHYS_RELOCATABLE 48 unsigned long core_mmu_tee_load_pa __nex_bss; 49 #else 50 const unsigned long core_mmu_tee_load_pa = TEE_LOAD_ADDR; 51 #endif 52 53 /* 54 * These variables are initialized before .bss is cleared. To avoid 55 * resetting them when .bss is cleared we're storing them in .data instead, 56 * even if they initially are zero. 57 */ 58 59 #ifdef CFG_CORE_RESERVED_SHM 60 /* Default NSec shared memory allocated from NSec world */ 61 unsigned long default_nsec_shm_size __nex_bss; 62 unsigned long default_nsec_shm_paddr __nex_bss; 63 #endif 64 65 #ifdef CFG_BOOT_MEM 66 static struct memory_map static_memory_map __nex_bss; 67 #else 68 static struct tee_mmap_region static_mmap_regions[CFG_MMAP_REGIONS 69 #if defined(CFG_CORE_ASLR) || defined(CFG_CORE_PHYS_RELOCATABLE) 70 + 1 71 #endif 72 + 4] __nex_bss; 73 static struct memory_map static_memory_map __nex_data = { 74 .map = static_mmap_regions, 75 .alloc_count = ARRAY_SIZE(static_mmap_regions), 76 }; 77 #endif 78 void (*memory_map_realloc_func)(struct memory_map *mem_map) __nex_bss; 79 80 /* Offset of the first TEE RAM mapping from start of secure RAM */ 81 static size_t tee_ram_initial_offs __nex_bss; 82 83 /* Define the platform's memory layout. */ 84 struct memaccess_area { 85 paddr_t paddr; 86 size_t size; 87 }; 88 89 #define MEMACCESS_AREA(a, s) { .paddr = a, .size = s } 90 91 static struct memaccess_area secure_only[] __nex_data = { 92 #ifdef CFG_CORE_PHYS_RELOCATABLE 93 MEMACCESS_AREA(0, 0), 94 #else 95 #ifdef TRUSTED_SRAM_BASE 96 MEMACCESS_AREA(TRUSTED_SRAM_BASE, TRUSTED_SRAM_SIZE), 97 #endif 98 MEMACCESS_AREA(TRUSTED_DRAM_BASE, TRUSTED_DRAM_SIZE), 99 #endif 100 }; 101 102 static struct memaccess_area nsec_shared[] __nex_data = { 103 #ifdef CFG_CORE_RESERVED_SHM 104 MEMACCESS_AREA(TEE_SHMEM_START, TEE_SHMEM_SIZE), 105 #endif 106 }; 107 108 #if defined(CFG_SECURE_DATA_PATH) 109 static const char *tz_sdp_match = "linaro,secure-heap"; 110 static struct memaccess_area sec_sdp; 111 #ifdef CFG_TEE_SDP_MEM_BASE 112 register_sdp_mem(CFG_TEE_SDP_MEM_BASE, CFG_TEE_SDP_MEM_SIZE); 113 #endif 114 #ifdef TEE_SDP_TEST_MEM_BASE 115 register_sdp_mem(TEE_SDP_TEST_MEM_BASE, TEE_SDP_TEST_MEM_SIZE); 116 #endif 117 #endif 118 119 #ifdef CFG_CORE_RESERVED_SHM 120 register_phys_mem(MEM_AREA_NSEC_SHM, TEE_SHMEM_START, TEE_SHMEM_SIZE); 121 #endif 122 static unsigned int mmu_spinlock; 123 124 static uint32_t mmu_lock(void) 125 { 126 return cpu_spin_lock_xsave(&mmu_spinlock); 127 } 128 129 static void mmu_unlock(uint32_t exceptions) 130 { 131 cpu_spin_unlock_xrestore(&mmu_spinlock, exceptions); 132 } 133 134 static void heap_realloc_memory_map(struct memory_map *mem_map) 135 { 136 struct tee_mmap_region *m = NULL; 137 struct tee_mmap_region *old = mem_map->map; 138 size_t old_sz = sizeof(*old) * mem_map->alloc_count; 139 size_t sz = old_sz + sizeof(*m); 140 141 assert(nex_malloc_buffer_is_within_alloced(old, old_sz)); 142 m = nex_realloc(old, sz); 143 if (!m) 144 panic(); 145 mem_map->map = m; 146 mem_map->alloc_count++; 147 } 148 149 static void boot_mem_realloc_memory_map(struct memory_map *mem_map) 150 { 151 struct tee_mmap_region *m = NULL; 152 struct tee_mmap_region *old = mem_map->map; 153 size_t old_sz = sizeof(*old) * mem_map->alloc_count; 154 size_t sz = old_sz * 2; 155 156 m = boot_mem_alloc_tmp(sz, alignof(*m)); 157 memcpy(m, old, old_sz); 158 mem_map->map = m; 159 mem_map->alloc_count *= 2; 160 } 161 162 static void grow_mem_map(struct memory_map *mem_map) 163 { 164 if (mem_map->count == mem_map->alloc_count) { 165 if (!memory_map_realloc_func) { 166 EMSG("Out of entries (%zu) in mem_map", 167 mem_map->alloc_count); 168 panic(); 169 } 170 memory_map_realloc_func(mem_map); 171 } 172 mem_map->count++; 173 } 174 175 void core_mmu_get_secure_memory(paddr_t *base, paddr_size_t *size) 176 { 177 /* 178 * The first range is always used to cover OP-TEE core memory, but 179 * depending on configuration it may cover more than that. 180 */ 181 *base = secure_only[0].paddr; 182 *size = secure_only[0].size; 183 } 184 185 void core_mmu_set_secure_memory(paddr_t base, size_t size) 186 { 187 #ifdef CFG_CORE_PHYS_RELOCATABLE 188 static_assert(ARRAY_SIZE(secure_only) == 1); 189 #endif 190 runtime_assert(IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE)); 191 assert(!secure_only[0].size); 192 assert(base && size); 193 194 DMSG("Physical secure memory base %#"PRIxPA" size %#zx", base, size); 195 secure_only[0].paddr = base; 196 secure_only[0].size = size; 197 } 198 199 static struct memory_map *get_memory_map(void) 200 { 201 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 202 struct memory_map *map = virt_get_memory_map(); 203 204 if (map) 205 return map; 206 } 207 208 return &static_memory_map; 209 } 210 211 static bool _pbuf_intersects(struct memaccess_area *a, size_t alen, 212 paddr_t pa, size_t size) 213 { 214 size_t n; 215 216 for (n = 0; n < alen; n++) 217 if (core_is_buffer_intersect(pa, size, a[n].paddr, a[n].size)) 218 return true; 219 return false; 220 } 221 222 #define pbuf_intersects(a, pa, size) \ 223 _pbuf_intersects((a), ARRAY_SIZE(a), (pa), (size)) 224 225 static bool _pbuf_is_inside(struct memaccess_area *a, size_t alen, 226 paddr_t pa, size_t size) 227 { 228 size_t n; 229 230 for (n = 0; n < alen; n++) 231 if (core_is_buffer_inside(pa, size, a[n].paddr, a[n].size)) 232 return true; 233 return false; 234 } 235 236 #define pbuf_is_inside(a, pa, size) \ 237 _pbuf_is_inside((a), ARRAY_SIZE(a), (pa), (size)) 238 239 static bool pa_is_in_map(struct tee_mmap_region *map, paddr_t pa, size_t len) 240 { 241 paddr_t end_pa = 0; 242 243 if (!map) 244 return false; 245 246 if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) 247 return false; 248 249 return (pa >= map->pa && end_pa <= map->pa + map->size - 1); 250 } 251 252 static bool va_is_in_map(struct tee_mmap_region *map, vaddr_t va) 253 { 254 if (!map) 255 return false; 256 return (va >= map->va && va <= (map->va + map->size - 1)); 257 } 258 259 /* check if target buffer fits in a core default map area */ 260 static bool pbuf_inside_map_area(unsigned long p, size_t l, 261 struct tee_mmap_region *map) 262 { 263 return core_is_buffer_inside(p, l, map->pa, map->size); 264 } 265 266 TEE_Result core_mmu_for_each_map(void *ptr, 267 TEE_Result (*fn)(struct tee_mmap_region *map, 268 void *ptr)) 269 { 270 struct memory_map *mem_map = get_memory_map(); 271 TEE_Result res = TEE_SUCCESS; 272 size_t n = 0; 273 274 for (n = 0; n < mem_map->count; n++) { 275 res = fn(mem_map->map + n, ptr); 276 if (res) 277 return res; 278 } 279 280 return TEE_SUCCESS; 281 } 282 283 static struct tee_mmap_region *find_map_by_type(enum teecore_memtypes type) 284 { 285 struct memory_map *mem_map = get_memory_map(); 286 size_t n = 0; 287 288 for (n = 0; n < mem_map->count; n++) { 289 if (mem_map->map[n].type == type) 290 return mem_map->map + n; 291 } 292 return NULL; 293 } 294 295 static struct tee_mmap_region * 296 find_map_by_type_and_pa(enum teecore_memtypes type, paddr_t pa, size_t len) 297 { 298 struct memory_map *mem_map = get_memory_map(); 299 size_t n = 0; 300 301 for (n = 0; n < mem_map->count; n++) { 302 if (mem_map->map[n].type != type) 303 continue; 304 if (pa_is_in_map(mem_map->map + n, pa, len)) 305 return mem_map->map + n; 306 } 307 return NULL; 308 } 309 310 static struct tee_mmap_region *find_map_by_va(void *va) 311 { 312 struct memory_map *mem_map = get_memory_map(); 313 vaddr_t a = (vaddr_t)va; 314 size_t n = 0; 315 316 for (n = 0; n < mem_map->count; n++) { 317 if (a >= mem_map->map[n].va && 318 a <= (mem_map->map[n].va - 1 + mem_map->map[n].size)) 319 return mem_map->map + n; 320 } 321 322 return NULL; 323 } 324 325 static struct tee_mmap_region *find_map_by_pa(unsigned long pa) 326 { 327 struct memory_map *mem_map = get_memory_map(); 328 size_t n = 0; 329 330 for (n = 0; n < mem_map->count; n++) { 331 /* Skip unmapped regions */ 332 if ((mem_map->map[n].attr & TEE_MATTR_VALID_BLOCK) && 333 pa >= mem_map->map[n].pa && 334 pa <= (mem_map->map[n].pa - 1 + mem_map->map[n].size)) 335 return mem_map->map + n; 336 } 337 338 return NULL; 339 } 340 341 #if defined(CFG_SECURE_DATA_PATH) 342 static bool dtb_get_sdp_region(void) 343 { 344 void *fdt = NULL; 345 int node = 0; 346 int tmp_node = 0; 347 paddr_t tmp_addr = 0; 348 size_t tmp_size = 0; 349 350 if (!IS_ENABLED(CFG_EMBED_DTB)) 351 return false; 352 353 fdt = get_embedded_dt(); 354 if (!fdt) 355 panic("No DTB found"); 356 357 node = fdt_node_offset_by_compatible(fdt, -1, tz_sdp_match); 358 if (node < 0) { 359 DMSG("No %s compatible node found", tz_sdp_match); 360 return false; 361 } 362 tmp_node = node; 363 while (tmp_node >= 0) { 364 tmp_node = fdt_node_offset_by_compatible(fdt, tmp_node, 365 tz_sdp_match); 366 if (tmp_node >= 0) 367 DMSG("Ignore SDP pool node %s, supports only 1 node", 368 fdt_get_name(fdt, tmp_node, NULL)); 369 } 370 371 if (fdt_reg_info(fdt, node, &tmp_addr, &tmp_size)) { 372 EMSG("%s: Unable to get base addr or size from DT", 373 tz_sdp_match); 374 return false; 375 } 376 377 sec_sdp.paddr = tmp_addr; 378 sec_sdp.size = tmp_size; 379 380 return true; 381 } 382 #endif 383 384 #if defined(CFG_CORE_DYN_SHM) || defined(CFG_SECURE_DATA_PATH) 385 static bool pbuf_is_special_mem(paddr_t pbuf, size_t len, 386 const struct core_mmu_phys_mem *start, 387 const struct core_mmu_phys_mem *end) 388 { 389 const struct core_mmu_phys_mem *mem; 390 391 for (mem = start; mem < end; mem++) { 392 if (core_is_buffer_inside(pbuf, len, mem->addr, mem->size)) 393 return true; 394 } 395 396 return false; 397 } 398 #endif 399 400 #ifdef CFG_CORE_DYN_SHM 401 static void carve_out_phys_mem(struct core_mmu_phys_mem **mem, size_t *nelems, 402 paddr_t pa, size_t size) 403 { 404 struct core_mmu_phys_mem *m = *mem; 405 size_t n = 0; 406 407 while (true) { 408 if (n >= *nelems) { 409 DMSG("No need to carve out %#" PRIxPA " size %#zx", 410 pa, size); 411 return; 412 } 413 if (core_is_buffer_inside(pa, size, m[n].addr, m[n].size)) 414 break; 415 if (!core_is_buffer_outside(pa, size, m[n].addr, m[n].size)) 416 panic(); 417 n++; 418 } 419 420 if (pa == m[n].addr && size == m[n].size) { 421 /* Remove this entry */ 422 (*nelems)--; 423 memmove(m + n, m + n + 1, sizeof(*m) * (*nelems - n)); 424 m = nex_realloc(m, sizeof(*m) * *nelems); 425 if (!m) 426 panic(); 427 *mem = m; 428 } else if (pa == m[n].addr) { 429 m[n].addr += size; 430 m[n].size -= size; 431 } else if ((pa + size) == (m[n].addr + m[n].size)) { 432 m[n].size -= size; 433 } else { 434 /* Need to split the memory entry */ 435 m = nex_realloc(m, sizeof(*m) * (*nelems + 1)); 436 if (!m) 437 panic(); 438 *mem = m; 439 memmove(m + n + 1, m + n, sizeof(*m) * (*nelems - n)); 440 (*nelems)++; 441 m[n].size = pa - m[n].addr; 442 m[n + 1].size -= size + m[n].size; 443 m[n + 1].addr = pa + size; 444 } 445 } 446 447 static void check_phys_mem_is_outside(struct core_mmu_phys_mem *start, 448 size_t nelems, 449 struct tee_mmap_region *map) 450 { 451 size_t n; 452 453 for (n = 0; n < nelems; n++) { 454 if (!core_is_buffer_outside(start[n].addr, start[n].size, 455 map->pa, map->size)) { 456 EMSG("Non-sec mem (%#" PRIxPA ":%#" PRIxPASZ 457 ") overlaps map (type %d %#" PRIxPA ":%#zx)", 458 start[n].addr, start[n].size, 459 map->type, map->pa, map->size); 460 panic(); 461 } 462 } 463 } 464 465 static const struct core_mmu_phys_mem *discovered_nsec_ddr_start __nex_bss; 466 static size_t discovered_nsec_ddr_nelems __nex_bss; 467 468 static int cmp_pmem_by_addr(const void *a, const void *b) 469 { 470 const struct core_mmu_phys_mem *pmem_a = a; 471 const struct core_mmu_phys_mem *pmem_b = b; 472 473 return CMP_TRILEAN(pmem_a->addr, pmem_b->addr); 474 } 475 476 void core_mmu_set_discovered_nsec_ddr(struct core_mmu_phys_mem *start, 477 size_t nelems) 478 { 479 struct core_mmu_phys_mem *m = start; 480 size_t num_elems = nelems; 481 struct memory_map *mem_map = &static_memory_map; 482 const struct core_mmu_phys_mem __maybe_unused *pmem; 483 size_t n = 0; 484 485 assert(!discovered_nsec_ddr_start); 486 assert(m && num_elems); 487 488 qsort(m, num_elems, sizeof(*m), cmp_pmem_by_addr); 489 490 /* 491 * Non-secure shared memory and also secure data 492 * path memory are supposed to reside inside 493 * non-secure memory. Since NSEC_SHM and SDP_MEM 494 * are used for a specific purpose make holes for 495 * those memory in the normal non-secure memory. 496 * 497 * This has to be done since for instance QEMU 498 * isn't aware of which memory range in the 499 * non-secure memory is used for NSEC_SHM. 500 */ 501 502 #ifdef CFG_SECURE_DATA_PATH 503 if (dtb_get_sdp_region()) 504 carve_out_phys_mem(&m, &num_elems, sec_sdp.paddr, sec_sdp.size); 505 506 for (pmem = phys_sdp_mem_begin; pmem < phys_sdp_mem_end; pmem++) 507 carve_out_phys_mem(&m, &num_elems, pmem->addr, pmem->size); 508 #endif 509 510 for (n = 0; n < ARRAY_SIZE(secure_only); n++) 511 carve_out_phys_mem(&m, &num_elems, secure_only[n].paddr, 512 secure_only[n].size); 513 514 for (n = 0; n < mem_map->count; n++) { 515 switch (mem_map->map[n].type) { 516 case MEM_AREA_NSEC_SHM: 517 carve_out_phys_mem(&m, &num_elems, mem_map->map[n].pa, 518 mem_map->map[n].size); 519 break; 520 case MEM_AREA_EXT_DT: 521 case MEM_AREA_MANIFEST_DT: 522 case MEM_AREA_RAM_NSEC: 523 case MEM_AREA_RES_VASPACE: 524 case MEM_AREA_SHM_VASPACE: 525 case MEM_AREA_TS_VASPACE: 526 case MEM_AREA_PAGER_VASPACE: 527 break; 528 default: 529 check_phys_mem_is_outside(m, num_elems, 530 mem_map->map + n); 531 } 532 } 533 534 discovered_nsec_ddr_start = m; 535 discovered_nsec_ddr_nelems = num_elems; 536 537 if (!core_mmu_check_end_pa(m[num_elems - 1].addr, 538 m[num_elems - 1].size)) 539 panic(); 540 } 541 542 static bool get_discovered_nsec_ddr(const struct core_mmu_phys_mem **start, 543 const struct core_mmu_phys_mem **end) 544 { 545 if (!discovered_nsec_ddr_start) 546 return false; 547 548 *start = discovered_nsec_ddr_start; 549 *end = discovered_nsec_ddr_start + discovered_nsec_ddr_nelems; 550 551 return true; 552 } 553 554 static bool pbuf_is_nsec_ddr(paddr_t pbuf, size_t len) 555 { 556 const struct core_mmu_phys_mem *start; 557 const struct core_mmu_phys_mem *end; 558 559 if (!get_discovered_nsec_ddr(&start, &end)) 560 return false; 561 562 return pbuf_is_special_mem(pbuf, len, start, end); 563 } 564 565 bool core_mmu_nsec_ddr_is_defined(void) 566 { 567 const struct core_mmu_phys_mem *start; 568 const struct core_mmu_phys_mem *end; 569 570 if (!get_discovered_nsec_ddr(&start, &end)) 571 return false; 572 573 return start != end; 574 } 575 #else 576 static bool pbuf_is_nsec_ddr(paddr_t pbuf __unused, size_t len __unused) 577 { 578 return false; 579 } 580 #endif /*CFG_CORE_DYN_SHM*/ 581 582 #define MSG_MEM_INSTERSECT(pa1, sz1, pa2, sz2) \ 583 EMSG("[%" PRIxPA " %" PRIx64 "] intersects [%" PRIxPA " %" PRIx64 "]", \ 584 pa1, (uint64_t)pa1 + (sz1), pa2, (uint64_t)pa2 + (sz2)) 585 586 #ifdef CFG_SECURE_DATA_PATH 587 static bool pbuf_is_sdp_mem(paddr_t pbuf, size_t len) 588 { 589 bool is_sdp_mem = false; 590 591 if (sec_sdp.size) 592 is_sdp_mem = core_is_buffer_inside(pbuf, len, sec_sdp.paddr, 593 sec_sdp.size); 594 595 if (!is_sdp_mem) 596 is_sdp_mem = pbuf_is_special_mem(pbuf, len, phys_sdp_mem_begin, 597 phys_sdp_mem_end); 598 599 return is_sdp_mem; 600 } 601 602 static struct mobj *core_sdp_mem_alloc_mobj(paddr_t pa, size_t size) 603 { 604 struct mobj *mobj = mobj_phys_alloc(pa, size, TEE_MATTR_MEM_TYPE_CACHED, 605 CORE_MEM_SDP_MEM); 606 607 if (!mobj) 608 panic("can't create SDP physical memory object"); 609 610 return mobj; 611 } 612 613 struct mobj **core_sdp_mem_create_mobjs(void) 614 { 615 const struct core_mmu_phys_mem *mem = NULL; 616 struct mobj **mobj_base = NULL; 617 struct mobj **mobj = NULL; 618 int cnt = phys_sdp_mem_end - phys_sdp_mem_begin; 619 620 if (sec_sdp.size) 621 cnt++; 622 623 /* SDP mobjs table must end with a NULL entry */ 624 mobj_base = calloc(cnt + 1, sizeof(struct mobj *)); 625 if (!mobj_base) 626 panic("Out of memory"); 627 628 mobj = mobj_base; 629 630 for (mem = phys_sdp_mem_begin; mem < phys_sdp_mem_end; mem++, mobj++) 631 *mobj = core_sdp_mem_alloc_mobj(mem->addr, mem->size); 632 633 if (sec_sdp.size) 634 *mobj = core_sdp_mem_alloc_mobj(sec_sdp.paddr, sec_sdp.size); 635 636 return mobj_base; 637 } 638 639 #else /* CFG_SECURE_DATA_PATH */ 640 static bool pbuf_is_sdp_mem(paddr_t pbuf __unused, size_t len __unused) 641 { 642 return false; 643 } 644 645 #endif /* CFG_SECURE_DATA_PATH */ 646 647 /* Check special memories comply with registered memories */ 648 static void verify_special_mem_areas(struct memory_map *mem_map, 649 const struct core_mmu_phys_mem *start, 650 const struct core_mmu_phys_mem *end, 651 const char *area_name __maybe_unused) 652 { 653 const struct core_mmu_phys_mem *mem = NULL; 654 const struct core_mmu_phys_mem *mem2 = NULL; 655 size_t n = 0; 656 657 if (start == end) { 658 DMSG("No %s memory area defined", area_name); 659 return; 660 } 661 662 for (mem = start; mem < end; mem++) 663 DMSG("%s memory [%" PRIxPA " %" PRIx64 "]", 664 area_name, mem->addr, (uint64_t)mem->addr + mem->size); 665 666 /* Check memories do not intersect each other */ 667 for (mem = start; mem + 1 < end; mem++) { 668 for (mem2 = mem + 1; mem2 < end; mem2++) { 669 if (core_is_buffer_intersect(mem2->addr, mem2->size, 670 mem->addr, mem->size)) { 671 MSG_MEM_INSTERSECT(mem2->addr, mem2->size, 672 mem->addr, mem->size); 673 panic("Special memory intersection"); 674 } 675 } 676 } 677 678 /* 679 * Check memories do not intersect any mapped memory. 680 * This is called before reserved VA space is loaded in mem_map. 681 */ 682 for (mem = start; mem < end; mem++) { 683 for (n = 0; n < mem_map->count; n++) { 684 if (core_is_buffer_intersect(mem->addr, mem->size, 685 mem_map->map[n].pa, 686 mem_map->map[n].size)) { 687 MSG_MEM_INSTERSECT(mem->addr, mem->size, 688 mem_map->map[n].pa, 689 mem_map->map[n].size); 690 panic("Special memory intersection"); 691 } 692 } 693 } 694 } 695 696 static void merge_mmaps(struct tee_mmap_region *dst, 697 const struct tee_mmap_region *src) 698 { 699 paddr_t end_pa = MAX(dst->pa + dst->size - 1, src->pa + src->size - 1); 700 paddr_t pa = MIN(dst->pa, src->pa); 701 702 DMSG("Merging %#"PRIxPA"..%#"PRIxPA" and %#"PRIxPA"..%#"PRIxPA, 703 dst->pa, dst->pa + dst->size - 1, src->pa, 704 src->pa + src->size - 1); 705 dst->pa = pa; 706 dst->size = end_pa - pa + 1; 707 } 708 709 static bool mmaps_are_mergeable(const struct tee_mmap_region *r1, 710 const struct tee_mmap_region *r2) 711 { 712 if (r1->type != r2->type) 713 return false; 714 715 if (r1->pa == r2->pa) 716 return true; 717 718 if (r1->pa < r2->pa) 719 return r1->pa + r1->size >= r2->pa; 720 else 721 return r2->pa + r2->size >= r1->pa; 722 } 723 724 static void add_phys_mem(struct memory_map *mem_map, 725 const char *mem_name __maybe_unused, 726 enum teecore_memtypes mem_type, 727 paddr_t mem_addr, paddr_size_t mem_size) 728 { 729 size_t n = 0; 730 const struct tee_mmap_region m0 = { 731 .type = mem_type, 732 .pa = mem_addr, 733 .size = mem_size, 734 }; 735 736 if (!mem_size) /* Discard null size entries */ 737 return; 738 739 /* 740 * If some ranges of memory of the same type do overlap 741 * each others they are coalesced into one entry. To help this 742 * added entries are sorted by increasing physical. 743 * 744 * Note that it's valid to have the same physical memory as several 745 * different memory types, for instance the same device memory 746 * mapped as both secure and non-secure. This will probably not 747 * happen often in practice. 748 */ 749 DMSG("%s type %s 0x%08" PRIxPA " size 0x%08" PRIxPASZ, 750 mem_name, teecore_memtype_name(mem_type), mem_addr, mem_size); 751 for (n = 0; n < mem_map->count; n++) { 752 if (mmaps_are_mergeable(mem_map->map + n, &m0)) { 753 merge_mmaps(mem_map->map + n, &m0); 754 /* 755 * The merged result might be mergeable with the 756 * next or previous entry. 757 */ 758 if (n + 1 < mem_map->count && 759 mmaps_are_mergeable(mem_map->map + n, 760 mem_map->map + n + 1)) { 761 merge_mmaps(mem_map->map + n, 762 mem_map->map + n + 1); 763 rem_array_elem(mem_map->map, mem_map->count, 764 sizeof(*mem_map->map), n + 1); 765 mem_map->count--; 766 } 767 if (n > 0 && mmaps_are_mergeable(mem_map->map + n - 1, 768 mem_map->map + n)) { 769 merge_mmaps(mem_map->map + n - 1, 770 mem_map->map + n); 771 rem_array_elem(mem_map->map, mem_map->count, 772 sizeof(*mem_map->map), n); 773 mem_map->count--; 774 } 775 return; 776 } 777 if (mem_type < mem_map->map[n].type || 778 (mem_type == mem_map->map[n].type && 779 mem_addr < mem_map->map[n].pa)) 780 break; /* found the spot where to insert this memory */ 781 } 782 783 grow_mem_map(mem_map); 784 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 785 n, &m0); 786 } 787 788 static void add_va_space(struct memory_map *mem_map, 789 enum teecore_memtypes type, size_t size) 790 { 791 size_t n = 0; 792 793 DMSG("type %s size 0x%08zx", teecore_memtype_name(type), size); 794 for (n = 0; n < mem_map->count; n++) { 795 if (type < mem_map->map[n].type) 796 break; 797 } 798 799 grow_mem_map(mem_map); 800 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 801 n, NULL); 802 mem_map->map[n] = (struct tee_mmap_region){ 803 .type = type, 804 .size = size, 805 }; 806 } 807 808 uint32_t core_mmu_type_to_attr(enum teecore_memtypes t) 809 { 810 const uint32_t attr = TEE_MATTR_VALID_BLOCK; 811 const uint32_t tagged = TEE_MATTR_MEM_TYPE_TAGGED << 812 TEE_MATTR_MEM_TYPE_SHIFT; 813 const uint32_t cached = TEE_MATTR_MEM_TYPE_CACHED << 814 TEE_MATTR_MEM_TYPE_SHIFT; 815 const uint32_t noncache = TEE_MATTR_MEM_TYPE_DEV << 816 TEE_MATTR_MEM_TYPE_SHIFT; 817 818 switch (t) { 819 case MEM_AREA_TEE_RAM: 820 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | tagged; 821 case MEM_AREA_TEE_RAM_RX: 822 case MEM_AREA_INIT_RAM_RX: 823 case MEM_AREA_IDENTITY_MAP_RX: 824 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRX | tagged; 825 case MEM_AREA_TEE_RAM_RO: 826 case MEM_AREA_INIT_RAM_RO: 827 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | tagged; 828 case MEM_AREA_TEE_RAM_RW: 829 case MEM_AREA_NEX_RAM_RO: /* This has to be r/w during init runtime */ 830 case MEM_AREA_NEX_RAM_RW: 831 case MEM_AREA_TEE_ASAN: 832 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | tagged; 833 case MEM_AREA_TEE_COHERENT: 834 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | noncache; 835 case MEM_AREA_NSEC_SHM: 836 case MEM_AREA_NEX_NSEC_SHM: 837 return attr | TEE_MATTR_PRW | cached; 838 case MEM_AREA_MANIFEST_DT: 839 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | cached; 840 case MEM_AREA_TRANSFER_LIST: 841 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 842 case MEM_AREA_EXT_DT: 843 /* 844 * If CFG_MAP_EXT_DT_SECURE is enabled map the external device 845 * tree as secure non-cached memory, otherwise, fall back to 846 * non-secure mapping. 847 */ 848 if (IS_ENABLED(CFG_MAP_EXT_DT_SECURE)) 849 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | 850 noncache; 851 fallthrough; 852 case MEM_AREA_IO_NSEC: 853 return attr | TEE_MATTR_PRW | noncache; 854 case MEM_AREA_IO_SEC: 855 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | noncache; 856 case MEM_AREA_RAM_NSEC: 857 return attr | TEE_MATTR_PRW | cached; 858 case MEM_AREA_RAM_SEC: 859 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 860 case MEM_AREA_SEC_RAM_OVERALL: 861 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | tagged; 862 case MEM_AREA_ROM_SEC: 863 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | cached; 864 case MEM_AREA_RES_VASPACE: 865 case MEM_AREA_SHM_VASPACE: 866 return 0; 867 case MEM_AREA_PAGER_VASPACE: 868 return TEE_MATTR_SECURE; 869 default: 870 panic("invalid type"); 871 } 872 } 873 874 static bool __maybe_unused map_is_tee_ram(const struct tee_mmap_region *mm) 875 { 876 switch (mm->type) { 877 case MEM_AREA_TEE_RAM: 878 case MEM_AREA_TEE_RAM_RX: 879 case MEM_AREA_TEE_RAM_RO: 880 case MEM_AREA_TEE_RAM_RW: 881 case MEM_AREA_INIT_RAM_RX: 882 case MEM_AREA_INIT_RAM_RO: 883 case MEM_AREA_NEX_RAM_RW: 884 case MEM_AREA_NEX_RAM_RO: 885 case MEM_AREA_TEE_ASAN: 886 return true; 887 default: 888 return false; 889 } 890 } 891 892 static bool __maybe_unused map_is_secure(const struct tee_mmap_region *mm) 893 { 894 return !!(core_mmu_type_to_attr(mm->type) & TEE_MATTR_SECURE); 895 } 896 897 static bool __maybe_unused map_is_pgdir(const struct tee_mmap_region *mm) 898 { 899 return mm->region_size == CORE_MMU_PGDIR_SIZE; 900 } 901 902 static int cmp_mmap_by_lower_va(const void *a, const void *b) 903 { 904 const struct tee_mmap_region *mm_a = a; 905 const struct tee_mmap_region *mm_b = b; 906 907 return CMP_TRILEAN(mm_a->va, mm_b->va); 908 } 909 910 static void dump_mmap_table(struct memory_map *mem_map) 911 { 912 size_t n = 0; 913 914 for (n = 0; n < mem_map->count; n++) { 915 struct tee_mmap_region *map __maybe_unused = mem_map->map + n; 916 917 DMSG("type %-12s va 0x%08" PRIxVA "..0x%08" PRIxVA 918 " pa 0x%08" PRIxPA "..0x%08" PRIxPA " size 0x%08zx (%s)", 919 teecore_memtype_name(map->type), map->va, 920 map->va + map->size - 1, map->pa, 921 (paddr_t)(map->pa + map->size - 1), map->size, 922 map->region_size == SMALL_PAGE_SIZE ? "smallpg" : "pgdir"); 923 } 924 } 925 926 #if DEBUG_XLAT_TABLE 927 928 static void dump_xlat_table(vaddr_t va, unsigned int level) 929 { 930 struct core_mmu_table_info tbl_info; 931 unsigned int idx = 0; 932 paddr_t pa; 933 uint32_t attr; 934 935 core_mmu_find_table(NULL, va, level, &tbl_info); 936 va = tbl_info.va_base; 937 for (idx = 0; idx < tbl_info.num_entries; idx++) { 938 core_mmu_get_entry(&tbl_info, idx, &pa, &attr); 939 if (attr || level > CORE_MMU_BASE_TABLE_LEVEL) { 940 const char *security_bit = ""; 941 942 if (core_mmu_entry_have_security_bit(attr)) { 943 if (attr & TEE_MATTR_SECURE) 944 security_bit = "S"; 945 else 946 security_bit = "NS"; 947 } 948 949 if (attr & TEE_MATTR_TABLE) { 950 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 951 " TBL:0x%010" PRIxPA " %s", 952 level * 2, "", level, va, pa, 953 security_bit); 954 dump_xlat_table(va, level + 1); 955 } else if (attr) { 956 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 957 " PA:0x%010" PRIxPA " %s-%s-%s-%s", 958 level * 2, "", level, va, pa, 959 mattr_is_cached(attr) ? "MEM" : 960 "DEV", 961 attr & TEE_MATTR_PW ? "RW" : "RO", 962 attr & TEE_MATTR_PX ? "X " : "XN", 963 security_bit); 964 } else { 965 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 966 " INVALID\n", 967 level * 2, "", level, va); 968 } 969 } 970 va += BIT64(tbl_info.shift); 971 } 972 } 973 974 #else 975 976 static void dump_xlat_table(vaddr_t va __unused, int level __unused) 977 { 978 } 979 980 #endif 981 982 /* 983 * Reserves virtual memory space for pager usage. 984 * 985 * From the start of the first memory used by the link script + 986 * TEE_RAM_VA_SIZE should be covered, either with a direct mapping or empty 987 * mapping for pager usage. This adds translation tables as needed for the 988 * pager to operate. 989 */ 990 static void add_pager_vaspace(struct memory_map *mem_map) 991 { 992 paddr_t begin = 0; 993 paddr_t end = 0; 994 size_t size = 0; 995 size_t pos = 0; 996 size_t n = 0; 997 998 999 for (n = 0; n < mem_map->count; n++) { 1000 if (map_is_tee_ram(mem_map->map + n)) { 1001 if (!begin) 1002 begin = mem_map->map[n].pa; 1003 pos = n + 1; 1004 } 1005 } 1006 1007 end = mem_map->map[pos - 1].pa + mem_map->map[pos - 1].size; 1008 assert(end - begin < TEE_RAM_VA_SIZE); 1009 size = TEE_RAM_VA_SIZE - (end - begin); 1010 1011 grow_mem_map(mem_map); 1012 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 1013 n, NULL); 1014 mem_map->map[n] = (struct tee_mmap_region){ 1015 .type = MEM_AREA_PAGER_VASPACE, 1016 .size = size, 1017 .region_size = SMALL_PAGE_SIZE, 1018 .attr = core_mmu_type_to_attr(MEM_AREA_PAGER_VASPACE), 1019 }; 1020 } 1021 1022 static void check_sec_nsec_mem_config(void) 1023 { 1024 size_t n = 0; 1025 1026 for (n = 0; n < ARRAY_SIZE(secure_only); n++) { 1027 if (pbuf_intersects(nsec_shared, secure_only[n].paddr, 1028 secure_only[n].size)) 1029 panic("Invalid memory access config: sec/nsec"); 1030 } 1031 } 1032 1033 static void collect_device_mem_ranges(struct memory_map *mem_map) 1034 { 1035 const char *compatible = "arm,ffa-manifest-device-regions"; 1036 void *fdt = get_manifest_dt(); 1037 const char *name = NULL; 1038 uint64_t page_count = 0; 1039 uint64_t base = 0; 1040 int subnode = 0; 1041 int node = 0; 1042 1043 assert(fdt); 1044 1045 node = fdt_node_offset_by_compatible(fdt, 0, compatible); 1046 if (node < 0) 1047 return; 1048 1049 fdt_for_each_subnode(subnode, fdt, node) { 1050 name = fdt_get_name(fdt, subnode, NULL); 1051 if (!name) 1052 continue; 1053 1054 if (dt_getprop_as_number(fdt, subnode, "base-address", 1055 &base)) { 1056 EMSG("Mandatory field is missing: base-address"); 1057 continue; 1058 } 1059 1060 if (base & SMALL_PAGE_MASK) { 1061 EMSG("base-address is not page aligned"); 1062 continue; 1063 } 1064 1065 if (dt_getprop_as_number(fdt, subnode, "pages-count", 1066 &page_count)) { 1067 EMSG("Mandatory field is missing: pages-count"); 1068 continue; 1069 } 1070 1071 add_phys_mem(mem_map, name, MEM_AREA_IO_SEC, 1072 base, base + page_count * SMALL_PAGE_SIZE); 1073 } 1074 } 1075 1076 static void collect_mem_ranges(struct memory_map *mem_map) 1077 { 1078 const struct core_mmu_phys_mem *mem = NULL; 1079 vaddr_t ram_start = secure_only[0].paddr; 1080 size_t n = 0; 1081 1082 #define ADD_PHYS_MEM(_type, _addr, _size) \ 1083 add_phys_mem(mem_map, #_addr, (_type), (_addr), (_size)) 1084 1085 if (IS_ENABLED(CFG_CORE_RWDATA_NOEXEC)) { 1086 paddr_t next_pa = 0; 1087 1088 /* 1089 * Read-only and read-execute physical memory areas must 1090 * not be mapped by MEM_AREA_SEC_RAM_OVERALL, but all the 1091 * read/write should. 1092 */ 1093 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, ram_start, 1094 VCORE_UNPG_RX_PA - ram_start); 1095 assert(VCORE_UNPG_RX_PA >= ram_start); 1096 tee_ram_initial_offs = VCORE_UNPG_RX_PA - ram_start; 1097 DMSG("tee_ram_initial_offs %#zx", tee_ram_initial_offs); 1098 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RX, VCORE_UNPG_RX_PA, 1099 VCORE_UNPG_RX_SZ); 1100 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, VCORE_UNPG_RO_PA, 1101 VCORE_UNPG_RO_SZ); 1102 1103 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1104 ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RO, VCORE_UNPG_RW_PA, 1105 VCORE_UNPG_RW_SZ); 1106 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_UNPG_RW_PA, 1107 VCORE_UNPG_RW_SZ); 1108 1109 ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RW, VCORE_NEX_RW_PA, 1110 VCORE_NEX_RW_SZ); 1111 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_NEX_RW_PA, 1112 VCORE_NEX_RW_SZ); 1113 1114 ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RW, VCORE_FREE_PA, 1115 VCORE_FREE_SZ); 1116 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_FREE_PA, 1117 VCORE_FREE_SZ); 1118 next_pa = VCORE_FREE_PA + VCORE_FREE_SZ; 1119 } else { 1120 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RW, VCORE_UNPG_RW_PA, 1121 VCORE_UNPG_RW_SZ); 1122 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_UNPG_RW_PA, 1123 VCORE_UNPG_RW_SZ); 1124 1125 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RW, VCORE_FREE_PA, 1126 VCORE_FREE_SZ); 1127 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, VCORE_FREE_PA, 1128 VCORE_FREE_SZ); 1129 next_pa = VCORE_FREE_PA + VCORE_FREE_SZ; 1130 } 1131 1132 if (IS_ENABLED(CFG_WITH_PAGER)) { 1133 paddr_t pa = 0; 1134 size_t sz = 0; 1135 1136 ADD_PHYS_MEM(MEM_AREA_INIT_RAM_RX, VCORE_INIT_RX_PA, 1137 VCORE_INIT_RX_SZ); 1138 ADD_PHYS_MEM(MEM_AREA_INIT_RAM_RO, VCORE_INIT_RO_PA, 1139 VCORE_INIT_RO_SZ); 1140 /* 1141 * Core init mapping shall cover up to end of the 1142 * physical RAM. This is required since the hash 1143 * table is appended to the binary data after the 1144 * firmware build sequence. 1145 */ 1146 pa = VCORE_INIT_RO_PA + VCORE_INIT_RO_SZ; 1147 sz = TEE_RAM_START + TEE_RAM_PH_SIZE - pa; 1148 ADD_PHYS_MEM(MEM_AREA_TEE_RAM, pa, sz); 1149 } else { 1150 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, next_pa, 1151 secure_only[0].paddr + 1152 secure_only[0].size - next_pa); 1153 } 1154 } else { 1155 ADD_PHYS_MEM(MEM_AREA_TEE_RAM, TEE_RAM_START, TEE_RAM_PH_SIZE); 1156 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, secure_only[n].paddr, 1157 secure_only[0].size); 1158 } 1159 1160 for (n = 1; n < ARRAY_SIZE(secure_only); n++) 1161 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, secure_only[n].paddr, 1162 secure_only[n].size); 1163 1164 if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS) && 1165 IS_ENABLED(CFG_WITH_PAGER)) { 1166 /* 1167 * Asan ram is part of MEM_AREA_TEE_RAM_RW when pager is 1168 * disabled. 1169 */ 1170 ADD_PHYS_MEM(MEM_AREA_TEE_ASAN, ASAN_MAP_PA, ASAN_MAP_SZ); 1171 } 1172 1173 #undef ADD_PHYS_MEM 1174 1175 /* Collect device memory info from SP manifest */ 1176 if (IS_ENABLED(CFG_CORE_SEL2_SPMC)) 1177 collect_device_mem_ranges(mem_map); 1178 1179 for (mem = phys_mem_map_begin; mem < phys_mem_map_end; mem++) { 1180 /* Only unmapped virtual range may have a null phys addr */ 1181 assert(mem->addr || !core_mmu_type_to_attr(mem->type)); 1182 1183 add_phys_mem(mem_map, mem->name, mem->type, 1184 mem->addr, mem->size); 1185 } 1186 1187 if (IS_ENABLED(CFG_SECURE_DATA_PATH)) 1188 verify_special_mem_areas(mem_map, phys_sdp_mem_begin, 1189 phys_sdp_mem_end, "SDP"); 1190 1191 add_va_space(mem_map, MEM_AREA_RES_VASPACE, CFG_RESERVED_VASPACE_SIZE); 1192 add_va_space(mem_map, MEM_AREA_SHM_VASPACE, SHM_VASPACE_SIZE); 1193 } 1194 1195 static void assign_mem_granularity(struct memory_map *mem_map) 1196 { 1197 size_t n = 0; 1198 1199 /* 1200 * Assign region sizes, note that MEM_AREA_TEE_RAM always uses 1201 * SMALL_PAGE_SIZE. 1202 */ 1203 for (n = 0; n < mem_map->count; n++) { 1204 paddr_t mask = mem_map->map[n].pa | mem_map->map[n].size; 1205 1206 if (mask & SMALL_PAGE_MASK) 1207 panic("Impossible memory alignment"); 1208 1209 if (map_is_tee_ram(mem_map->map + n)) 1210 mem_map->map[n].region_size = SMALL_PAGE_SIZE; 1211 else 1212 mem_map->map[n].region_size = CORE_MMU_PGDIR_SIZE; 1213 } 1214 } 1215 1216 static bool place_tee_ram_at_top(paddr_t paddr) 1217 { 1218 return paddr > BIT64(core_mmu_get_va_width()) / 2; 1219 } 1220 1221 /* 1222 * MMU arch driver shall override this function if it helps 1223 * optimizing the memory footprint of the address translation tables. 1224 */ 1225 bool __weak core_mmu_prefer_tee_ram_at_top(paddr_t paddr) 1226 { 1227 return place_tee_ram_at_top(paddr); 1228 } 1229 1230 static bool assign_mem_va_dir(vaddr_t tee_ram_va, struct memory_map *mem_map, 1231 bool tee_ram_at_top) 1232 { 1233 struct tee_mmap_region *map = NULL; 1234 vaddr_t va = 0; 1235 bool va_is_secure = true; 1236 size_t n = 0; 1237 1238 /* 1239 * tee_ram_va might equals 0 when CFG_CORE_ASLR=y. 1240 * 0 is by design an invalid va, so return false directly. 1241 */ 1242 if (!tee_ram_va) 1243 return false; 1244 1245 /* Clear eventual previous assignments */ 1246 for (n = 0; n < mem_map->count; n++) 1247 mem_map->map[n].va = 0; 1248 1249 /* 1250 * TEE RAM regions are always aligned with region_size. 1251 * 1252 * Note that MEM_AREA_PAGER_VASPACE also counts as TEE RAM here 1253 * since it handles virtual memory which covers the part of the ELF 1254 * that cannot fit directly into memory. 1255 */ 1256 va = tee_ram_va + tee_ram_initial_offs; 1257 for (n = 0; n < mem_map->count; n++) { 1258 map = mem_map->map + n; 1259 if (map_is_tee_ram(map) || 1260 map->type == MEM_AREA_PAGER_VASPACE) { 1261 assert(!(va & (map->region_size - 1))); 1262 assert(!(map->size & (map->region_size - 1))); 1263 map->va = va; 1264 if (ADD_OVERFLOW(va, map->size, &va)) 1265 return false; 1266 if (va >= BIT64(core_mmu_get_va_width())) 1267 return false; 1268 } 1269 } 1270 1271 if (tee_ram_at_top) { 1272 /* 1273 * Map non-tee ram regions at addresses lower than the tee 1274 * ram region. 1275 */ 1276 va = tee_ram_va; 1277 for (n = 0; n < mem_map->count; n++) { 1278 map = mem_map->map + n; 1279 map->attr = core_mmu_type_to_attr(map->type); 1280 if (map->va) 1281 continue; 1282 1283 if (!IS_ENABLED(CFG_WITH_LPAE) && 1284 va_is_secure != map_is_secure(map)) { 1285 va_is_secure = !va_is_secure; 1286 va = ROUNDDOWN(va, CORE_MMU_PGDIR_SIZE); 1287 } 1288 1289 if (SUB_OVERFLOW(va, map->size, &va)) 1290 return false; 1291 va = ROUNDDOWN(va, map->region_size); 1292 /* 1293 * Make sure that va is aligned with pa for 1294 * efficient pgdir mapping. Basically pa & 1295 * pgdir_mask should be == va & pgdir_mask 1296 */ 1297 if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { 1298 if (SUB_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, &va)) 1299 return false; 1300 va += (map->pa - va) & CORE_MMU_PGDIR_MASK; 1301 } 1302 map->va = va; 1303 } 1304 } else { 1305 /* 1306 * Map non-tee ram regions at addresses higher than the tee 1307 * ram region. 1308 */ 1309 for (n = 0; n < mem_map->count; n++) { 1310 map = mem_map->map + n; 1311 map->attr = core_mmu_type_to_attr(map->type); 1312 if (map->va) 1313 continue; 1314 1315 if (!IS_ENABLED(CFG_WITH_LPAE) && 1316 va_is_secure != map_is_secure(map)) { 1317 va_is_secure = !va_is_secure; 1318 if (ROUNDUP_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, 1319 &va)) 1320 return false; 1321 } 1322 1323 if (ROUNDUP_OVERFLOW(va, map->region_size, &va)) 1324 return false; 1325 /* 1326 * Make sure that va is aligned with pa for 1327 * efficient pgdir mapping. Basically pa & 1328 * pgdir_mask should be == va & pgdir_mask 1329 */ 1330 if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { 1331 vaddr_t offs = (map->pa - va) & 1332 CORE_MMU_PGDIR_MASK; 1333 1334 if (ADD_OVERFLOW(va, offs, &va)) 1335 return false; 1336 } 1337 1338 map->va = va; 1339 if (ADD_OVERFLOW(va, map->size, &va)) 1340 return false; 1341 if (va >= BIT64(core_mmu_get_va_width())) 1342 return false; 1343 } 1344 } 1345 1346 return true; 1347 } 1348 1349 static bool assign_mem_va(vaddr_t tee_ram_va, struct memory_map *mem_map) 1350 { 1351 bool tee_ram_at_top = place_tee_ram_at_top(tee_ram_va); 1352 1353 /* 1354 * Check that we're not overlapping with the user VA range. 1355 */ 1356 if (IS_ENABLED(CFG_WITH_LPAE)) { 1357 /* 1358 * User VA range is supposed to be defined after these 1359 * mappings have been established. 1360 */ 1361 assert(!core_mmu_user_va_range_is_defined()); 1362 } else { 1363 vaddr_t user_va_base = 0; 1364 size_t user_va_size = 0; 1365 1366 assert(core_mmu_user_va_range_is_defined()); 1367 core_mmu_get_user_va_range(&user_va_base, &user_va_size); 1368 if (tee_ram_va < (user_va_base + user_va_size)) 1369 return false; 1370 } 1371 1372 if (IS_ENABLED(CFG_WITH_PAGER)) { 1373 bool prefered_dir = core_mmu_prefer_tee_ram_at_top(tee_ram_va); 1374 1375 /* Try whole mapping covered by a single base xlat entry */ 1376 if (prefered_dir != tee_ram_at_top && 1377 assign_mem_va_dir(tee_ram_va, mem_map, prefered_dir)) 1378 return true; 1379 } 1380 1381 return assign_mem_va_dir(tee_ram_va, mem_map, tee_ram_at_top); 1382 } 1383 1384 static int cmp_init_mem_map(const void *a, const void *b) 1385 { 1386 const struct tee_mmap_region *mm_a = a; 1387 const struct tee_mmap_region *mm_b = b; 1388 int rc = 0; 1389 1390 rc = CMP_TRILEAN(mm_a->region_size, mm_b->region_size); 1391 if (!rc) 1392 rc = CMP_TRILEAN(mm_a->pa, mm_b->pa); 1393 /* 1394 * 32bit MMU descriptors cannot mix secure and non-secure mapping in 1395 * the same level2 table. Hence sort secure mapping from non-secure 1396 * mapping. 1397 */ 1398 if (!rc && !IS_ENABLED(CFG_WITH_LPAE)) 1399 rc = CMP_TRILEAN(map_is_secure(mm_a), map_is_secure(mm_b)); 1400 1401 return rc; 1402 } 1403 1404 static bool mem_map_add_id_map(struct memory_map *mem_map, 1405 vaddr_t id_map_start, vaddr_t id_map_end) 1406 { 1407 vaddr_t start = ROUNDDOWN(id_map_start, SMALL_PAGE_SIZE); 1408 vaddr_t end = ROUNDUP(id_map_end, SMALL_PAGE_SIZE); 1409 size_t len = end - start; 1410 size_t n = 0; 1411 1412 1413 for (n = 0; n < mem_map->count; n++) 1414 if (core_is_buffer_intersect(mem_map->map[n].va, 1415 mem_map->map[n].size, start, len)) 1416 return false; 1417 1418 grow_mem_map(mem_map); 1419 mem_map->map[mem_map->count - 1] = (struct tee_mmap_region){ 1420 .type = MEM_AREA_IDENTITY_MAP_RX, 1421 /* 1422 * Could use CORE_MMU_PGDIR_SIZE to potentially save a 1423 * translation table, at the increased risk of clashes with 1424 * the rest of the memory map. 1425 */ 1426 .region_size = SMALL_PAGE_SIZE, 1427 .pa = start, 1428 .va = start, 1429 .size = len, 1430 .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), 1431 }; 1432 1433 return true; 1434 } 1435 1436 static struct memory_map *init_mem_map(struct memory_map *mem_map, 1437 unsigned long seed, 1438 unsigned long *ret_offs) 1439 { 1440 /* 1441 * @id_map_start and @id_map_end describes a physical memory range 1442 * that must be mapped Read-Only eXecutable at identical virtual 1443 * addresses. 1444 */ 1445 vaddr_t id_map_start = (vaddr_t)__identity_map_init_start; 1446 vaddr_t id_map_end = (vaddr_t)__identity_map_init_end; 1447 vaddr_t start_addr = secure_only[0].paddr; 1448 unsigned long offs = 0; 1449 1450 collect_mem_ranges(mem_map); 1451 assign_mem_granularity(mem_map); 1452 1453 /* 1454 * To ease mapping and lower use of xlat tables, sort mapping 1455 * description moving small-page regions after the pgdir regions. 1456 */ 1457 qsort(mem_map->map, mem_map->count, sizeof(struct tee_mmap_region), 1458 cmp_init_mem_map); 1459 1460 if (IS_ENABLED(CFG_WITH_PAGER)) 1461 add_pager_vaspace(mem_map); 1462 1463 if (IS_ENABLED(CFG_CORE_ASLR) && seed) { 1464 vaddr_t base_addr = start_addr + seed; 1465 const unsigned int va_width = core_mmu_get_va_width(); 1466 const vaddr_t va_mask = GENMASK_64(va_width - 1, 1467 SMALL_PAGE_SHIFT); 1468 vaddr_t ba = base_addr; 1469 size_t n = 0; 1470 1471 for (n = 0; n < 3; n++) { 1472 if (n) 1473 ba = base_addr ^ BIT64(va_width - n); 1474 ba &= va_mask; 1475 if (assign_mem_va(ba, mem_map) && 1476 mem_map_add_id_map(mem_map, id_map_start, 1477 id_map_end)) { 1478 offs = ba - start_addr; 1479 DMSG("Mapping core at %#"PRIxVA" offs %#lx", 1480 ba, offs); 1481 goto out; 1482 } else { 1483 DMSG("Failed to map core at %#"PRIxVA, ba); 1484 } 1485 } 1486 EMSG("Failed to map core with seed %#lx", seed); 1487 } 1488 1489 if (!assign_mem_va(start_addr, mem_map)) 1490 panic(); 1491 1492 out: 1493 qsort(mem_map->map, mem_map->count, sizeof(struct tee_mmap_region), 1494 cmp_mmap_by_lower_va); 1495 1496 dump_mmap_table(mem_map); 1497 1498 *ret_offs = offs; 1499 return mem_map; 1500 } 1501 1502 static void check_mem_map(struct memory_map *mem_map) 1503 { 1504 struct tee_mmap_region *m = NULL; 1505 size_t n = 0; 1506 1507 for (n = 0; n < mem_map->count; n++) { 1508 m = mem_map->map + n; 1509 switch (m->type) { 1510 case MEM_AREA_TEE_RAM: 1511 case MEM_AREA_TEE_RAM_RX: 1512 case MEM_AREA_TEE_RAM_RO: 1513 case MEM_AREA_TEE_RAM_RW: 1514 case MEM_AREA_INIT_RAM_RX: 1515 case MEM_AREA_INIT_RAM_RO: 1516 case MEM_AREA_NEX_RAM_RW: 1517 case MEM_AREA_NEX_RAM_RO: 1518 case MEM_AREA_IDENTITY_MAP_RX: 1519 if (!pbuf_is_inside(secure_only, m->pa, m->size)) 1520 panic("TEE_RAM can't fit in secure_only"); 1521 break; 1522 case MEM_AREA_SEC_RAM_OVERALL: 1523 if (!pbuf_is_inside(secure_only, m->pa, m->size)) 1524 panic("SEC_RAM_OVERALL can't fit in secure_only"); 1525 break; 1526 case MEM_AREA_NSEC_SHM: 1527 if (!pbuf_is_inside(nsec_shared, m->pa, m->size)) 1528 panic("NS_SHM can't fit in nsec_shared"); 1529 break; 1530 case MEM_AREA_TEE_COHERENT: 1531 case MEM_AREA_TEE_ASAN: 1532 case MEM_AREA_IO_SEC: 1533 case MEM_AREA_IO_NSEC: 1534 case MEM_AREA_EXT_DT: 1535 case MEM_AREA_MANIFEST_DT: 1536 case MEM_AREA_TRANSFER_LIST: 1537 case MEM_AREA_RAM_SEC: 1538 case MEM_AREA_RAM_NSEC: 1539 case MEM_AREA_ROM_SEC: 1540 case MEM_AREA_RES_VASPACE: 1541 case MEM_AREA_SHM_VASPACE: 1542 case MEM_AREA_PAGER_VASPACE: 1543 break; 1544 default: 1545 EMSG("Uhandled memtype %d", m->type); 1546 panic(); 1547 } 1548 } 1549 } 1550 1551 /* 1552 * core_init_mmu_map() - init tee core default memory mapping 1553 * 1554 * This routine sets the static default TEE core mapping. If @seed is > 0 1555 * and configured with CFG_CORE_ASLR it will map tee core at a location 1556 * based on the seed and return the offset from the link address. 1557 * 1558 * If an error happened: core_init_mmu_map is expected to panic. 1559 * 1560 * Note: this function is weak just to make it possible to exclude it from 1561 * the unpaged area. 1562 */ 1563 void __weak core_init_mmu_map(unsigned long seed, struct core_mmu_config *cfg) 1564 { 1565 #ifndef CFG_NS_VIRTUALIZATION 1566 vaddr_t start = ROUNDDOWN((vaddr_t)__nozi_start, SMALL_PAGE_SIZE); 1567 #else 1568 vaddr_t start = ROUNDDOWN((vaddr_t)__vcore_nex_rw_start, 1569 SMALL_PAGE_SIZE); 1570 #endif 1571 vaddr_t len = ROUNDUP((vaddr_t)__nozi_end, SMALL_PAGE_SIZE) - start; 1572 struct tee_mmap_region tmp_mmap_region = { }; 1573 struct memory_map mem_map = { }; 1574 unsigned long offs = 0; 1575 1576 if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) && 1577 (core_mmu_tee_load_pa & SMALL_PAGE_MASK)) 1578 panic("OP-TEE load address is not page aligned"); 1579 1580 check_sec_nsec_mem_config(); 1581 1582 if (IS_ENABLED(CFG_BOOT_MEM)) { 1583 mem_map.alloc_count = CFG_MMAP_REGIONS; 1584 mem_map.map = boot_mem_alloc_tmp(mem_map.alloc_count * 1585 sizeof(*mem_map.map), 1586 alignof(*mem_map.map)); 1587 memory_map_realloc_func = boot_mem_realloc_memory_map; 1588 } else { 1589 mem_map = static_memory_map; 1590 } 1591 1592 static_memory_map = (struct memory_map){ 1593 .map = &tmp_mmap_region, 1594 .alloc_count = 1, 1595 .count = 1, 1596 }; 1597 /* 1598 * Add a entry covering the translation tables which will be 1599 * involved in some virt_to_phys() and phys_to_virt() conversions. 1600 */ 1601 static_memory_map.map[0] = (struct tee_mmap_region){ 1602 .type = MEM_AREA_TEE_RAM, 1603 .region_size = SMALL_PAGE_SIZE, 1604 .pa = start, 1605 .va = start, 1606 .size = len, 1607 .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), 1608 }; 1609 1610 init_mem_map(&mem_map, seed, &offs); 1611 1612 check_mem_map(&mem_map); 1613 core_init_mmu(&mem_map); 1614 dump_xlat_table(0x0, CORE_MMU_BASE_TABLE_LEVEL); 1615 core_init_mmu_regs(cfg); 1616 cfg->map_offset = offs; 1617 static_memory_map = mem_map; 1618 boot_mem_add_reloc(&static_memory_map.map); 1619 } 1620 1621 void core_mmu_save_mem_map(void) 1622 { 1623 if (IS_ENABLED(CFG_BOOT_MEM)) { 1624 size_t alloc_count = static_memory_map.count + 5; 1625 size_t elem_sz = sizeof(*static_memory_map.map); 1626 void *p = NULL; 1627 1628 p = nex_calloc(alloc_count, elem_sz); 1629 if (!p) 1630 panic(); 1631 memcpy(p, static_memory_map.map, 1632 static_memory_map.count * elem_sz); 1633 static_memory_map.map = p; 1634 static_memory_map.alloc_count = alloc_count; 1635 memory_map_realloc_func = heap_realloc_memory_map; 1636 } 1637 } 1638 1639 bool core_mmu_mattr_is_ok(uint32_t mattr) 1640 { 1641 /* 1642 * Keep in sync with core_mmu_lpae.c:mattr_to_desc and 1643 * core_mmu_v7.c:mattr_to_texcb 1644 */ 1645 1646 switch ((mattr >> TEE_MATTR_MEM_TYPE_SHIFT) & TEE_MATTR_MEM_TYPE_MASK) { 1647 case TEE_MATTR_MEM_TYPE_DEV: 1648 case TEE_MATTR_MEM_TYPE_STRONGLY_O: 1649 case TEE_MATTR_MEM_TYPE_CACHED: 1650 case TEE_MATTR_MEM_TYPE_TAGGED: 1651 return true; 1652 default: 1653 return false; 1654 } 1655 } 1656 1657 /* 1658 * test attributes of target physical buffer 1659 * 1660 * Flags: pbuf_is(SECURE, NOT_SECURE, RAM, IOMEM, KEYVAULT). 1661 * 1662 */ 1663 bool core_pbuf_is(uint32_t attr, paddr_t pbuf, size_t len) 1664 { 1665 struct tee_mmap_region *map; 1666 1667 /* Empty buffers complies with anything */ 1668 if (len == 0) 1669 return true; 1670 1671 switch (attr) { 1672 case CORE_MEM_SEC: 1673 return pbuf_is_inside(secure_only, pbuf, len); 1674 case CORE_MEM_NON_SEC: 1675 return pbuf_is_inside(nsec_shared, pbuf, len) || 1676 pbuf_is_nsec_ddr(pbuf, len); 1677 case CORE_MEM_TEE_RAM: 1678 return core_is_buffer_inside(pbuf, len, TEE_RAM_START, 1679 TEE_RAM_PH_SIZE); 1680 #ifdef CFG_CORE_RESERVED_SHM 1681 case CORE_MEM_NSEC_SHM: 1682 return core_is_buffer_inside(pbuf, len, TEE_SHMEM_START, 1683 TEE_SHMEM_SIZE); 1684 #endif 1685 case CORE_MEM_SDP_MEM: 1686 return pbuf_is_sdp_mem(pbuf, len); 1687 case CORE_MEM_CACHED: 1688 map = find_map_by_pa(pbuf); 1689 if (!map || !pbuf_inside_map_area(pbuf, len, map)) 1690 return false; 1691 return mattr_is_cached(map->attr); 1692 default: 1693 return false; 1694 } 1695 } 1696 1697 /* test attributes of target virtual buffer (in core mapping) */ 1698 bool core_vbuf_is(uint32_t attr, const void *vbuf, size_t len) 1699 { 1700 paddr_t p; 1701 1702 /* Empty buffers complies with anything */ 1703 if (len == 0) 1704 return true; 1705 1706 p = virt_to_phys((void *)vbuf); 1707 if (!p) 1708 return false; 1709 1710 return core_pbuf_is(attr, p, len); 1711 } 1712 1713 /* core_va2pa - teecore exported service */ 1714 static int __maybe_unused core_va2pa_helper(void *va, paddr_t *pa) 1715 { 1716 struct tee_mmap_region *map; 1717 1718 map = find_map_by_va(va); 1719 if (!va_is_in_map(map, (vaddr_t)va)) 1720 return -1; 1721 1722 /* 1723 * We can calculate PA for static map. Virtual address ranges 1724 * reserved to core dynamic mapping return a 'match' (return 0;) 1725 * together with an invalid null physical address. 1726 */ 1727 if (map->pa) 1728 *pa = map->pa + (vaddr_t)va - map->va; 1729 else 1730 *pa = 0; 1731 1732 return 0; 1733 } 1734 1735 static void *map_pa2va(struct tee_mmap_region *map, paddr_t pa, size_t len) 1736 { 1737 if (!pa_is_in_map(map, pa, len)) 1738 return NULL; 1739 1740 return (void *)(vaddr_t)(map->va + pa - map->pa); 1741 } 1742 1743 /* 1744 * teecore gets some memory area definitions 1745 */ 1746 void core_mmu_get_mem_by_type(enum teecore_memtypes type, vaddr_t *s, 1747 vaddr_t *e) 1748 { 1749 struct tee_mmap_region *map = find_map_by_type(type); 1750 1751 if (map) { 1752 *s = map->va; 1753 *e = map->va + map->size; 1754 } else { 1755 *s = 0; 1756 *e = 0; 1757 } 1758 } 1759 1760 enum teecore_memtypes core_mmu_get_type_by_pa(paddr_t pa) 1761 { 1762 struct tee_mmap_region *map = find_map_by_pa(pa); 1763 1764 if (!map) 1765 return MEM_AREA_MAXTYPE; 1766 return map->type; 1767 } 1768 1769 void core_mmu_set_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, 1770 paddr_t pa, uint32_t attr) 1771 { 1772 assert(idx < tbl_info->num_entries); 1773 core_mmu_set_entry_primitive(tbl_info->table, tbl_info->level, 1774 idx, pa, attr); 1775 } 1776 1777 void core_mmu_get_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, 1778 paddr_t *pa, uint32_t *attr) 1779 { 1780 assert(idx < tbl_info->num_entries); 1781 core_mmu_get_entry_primitive(tbl_info->table, tbl_info->level, 1782 idx, pa, attr); 1783 } 1784 1785 static void clear_region(struct core_mmu_table_info *tbl_info, 1786 struct tee_mmap_region *region) 1787 { 1788 unsigned int end = 0; 1789 unsigned int idx = 0; 1790 1791 /* va, len and pa should be block aligned */ 1792 assert(!core_mmu_get_block_offset(tbl_info, region->va)); 1793 assert(!core_mmu_get_block_offset(tbl_info, region->size)); 1794 assert(!core_mmu_get_block_offset(tbl_info, region->pa)); 1795 1796 idx = core_mmu_va2idx(tbl_info, region->va); 1797 end = core_mmu_va2idx(tbl_info, region->va + region->size); 1798 1799 while (idx < end) { 1800 core_mmu_set_entry(tbl_info, idx, 0, 0); 1801 idx++; 1802 } 1803 } 1804 1805 static void set_region(struct core_mmu_table_info *tbl_info, 1806 struct tee_mmap_region *region) 1807 { 1808 unsigned int end; 1809 unsigned int idx; 1810 paddr_t pa; 1811 1812 /* va, len and pa should be block aligned */ 1813 assert(!core_mmu_get_block_offset(tbl_info, region->va)); 1814 assert(!core_mmu_get_block_offset(tbl_info, region->size)); 1815 assert(!core_mmu_get_block_offset(tbl_info, region->pa)); 1816 1817 idx = core_mmu_va2idx(tbl_info, region->va); 1818 end = core_mmu_va2idx(tbl_info, region->va + region->size); 1819 pa = region->pa; 1820 1821 while (idx < end) { 1822 core_mmu_set_entry(tbl_info, idx, pa, region->attr); 1823 idx++; 1824 pa += BIT64(tbl_info->shift); 1825 } 1826 } 1827 1828 static void set_pg_region(struct core_mmu_table_info *dir_info, 1829 struct vm_region *region, struct pgt **pgt, 1830 struct core_mmu_table_info *pg_info) 1831 { 1832 struct tee_mmap_region r = { 1833 .va = region->va, 1834 .size = region->size, 1835 .attr = region->attr, 1836 }; 1837 vaddr_t end = r.va + r.size; 1838 uint32_t pgt_attr = (r.attr & TEE_MATTR_SECURE) | TEE_MATTR_TABLE; 1839 1840 while (r.va < end) { 1841 if (!pg_info->table || 1842 r.va >= (pg_info->va_base + CORE_MMU_PGDIR_SIZE)) { 1843 /* 1844 * We're assigning a new translation table. 1845 */ 1846 unsigned int idx; 1847 1848 /* Virtual addresses must grow */ 1849 assert(r.va > pg_info->va_base); 1850 1851 idx = core_mmu_va2idx(dir_info, r.va); 1852 pg_info->va_base = core_mmu_idx2va(dir_info, idx); 1853 1854 /* 1855 * Advance pgt to va_base, note that we may need to 1856 * skip multiple page tables if there are large 1857 * holes in the vm map. 1858 */ 1859 while ((*pgt)->vabase < pg_info->va_base) { 1860 *pgt = SLIST_NEXT(*pgt, link); 1861 /* We should have allocated enough */ 1862 assert(*pgt); 1863 } 1864 assert((*pgt)->vabase == pg_info->va_base); 1865 pg_info->table = (*pgt)->tbl; 1866 1867 core_mmu_set_entry(dir_info, idx, 1868 virt_to_phys(pg_info->table), 1869 pgt_attr); 1870 } 1871 1872 r.size = MIN(CORE_MMU_PGDIR_SIZE - (r.va - pg_info->va_base), 1873 end - r.va); 1874 1875 if (!(*pgt)->populated && !mobj_is_paged(region->mobj)) { 1876 size_t granule = BIT(pg_info->shift); 1877 size_t offset = r.va - region->va + region->offset; 1878 1879 r.size = MIN(r.size, 1880 mobj_get_phys_granule(region->mobj)); 1881 r.size = ROUNDUP(r.size, SMALL_PAGE_SIZE); 1882 1883 if (mobj_get_pa(region->mobj, offset, granule, 1884 &r.pa) != TEE_SUCCESS) 1885 panic("Failed to get PA of unpaged mobj"); 1886 set_region(pg_info, &r); 1887 } 1888 r.va += r.size; 1889 } 1890 } 1891 1892 static bool can_map_at_level(paddr_t paddr, vaddr_t vaddr, 1893 size_t size_left, paddr_t block_size, 1894 struct tee_mmap_region *mm) 1895 { 1896 /* VA and PA are aligned to block size at current level */ 1897 if ((vaddr | paddr) & (block_size - 1)) 1898 return false; 1899 1900 /* Remainder fits into block at current level */ 1901 if (size_left < block_size) 1902 return false; 1903 1904 /* 1905 * The required block size of the region is compatible with the 1906 * block size of the current level. 1907 */ 1908 if (mm->region_size < block_size) 1909 return false; 1910 1911 #ifdef CFG_WITH_PAGER 1912 /* 1913 * If pager is enabled, we need to map TEE RAM and the whole pager 1914 * regions with small pages only 1915 */ 1916 if ((map_is_tee_ram(mm) || mm->type == MEM_AREA_PAGER_VASPACE) && 1917 block_size != SMALL_PAGE_SIZE) 1918 return false; 1919 #endif 1920 1921 return true; 1922 } 1923 1924 void core_mmu_map_region(struct mmu_partition *prtn, struct tee_mmap_region *mm) 1925 { 1926 struct core_mmu_table_info tbl_info; 1927 unsigned int idx; 1928 vaddr_t vaddr = mm->va; 1929 paddr_t paddr = mm->pa; 1930 ssize_t size_left = mm->size; 1931 unsigned int level; 1932 bool table_found; 1933 uint32_t old_attr; 1934 1935 assert(!((vaddr | paddr) & SMALL_PAGE_MASK)); 1936 1937 while (size_left > 0) { 1938 level = CORE_MMU_BASE_TABLE_LEVEL; 1939 1940 while (true) { 1941 paddr_t block_size = 0; 1942 1943 assert(core_mmu_level_in_range(level)); 1944 1945 table_found = core_mmu_find_table(prtn, vaddr, level, 1946 &tbl_info); 1947 if (!table_found) 1948 panic("can't find table for mapping"); 1949 1950 block_size = BIT64(tbl_info.shift); 1951 1952 idx = core_mmu_va2idx(&tbl_info, vaddr); 1953 if (!can_map_at_level(paddr, vaddr, size_left, 1954 block_size, mm)) { 1955 bool secure = mm->attr & TEE_MATTR_SECURE; 1956 1957 /* 1958 * This part of the region can't be mapped at 1959 * this level. Need to go deeper. 1960 */ 1961 if (!core_mmu_entry_to_finer_grained(&tbl_info, 1962 idx, 1963 secure)) 1964 panic("Can't divide MMU entry"); 1965 level = tbl_info.next_level; 1966 continue; 1967 } 1968 1969 /* We can map part of the region at current level */ 1970 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 1971 if (old_attr) 1972 panic("Page is already mapped"); 1973 1974 core_mmu_set_entry(&tbl_info, idx, paddr, mm->attr); 1975 paddr += block_size; 1976 vaddr += block_size; 1977 size_left -= block_size; 1978 1979 break; 1980 } 1981 } 1982 } 1983 1984 TEE_Result core_mmu_map_pages(vaddr_t vstart, paddr_t *pages, size_t num_pages, 1985 enum teecore_memtypes memtype) 1986 { 1987 TEE_Result ret; 1988 struct core_mmu_table_info tbl_info; 1989 struct tee_mmap_region *mm; 1990 unsigned int idx; 1991 uint32_t old_attr; 1992 uint32_t exceptions; 1993 vaddr_t vaddr = vstart; 1994 size_t i; 1995 bool secure; 1996 1997 assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); 1998 1999 secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; 2000 2001 if (vaddr & SMALL_PAGE_MASK) 2002 return TEE_ERROR_BAD_PARAMETERS; 2003 2004 exceptions = mmu_lock(); 2005 2006 mm = find_map_by_va((void *)vaddr); 2007 if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) 2008 panic("VA does not belong to any known mm region"); 2009 2010 if (!core_mmu_is_dynamic_vaspace(mm)) 2011 panic("Trying to map into static region"); 2012 2013 for (i = 0; i < num_pages; i++) { 2014 if (pages[i] & SMALL_PAGE_MASK) { 2015 ret = TEE_ERROR_BAD_PARAMETERS; 2016 goto err; 2017 } 2018 2019 while (true) { 2020 if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, 2021 &tbl_info)) 2022 panic("Can't find pagetable for vaddr "); 2023 2024 idx = core_mmu_va2idx(&tbl_info, vaddr); 2025 if (tbl_info.shift == SMALL_PAGE_SHIFT) 2026 break; 2027 2028 /* This is supertable. Need to divide it. */ 2029 if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, 2030 secure)) 2031 panic("Failed to spread pgdir on small tables"); 2032 } 2033 2034 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 2035 if (old_attr) 2036 panic("Page is already mapped"); 2037 2038 core_mmu_set_entry(&tbl_info, idx, pages[i], 2039 core_mmu_type_to_attr(memtype)); 2040 vaddr += SMALL_PAGE_SIZE; 2041 } 2042 2043 /* 2044 * Make sure all the changes to translation tables are visible 2045 * before returning. TLB doesn't need to be invalidated as we are 2046 * guaranteed that there's no valid mapping in this range. 2047 */ 2048 core_mmu_table_write_barrier(); 2049 mmu_unlock(exceptions); 2050 2051 return TEE_SUCCESS; 2052 err: 2053 mmu_unlock(exceptions); 2054 2055 if (i) 2056 core_mmu_unmap_pages(vstart, i); 2057 2058 return ret; 2059 } 2060 2061 TEE_Result core_mmu_map_contiguous_pages(vaddr_t vstart, paddr_t pstart, 2062 size_t num_pages, 2063 enum teecore_memtypes memtype) 2064 { 2065 struct core_mmu_table_info tbl_info = { }; 2066 struct tee_mmap_region *mm = NULL; 2067 unsigned int idx = 0; 2068 uint32_t old_attr = 0; 2069 uint32_t exceptions = 0; 2070 vaddr_t vaddr = vstart; 2071 paddr_t paddr = pstart; 2072 size_t i = 0; 2073 bool secure = false; 2074 2075 assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); 2076 2077 secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; 2078 2079 if ((vaddr | paddr) & SMALL_PAGE_MASK) 2080 return TEE_ERROR_BAD_PARAMETERS; 2081 2082 exceptions = mmu_lock(); 2083 2084 mm = find_map_by_va((void *)vaddr); 2085 if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) 2086 panic("VA does not belong to any known mm region"); 2087 2088 if (!core_mmu_is_dynamic_vaspace(mm)) 2089 panic("Trying to map into static region"); 2090 2091 for (i = 0; i < num_pages; i++) { 2092 while (true) { 2093 if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, 2094 &tbl_info)) 2095 panic("Can't find pagetable for vaddr "); 2096 2097 idx = core_mmu_va2idx(&tbl_info, vaddr); 2098 if (tbl_info.shift == SMALL_PAGE_SHIFT) 2099 break; 2100 2101 /* This is supertable. Need to divide it. */ 2102 if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, 2103 secure)) 2104 panic("Failed to spread pgdir on small tables"); 2105 } 2106 2107 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 2108 if (old_attr) 2109 panic("Page is already mapped"); 2110 2111 core_mmu_set_entry(&tbl_info, idx, paddr, 2112 core_mmu_type_to_attr(memtype)); 2113 paddr += SMALL_PAGE_SIZE; 2114 vaddr += SMALL_PAGE_SIZE; 2115 } 2116 2117 /* 2118 * Make sure all the changes to translation tables are visible 2119 * before returning. TLB doesn't need to be invalidated as we are 2120 * guaranteed that there's no valid mapping in this range. 2121 */ 2122 core_mmu_table_write_barrier(); 2123 mmu_unlock(exceptions); 2124 2125 return TEE_SUCCESS; 2126 } 2127 2128 static bool mem_range_is_in_vcore_free(vaddr_t vstart, size_t num_pages) 2129 { 2130 return core_is_buffer_inside(vstart, num_pages * SMALL_PAGE_SIZE, 2131 VCORE_FREE_PA, VCORE_FREE_SZ); 2132 } 2133 2134 void core_mmu_unmap_pages(vaddr_t vstart, size_t num_pages) 2135 { 2136 struct core_mmu_table_info tbl_info; 2137 struct tee_mmap_region *mm; 2138 size_t i; 2139 unsigned int idx; 2140 uint32_t exceptions; 2141 2142 exceptions = mmu_lock(); 2143 2144 mm = find_map_by_va((void *)vstart); 2145 if (!mm || !va_is_in_map(mm, vstart + num_pages * SMALL_PAGE_SIZE - 1)) 2146 panic("VA does not belong to any known mm region"); 2147 2148 if (!core_mmu_is_dynamic_vaspace(mm) && 2149 !mem_range_is_in_vcore_free(vstart, num_pages)) 2150 panic("Trying to unmap static region"); 2151 2152 for (i = 0; i < num_pages; i++, vstart += SMALL_PAGE_SIZE) { 2153 if (!core_mmu_find_table(NULL, vstart, UINT_MAX, &tbl_info)) 2154 panic("Can't find pagetable"); 2155 2156 if (tbl_info.shift != SMALL_PAGE_SHIFT) 2157 panic("Invalid pagetable level"); 2158 2159 idx = core_mmu_va2idx(&tbl_info, vstart); 2160 core_mmu_set_entry(&tbl_info, idx, 0, 0); 2161 } 2162 tlbi_all(); 2163 2164 mmu_unlock(exceptions); 2165 } 2166 2167 void core_mmu_populate_user_map(struct core_mmu_table_info *dir_info, 2168 struct user_mode_ctx *uctx) 2169 { 2170 struct core_mmu_table_info pg_info = { }; 2171 struct pgt_cache *pgt_cache = &uctx->pgt_cache; 2172 struct pgt *pgt = NULL; 2173 struct pgt *p = NULL; 2174 struct vm_region *r = NULL; 2175 2176 if (TAILQ_EMPTY(&uctx->vm_info.regions)) 2177 return; /* Nothing to map */ 2178 2179 /* 2180 * Allocate all page tables in advance. 2181 */ 2182 pgt_get_all(uctx); 2183 pgt = SLIST_FIRST(pgt_cache); 2184 2185 core_mmu_set_info_table(&pg_info, dir_info->next_level, 0, NULL); 2186 2187 TAILQ_FOREACH(r, &uctx->vm_info.regions, link) 2188 set_pg_region(dir_info, r, &pgt, &pg_info); 2189 /* Record that the translation tables now are populated. */ 2190 SLIST_FOREACH(p, pgt_cache, link) { 2191 p->populated = true; 2192 if (p == pgt) 2193 break; 2194 } 2195 assert(p == pgt); 2196 } 2197 2198 TEE_Result core_mmu_remove_mapping(enum teecore_memtypes type, void *addr, 2199 size_t len) 2200 { 2201 struct core_mmu_table_info tbl_info = { }; 2202 struct tee_mmap_region *res_map = NULL; 2203 struct tee_mmap_region *map = NULL; 2204 paddr_t pa = virt_to_phys(addr); 2205 size_t granule = 0; 2206 ptrdiff_t i = 0; 2207 paddr_t p = 0; 2208 size_t l = 0; 2209 2210 map = find_map_by_type_and_pa(type, pa, len); 2211 if (!map) 2212 return TEE_ERROR_GENERIC; 2213 2214 res_map = find_map_by_type(MEM_AREA_RES_VASPACE); 2215 if (!res_map) 2216 return TEE_ERROR_GENERIC; 2217 if (!core_mmu_find_table(NULL, res_map->va, UINT_MAX, &tbl_info)) 2218 return TEE_ERROR_GENERIC; 2219 granule = BIT(tbl_info.shift); 2220 2221 if (map < static_memory_map.map || 2222 map >= static_memory_map.map + static_memory_map.count) 2223 return TEE_ERROR_GENERIC; 2224 i = map - static_memory_map.map; 2225 2226 /* Check that we have a full match */ 2227 p = ROUNDDOWN(pa, granule); 2228 l = ROUNDUP(len + pa - p, granule); 2229 if (map->pa != p || map->size != l) 2230 return TEE_ERROR_GENERIC; 2231 2232 clear_region(&tbl_info, map); 2233 tlbi_all(); 2234 2235 /* If possible remove the va range from res_map */ 2236 if (res_map->va - map->size == map->va) { 2237 res_map->va -= map->size; 2238 res_map->size += map->size; 2239 } 2240 2241 /* Remove the entry. */ 2242 rem_array_elem(static_memory_map.map, static_memory_map.count, 2243 sizeof(*static_memory_map.map), i); 2244 static_memory_map.count--; 2245 2246 return TEE_SUCCESS; 2247 } 2248 2249 struct tee_mmap_region * 2250 core_mmu_find_mapping_exclusive(enum teecore_memtypes type, size_t len) 2251 { 2252 struct memory_map *mem_map = get_memory_map(); 2253 struct tee_mmap_region *map_found = NULL; 2254 size_t n = 0; 2255 2256 if (!len) 2257 return NULL; 2258 2259 for (n = 0; n < mem_map->count; n++) { 2260 if (mem_map->map[n].type != type) 2261 continue; 2262 2263 if (map_found) 2264 return NULL; 2265 2266 map_found = mem_map->map + n; 2267 } 2268 2269 if (!map_found || map_found->size < len) 2270 return NULL; 2271 2272 return map_found; 2273 } 2274 2275 void *core_mmu_add_mapping(enum teecore_memtypes type, paddr_t addr, size_t len) 2276 { 2277 struct memory_map *mem_map = &static_memory_map; 2278 struct core_mmu_table_info tbl_info = { }; 2279 struct tee_mmap_region *map = NULL; 2280 size_t granule = 0; 2281 paddr_t p = 0; 2282 size_t l = 0; 2283 2284 if (!len) 2285 return NULL; 2286 2287 if (!core_mmu_check_end_pa(addr, len)) 2288 return NULL; 2289 2290 /* Check if the memory is already mapped */ 2291 map = find_map_by_type_and_pa(type, addr, len); 2292 if (map && pbuf_inside_map_area(addr, len, map)) 2293 return (void *)(vaddr_t)(map->va + addr - map->pa); 2294 2295 /* Find the reserved va space used for late mappings */ 2296 map = find_map_by_type(MEM_AREA_RES_VASPACE); 2297 if (!map) 2298 return NULL; 2299 2300 if (!core_mmu_find_table(NULL, map->va, UINT_MAX, &tbl_info)) 2301 return NULL; 2302 2303 granule = BIT64(tbl_info.shift); 2304 p = ROUNDDOWN(addr, granule); 2305 l = ROUNDUP(len + addr - p, granule); 2306 2307 /* Ban overflowing virtual addresses */ 2308 if (map->size < l) 2309 return NULL; 2310 2311 /* 2312 * Something is wrong, we can't fit the va range into the selected 2313 * table. The reserved va range is possibly missaligned with 2314 * granule. 2315 */ 2316 if (core_mmu_va2idx(&tbl_info, map->va + len) >= tbl_info.num_entries) 2317 return NULL; 2318 2319 if (static_memory_map.count >= static_memory_map.alloc_count) 2320 return NULL; 2321 2322 mem_map->map[mem_map->count] = (struct tee_mmap_region){ 2323 .va = map->va, 2324 .size = l, 2325 .type = type, 2326 .region_size = granule, 2327 .attr = core_mmu_type_to_attr(type), 2328 .pa = p, 2329 }; 2330 map->va += l; 2331 map->size -= l; 2332 map = mem_map->map + mem_map->count; 2333 mem_map->count++; 2334 2335 set_region(&tbl_info, map); 2336 2337 /* Make sure the new entry is visible before continuing. */ 2338 core_mmu_table_write_barrier(); 2339 2340 return (void *)(vaddr_t)(map->va + addr - map->pa); 2341 } 2342 2343 #ifdef CFG_WITH_PAGER 2344 static vaddr_t get_linear_map_end_va(void) 2345 { 2346 /* this is synced with the generic linker file kern.ld.S */ 2347 return (vaddr_t)__heap2_end; 2348 } 2349 2350 static paddr_t get_linear_map_end_pa(void) 2351 { 2352 return get_linear_map_end_va() - boot_mmu_config.map_offset; 2353 } 2354 #endif 2355 2356 #if defined(CFG_TEE_CORE_DEBUG) 2357 static void check_pa_matches_va(void *va, paddr_t pa) 2358 { 2359 TEE_Result res = TEE_ERROR_GENERIC; 2360 vaddr_t v = (vaddr_t)va; 2361 paddr_t p = 0; 2362 struct core_mmu_table_info ti __maybe_unused = { }; 2363 2364 if (core_mmu_user_va_range_is_defined()) { 2365 vaddr_t user_va_base = 0; 2366 size_t user_va_size = 0; 2367 2368 core_mmu_get_user_va_range(&user_va_base, &user_va_size); 2369 if (v >= user_va_base && 2370 v <= (user_va_base - 1 + user_va_size)) { 2371 if (!core_mmu_user_mapping_is_active()) { 2372 if (pa) 2373 panic("issue in linear address space"); 2374 return; 2375 } 2376 2377 res = vm_va2pa(to_user_mode_ctx(thread_get_tsd()->ctx), 2378 va, &p); 2379 if (res == TEE_ERROR_NOT_SUPPORTED) 2380 return; 2381 if (res == TEE_SUCCESS && pa != p) 2382 panic("bad pa"); 2383 if (res != TEE_SUCCESS && pa) 2384 panic("false pa"); 2385 return; 2386 } 2387 } 2388 #ifdef CFG_WITH_PAGER 2389 if (is_unpaged(va)) { 2390 if (v - boot_mmu_config.map_offset != pa) 2391 panic("issue in linear address space"); 2392 return; 2393 } 2394 2395 if (tee_pager_get_table_info(v, &ti)) { 2396 uint32_t a; 2397 2398 /* 2399 * Lookups in the page table managed by the pager is 2400 * dangerous for addresses in the paged area as those pages 2401 * changes all the time. But some ranges are safe, 2402 * rw-locked areas when the page is populated for instance. 2403 */ 2404 core_mmu_get_entry(&ti, core_mmu_va2idx(&ti, v), &p, &a); 2405 if (a & TEE_MATTR_VALID_BLOCK) { 2406 paddr_t mask = BIT64(ti.shift) - 1; 2407 2408 p |= v & mask; 2409 if (pa != p) 2410 panic(); 2411 } else { 2412 if (pa) 2413 panic(); 2414 } 2415 return; 2416 } 2417 #endif 2418 2419 if (!core_va2pa_helper(va, &p)) { 2420 /* Verfiy only the static mapping (case non null phys addr) */ 2421 if (p && pa != p) { 2422 DMSG("va %p maps 0x%" PRIxPA ", expect 0x%" PRIxPA, 2423 va, p, pa); 2424 panic(); 2425 } 2426 } else { 2427 if (pa) { 2428 DMSG("va %p unmapped, expect 0x%" PRIxPA, va, pa); 2429 panic(); 2430 } 2431 } 2432 } 2433 #else 2434 static void check_pa_matches_va(void *va __unused, paddr_t pa __unused) 2435 { 2436 } 2437 #endif 2438 2439 paddr_t virt_to_phys(void *va) 2440 { 2441 paddr_t pa = 0; 2442 2443 if (!arch_va2pa_helper(va, &pa)) 2444 pa = 0; 2445 check_pa_matches_va(memtag_strip_tag(va), pa); 2446 return pa; 2447 } 2448 2449 /* 2450 * Don't use check_va_matches_pa() for RISC-V, as its callee 2451 * arch_va2pa_helper() will call it eventually, this creates 2452 * indirect recursion and can lead to a stack overflow. 2453 * Moreover, if arch_va2pa_helper() returns true, it implies 2454 * the va2pa mapping is matched, no need to check it again. 2455 */ 2456 #if defined(CFG_TEE_CORE_DEBUG) && !defined(__riscv) 2457 static void check_va_matches_pa(paddr_t pa, void *va) 2458 { 2459 paddr_t p = 0; 2460 2461 if (!va) 2462 return; 2463 2464 p = virt_to_phys(va); 2465 if (p != pa) { 2466 DMSG("va %p maps 0x%" PRIxPA " expect 0x%" PRIxPA, va, p, pa); 2467 panic(); 2468 } 2469 } 2470 #else 2471 static void check_va_matches_pa(paddr_t pa __unused, void *va __unused) 2472 { 2473 } 2474 #endif 2475 2476 static void *phys_to_virt_ts_vaspace(paddr_t pa, size_t len) 2477 { 2478 if (!core_mmu_user_mapping_is_active()) 2479 return NULL; 2480 2481 return vm_pa2va(to_user_mode_ctx(thread_get_tsd()->ctx), pa, len); 2482 } 2483 2484 #ifdef CFG_WITH_PAGER 2485 static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) 2486 { 2487 paddr_t end_pa = 0; 2488 2489 if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) 2490 return NULL; 2491 2492 if (pa >= TEE_LOAD_ADDR && pa < get_linear_map_end_pa()) { 2493 if (end_pa > get_linear_map_end_pa()) 2494 return NULL; 2495 return (void *)(vaddr_t)(pa + boot_mmu_config.map_offset); 2496 } 2497 2498 return tee_pager_phys_to_virt(pa, len); 2499 } 2500 #else 2501 static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) 2502 { 2503 struct tee_mmap_region *mmap = NULL; 2504 2505 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM, pa, len); 2506 if (!mmap) 2507 mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RW, pa, len); 2508 if (!mmap) 2509 mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RO, pa, len); 2510 if (!mmap) 2511 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RW, pa, len); 2512 if (!mmap) 2513 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RO, pa, len); 2514 if (!mmap) 2515 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RX, pa, len); 2516 /* 2517 * Note that MEM_AREA_INIT_RAM_RO and MEM_AREA_INIT_RAM_RX are only 2518 * used with pager and not needed here. 2519 */ 2520 return map_pa2va(mmap, pa, len); 2521 } 2522 #endif 2523 2524 void *phys_to_virt(paddr_t pa, enum teecore_memtypes m, size_t len) 2525 { 2526 void *va = NULL; 2527 2528 switch (m) { 2529 case MEM_AREA_TS_VASPACE: 2530 va = phys_to_virt_ts_vaspace(pa, len); 2531 break; 2532 case MEM_AREA_TEE_RAM: 2533 case MEM_AREA_TEE_RAM_RX: 2534 case MEM_AREA_TEE_RAM_RO: 2535 case MEM_AREA_TEE_RAM_RW: 2536 case MEM_AREA_NEX_RAM_RO: 2537 case MEM_AREA_NEX_RAM_RW: 2538 va = phys_to_virt_tee_ram(pa, len); 2539 break; 2540 case MEM_AREA_SHM_VASPACE: 2541 /* Find VA from PA in dynamic SHM is not yet supported */ 2542 va = NULL; 2543 break; 2544 default: 2545 va = map_pa2va(find_map_by_type_and_pa(m, pa, len), pa, len); 2546 } 2547 if (m != MEM_AREA_SEC_RAM_OVERALL) 2548 check_va_matches_pa(pa, va); 2549 return va; 2550 } 2551 2552 void *phys_to_virt_io(paddr_t pa, size_t len) 2553 { 2554 struct tee_mmap_region *map = NULL; 2555 void *va = NULL; 2556 2557 map = find_map_by_type_and_pa(MEM_AREA_IO_SEC, pa, len); 2558 if (!map) 2559 map = find_map_by_type_and_pa(MEM_AREA_IO_NSEC, pa, len); 2560 if (!map) 2561 return NULL; 2562 va = map_pa2va(map, pa, len); 2563 check_va_matches_pa(pa, va); 2564 return va; 2565 } 2566 2567 vaddr_t core_mmu_get_va(paddr_t pa, enum teecore_memtypes type, size_t len) 2568 { 2569 if (cpu_mmu_enabled()) 2570 return (vaddr_t)phys_to_virt(pa, type, len); 2571 2572 return (vaddr_t)pa; 2573 } 2574 2575 #ifdef CFG_WITH_PAGER 2576 bool is_unpaged(const void *va) 2577 { 2578 vaddr_t v = (vaddr_t)va; 2579 2580 return v >= VCORE_START_VA && v < get_linear_map_end_va(); 2581 } 2582 #endif 2583 2584 #ifdef CFG_NS_VIRTUALIZATION 2585 bool is_nexus(const void *va) 2586 { 2587 vaddr_t v = (vaddr_t)va; 2588 2589 return v >= VCORE_START_VA && v < VCORE_NEX_RW_PA + VCORE_NEX_RW_SZ; 2590 } 2591 #endif 2592 2593 vaddr_t io_pa_or_va(struct io_pa_va *p, size_t len) 2594 { 2595 assert(p->pa); 2596 if (cpu_mmu_enabled()) { 2597 if (!p->va) 2598 p->va = (vaddr_t)phys_to_virt_io(p->pa, len); 2599 assert(p->va); 2600 return p->va; 2601 } 2602 return p->pa; 2603 } 2604 2605 vaddr_t io_pa_or_va_secure(struct io_pa_va *p, size_t len) 2606 { 2607 assert(p->pa); 2608 if (cpu_mmu_enabled()) { 2609 if (!p->va) 2610 p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_SEC, 2611 len); 2612 assert(p->va); 2613 return p->va; 2614 } 2615 return p->pa; 2616 } 2617 2618 vaddr_t io_pa_or_va_nsec(struct io_pa_va *p, size_t len) 2619 { 2620 assert(p->pa); 2621 if (cpu_mmu_enabled()) { 2622 if (!p->va) 2623 p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_NSEC, 2624 len); 2625 assert(p->va); 2626 return p->va; 2627 } 2628 return p->pa; 2629 } 2630 2631 #ifdef CFG_CORE_RESERVED_SHM 2632 static TEE_Result teecore_init_pub_ram(void) 2633 { 2634 vaddr_t s = 0; 2635 vaddr_t e = 0; 2636 2637 /* get virtual addr/size of NSec shared mem allocated from teecore */ 2638 core_mmu_get_mem_by_type(MEM_AREA_NSEC_SHM, &s, &e); 2639 2640 if (s >= e || s & SMALL_PAGE_MASK || e & SMALL_PAGE_MASK) 2641 panic("invalid PUB RAM"); 2642 2643 /* extra check: we could rely on core_mmu_get_mem_by_type() */ 2644 if (!tee_vbuf_is_non_sec(s, e - s)) 2645 panic("PUB RAM is not non-secure"); 2646 2647 #ifdef CFG_PL310 2648 /* Allocate statically the l2cc mutex */ 2649 tee_l2cc_store_mutex_boot_pa(virt_to_phys((void *)s)); 2650 s += sizeof(uint32_t); /* size of a pl310 mutex */ 2651 s = ROUNDUP(s, SMALL_PAGE_SIZE); /* keep required alignment */ 2652 #endif 2653 2654 default_nsec_shm_paddr = virt_to_phys((void *)s); 2655 default_nsec_shm_size = e - s; 2656 2657 return TEE_SUCCESS; 2658 } 2659 early_init(teecore_init_pub_ram); 2660 #endif /*CFG_CORE_RESERVED_SHM*/ 2661 2662 static void __maybe_unused carve_out_core_mem(paddr_t pa, paddr_t end_pa) 2663 { 2664 tee_mm_entry_t *mm __maybe_unused = NULL; 2665 2666 DMSG("%#"PRIxPA" .. %#"PRIxPA, pa, end_pa); 2667 mm = phys_mem_alloc2(pa, end_pa - pa); 2668 assert(mm); 2669 } 2670 2671 void core_mmu_init_phys_mem(void) 2672 { 2673 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 2674 paddr_t b1 = 0; 2675 paddr_size_t s1 = 0; 2676 2677 static_assert(ARRAY_SIZE(secure_only) <= 2); 2678 2679 if (ARRAY_SIZE(secure_only) == 2) { 2680 b1 = secure_only[1].paddr; 2681 s1 = secure_only[1].size; 2682 } 2683 virt_init_memory(&static_memory_map, secure_only[0].paddr, 2684 secure_only[0].size, b1, s1); 2685 } else { 2686 #ifdef CFG_WITH_PAGER 2687 /* 2688 * The pager uses all core memory so there's no need to add 2689 * it to the pool. 2690 */ 2691 static_assert(ARRAY_SIZE(secure_only) == 2); 2692 phys_mem_init(0, 0, secure_only[1].paddr, secure_only[1].size); 2693 #else /*!CFG_WITH_PAGER*/ 2694 size_t align = BIT(CORE_MMU_USER_CODE_SHIFT); 2695 paddr_t end_pa = 0; 2696 size_t size = 0; 2697 paddr_t ps = 0; 2698 paddr_t pa = 0; 2699 2700 static_assert(ARRAY_SIZE(secure_only) <= 2); 2701 if (ARRAY_SIZE(secure_only) == 2) { 2702 ps = secure_only[1].paddr; 2703 size = secure_only[1].size; 2704 } 2705 phys_mem_init(secure_only[0].paddr, secure_only[0].size, 2706 ps, size); 2707 2708 /* 2709 * The VCORE macros are relocatable so we need to translate 2710 * the addresses now that the MMU is enabled. 2711 */ 2712 end_pa = vaddr_to_phys(ROUNDUP(VCORE_FREE_END_PA, 2713 align) - 1) + 1; 2714 /* Carve out the part used by OP-TEE core */ 2715 carve_out_core_mem(vaddr_to_phys(VCORE_UNPG_RX_PA), end_pa); 2716 if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS)) { 2717 pa = vaddr_to_phys(ROUNDUP(ASAN_MAP_PA, align)); 2718 carve_out_core_mem(pa, pa + ASAN_MAP_SZ); 2719 } 2720 2721 /* Carve out test SDP memory */ 2722 #ifdef TEE_SDP_TEST_MEM_BASE 2723 if (TEE_SDP_TEST_MEM_SIZE) { 2724 pa = vaddr_to_phys(TEE_SDP_TEST_MEM_BASE); 2725 carve_out_core_mem(pa, pa + TEE_SDP_TEST_MEM_SIZE); 2726 } 2727 #endif 2728 #endif /*!CFG_WITH_PAGER*/ 2729 } 2730 } 2731