1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016, 2022 Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * Copyright (c) 2022, Arm Limited and Contributors. All rights reserved. 6 */ 7 8 #include <assert.h> 9 #include <config.h> 10 #include <kernel/boot.h> 11 #include <kernel/dt.h> 12 #include <kernel/linker.h> 13 #include <kernel/panic.h> 14 #include <kernel/spinlock.h> 15 #include <kernel/tee_l2cc_mutex.h> 16 #include <kernel/tee_misc.h> 17 #include <kernel/tlb_helpers.h> 18 #include <kernel/user_mode_ctx.h> 19 #include <kernel/virtualization.h> 20 #include <libfdt.h> 21 #include <memtag.h> 22 #include <mm/core_memprot.h> 23 #include <mm/core_mmu.h> 24 #include <mm/mobj.h> 25 #include <mm/pgt_cache.h> 26 #include <mm/tee_pager.h> 27 #include <mm/vm.h> 28 #include <platform_config.h> 29 #include <stdalign.h> 30 #include <string.h> 31 #include <trace.h> 32 #include <util.h> 33 34 #ifndef DEBUG_XLAT_TABLE 35 #define DEBUG_XLAT_TABLE 0 36 #endif 37 38 #define SHM_VASPACE_SIZE (1024 * 1024 * 32) 39 40 /* Physical Secure DDR pool */ 41 tee_mm_pool_t tee_mm_sec_ddr; 42 43 /* Virtual memory pool for core mappings */ 44 tee_mm_pool_t core_virt_mem_pool; 45 46 /* Virtual memory pool for shared memory mappings */ 47 tee_mm_pool_t core_virt_shm_pool; 48 49 #ifdef CFG_CORE_PHYS_RELOCATABLE 50 unsigned long core_mmu_tee_load_pa __nex_bss; 51 #else 52 const unsigned long core_mmu_tee_load_pa = TEE_LOAD_ADDR; 53 #endif 54 55 /* 56 * These variables are initialized before .bss is cleared. To avoid 57 * resetting them when .bss is cleared we're storing them in .data instead, 58 * even if they initially are zero. 59 */ 60 61 #ifdef CFG_CORE_RESERVED_SHM 62 /* Default NSec shared memory allocated from NSec world */ 63 unsigned long default_nsec_shm_size __nex_bss; 64 unsigned long default_nsec_shm_paddr __nex_bss; 65 #endif 66 67 static struct tee_mmap_region static_mmap_regions[CFG_MMAP_REGIONS 68 #if defined(CFG_CORE_ASLR) || defined(CFG_CORE_PHYS_RELOCATABLE) 69 + 1 70 #endif 71 + 1] __nex_bss; 72 static struct memory_map static_memory_map __nex_data = { 73 .map = static_mmap_regions, 74 .alloc_count = ARRAY_SIZE(static_mmap_regions), 75 }; 76 77 /* Define the platform's memory layout. */ 78 struct memaccess_area { 79 paddr_t paddr; 80 size_t size; 81 }; 82 83 #define MEMACCESS_AREA(a, s) { .paddr = a, .size = s } 84 85 static struct memaccess_area secure_only[] __nex_data = { 86 #ifdef CFG_CORE_PHYS_RELOCATABLE 87 MEMACCESS_AREA(0, 0), 88 #else 89 #ifdef TRUSTED_SRAM_BASE 90 MEMACCESS_AREA(TRUSTED_SRAM_BASE, TRUSTED_SRAM_SIZE), 91 #endif 92 MEMACCESS_AREA(TRUSTED_DRAM_BASE, TRUSTED_DRAM_SIZE), 93 #endif 94 }; 95 96 static struct memaccess_area nsec_shared[] __nex_data = { 97 #ifdef CFG_CORE_RESERVED_SHM 98 MEMACCESS_AREA(TEE_SHMEM_START, TEE_SHMEM_SIZE), 99 #endif 100 }; 101 102 #if defined(CFG_SECURE_DATA_PATH) 103 static const char *tz_sdp_match = "linaro,secure-heap"; 104 static struct memaccess_area sec_sdp; 105 #ifdef CFG_TEE_SDP_MEM_BASE 106 register_sdp_mem(CFG_TEE_SDP_MEM_BASE, CFG_TEE_SDP_MEM_SIZE); 107 #endif 108 #ifdef TEE_SDP_TEST_MEM_BASE 109 register_sdp_mem(TEE_SDP_TEST_MEM_BASE, TEE_SDP_TEST_MEM_SIZE); 110 #endif 111 #endif 112 113 #ifdef CFG_CORE_RESERVED_SHM 114 register_phys_mem(MEM_AREA_NSEC_SHM, TEE_SHMEM_START, TEE_SHMEM_SIZE); 115 #endif 116 static unsigned int mmu_spinlock; 117 118 static uint32_t mmu_lock(void) 119 { 120 return cpu_spin_lock_xsave(&mmu_spinlock); 121 } 122 123 static void mmu_unlock(uint32_t exceptions) 124 { 125 cpu_spin_unlock_xrestore(&mmu_spinlock, exceptions); 126 } 127 128 static void grow_mem_map(struct memory_map *mem_map) 129 { 130 if (mem_map->count == mem_map->alloc_count) { 131 EMSG("Out of entries (%zu) in mem_map", mem_map->alloc_count); 132 panic(); 133 } 134 mem_map->count++; 135 } 136 137 void core_mmu_get_secure_memory(paddr_t *base, paddr_size_t *size) 138 { 139 /* 140 * The first range is always used to cover OP-TEE core memory, but 141 * depending on configuration it may cover more than that. 142 */ 143 *base = secure_only[0].paddr; 144 *size = secure_only[0].size; 145 } 146 147 void core_mmu_set_secure_memory(paddr_t base, size_t size) 148 { 149 #ifdef CFG_CORE_PHYS_RELOCATABLE 150 static_assert(ARRAY_SIZE(secure_only) == 1); 151 #endif 152 runtime_assert(IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE)); 153 assert(!secure_only[0].size); 154 assert(base && size); 155 156 DMSG("Physical secure memory base %#"PRIxPA" size %#zx", base, size); 157 secure_only[0].paddr = base; 158 secure_only[0].size = size; 159 } 160 161 void core_mmu_get_ta_range(paddr_t *base, size_t *size) 162 { 163 paddr_t b = 0; 164 size_t s = 0; 165 166 static_assert(!(TEE_RAM_VA_SIZE % SMALL_PAGE_SIZE)); 167 #ifdef TA_RAM_START 168 b = TA_RAM_START; 169 s = TA_RAM_SIZE; 170 #else 171 static_assert(ARRAY_SIZE(secure_only) <= 2); 172 if (ARRAY_SIZE(secure_only) == 1) { 173 vaddr_t load_offs = 0; 174 175 assert(core_mmu_tee_load_pa >= secure_only[0].paddr); 176 load_offs = core_mmu_tee_load_pa - secure_only[0].paddr; 177 178 assert(secure_only[0].size > 179 load_offs + TEE_RAM_VA_SIZE + TEE_SDP_TEST_MEM_SIZE); 180 b = secure_only[0].paddr + load_offs + TEE_RAM_VA_SIZE; 181 s = secure_only[0].size - load_offs - TEE_RAM_VA_SIZE - 182 TEE_SDP_TEST_MEM_SIZE; 183 } else { 184 assert(secure_only[1].size > TEE_SDP_TEST_MEM_SIZE); 185 b = secure_only[1].paddr; 186 s = secure_only[1].size - TEE_SDP_TEST_MEM_SIZE; 187 } 188 #endif 189 if (base) 190 *base = b; 191 if (size) 192 *size = s; 193 } 194 195 static struct memory_map *get_memory_map(void) 196 { 197 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 198 struct memory_map *map = virt_get_memory_map(); 199 200 if (map) 201 return map; 202 } 203 204 return &static_memory_map; 205 } 206 207 static bool _pbuf_intersects(struct memaccess_area *a, size_t alen, 208 paddr_t pa, size_t size) 209 { 210 size_t n; 211 212 for (n = 0; n < alen; n++) 213 if (core_is_buffer_intersect(pa, size, a[n].paddr, a[n].size)) 214 return true; 215 return false; 216 } 217 218 #define pbuf_intersects(a, pa, size) \ 219 _pbuf_intersects((a), ARRAY_SIZE(a), (pa), (size)) 220 221 static bool _pbuf_is_inside(struct memaccess_area *a, size_t alen, 222 paddr_t pa, size_t size) 223 { 224 size_t n; 225 226 for (n = 0; n < alen; n++) 227 if (core_is_buffer_inside(pa, size, a[n].paddr, a[n].size)) 228 return true; 229 return false; 230 } 231 232 #define pbuf_is_inside(a, pa, size) \ 233 _pbuf_is_inside((a), ARRAY_SIZE(a), (pa), (size)) 234 235 static bool pa_is_in_map(struct tee_mmap_region *map, paddr_t pa, size_t len) 236 { 237 paddr_t end_pa = 0; 238 239 if (!map) 240 return false; 241 242 if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) 243 return false; 244 245 return (pa >= map->pa && end_pa <= map->pa + map->size - 1); 246 } 247 248 static bool va_is_in_map(struct tee_mmap_region *map, vaddr_t va) 249 { 250 if (!map) 251 return false; 252 return (va >= map->va && va <= (map->va + map->size - 1)); 253 } 254 255 /* check if target buffer fits in a core default map area */ 256 static bool pbuf_inside_map_area(unsigned long p, size_t l, 257 struct tee_mmap_region *map) 258 { 259 return core_is_buffer_inside(p, l, map->pa, map->size); 260 } 261 262 static struct tee_mmap_region *find_map_by_type(enum teecore_memtypes type) 263 { 264 struct memory_map *mem_map = get_memory_map(); 265 size_t n = 0; 266 267 for (n = 0; n < mem_map->count; n++) { 268 if (mem_map->map[n].type == type) 269 return mem_map->map + n; 270 } 271 return NULL; 272 } 273 274 static struct tee_mmap_region * 275 find_map_by_type_and_pa(enum teecore_memtypes type, paddr_t pa, size_t len) 276 { 277 struct memory_map *mem_map = get_memory_map(); 278 size_t n = 0; 279 280 for (n = 0; n < mem_map->count; n++) { 281 if (mem_map->map[n].type != type) 282 continue; 283 if (pa_is_in_map(mem_map->map + n, pa, len)) 284 return mem_map->map + n; 285 } 286 return NULL; 287 } 288 289 static struct tee_mmap_region *find_map_by_va(void *va) 290 { 291 struct memory_map *mem_map = get_memory_map(); 292 vaddr_t a = (vaddr_t)va; 293 size_t n = 0; 294 295 for (n = 0; n < mem_map->count; n++) { 296 if (a >= mem_map->map[n].va && 297 a <= (mem_map->map[n].va - 1 + mem_map->map[n].size)) 298 return mem_map->map + n; 299 } 300 301 return NULL; 302 } 303 304 static struct tee_mmap_region *find_map_by_pa(unsigned long pa) 305 { 306 struct memory_map *mem_map = get_memory_map(); 307 size_t n = 0; 308 309 for (n = 0; n < mem_map->count; n++) { 310 /* Skip unmapped regions */ 311 if ((mem_map->map[n].attr & TEE_MATTR_VALID_BLOCK) && 312 pa >= mem_map->map[n].pa && 313 pa <= (mem_map->map[n].pa - 1 + mem_map->map[n].size)) 314 return mem_map->map + n; 315 } 316 317 return NULL; 318 } 319 320 #if defined(CFG_SECURE_DATA_PATH) 321 static bool dtb_get_sdp_region(void) 322 { 323 void *fdt = NULL; 324 int node = 0; 325 int tmp_node = 0; 326 paddr_t tmp_addr = 0; 327 size_t tmp_size = 0; 328 329 if (!IS_ENABLED(CFG_EMBED_DTB)) 330 return false; 331 332 fdt = get_embedded_dt(); 333 if (!fdt) 334 panic("No DTB found"); 335 336 node = fdt_node_offset_by_compatible(fdt, -1, tz_sdp_match); 337 if (node < 0) { 338 DMSG("No %s compatible node found", tz_sdp_match); 339 return false; 340 } 341 tmp_node = node; 342 while (tmp_node >= 0) { 343 tmp_node = fdt_node_offset_by_compatible(fdt, tmp_node, 344 tz_sdp_match); 345 if (tmp_node >= 0) 346 DMSG("Ignore SDP pool node %s, supports only 1 node", 347 fdt_get_name(fdt, tmp_node, NULL)); 348 } 349 350 tmp_addr = fdt_reg_base_address(fdt, node); 351 if (tmp_addr == DT_INFO_INVALID_REG) { 352 EMSG("%s: Unable to get base addr from DT", tz_sdp_match); 353 return false; 354 } 355 356 tmp_size = fdt_reg_size(fdt, node); 357 if (tmp_size == DT_INFO_INVALID_REG_SIZE) { 358 EMSG("%s: Unable to get size of base addr from DT", 359 tz_sdp_match); 360 return false; 361 } 362 363 sec_sdp.paddr = tmp_addr; 364 sec_sdp.size = tmp_size; 365 366 return true; 367 } 368 #endif 369 370 #if defined(CFG_CORE_DYN_SHM) || defined(CFG_SECURE_DATA_PATH) 371 static bool pbuf_is_special_mem(paddr_t pbuf, size_t len, 372 const struct core_mmu_phys_mem *start, 373 const struct core_mmu_phys_mem *end) 374 { 375 const struct core_mmu_phys_mem *mem; 376 377 for (mem = start; mem < end; mem++) { 378 if (core_is_buffer_inside(pbuf, len, mem->addr, mem->size)) 379 return true; 380 } 381 382 return false; 383 } 384 #endif 385 386 #ifdef CFG_CORE_DYN_SHM 387 static void carve_out_phys_mem(struct core_mmu_phys_mem **mem, size_t *nelems, 388 paddr_t pa, size_t size) 389 { 390 struct core_mmu_phys_mem *m = *mem; 391 size_t n = 0; 392 393 while (true) { 394 if (n >= *nelems) { 395 DMSG("No need to carve out %#" PRIxPA " size %#zx", 396 pa, size); 397 return; 398 } 399 if (core_is_buffer_inside(pa, size, m[n].addr, m[n].size)) 400 break; 401 if (!core_is_buffer_outside(pa, size, m[n].addr, m[n].size)) 402 panic(); 403 n++; 404 } 405 406 if (pa == m[n].addr && size == m[n].size) { 407 /* Remove this entry */ 408 (*nelems)--; 409 memmove(m + n, m + n + 1, sizeof(*m) * (*nelems - n)); 410 m = nex_realloc(m, sizeof(*m) * *nelems); 411 if (!m) 412 panic(); 413 *mem = m; 414 } else if (pa == m[n].addr) { 415 m[n].addr += size; 416 m[n].size -= size; 417 } else if ((pa + size) == (m[n].addr + m[n].size)) { 418 m[n].size -= size; 419 } else { 420 /* Need to split the memory entry */ 421 m = nex_realloc(m, sizeof(*m) * (*nelems + 1)); 422 if (!m) 423 panic(); 424 *mem = m; 425 memmove(m + n + 1, m + n, sizeof(*m) * (*nelems - n)); 426 (*nelems)++; 427 m[n].size = pa - m[n].addr; 428 m[n + 1].size -= size + m[n].size; 429 m[n + 1].addr = pa + size; 430 } 431 } 432 433 static void check_phys_mem_is_outside(struct core_mmu_phys_mem *start, 434 size_t nelems, 435 struct tee_mmap_region *map) 436 { 437 size_t n; 438 439 for (n = 0; n < nelems; n++) { 440 if (!core_is_buffer_outside(start[n].addr, start[n].size, 441 map->pa, map->size)) { 442 EMSG("Non-sec mem (%#" PRIxPA ":%#" PRIxPASZ 443 ") overlaps map (type %d %#" PRIxPA ":%#zx)", 444 start[n].addr, start[n].size, 445 map->type, map->pa, map->size); 446 panic(); 447 } 448 } 449 } 450 451 static const struct core_mmu_phys_mem *discovered_nsec_ddr_start __nex_bss; 452 static size_t discovered_nsec_ddr_nelems __nex_bss; 453 454 static int cmp_pmem_by_addr(const void *a, const void *b) 455 { 456 const struct core_mmu_phys_mem *pmem_a = a; 457 const struct core_mmu_phys_mem *pmem_b = b; 458 459 return CMP_TRILEAN(pmem_a->addr, pmem_b->addr); 460 } 461 462 void core_mmu_set_discovered_nsec_ddr(struct core_mmu_phys_mem *start, 463 size_t nelems) 464 { 465 struct core_mmu_phys_mem *m = start; 466 size_t num_elems = nelems; 467 struct memory_map *mem_map = &static_memory_map; 468 const struct core_mmu_phys_mem __maybe_unused *pmem; 469 size_t n = 0; 470 471 assert(!discovered_nsec_ddr_start); 472 assert(m && num_elems); 473 474 qsort(m, num_elems, sizeof(*m), cmp_pmem_by_addr); 475 476 /* 477 * Non-secure shared memory and also secure data 478 * path memory are supposed to reside inside 479 * non-secure memory. Since NSEC_SHM and SDP_MEM 480 * are used for a specific purpose make holes for 481 * those memory in the normal non-secure memory. 482 * 483 * This has to be done since for instance QEMU 484 * isn't aware of which memory range in the 485 * non-secure memory is used for NSEC_SHM. 486 */ 487 488 #ifdef CFG_SECURE_DATA_PATH 489 if (dtb_get_sdp_region()) 490 carve_out_phys_mem(&m, &num_elems, sec_sdp.paddr, sec_sdp.size); 491 492 for (pmem = phys_sdp_mem_begin; pmem < phys_sdp_mem_end; pmem++) 493 carve_out_phys_mem(&m, &num_elems, pmem->addr, pmem->size); 494 #endif 495 496 for (n = 0; n < ARRAY_SIZE(secure_only); n++) 497 carve_out_phys_mem(&m, &num_elems, secure_only[n].paddr, 498 secure_only[n].size); 499 500 for (n = 0; n < mem_map->count; n++) { 501 switch (mem_map->map[n].type) { 502 case MEM_AREA_NSEC_SHM: 503 carve_out_phys_mem(&m, &num_elems, mem_map->map[n].pa, 504 mem_map->map[n].size); 505 break; 506 case MEM_AREA_EXT_DT: 507 case MEM_AREA_MANIFEST_DT: 508 case MEM_AREA_RAM_NSEC: 509 case MEM_AREA_RES_VASPACE: 510 case MEM_AREA_SHM_VASPACE: 511 case MEM_AREA_TS_VASPACE: 512 case MEM_AREA_PAGER_VASPACE: 513 break; 514 default: 515 check_phys_mem_is_outside(m, num_elems, 516 mem_map->map + n); 517 } 518 } 519 520 discovered_nsec_ddr_start = m; 521 discovered_nsec_ddr_nelems = num_elems; 522 523 if (!core_mmu_check_end_pa(m[num_elems - 1].addr, 524 m[num_elems - 1].size)) 525 panic(); 526 } 527 528 static bool get_discovered_nsec_ddr(const struct core_mmu_phys_mem **start, 529 const struct core_mmu_phys_mem **end) 530 { 531 if (!discovered_nsec_ddr_start) 532 return false; 533 534 *start = discovered_nsec_ddr_start; 535 *end = discovered_nsec_ddr_start + discovered_nsec_ddr_nelems; 536 537 return true; 538 } 539 540 static bool pbuf_is_nsec_ddr(paddr_t pbuf, size_t len) 541 { 542 const struct core_mmu_phys_mem *start; 543 const struct core_mmu_phys_mem *end; 544 545 if (!get_discovered_nsec_ddr(&start, &end)) 546 return false; 547 548 return pbuf_is_special_mem(pbuf, len, start, end); 549 } 550 551 bool core_mmu_nsec_ddr_is_defined(void) 552 { 553 const struct core_mmu_phys_mem *start; 554 const struct core_mmu_phys_mem *end; 555 556 if (!get_discovered_nsec_ddr(&start, &end)) 557 return false; 558 559 return start != end; 560 } 561 #else 562 static bool pbuf_is_nsec_ddr(paddr_t pbuf __unused, size_t len __unused) 563 { 564 return false; 565 } 566 #endif /*CFG_CORE_DYN_SHM*/ 567 568 #define MSG_MEM_INSTERSECT(pa1, sz1, pa2, sz2) \ 569 EMSG("[%" PRIxPA " %" PRIx64 "] intersects [%" PRIxPA " %" PRIx64 "]", \ 570 pa1, (uint64_t)pa1 + (sz1), pa2, (uint64_t)pa2 + (sz2)) 571 572 #ifdef CFG_SECURE_DATA_PATH 573 static bool pbuf_is_sdp_mem(paddr_t pbuf, size_t len) 574 { 575 bool is_sdp_mem = false; 576 577 if (sec_sdp.size) 578 is_sdp_mem = core_is_buffer_inside(pbuf, len, sec_sdp.paddr, 579 sec_sdp.size); 580 581 if (!is_sdp_mem) 582 is_sdp_mem = pbuf_is_special_mem(pbuf, len, phys_sdp_mem_begin, 583 phys_sdp_mem_end); 584 585 return is_sdp_mem; 586 } 587 588 static struct mobj *core_sdp_mem_alloc_mobj(paddr_t pa, size_t size) 589 { 590 struct mobj *mobj = mobj_phys_alloc(pa, size, TEE_MATTR_MEM_TYPE_CACHED, 591 CORE_MEM_SDP_MEM); 592 593 if (!mobj) 594 panic("can't create SDP physical memory object"); 595 596 return mobj; 597 } 598 599 struct mobj **core_sdp_mem_create_mobjs(void) 600 { 601 const struct core_mmu_phys_mem *mem = NULL; 602 struct mobj **mobj_base = NULL; 603 struct mobj **mobj = NULL; 604 int cnt = phys_sdp_mem_end - phys_sdp_mem_begin; 605 606 if (sec_sdp.size) 607 cnt++; 608 609 /* SDP mobjs table must end with a NULL entry */ 610 mobj_base = calloc(cnt + 1, sizeof(struct mobj *)); 611 if (!mobj_base) 612 panic("Out of memory"); 613 614 mobj = mobj_base; 615 616 for (mem = phys_sdp_mem_begin; mem < phys_sdp_mem_end; mem++, mobj++) 617 *mobj = core_sdp_mem_alloc_mobj(mem->addr, mem->size); 618 619 if (sec_sdp.size) 620 *mobj = core_sdp_mem_alloc_mobj(sec_sdp.paddr, sec_sdp.size); 621 622 return mobj_base; 623 } 624 625 #else /* CFG_SECURE_DATA_PATH */ 626 static bool pbuf_is_sdp_mem(paddr_t pbuf __unused, size_t len __unused) 627 { 628 return false; 629 } 630 631 #endif /* CFG_SECURE_DATA_PATH */ 632 633 /* Check special memories comply with registered memories */ 634 static void verify_special_mem_areas(struct memory_map *mem_map, 635 const struct core_mmu_phys_mem *start, 636 const struct core_mmu_phys_mem *end, 637 const char *area_name __maybe_unused) 638 { 639 const struct core_mmu_phys_mem *mem = NULL; 640 const struct core_mmu_phys_mem *mem2 = NULL; 641 size_t n = 0; 642 643 if (start == end) { 644 DMSG("No %s memory area defined", area_name); 645 return; 646 } 647 648 for (mem = start; mem < end; mem++) 649 DMSG("%s memory [%" PRIxPA " %" PRIx64 "]", 650 area_name, mem->addr, (uint64_t)mem->addr + mem->size); 651 652 /* Check memories do not intersect each other */ 653 for (mem = start; mem + 1 < end; mem++) { 654 for (mem2 = mem + 1; mem2 < end; mem2++) { 655 if (core_is_buffer_intersect(mem2->addr, mem2->size, 656 mem->addr, mem->size)) { 657 MSG_MEM_INSTERSECT(mem2->addr, mem2->size, 658 mem->addr, mem->size); 659 panic("Special memory intersection"); 660 } 661 } 662 } 663 664 /* 665 * Check memories do not intersect any mapped memory. 666 * This is called before reserved VA space is loaded in mem_map. 667 */ 668 for (mem = start; mem < end; mem++) { 669 for (n = 0; n < mem_map->count; n++) { 670 if (core_is_buffer_intersect(mem->addr, mem->size, 671 mem_map->map[n].pa, 672 mem_map->map[n].size)) { 673 MSG_MEM_INSTERSECT(mem->addr, mem->size, 674 mem_map->map[n].pa, 675 mem_map->map[n].size); 676 panic("Special memory intersection"); 677 } 678 } 679 } 680 } 681 682 static void merge_mmaps(struct tee_mmap_region *dst, 683 const struct tee_mmap_region *src) 684 { 685 paddr_t end_pa = MAX(dst->pa + dst->size - 1, src->pa + src->size - 1); 686 paddr_t pa = MIN(dst->pa, src->pa); 687 688 DMSG("Merging %#"PRIxPA"..%#"PRIxPA" and %#"PRIxPA"..%#"PRIxPA, 689 dst->pa, dst->pa + dst->size - 1, src->pa, 690 src->pa + src->size - 1); 691 dst->pa = pa; 692 dst->size = end_pa - pa + 1; 693 } 694 695 static bool mmaps_are_mergeable(const struct tee_mmap_region *r1, 696 const struct tee_mmap_region *r2) 697 { 698 if (r1->type != r2->type) 699 return false; 700 701 if (r1->pa == r2->pa) 702 return true; 703 704 if (r1->pa < r2->pa) 705 return r1->pa + r1->size >= r2->pa; 706 else 707 return r2->pa + r2->size >= r1->pa; 708 } 709 710 static void add_phys_mem(struct memory_map *mem_map, 711 const char *mem_name __maybe_unused, 712 enum teecore_memtypes mem_type, 713 paddr_t mem_addr, paddr_size_t mem_size) 714 { 715 size_t n = 0; 716 const struct tee_mmap_region m0 = { 717 .type = mem_type, 718 .pa = mem_addr, 719 .size = mem_size, 720 }; 721 722 if (!mem_size) /* Discard null size entries */ 723 return; 724 725 /* 726 * If some ranges of memory of the same type do overlap 727 * each others they are coalesced into one entry. To help this 728 * added entries are sorted by increasing physical. 729 * 730 * Note that it's valid to have the same physical memory as several 731 * different memory types, for instance the same device memory 732 * mapped as both secure and non-secure. This will probably not 733 * happen often in practice. 734 */ 735 DMSG("%s type %s 0x%08" PRIxPA " size 0x%08" PRIxPASZ, 736 mem_name, teecore_memtype_name(mem_type), mem_addr, mem_size); 737 for (n = 0; n < mem_map->count; n++) { 738 if (mmaps_are_mergeable(mem_map->map + n, &m0)) { 739 merge_mmaps(mem_map->map + n, &m0); 740 /* 741 * The merged result might be mergeable with the 742 * next or previous entry. 743 */ 744 if (n + 1 < mem_map->count && 745 mmaps_are_mergeable(mem_map->map + n, 746 mem_map->map + n + 1)) { 747 merge_mmaps(mem_map->map + n, 748 mem_map->map + n + 1); 749 rem_array_elem(mem_map->map, mem_map->count, 750 sizeof(*mem_map->map), n + 1); 751 mem_map->count--; 752 } 753 if (n > 0 && mmaps_are_mergeable(mem_map->map + n - 1, 754 mem_map->map + n)) { 755 merge_mmaps(mem_map->map + n - 1, 756 mem_map->map + n); 757 rem_array_elem(mem_map->map, mem_map->count, 758 sizeof(*mem_map->map), n); 759 mem_map->count--; 760 } 761 return; 762 } 763 if (mem_type < mem_map->map[n].type || 764 (mem_type == mem_map->map[n].type && 765 mem_addr < mem_map->map[n].pa)) 766 break; /* found the spot where to insert this memory */ 767 } 768 769 grow_mem_map(mem_map); 770 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 771 n, &m0); 772 } 773 774 static void add_va_space(struct memory_map *mem_map, 775 enum teecore_memtypes type, size_t size) 776 { 777 size_t n = 0; 778 779 DMSG("type %s size 0x%08zx", teecore_memtype_name(type), size); 780 for (n = 0; n < mem_map->count; n++) { 781 if (type < mem_map->map[n].type) 782 break; 783 } 784 785 grow_mem_map(mem_map); 786 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 787 n, NULL); 788 mem_map->map[n] = (struct tee_mmap_region){ 789 .type = type, 790 .size = size, 791 }; 792 } 793 794 uint32_t core_mmu_type_to_attr(enum teecore_memtypes t) 795 { 796 const uint32_t attr = TEE_MATTR_VALID_BLOCK; 797 const uint32_t tagged = TEE_MATTR_MEM_TYPE_TAGGED << 798 TEE_MATTR_MEM_TYPE_SHIFT; 799 const uint32_t cached = TEE_MATTR_MEM_TYPE_CACHED << 800 TEE_MATTR_MEM_TYPE_SHIFT; 801 const uint32_t noncache = TEE_MATTR_MEM_TYPE_DEV << 802 TEE_MATTR_MEM_TYPE_SHIFT; 803 804 switch (t) { 805 case MEM_AREA_TEE_RAM: 806 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | tagged; 807 case MEM_AREA_TEE_RAM_RX: 808 case MEM_AREA_INIT_RAM_RX: 809 case MEM_AREA_IDENTITY_MAP_RX: 810 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRX | tagged; 811 case MEM_AREA_TEE_RAM_RO: 812 case MEM_AREA_INIT_RAM_RO: 813 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | tagged; 814 case MEM_AREA_TEE_RAM_RW: 815 case MEM_AREA_NEX_RAM_RO: /* This has to be r/w during init runtime */ 816 case MEM_AREA_NEX_RAM_RW: 817 case MEM_AREA_TEE_ASAN: 818 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | tagged; 819 case MEM_AREA_TEE_COHERENT: 820 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | noncache; 821 case MEM_AREA_TA_RAM: 822 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | tagged; 823 case MEM_AREA_NSEC_SHM: 824 case MEM_AREA_NEX_NSEC_SHM: 825 return attr | TEE_MATTR_PRW | cached; 826 case MEM_AREA_MANIFEST_DT: 827 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | cached; 828 case MEM_AREA_TRANSFER_LIST: 829 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 830 case MEM_AREA_EXT_DT: 831 /* 832 * If CFG_MAP_EXT_DT_SECURE is enabled map the external device 833 * tree as secure non-cached memory, otherwise, fall back to 834 * non-secure mapping. 835 */ 836 if (IS_ENABLED(CFG_MAP_EXT_DT_SECURE)) 837 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | 838 noncache; 839 fallthrough; 840 case MEM_AREA_IO_NSEC: 841 return attr | TEE_MATTR_PRW | noncache; 842 case MEM_AREA_IO_SEC: 843 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | noncache; 844 case MEM_AREA_RAM_NSEC: 845 return attr | TEE_MATTR_PRW | cached; 846 case MEM_AREA_RAM_SEC: 847 case MEM_AREA_SEC_RAM_OVERALL: 848 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 849 case MEM_AREA_ROM_SEC: 850 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | cached; 851 case MEM_AREA_RES_VASPACE: 852 case MEM_AREA_SHM_VASPACE: 853 return 0; 854 case MEM_AREA_PAGER_VASPACE: 855 return TEE_MATTR_SECURE; 856 default: 857 panic("invalid type"); 858 } 859 } 860 861 static bool __maybe_unused map_is_tee_ram(const struct tee_mmap_region *mm) 862 { 863 switch (mm->type) { 864 case MEM_AREA_TEE_RAM: 865 case MEM_AREA_TEE_RAM_RX: 866 case MEM_AREA_TEE_RAM_RO: 867 case MEM_AREA_TEE_RAM_RW: 868 case MEM_AREA_INIT_RAM_RX: 869 case MEM_AREA_INIT_RAM_RO: 870 case MEM_AREA_NEX_RAM_RW: 871 case MEM_AREA_NEX_RAM_RO: 872 case MEM_AREA_TEE_ASAN: 873 return true; 874 default: 875 return false; 876 } 877 } 878 879 static bool __maybe_unused map_is_secure(const struct tee_mmap_region *mm) 880 { 881 return !!(core_mmu_type_to_attr(mm->type) & TEE_MATTR_SECURE); 882 } 883 884 static bool __maybe_unused map_is_pgdir(const struct tee_mmap_region *mm) 885 { 886 return mm->region_size == CORE_MMU_PGDIR_SIZE; 887 } 888 889 static int cmp_mmap_by_lower_va(const void *a, const void *b) 890 { 891 const struct tee_mmap_region *mm_a = a; 892 const struct tee_mmap_region *mm_b = b; 893 894 return CMP_TRILEAN(mm_a->va, mm_b->va); 895 } 896 897 static void dump_mmap_table(struct memory_map *mem_map) 898 { 899 size_t n = 0; 900 901 for (n = 0; n < mem_map->count; n++) { 902 struct tee_mmap_region *map = mem_map->map + n; 903 vaddr_t __maybe_unused vstart; 904 905 vstart = map->va + ((vaddr_t)map->pa & (map->region_size - 1)); 906 DMSG("type %-12s va 0x%08" PRIxVA "..0x%08" PRIxVA 907 " pa 0x%08" PRIxPA "..0x%08" PRIxPA " size 0x%08zx (%s)", 908 teecore_memtype_name(map->type), vstart, 909 vstart + map->size - 1, map->pa, 910 (paddr_t)(map->pa + map->size - 1), map->size, 911 map->region_size == SMALL_PAGE_SIZE ? "smallpg" : "pgdir"); 912 } 913 } 914 915 #if DEBUG_XLAT_TABLE 916 917 static void dump_xlat_table(vaddr_t va, unsigned int level) 918 { 919 struct core_mmu_table_info tbl_info; 920 unsigned int idx = 0; 921 paddr_t pa; 922 uint32_t attr; 923 924 core_mmu_find_table(NULL, va, level, &tbl_info); 925 va = tbl_info.va_base; 926 for (idx = 0; idx < tbl_info.num_entries; idx++) { 927 core_mmu_get_entry(&tbl_info, idx, &pa, &attr); 928 if (attr || level > CORE_MMU_BASE_TABLE_LEVEL) { 929 const char *security_bit = ""; 930 931 if (core_mmu_entry_have_security_bit(attr)) { 932 if (attr & TEE_MATTR_SECURE) 933 security_bit = "S"; 934 else 935 security_bit = "NS"; 936 } 937 938 if (attr & TEE_MATTR_TABLE) { 939 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 940 " TBL:0x%010" PRIxPA " %s", 941 level * 2, "", level, va, pa, 942 security_bit); 943 dump_xlat_table(va, level + 1); 944 } else if (attr) { 945 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 946 " PA:0x%010" PRIxPA " %s-%s-%s-%s", 947 level * 2, "", level, va, pa, 948 mattr_is_cached(attr) ? "MEM" : 949 "DEV", 950 attr & TEE_MATTR_PW ? "RW" : "RO", 951 attr & TEE_MATTR_PX ? "X " : "XN", 952 security_bit); 953 } else { 954 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 955 " INVALID\n", 956 level * 2, "", level, va); 957 } 958 } 959 va += BIT64(tbl_info.shift); 960 } 961 } 962 963 #else 964 965 static void dump_xlat_table(vaddr_t va __unused, int level __unused) 966 { 967 } 968 969 #endif 970 971 /* 972 * Reserves virtual memory space for pager usage. 973 * 974 * From the start of the first memory used by the link script + 975 * TEE_RAM_VA_SIZE should be covered, either with a direct mapping or empty 976 * mapping for pager usage. This adds translation tables as needed for the 977 * pager to operate. 978 */ 979 static void add_pager_vaspace(struct memory_map *mem_map) 980 { 981 paddr_t begin = 0; 982 paddr_t end = 0; 983 size_t size = 0; 984 size_t pos = 0; 985 size_t n = 0; 986 987 988 for (n = 0; n < mem_map->count; n++) { 989 if (map_is_tee_ram(mem_map->map + n)) { 990 if (!begin) 991 begin = mem_map->map[n].pa; 992 pos = n + 1; 993 } 994 } 995 996 end = mem_map->map[pos - 1].pa + mem_map->map[pos - 1].size; 997 assert(end - begin < TEE_RAM_VA_SIZE); 998 size = TEE_RAM_VA_SIZE - (end - begin); 999 1000 grow_mem_map(mem_map); 1001 ins_array_elem(mem_map->map, mem_map->count, sizeof(*mem_map->map), 1002 n, NULL); 1003 mem_map->map[n] = (struct tee_mmap_region){ 1004 .type = MEM_AREA_PAGER_VASPACE, 1005 .size = size, 1006 .region_size = SMALL_PAGE_SIZE, 1007 .attr = core_mmu_type_to_attr(MEM_AREA_PAGER_VASPACE), 1008 }; 1009 } 1010 1011 static void check_sec_nsec_mem_config(void) 1012 { 1013 size_t n = 0; 1014 1015 for (n = 0; n < ARRAY_SIZE(secure_only); n++) { 1016 if (pbuf_intersects(nsec_shared, secure_only[n].paddr, 1017 secure_only[n].size)) 1018 panic("Invalid memory access config: sec/nsec"); 1019 } 1020 } 1021 1022 static void collect_device_mem_ranges(struct memory_map *mem_map) 1023 { 1024 const char *compatible = "arm,ffa-manifest-device-regions"; 1025 void *fdt = get_manifest_dt(); 1026 const char *name = NULL; 1027 uint64_t page_count = 0; 1028 uint64_t base = 0; 1029 int subnode = 0; 1030 int node = 0; 1031 1032 assert(fdt); 1033 1034 node = fdt_node_offset_by_compatible(fdt, 0, compatible); 1035 if (node < 0) 1036 return; 1037 1038 fdt_for_each_subnode(subnode, fdt, node) { 1039 name = fdt_get_name(fdt, subnode, NULL); 1040 if (!name) 1041 continue; 1042 1043 if (dt_getprop_as_number(fdt, subnode, "base-address", 1044 &base)) { 1045 EMSG("Mandatory field is missing: base-address"); 1046 continue; 1047 } 1048 1049 if (base & SMALL_PAGE_MASK) { 1050 EMSG("base-address is not page aligned"); 1051 continue; 1052 } 1053 1054 if (dt_getprop_as_number(fdt, subnode, "pages-count", 1055 &page_count)) { 1056 EMSG("Mandatory field is missing: pages-count"); 1057 continue; 1058 } 1059 1060 add_phys_mem(mem_map, name, MEM_AREA_IO_SEC, 1061 base, base + page_count * SMALL_PAGE_SIZE); 1062 } 1063 } 1064 1065 static void collect_mem_ranges(struct memory_map *mem_map) 1066 { 1067 const struct core_mmu_phys_mem *mem = NULL; 1068 vaddr_t ram_start = secure_only[0].paddr; 1069 1070 #define ADD_PHYS_MEM(_type, _addr, _size) \ 1071 add_phys_mem(mem_map, #_addr, (_type), (_addr), (_size)) 1072 1073 if (IS_ENABLED(CFG_CORE_RWDATA_NOEXEC)) { 1074 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, ram_start, 1075 VCORE_UNPG_RX_PA - ram_start); 1076 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RX, VCORE_UNPG_RX_PA, 1077 VCORE_UNPG_RX_SZ); 1078 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, VCORE_UNPG_RO_PA, 1079 VCORE_UNPG_RO_SZ); 1080 1081 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1082 ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RO, VCORE_UNPG_RW_PA, 1083 VCORE_UNPG_RW_SZ); 1084 ADD_PHYS_MEM(MEM_AREA_NEX_RAM_RW, VCORE_NEX_RW_PA, 1085 VCORE_NEX_RW_SZ); 1086 } else { 1087 ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RW, VCORE_UNPG_RW_PA, 1088 VCORE_UNPG_RW_SZ); 1089 } 1090 1091 if (IS_ENABLED(CFG_WITH_PAGER)) { 1092 ADD_PHYS_MEM(MEM_AREA_INIT_RAM_RX, VCORE_INIT_RX_PA, 1093 VCORE_INIT_RX_SZ); 1094 ADD_PHYS_MEM(MEM_AREA_INIT_RAM_RO, VCORE_INIT_RO_PA, 1095 VCORE_INIT_RO_SZ); 1096 } 1097 } else { 1098 ADD_PHYS_MEM(MEM_AREA_TEE_RAM, TEE_RAM_START, TEE_RAM_PH_SIZE); 1099 } 1100 1101 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) { 1102 ADD_PHYS_MEM(MEM_AREA_SEC_RAM_OVERALL, TRUSTED_DRAM_BASE, 1103 TRUSTED_DRAM_SIZE); 1104 } else { 1105 /* 1106 * Every guest will have own TA RAM if virtualization 1107 * support is enabled. 1108 */ 1109 paddr_t ta_base = 0; 1110 size_t ta_size = 0; 1111 1112 core_mmu_get_ta_range(&ta_base, &ta_size); 1113 ADD_PHYS_MEM(MEM_AREA_TA_RAM, ta_base, ta_size); 1114 } 1115 1116 if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS) && 1117 IS_ENABLED(CFG_WITH_PAGER)) { 1118 /* 1119 * Asan ram is part of MEM_AREA_TEE_RAM_RW when pager is 1120 * disabled. 1121 */ 1122 ADD_PHYS_MEM(MEM_AREA_TEE_ASAN, ASAN_MAP_PA, ASAN_MAP_SZ); 1123 } 1124 1125 #undef ADD_PHYS_MEM 1126 1127 /* Collect device memory info from SP manifest */ 1128 if (IS_ENABLED(CFG_CORE_SEL2_SPMC)) 1129 collect_device_mem_ranges(mem_map); 1130 1131 for (mem = phys_mem_map_begin; mem < phys_mem_map_end; mem++) { 1132 /* Only unmapped virtual range may have a null phys addr */ 1133 assert(mem->addr || !core_mmu_type_to_attr(mem->type)); 1134 1135 add_phys_mem(mem_map, mem->name, mem->type, 1136 mem->addr, mem->size); 1137 } 1138 1139 if (IS_ENABLED(CFG_SECURE_DATA_PATH)) 1140 verify_special_mem_areas(mem_map, phys_sdp_mem_begin, 1141 phys_sdp_mem_end, "SDP"); 1142 1143 add_va_space(mem_map, MEM_AREA_RES_VASPACE, CFG_RESERVED_VASPACE_SIZE); 1144 add_va_space(mem_map, MEM_AREA_SHM_VASPACE, SHM_VASPACE_SIZE); 1145 } 1146 1147 static void assign_mem_granularity(struct memory_map *mem_map) 1148 { 1149 size_t n = 0; 1150 1151 /* 1152 * Assign region sizes, note that MEM_AREA_TEE_RAM always uses 1153 * SMALL_PAGE_SIZE. 1154 */ 1155 for (n = 0; n < mem_map->count; n++) { 1156 paddr_t mask = mem_map->map[n].pa | mem_map->map[n].size; 1157 1158 if (!(mask & CORE_MMU_PGDIR_MASK)) 1159 mem_map->map[n].region_size = CORE_MMU_PGDIR_SIZE; 1160 else if (!(mask & SMALL_PAGE_MASK)) 1161 mem_map->map[n].region_size = SMALL_PAGE_SIZE; 1162 else 1163 panic("Impossible memory alignment"); 1164 1165 if (map_is_tee_ram(mem_map->map + n)) 1166 mem_map->map[n].region_size = SMALL_PAGE_SIZE; 1167 } 1168 } 1169 1170 static bool place_tee_ram_at_top(paddr_t paddr) 1171 { 1172 return paddr > BIT64(core_mmu_get_va_width()) / 2; 1173 } 1174 1175 /* 1176 * MMU arch driver shall override this function if it helps 1177 * optimizing the memory footprint of the address translation tables. 1178 */ 1179 bool __weak core_mmu_prefer_tee_ram_at_top(paddr_t paddr) 1180 { 1181 return place_tee_ram_at_top(paddr); 1182 } 1183 1184 static bool assign_mem_va_dir(vaddr_t tee_ram_va, struct memory_map *mem_map, 1185 bool tee_ram_at_top) 1186 { 1187 struct tee_mmap_region *map = NULL; 1188 vaddr_t va = 0; 1189 bool va_is_secure = true; 1190 size_t n = 0; 1191 1192 /* 1193 * tee_ram_va might equals 0 when CFG_CORE_ASLR=y. 1194 * 0 is by design an invalid va, so return false directly. 1195 */ 1196 if (!tee_ram_va) 1197 return false; 1198 1199 /* Clear eventual previous assignments */ 1200 for (n = 0; n < mem_map->count; n++) 1201 mem_map->map[n].va = 0; 1202 1203 /* 1204 * TEE RAM regions are always aligned with region_size. 1205 * 1206 * Note that MEM_AREA_PAGER_VASPACE also counts as TEE RAM here 1207 * since it handles virtual memory which covers the part of the ELF 1208 * that cannot fit directly into memory. 1209 */ 1210 va = tee_ram_va; 1211 for (n = 0; n < mem_map->count; n++) { 1212 map = mem_map->map + n; 1213 if (map_is_tee_ram(map) || 1214 map->type == MEM_AREA_PAGER_VASPACE) { 1215 assert(!(va & (map->region_size - 1))); 1216 assert(!(map->size & (map->region_size - 1))); 1217 map->va = va; 1218 if (ADD_OVERFLOW(va, map->size, &va)) 1219 return false; 1220 if (va >= BIT64(core_mmu_get_va_width())) 1221 return false; 1222 } 1223 } 1224 1225 if (tee_ram_at_top) { 1226 /* 1227 * Map non-tee ram regions at addresses lower than the tee 1228 * ram region. 1229 */ 1230 va = tee_ram_va; 1231 for (n = 0; n < mem_map->count; n++) { 1232 map = mem_map->map + n; 1233 map->attr = core_mmu_type_to_attr(map->type); 1234 if (map->va) 1235 continue; 1236 1237 if (!IS_ENABLED(CFG_WITH_LPAE) && 1238 va_is_secure != map_is_secure(map)) { 1239 va_is_secure = !va_is_secure; 1240 va = ROUNDDOWN(va, CORE_MMU_PGDIR_SIZE); 1241 } 1242 1243 if (SUB_OVERFLOW(va, map->size, &va)) 1244 return false; 1245 va = ROUNDDOWN(va, map->region_size); 1246 /* 1247 * Make sure that va is aligned with pa for 1248 * efficient pgdir mapping. Basically pa & 1249 * pgdir_mask should be == va & pgdir_mask 1250 */ 1251 if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { 1252 if (SUB_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, &va)) 1253 return false; 1254 va += (map->pa - va) & CORE_MMU_PGDIR_MASK; 1255 } 1256 map->va = va; 1257 } 1258 } else { 1259 /* 1260 * Map non-tee ram regions at addresses higher than the tee 1261 * ram region. 1262 */ 1263 for (n = 0; n < mem_map->count; n++) { 1264 map = mem_map->map + n; 1265 map->attr = core_mmu_type_to_attr(map->type); 1266 if (map->va) 1267 continue; 1268 1269 if (!IS_ENABLED(CFG_WITH_LPAE) && 1270 va_is_secure != map_is_secure(map)) { 1271 va_is_secure = !va_is_secure; 1272 if (ROUNDUP_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, 1273 &va)) 1274 return false; 1275 } 1276 1277 if (ROUNDUP_OVERFLOW(va, map->region_size, &va)) 1278 return false; 1279 /* 1280 * Make sure that va is aligned with pa for 1281 * efficient pgdir mapping. Basically pa & 1282 * pgdir_mask should be == va & pgdir_mask 1283 */ 1284 if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { 1285 vaddr_t offs = (map->pa - va) & 1286 CORE_MMU_PGDIR_MASK; 1287 1288 if (ADD_OVERFLOW(va, offs, &va)) 1289 return false; 1290 } 1291 1292 map->va = va; 1293 if (ADD_OVERFLOW(va, map->size, &va)) 1294 return false; 1295 if (va >= BIT64(core_mmu_get_va_width())) 1296 return false; 1297 } 1298 } 1299 1300 return true; 1301 } 1302 1303 static bool assign_mem_va(vaddr_t tee_ram_va, struct memory_map *mem_map) 1304 { 1305 bool tee_ram_at_top = place_tee_ram_at_top(tee_ram_va); 1306 1307 /* 1308 * Check that we're not overlapping with the user VA range. 1309 */ 1310 if (IS_ENABLED(CFG_WITH_LPAE)) { 1311 /* 1312 * User VA range is supposed to be defined after these 1313 * mappings have been established. 1314 */ 1315 assert(!core_mmu_user_va_range_is_defined()); 1316 } else { 1317 vaddr_t user_va_base = 0; 1318 size_t user_va_size = 0; 1319 1320 assert(core_mmu_user_va_range_is_defined()); 1321 core_mmu_get_user_va_range(&user_va_base, &user_va_size); 1322 if (tee_ram_va < (user_va_base + user_va_size)) 1323 return false; 1324 } 1325 1326 if (IS_ENABLED(CFG_WITH_PAGER)) { 1327 bool prefered_dir = core_mmu_prefer_tee_ram_at_top(tee_ram_va); 1328 1329 /* Try whole mapping covered by a single base xlat entry */ 1330 if (prefered_dir != tee_ram_at_top && 1331 assign_mem_va_dir(tee_ram_va, mem_map, prefered_dir)) 1332 return true; 1333 } 1334 1335 return assign_mem_va_dir(tee_ram_va, mem_map, tee_ram_at_top); 1336 } 1337 1338 static int cmp_init_mem_map(const void *a, const void *b) 1339 { 1340 const struct tee_mmap_region *mm_a = a; 1341 const struct tee_mmap_region *mm_b = b; 1342 int rc = 0; 1343 1344 rc = CMP_TRILEAN(mm_a->region_size, mm_b->region_size); 1345 if (!rc) 1346 rc = CMP_TRILEAN(mm_a->pa, mm_b->pa); 1347 /* 1348 * 32bit MMU descriptors cannot mix secure and non-secure mapping in 1349 * the same level2 table. Hence sort secure mapping from non-secure 1350 * mapping. 1351 */ 1352 if (!rc && !IS_ENABLED(CFG_WITH_LPAE)) 1353 rc = CMP_TRILEAN(map_is_secure(mm_a), map_is_secure(mm_b)); 1354 1355 return rc; 1356 } 1357 1358 static bool mem_map_add_id_map(struct memory_map *mem_map, 1359 vaddr_t id_map_start, vaddr_t id_map_end) 1360 { 1361 vaddr_t start = ROUNDDOWN(id_map_start, SMALL_PAGE_SIZE); 1362 vaddr_t end = ROUNDUP(id_map_end, SMALL_PAGE_SIZE); 1363 size_t len = end - start; 1364 size_t n = 0; 1365 1366 1367 for (n = 0; n < mem_map->count; n++) 1368 if (core_is_buffer_intersect(mem_map->map[n].va, 1369 mem_map->map[n].size, start, len)) 1370 return false; 1371 1372 grow_mem_map(mem_map); 1373 mem_map->map[mem_map->count - 1] = (struct tee_mmap_region){ 1374 .type = MEM_AREA_IDENTITY_MAP_RX, 1375 /* 1376 * Could use CORE_MMU_PGDIR_SIZE to potentially save a 1377 * translation table, at the increased risk of clashes with 1378 * the rest of the memory map. 1379 */ 1380 .region_size = SMALL_PAGE_SIZE, 1381 .pa = start, 1382 .va = start, 1383 .size = len, 1384 .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), 1385 }; 1386 1387 return true; 1388 } 1389 1390 static struct memory_map *init_mem_map(struct memory_map *mem_map, 1391 unsigned long seed, 1392 unsigned long *ret_offs) 1393 { 1394 /* 1395 * @id_map_start and @id_map_end describes a physical memory range 1396 * that must be mapped Read-Only eXecutable at identical virtual 1397 * addresses. 1398 */ 1399 vaddr_t id_map_start = (vaddr_t)__identity_map_init_start; 1400 vaddr_t id_map_end = (vaddr_t)__identity_map_init_end; 1401 vaddr_t start_addr = secure_only[0].paddr; 1402 unsigned long offs = 0; 1403 1404 collect_mem_ranges(mem_map); 1405 assign_mem_granularity(mem_map); 1406 1407 /* 1408 * To ease mapping and lower use of xlat tables, sort mapping 1409 * description moving small-page regions after the pgdir regions. 1410 */ 1411 qsort(mem_map->map, mem_map->count, sizeof(struct tee_mmap_region), 1412 cmp_init_mem_map); 1413 1414 if (IS_ENABLED(CFG_WITH_PAGER)) 1415 add_pager_vaspace(mem_map); 1416 1417 if (IS_ENABLED(CFG_CORE_ASLR) && seed) { 1418 vaddr_t base_addr = start_addr + seed; 1419 const unsigned int va_width = core_mmu_get_va_width(); 1420 const vaddr_t va_mask = GENMASK_64(va_width - 1, 1421 SMALL_PAGE_SHIFT); 1422 vaddr_t ba = base_addr; 1423 size_t n = 0; 1424 1425 for (n = 0; n < 3; n++) { 1426 if (n) 1427 ba = base_addr ^ BIT64(va_width - n); 1428 ba &= va_mask; 1429 if (assign_mem_va(ba, mem_map) && 1430 mem_map_add_id_map(mem_map, id_map_start, 1431 id_map_end)) { 1432 offs = ba - start_addr; 1433 DMSG("Mapping core at %#"PRIxVA" offs %#lx", 1434 ba, offs); 1435 goto out; 1436 } else { 1437 DMSG("Failed to map core at %#"PRIxVA, ba); 1438 } 1439 } 1440 EMSG("Failed to map core with seed %#lx", seed); 1441 } 1442 1443 if (!assign_mem_va(start_addr, mem_map)) 1444 panic(); 1445 1446 out: 1447 qsort(mem_map->map, mem_map->count, sizeof(struct tee_mmap_region), 1448 cmp_mmap_by_lower_va); 1449 1450 dump_mmap_table(mem_map); 1451 1452 *ret_offs = offs; 1453 return mem_map; 1454 } 1455 1456 static void check_mem_map(struct memory_map *mem_map) 1457 { 1458 struct tee_mmap_region *m = NULL; 1459 size_t n = 0; 1460 1461 for (n = 0; n < mem_map->count; n++) { 1462 m = mem_map->map + n; 1463 switch (m->type) { 1464 case MEM_AREA_TEE_RAM: 1465 case MEM_AREA_TEE_RAM_RX: 1466 case MEM_AREA_TEE_RAM_RO: 1467 case MEM_AREA_TEE_RAM_RW: 1468 case MEM_AREA_INIT_RAM_RX: 1469 case MEM_AREA_INIT_RAM_RO: 1470 case MEM_AREA_NEX_RAM_RW: 1471 case MEM_AREA_NEX_RAM_RO: 1472 case MEM_AREA_IDENTITY_MAP_RX: 1473 if (!pbuf_is_inside(secure_only, m->pa, m->size)) 1474 panic("TEE_RAM can't fit in secure_only"); 1475 break; 1476 case MEM_AREA_TA_RAM: 1477 if (!pbuf_is_inside(secure_only, m->pa, m->size)) 1478 panic("TA_RAM can't fit in secure_only"); 1479 break; 1480 case MEM_AREA_NSEC_SHM: 1481 if (!pbuf_is_inside(nsec_shared, m->pa, m->size)) 1482 panic("NS_SHM can't fit in nsec_shared"); 1483 break; 1484 case MEM_AREA_SEC_RAM_OVERALL: 1485 case MEM_AREA_TEE_COHERENT: 1486 case MEM_AREA_TEE_ASAN: 1487 case MEM_AREA_IO_SEC: 1488 case MEM_AREA_IO_NSEC: 1489 case MEM_AREA_EXT_DT: 1490 case MEM_AREA_MANIFEST_DT: 1491 case MEM_AREA_TRANSFER_LIST: 1492 case MEM_AREA_RAM_SEC: 1493 case MEM_AREA_RAM_NSEC: 1494 case MEM_AREA_ROM_SEC: 1495 case MEM_AREA_RES_VASPACE: 1496 case MEM_AREA_SHM_VASPACE: 1497 case MEM_AREA_PAGER_VASPACE: 1498 break; 1499 default: 1500 EMSG("Uhandled memtype %d", m->type); 1501 panic(); 1502 } 1503 } 1504 } 1505 1506 /* 1507 * core_init_mmu_map() - init tee core default memory mapping 1508 * 1509 * This routine sets the static default TEE core mapping. If @seed is > 0 1510 * and configured with CFG_CORE_ASLR it will map tee core at a location 1511 * based on the seed and return the offset from the link address. 1512 * 1513 * If an error happened: core_init_mmu_map is expected to panic. 1514 * 1515 * Note: this function is weak just to make it possible to exclude it from 1516 * the unpaged area. 1517 */ 1518 void __weak core_init_mmu_map(unsigned long seed, struct core_mmu_config *cfg) 1519 { 1520 #ifndef CFG_NS_VIRTUALIZATION 1521 vaddr_t start = ROUNDDOWN((vaddr_t)__nozi_start, SMALL_PAGE_SIZE); 1522 #else 1523 vaddr_t start = ROUNDDOWN((vaddr_t)__vcore_nex_rw_start, 1524 SMALL_PAGE_SIZE); 1525 #endif 1526 vaddr_t len = ROUNDUP((vaddr_t)__nozi_end, SMALL_PAGE_SIZE) - start; 1527 struct tee_mmap_region tmp_mmap_region = { }; 1528 struct memory_map mem_map = { }; 1529 unsigned long offs = 0; 1530 1531 if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) && 1532 (core_mmu_tee_load_pa & SMALL_PAGE_MASK)) 1533 panic("OP-TEE load address is not page aligned"); 1534 1535 check_sec_nsec_mem_config(); 1536 1537 mem_map = static_memory_map; 1538 static_memory_map = (struct memory_map){ 1539 .map = &tmp_mmap_region, 1540 .alloc_count = 1, 1541 .count = 1, 1542 }; 1543 /* 1544 * Add a entry covering the translation tables which will be 1545 * involved in some virt_to_phys() and phys_to_virt() conversions. 1546 */ 1547 static_memory_map.map[0] = (struct tee_mmap_region){ 1548 .type = MEM_AREA_TEE_RAM, 1549 .region_size = SMALL_PAGE_SIZE, 1550 .pa = start, 1551 .va = start, 1552 .size = len, 1553 .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), 1554 }; 1555 1556 init_mem_map(&mem_map, seed, &offs); 1557 1558 check_mem_map(&mem_map); 1559 core_init_mmu(&mem_map); 1560 dump_xlat_table(0x0, CORE_MMU_BASE_TABLE_LEVEL); 1561 core_init_mmu_regs(cfg); 1562 cfg->map_offset = offs; 1563 static_memory_map = mem_map; 1564 } 1565 1566 bool core_mmu_mattr_is_ok(uint32_t mattr) 1567 { 1568 /* 1569 * Keep in sync with core_mmu_lpae.c:mattr_to_desc and 1570 * core_mmu_v7.c:mattr_to_texcb 1571 */ 1572 1573 switch ((mattr >> TEE_MATTR_MEM_TYPE_SHIFT) & TEE_MATTR_MEM_TYPE_MASK) { 1574 case TEE_MATTR_MEM_TYPE_DEV: 1575 case TEE_MATTR_MEM_TYPE_STRONGLY_O: 1576 case TEE_MATTR_MEM_TYPE_CACHED: 1577 case TEE_MATTR_MEM_TYPE_TAGGED: 1578 return true; 1579 default: 1580 return false; 1581 } 1582 } 1583 1584 /* 1585 * test attributes of target physical buffer 1586 * 1587 * Flags: pbuf_is(SECURE, NOT_SECURE, RAM, IOMEM, KEYVAULT). 1588 * 1589 */ 1590 bool core_pbuf_is(uint32_t attr, paddr_t pbuf, size_t len) 1591 { 1592 paddr_t ta_base = 0; 1593 size_t ta_size = 0; 1594 struct tee_mmap_region *map; 1595 1596 /* Empty buffers complies with anything */ 1597 if (len == 0) 1598 return true; 1599 1600 switch (attr) { 1601 case CORE_MEM_SEC: 1602 return pbuf_is_inside(secure_only, pbuf, len); 1603 case CORE_MEM_NON_SEC: 1604 return pbuf_is_inside(nsec_shared, pbuf, len) || 1605 pbuf_is_nsec_ddr(pbuf, len); 1606 case CORE_MEM_TEE_RAM: 1607 return core_is_buffer_inside(pbuf, len, TEE_RAM_START, 1608 TEE_RAM_PH_SIZE); 1609 case CORE_MEM_TA_RAM: 1610 core_mmu_get_ta_range(&ta_base, &ta_size); 1611 return core_is_buffer_inside(pbuf, len, ta_base, ta_size); 1612 #ifdef CFG_CORE_RESERVED_SHM 1613 case CORE_MEM_NSEC_SHM: 1614 return core_is_buffer_inside(pbuf, len, TEE_SHMEM_START, 1615 TEE_SHMEM_SIZE); 1616 #endif 1617 case CORE_MEM_SDP_MEM: 1618 return pbuf_is_sdp_mem(pbuf, len); 1619 case CORE_MEM_CACHED: 1620 map = find_map_by_pa(pbuf); 1621 if (!map || !pbuf_inside_map_area(pbuf, len, map)) 1622 return false; 1623 return mattr_is_cached(map->attr); 1624 default: 1625 return false; 1626 } 1627 } 1628 1629 /* test attributes of target virtual buffer (in core mapping) */ 1630 bool core_vbuf_is(uint32_t attr, const void *vbuf, size_t len) 1631 { 1632 paddr_t p; 1633 1634 /* Empty buffers complies with anything */ 1635 if (len == 0) 1636 return true; 1637 1638 p = virt_to_phys((void *)vbuf); 1639 if (!p) 1640 return false; 1641 1642 return core_pbuf_is(attr, p, len); 1643 } 1644 1645 /* core_va2pa - teecore exported service */ 1646 static int __maybe_unused core_va2pa_helper(void *va, paddr_t *pa) 1647 { 1648 struct tee_mmap_region *map; 1649 1650 map = find_map_by_va(va); 1651 if (!va_is_in_map(map, (vaddr_t)va)) 1652 return -1; 1653 1654 /* 1655 * We can calculate PA for static map. Virtual address ranges 1656 * reserved to core dynamic mapping return a 'match' (return 0;) 1657 * together with an invalid null physical address. 1658 */ 1659 if (map->pa) 1660 *pa = map->pa + (vaddr_t)va - map->va; 1661 else 1662 *pa = 0; 1663 1664 return 0; 1665 } 1666 1667 static void *map_pa2va(struct tee_mmap_region *map, paddr_t pa, size_t len) 1668 { 1669 if (!pa_is_in_map(map, pa, len)) 1670 return NULL; 1671 1672 return (void *)(vaddr_t)(map->va + pa - map->pa); 1673 } 1674 1675 /* 1676 * teecore gets some memory area definitions 1677 */ 1678 void core_mmu_get_mem_by_type(enum teecore_memtypes type, vaddr_t *s, 1679 vaddr_t *e) 1680 { 1681 struct tee_mmap_region *map = find_map_by_type(type); 1682 1683 if (map) { 1684 *s = map->va; 1685 *e = map->va + map->size; 1686 } else { 1687 *s = 0; 1688 *e = 0; 1689 } 1690 } 1691 1692 enum teecore_memtypes core_mmu_get_type_by_pa(paddr_t pa) 1693 { 1694 struct tee_mmap_region *map = find_map_by_pa(pa); 1695 1696 if (!map) 1697 return MEM_AREA_MAXTYPE; 1698 return map->type; 1699 } 1700 1701 void core_mmu_set_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, 1702 paddr_t pa, uint32_t attr) 1703 { 1704 assert(idx < tbl_info->num_entries); 1705 core_mmu_set_entry_primitive(tbl_info->table, tbl_info->level, 1706 idx, pa, attr); 1707 } 1708 1709 void core_mmu_get_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, 1710 paddr_t *pa, uint32_t *attr) 1711 { 1712 assert(idx < tbl_info->num_entries); 1713 core_mmu_get_entry_primitive(tbl_info->table, tbl_info->level, 1714 idx, pa, attr); 1715 } 1716 1717 static void clear_region(struct core_mmu_table_info *tbl_info, 1718 struct tee_mmap_region *region) 1719 { 1720 unsigned int end = 0; 1721 unsigned int idx = 0; 1722 1723 /* va, len and pa should be block aligned */ 1724 assert(!core_mmu_get_block_offset(tbl_info, region->va)); 1725 assert(!core_mmu_get_block_offset(tbl_info, region->size)); 1726 assert(!core_mmu_get_block_offset(tbl_info, region->pa)); 1727 1728 idx = core_mmu_va2idx(tbl_info, region->va); 1729 end = core_mmu_va2idx(tbl_info, region->va + region->size); 1730 1731 while (idx < end) { 1732 core_mmu_set_entry(tbl_info, idx, 0, 0); 1733 idx++; 1734 } 1735 } 1736 1737 static void set_region(struct core_mmu_table_info *tbl_info, 1738 struct tee_mmap_region *region) 1739 { 1740 unsigned int end; 1741 unsigned int idx; 1742 paddr_t pa; 1743 1744 /* va, len and pa should be block aligned */ 1745 assert(!core_mmu_get_block_offset(tbl_info, region->va)); 1746 assert(!core_mmu_get_block_offset(tbl_info, region->size)); 1747 assert(!core_mmu_get_block_offset(tbl_info, region->pa)); 1748 1749 idx = core_mmu_va2idx(tbl_info, region->va); 1750 end = core_mmu_va2idx(tbl_info, region->va + region->size); 1751 pa = region->pa; 1752 1753 while (idx < end) { 1754 core_mmu_set_entry(tbl_info, idx, pa, region->attr); 1755 idx++; 1756 pa += BIT64(tbl_info->shift); 1757 } 1758 } 1759 1760 static void set_pg_region(struct core_mmu_table_info *dir_info, 1761 struct vm_region *region, struct pgt **pgt, 1762 struct core_mmu_table_info *pg_info) 1763 { 1764 struct tee_mmap_region r = { 1765 .va = region->va, 1766 .size = region->size, 1767 .attr = region->attr, 1768 }; 1769 vaddr_t end = r.va + r.size; 1770 uint32_t pgt_attr = (r.attr & TEE_MATTR_SECURE) | TEE_MATTR_TABLE; 1771 1772 while (r.va < end) { 1773 if (!pg_info->table || 1774 r.va >= (pg_info->va_base + CORE_MMU_PGDIR_SIZE)) { 1775 /* 1776 * We're assigning a new translation table. 1777 */ 1778 unsigned int idx; 1779 1780 /* Virtual addresses must grow */ 1781 assert(r.va > pg_info->va_base); 1782 1783 idx = core_mmu_va2idx(dir_info, r.va); 1784 pg_info->va_base = core_mmu_idx2va(dir_info, idx); 1785 1786 /* 1787 * Advance pgt to va_base, note that we may need to 1788 * skip multiple page tables if there are large 1789 * holes in the vm map. 1790 */ 1791 while ((*pgt)->vabase < pg_info->va_base) { 1792 *pgt = SLIST_NEXT(*pgt, link); 1793 /* We should have allocated enough */ 1794 assert(*pgt); 1795 } 1796 assert((*pgt)->vabase == pg_info->va_base); 1797 pg_info->table = (*pgt)->tbl; 1798 1799 core_mmu_set_entry(dir_info, idx, 1800 virt_to_phys(pg_info->table), 1801 pgt_attr); 1802 } 1803 1804 r.size = MIN(CORE_MMU_PGDIR_SIZE - (r.va - pg_info->va_base), 1805 end - r.va); 1806 1807 if (!(*pgt)->populated && !mobj_is_paged(region->mobj)) { 1808 size_t granule = BIT(pg_info->shift); 1809 size_t offset = r.va - region->va + region->offset; 1810 1811 r.size = MIN(r.size, 1812 mobj_get_phys_granule(region->mobj)); 1813 r.size = ROUNDUP(r.size, SMALL_PAGE_SIZE); 1814 1815 if (mobj_get_pa(region->mobj, offset, granule, 1816 &r.pa) != TEE_SUCCESS) 1817 panic("Failed to get PA of unpaged mobj"); 1818 set_region(pg_info, &r); 1819 } 1820 r.va += r.size; 1821 } 1822 } 1823 1824 static bool can_map_at_level(paddr_t paddr, vaddr_t vaddr, 1825 size_t size_left, paddr_t block_size, 1826 struct tee_mmap_region *mm __maybe_unused) 1827 { 1828 /* VA and PA are aligned to block size at current level */ 1829 if ((vaddr | paddr) & (block_size - 1)) 1830 return false; 1831 1832 /* Remainder fits into block at current level */ 1833 if (size_left < block_size) 1834 return false; 1835 1836 #ifdef CFG_WITH_PAGER 1837 /* 1838 * If pager is enabled, we need to map TEE RAM and the whole pager 1839 * regions with small pages only 1840 */ 1841 if ((map_is_tee_ram(mm) || mm->type == MEM_AREA_PAGER_VASPACE) && 1842 block_size != SMALL_PAGE_SIZE) 1843 return false; 1844 #endif 1845 1846 return true; 1847 } 1848 1849 void core_mmu_map_region(struct mmu_partition *prtn, struct tee_mmap_region *mm) 1850 { 1851 struct core_mmu_table_info tbl_info; 1852 unsigned int idx; 1853 vaddr_t vaddr = mm->va; 1854 paddr_t paddr = mm->pa; 1855 ssize_t size_left = mm->size; 1856 unsigned int level; 1857 bool table_found; 1858 uint32_t old_attr; 1859 1860 assert(!((vaddr | paddr) & SMALL_PAGE_MASK)); 1861 1862 while (size_left > 0) { 1863 level = CORE_MMU_BASE_TABLE_LEVEL; 1864 1865 while (true) { 1866 paddr_t block_size = 0; 1867 1868 assert(core_mmu_level_in_range(level)); 1869 1870 table_found = core_mmu_find_table(prtn, vaddr, level, 1871 &tbl_info); 1872 if (!table_found) 1873 panic("can't find table for mapping"); 1874 1875 block_size = BIT64(tbl_info.shift); 1876 1877 idx = core_mmu_va2idx(&tbl_info, vaddr); 1878 if (!can_map_at_level(paddr, vaddr, size_left, 1879 block_size, mm)) { 1880 bool secure = mm->attr & TEE_MATTR_SECURE; 1881 1882 /* 1883 * This part of the region can't be mapped at 1884 * this level. Need to go deeper. 1885 */ 1886 if (!core_mmu_entry_to_finer_grained(&tbl_info, 1887 idx, 1888 secure)) 1889 panic("Can't divide MMU entry"); 1890 level = tbl_info.next_level; 1891 continue; 1892 } 1893 1894 /* We can map part of the region at current level */ 1895 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 1896 if (old_attr) 1897 panic("Page is already mapped"); 1898 1899 core_mmu_set_entry(&tbl_info, idx, paddr, mm->attr); 1900 paddr += block_size; 1901 vaddr += block_size; 1902 size_left -= block_size; 1903 1904 break; 1905 } 1906 } 1907 } 1908 1909 TEE_Result core_mmu_map_pages(vaddr_t vstart, paddr_t *pages, size_t num_pages, 1910 enum teecore_memtypes memtype) 1911 { 1912 TEE_Result ret; 1913 struct core_mmu_table_info tbl_info; 1914 struct tee_mmap_region *mm; 1915 unsigned int idx; 1916 uint32_t old_attr; 1917 uint32_t exceptions; 1918 vaddr_t vaddr = vstart; 1919 size_t i; 1920 bool secure; 1921 1922 assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); 1923 1924 secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; 1925 1926 if (vaddr & SMALL_PAGE_MASK) 1927 return TEE_ERROR_BAD_PARAMETERS; 1928 1929 exceptions = mmu_lock(); 1930 1931 mm = find_map_by_va((void *)vaddr); 1932 if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) 1933 panic("VA does not belong to any known mm region"); 1934 1935 if (!core_mmu_is_dynamic_vaspace(mm)) 1936 panic("Trying to map into static region"); 1937 1938 for (i = 0; i < num_pages; i++) { 1939 if (pages[i] & SMALL_PAGE_MASK) { 1940 ret = TEE_ERROR_BAD_PARAMETERS; 1941 goto err; 1942 } 1943 1944 while (true) { 1945 if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, 1946 &tbl_info)) 1947 panic("Can't find pagetable for vaddr "); 1948 1949 idx = core_mmu_va2idx(&tbl_info, vaddr); 1950 if (tbl_info.shift == SMALL_PAGE_SHIFT) 1951 break; 1952 1953 /* This is supertable. Need to divide it. */ 1954 if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, 1955 secure)) 1956 panic("Failed to spread pgdir on small tables"); 1957 } 1958 1959 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 1960 if (old_attr) 1961 panic("Page is already mapped"); 1962 1963 core_mmu_set_entry(&tbl_info, idx, pages[i], 1964 core_mmu_type_to_attr(memtype)); 1965 vaddr += SMALL_PAGE_SIZE; 1966 } 1967 1968 /* 1969 * Make sure all the changes to translation tables are visible 1970 * before returning. TLB doesn't need to be invalidated as we are 1971 * guaranteed that there's no valid mapping in this range. 1972 */ 1973 core_mmu_table_write_barrier(); 1974 mmu_unlock(exceptions); 1975 1976 return TEE_SUCCESS; 1977 err: 1978 mmu_unlock(exceptions); 1979 1980 if (i) 1981 core_mmu_unmap_pages(vstart, i); 1982 1983 return ret; 1984 } 1985 1986 TEE_Result core_mmu_map_contiguous_pages(vaddr_t vstart, paddr_t pstart, 1987 size_t num_pages, 1988 enum teecore_memtypes memtype) 1989 { 1990 struct core_mmu_table_info tbl_info = { }; 1991 struct tee_mmap_region *mm = NULL; 1992 unsigned int idx = 0; 1993 uint32_t old_attr = 0; 1994 uint32_t exceptions = 0; 1995 vaddr_t vaddr = vstart; 1996 paddr_t paddr = pstart; 1997 size_t i = 0; 1998 bool secure = false; 1999 2000 assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); 2001 2002 secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; 2003 2004 if ((vaddr | paddr) & SMALL_PAGE_MASK) 2005 return TEE_ERROR_BAD_PARAMETERS; 2006 2007 exceptions = mmu_lock(); 2008 2009 mm = find_map_by_va((void *)vaddr); 2010 if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) 2011 panic("VA does not belong to any known mm region"); 2012 2013 if (!core_mmu_is_dynamic_vaspace(mm)) 2014 panic("Trying to map into static region"); 2015 2016 for (i = 0; i < num_pages; i++) { 2017 while (true) { 2018 if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, 2019 &tbl_info)) 2020 panic("Can't find pagetable for vaddr "); 2021 2022 idx = core_mmu_va2idx(&tbl_info, vaddr); 2023 if (tbl_info.shift == SMALL_PAGE_SHIFT) 2024 break; 2025 2026 /* This is supertable. Need to divide it. */ 2027 if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, 2028 secure)) 2029 panic("Failed to spread pgdir on small tables"); 2030 } 2031 2032 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 2033 if (old_attr) 2034 panic("Page is already mapped"); 2035 2036 core_mmu_set_entry(&tbl_info, idx, paddr, 2037 core_mmu_type_to_attr(memtype)); 2038 paddr += SMALL_PAGE_SIZE; 2039 vaddr += SMALL_PAGE_SIZE; 2040 } 2041 2042 /* 2043 * Make sure all the changes to translation tables are visible 2044 * before returning. TLB doesn't need to be invalidated as we are 2045 * guaranteed that there's no valid mapping in this range. 2046 */ 2047 core_mmu_table_write_barrier(); 2048 mmu_unlock(exceptions); 2049 2050 return TEE_SUCCESS; 2051 } 2052 2053 void core_mmu_unmap_pages(vaddr_t vstart, size_t num_pages) 2054 { 2055 struct core_mmu_table_info tbl_info; 2056 struct tee_mmap_region *mm; 2057 size_t i; 2058 unsigned int idx; 2059 uint32_t exceptions; 2060 2061 exceptions = mmu_lock(); 2062 2063 mm = find_map_by_va((void *)vstart); 2064 if (!mm || !va_is_in_map(mm, vstart + num_pages * SMALL_PAGE_SIZE - 1)) 2065 panic("VA does not belong to any known mm region"); 2066 2067 if (!core_mmu_is_dynamic_vaspace(mm)) 2068 panic("Trying to unmap static region"); 2069 2070 for (i = 0; i < num_pages; i++, vstart += SMALL_PAGE_SIZE) { 2071 if (!core_mmu_find_table(NULL, vstart, UINT_MAX, &tbl_info)) 2072 panic("Can't find pagetable"); 2073 2074 if (tbl_info.shift != SMALL_PAGE_SHIFT) 2075 panic("Invalid pagetable level"); 2076 2077 idx = core_mmu_va2idx(&tbl_info, vstart); 2078 core_mmu_set_entry(&tbl_info, idx, 0, 0); 2079 } 2080 tlbi_all(); 2081 2082 mmu_unlock(exceptions); 2083 } 2084 2085 void core_mmu_populate_user_map(struct core_mmu_table_info *dir_info, 2086 struct user_mode_ctx *uctx) 2087 { 2088 struct core_mmu_table_info pg_info = { }; 2089 struct pgt_cache *pgt_cache = &uctx->pgt_cache; 2090 struct pgt *pgt = NULL; 2091 struct pgt *p = NULL; 2092 struct vm_region *r = NULL; 2093 2094 if (TAILQ_EMPTY(&uctx->vm_info.regions)) 2095 return; /* Nothing to map */ 2096 2097 /* 2098 * Allocate all page tables in advance. 2099 */ 2100 pgt_get_all(uctx); 2101 pgt = SLIST_FIRST(pgt_cache); 2102 2103 core_mmu_set_info_table(&pg_info, dir_info->next_level, 0, NULL); 2104 2105 TAILQ_FOREACH(r, &uctx->vm_info.regions, link) 2106 set_pg_region(dir_info, r, &pgt, &pg_info); 2107 /* Record that the translation tables now are populated. */ 2108 SLIST_FOREACH(p, pgt_cache, link) { 2109 p->populated = true; 2110 if (p == pgt) 2111 break; 2112 } 2113 assert(p == pgt); 2114 } 2115 2116 TEE_Result core_mmu_remove_mapping(enum teecore_memtypes type, void *addr, 2117 size_t len) 2118 { 2119 struct core_mmu_table_info tbl_info = { }; 2120 struct tee_mmap_region *res_map = NULL; 2121 struct tee_mmap_region *map = NULL; 2122 paddr_t pa = virt_to_phys(addr); 2123 size_t granule = 0; 2124 ptrdiff_t i = 0; 2125 paddr_t p = 0; 2126 size_t l = 0; 2127 2128 map = find_map_by_type_and_pa(type, pa, len); 2129 if (!map) 2130 return TEE_ERROR_GENERIC; 2131 2132 res_map = find_map_by_type(MEM_AREA_RES_VASPACE); 2133 if (!res_map) 2134 return TEE_ERROR_GENERIC; 2135 if (!core_mmu_find_table(NULL, res_map->va, UINT_MAX, &tbl_info)) 2136 return TEE_ERROR_GENERIC; 2137 granule = BIT(tbl_info.shift); 2138 2139 if (map < static_memory_map.map || 2140 map >= static_memory_map.map + static_memory_map.count) 2141 return TEE_ERROR_GENERIC; 2142 i = map - static_memory_map.map; 2143 2144 /* Check that we have a full match */ 2145 p = ROUNDDOWN(pa, granule); 2146 l = ROUNDUP(len + pa - p, granule); 2147 if (map->pa != p || map->size != l) 2148 return TEE_ERROR_GENERIC; 2149 2150 clear_region(&tbl_info, map); 2151 tlbi_all(); 2152 2153 /* If possible remove the va range from res_map */ 2154 if (res_map->va - map->size == map->va) { 2155 res_map->va -= map->size; 2156 res_map->size += map->size; 2157 } 2158 2159 /* Remove the entry. */ 2160 rem_array_elem(static_memory_map.map, static_memory_map.count, 2161 sizeof(*static_memory_map.map), i); 2162 static_memory_map.count--; 2163 2164 return TEE_SUCCESS; 2165 } 2166 2167 struct tee_mmap_region * 2168 core_mmu_find_mapping_exclusive(enum teecore_memtypes type, size_t len) 2169 { 2170 struct memory_map *mem_map = get_memory_map(); 2171 struct tee_mmap_region *map_found = NULL; 2172 size_t n = 0; 2173 2174 if (!len) 2175 return NULL; 2176 2177 for (n = 0; n < mem_map->count; n++) { 2178 if (mem_map->map[n].type != type) 2179 continue; 2180 2181 if (map_found) 2182 return NULL; 2183 2184 map_found = mem_map->map + n; 2185 } 2186 2187 if (!map_found || map_found->size < len) 2188 return NULL; 2189 2190 return map_found; 2191 } 2192 2193 void *core_mmu_add_mapping(enum teecore_memtypes type, paddr_t addr, size_t len) 2194 { 2195 struct memory_map *mem_map = &static_memory_map; 2196 struct core_mmu_table_info tbl_info = { }; 2197 struct tee_mmap_region *map = NULL; 2198 size_t granule = 0; 2199 paddr_t p = 0; 2200 size_t l = 0; 2201 2202 if (!len) 2203 return NULL; 2204 2205 if (!core_mmu_check_end_pa(addr, len)) 2206 return NULL; 2207 2208 /* Check if the memory is already mapped */ 2209 map = find_map_by_type_and_pa(type, addr, len); 2210 if (map && pbuf_inside_map_area(addr, len, map)) 2211 return (void *)(vaddr_t)(map->va + addr - map->pa); 2212 2213 /* Find the reserved va space used for late mappings */ 2214 map = find_map_by_type(MEM_AREA_RES_VASPACE); 2215 if (!map) 2216 return NULL; 2217 2218 if (!core_mmu_find_table(NULL, map->va, UINT_MAX, &tbl_info)) 2219 return NULL; 2220 2221 granule = BIT64(tbl_info.shift); 2222 p = ROUNDDOWN(addr, granule); 2223 l = ROUNDUP(len + addr - p, granule); 2224 2225 /* Ban overflowing virtual addresses */ 2226 if (map->size < l) 2227 return NULL; 2228 2229 /* 2230 * Something is wrong, we can't fit the va range into the selected 2231 * table. The reserved va range is possibly missaligned with 2232 * granule. 2233 */ 2234 if (core_mmu_va2idx(&tbl_info, map->va + len) >= tbl_info.num_entries) 2235 return NULL; 2236 2237 if (static_memory_map.count >= static_memory_map.alloc_count) 2238 return NULL; 2239 2240 mem_map->map[mem_map->count] = (struct tee_mmap_region){ 2241 .va = map->va, 2242 .size = l, 2243 .type = type, 2244 .region_size = granule, 2245 .attr = core_mmu_type_to_attr(type), 2246 .pa = p, 2247 }; 2248 map->va += l; 2249 map->size -= l; 2250 map = mem_map->map + mem_map->count; 2251 mem_map->count++; 2252 2253 set_region(&tbl_info, map); 2254 2255 /* Make sure the new entry is visible before continuing. */ 2256 core_mmu_table_write_barrier(); 2257 2258 return (void *)(vaddr_t)(map->va + addr - map->pa); 2259 } 2260 2261 #ifdef CFG_WITH_PAGER 2262 static vaddr_t get_linear_map_end_va(void) 2263 { 2264 /* this is synced with the generic linker file kern.ld.S */ 2265 return (vaddr_t)__heap2_end; 2266 } 2267 2268 static paddr_t get_linear_map_end_pa(void) 2269 { 2270 return get_linear_map_end_va() - boot_mmu_config.map_offset; 2271 } 2272 #endif 2273 2274 #if defined(CFG_TEE_CORE_DEBUG) 2275 static void check_pa_matches_va(void *va, paddr_t pa) 2276 { 2277 TEE_Result res = TEE_ERROR_GENERIC; 2278 vaddr_t v = (vaddr_t)va; 2279 paddr_t p = 0; 2280 struct core_mmu_table_info ti __maybe_unused = { }; 2281 2282 if (core_mmu_user_va_range_is_defined()) { 2283 vaddr_t user_va_base = 0; 2284 size_t user_va_size = 0; 2285 2286 core_mmu_get_user_va_range(&user_va_base, &user_va_size); 2287 if (v >= user_va_base && 2288 v <= (user_va_base - 1 + user_va_size)) { 2289 if (!core_mmu_user_mapping_is_active()) { 2290 if (pa) 2291 panic("issue in linear address space"); 2292 return; 2293 } 2294 2295 res = vm_va2pa(to_user_mode_ctx(thread_get_tsd()->ctx), 2296 va, &p); 2297 if (res == TEE_ERROR_NOT_SUPPORTED) 2298 return; 2299 if (res == TEE_SUCCESS && pa != p) 2300 panic("bad pa"); 2301 if (res != TEE_SUCCESS && pa) 2302 panic("false pa"); 2303 return; 2304 } 2305 } 2306 #ifdef CFG_WITH_PAGER 2307 if (is_unpaged(va)) { 2308 if (v - boot_mmu_config.map_offset != pa) 2309 panic("issue in linear address space"); 2310 return; 2311 } 2312 2313 if (tee_pager_get_table_info(v, &ti)) { 2314 uint32_t a; 2315 2316 /* 2317 * Lookups in the page table managed by the pager is 2318 * dangerous for addresses in the paged area as those pages 2319 * changes all the time. But some ranges are safe, 2320 * rw-locked areas when the page is populated for instance. 2321 */ 2322 core_mmu_get_entry(&ti, core_mmu_va2idx(&ti, v), &p, &a); 2323 if (a & TEE_MATTR_VALID_BLOCK) { 2324 paddr_t mask = BIT64(ti.shift) - 1; 2325 2326 p |= v & mask; 2327 if (pa != p) 2328 panic(); 2329 } else { 2330 if (pa) 2331 panic(); 2332 } 2333 return; 2334 } 2335 #endif 2336 2337 if (!core_va2pa_helper(va, &p)) { 2338 /* Verfiy only the static mapping (case non null phys addr) */ 2339 if (p && pa != p) { 2340 DMSG("va %p maps 0x%" PRIxPA ", expect 0x%" PRIxPA, 2341 va, p, pa); 2342 panic(); 2343 } 2344 } else { 2345 if (pa) { 2346 DMSG("va %p unmapped, expect 0x%" PRIxPA, va, pa); 2347 panic(); 2348 } 2349 } 2350 } 2351 #else 2352 static void check_pa_matches_va(void *va __unused, paddr_t pa __unused) 2353 { 2354 } 2355 #endif 2356 2357 paddr_t virt_to_phys(void *va) 2358 { 2359 paddr_t pa = 0; 2360 2361 if (!arch_va2pa_helper(va, &pa)) 2362 pa = 0; 2363 check_pa_matches_va(memtag_strip_tag(va), pa); 2364 return pa; 2365 } 2366 2367 #if defined(CFG_TEE_CORE_DEBUG) 2368 static void check_va_matches_pa(paddr_t pa, void *va) 2369 { 2370 paddr_t p = 0; 2371 2372 if (!va) 2373 return; 2374 2375 p = virt_to_phys(va); 2376 if (p != pa) { 2377 DMSG("va %p maps 0x%" PRIxPA " expect 0x%" PRIxPA, va, p, pa); 2378 panic(); 2379 } 2380 } 2381 #else 2382 static void check_va_matches_pa(paddr_t pa __unused, void *va __unused) 2383 { 2384 } 2385 #endif 2386 2387 static void *phys_to_virt_ts_vaspace(paddr_t pa, size_t len) 2388 { 2389 if (!core_mmu_user_mapping_is_active()) 2390 return NULL; 2391 2392 return vm_pa2va(to_user_mode_ctx(thread_get_tsd()->ctx), pa, len); 2393 } 2394 2395 #ifdef CFG_WITH_PAGER 2396 static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) 2397 { 2398 paddr_t end_pa = 0; 2399 2400 if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) 2401 return NULL; 2402 2403 if (pa >= TEE_LOAD_ADDR && pa < get_linear_map_end_pa()) { 2404 if (end_pa > get_linear_map_end_pa()) 2405 return NULL; 2406 return (void *)(vaddr_t)(pa + boot_mmu_config.map_offset); 2407 } 2408 2409 return tee_pager_phys_to_virt(pa, len); 2410 } 2411 #else 2412 static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) 2413 { 2414 struct tee_mmap_region *mmap = NULL; 2415 2416 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM, pa, len); 2417 if (!mmap) 2418 mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RW, pa, len); 2419 if (!mmap) 2420 mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RO, pa, len); 2421 if (!mmap) 2422 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RW, pa, len); 2423 if (!mmap) 2424 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RO, pa, len); 2425 if (!mmap) 2426 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RX, pa, len); 2427 /* 2428 * Note that MEM_AREA_INIT_RAM_RO and MEM_AREA_INIT_RAM_RX are only 2429 * used with pager and not needed here. 2430 */ 2431 return map_pa2va(mmap, pa, len); 2432 } 2433 #endif 2434 2435 void *phys_to_virt(paddr_t pa, enum teecore_memtypes m, size_t len) 2436 { 2437 void *va = NULL; 2438 2439 switch (m) { 2440 case MEM_AREA_TS_VASPACE: 2441 va = phys_to_virt_ts_vaspace(pa, len); 2442 break; 2443 case MEM_AREA_TEE_RAM: 2444 case MEM_AREA_TEE_RAM_RX: 2445 case MEM_AREA_TEE_RAM_RO: 2446 case MEM_AREA_TEE_RAM_RW: 2447 case MEM_AREA_NEX_RAM_RO: 2448 case MEM_AREA_NEX_RAM_RW: 2449 va = phys_to_virt_tee_ram(pa, len); 2450 break; 2451 case MEM_AREA_SHM_VASPACE: 2452 /* Find VA from PA in dynamic SHM is not yet supported */ 2453 va = NULL; 2454 break; 2455 default: 2456 va = map_pa2va(find_map_by_type_and_pa(m, pa, len), pa, len); 2457 } 2458 if (m != MEM_AREA_SEC_RAM_OVERALL) 2459 check_va_matches_pa(pa, va); 2460 return va; 2461 } 2462 2463 void *phys_to_virt_io(paddr_t pa, size_t len) 2464 { 2465 struct tee_mmap_region *map = NULL; 2466 void *va = NULL; 2467 2468 map = find_map_by_type_and_pa(MEM_AREA_IO_SEC, pa, len); 2469 if (!map) 2470 map = find_map_by_type_and_pa(MEM_AREA_IO_NSEC, pa, len); 2471 if (!map) 2472 return NULL; 2473 va = map_pa2va(map, pa, len); 2474 check_va_matches_pa(pa, va); 2475 return va; 2476 } 2477 2478 vaddr_t core_mmu_get_va(paddr_t pa, enum teecore_memtypes type, size_t len) 2479 { 2480 if (cpu_mmu_enabled()) 2481 return (vaddr_t)phys_to_virt(pa, type, len); 2482 2483 return (vaddr_t)pa; 2484 } 2485 2486 #ifdef CFG_WITH_PAGER 2487 bool is_unpaged(const void *va) 2488 { 2489 vaddr_t v = (vaddr_t)va; 2490 2491 return v >= VCORE_START_VA && v < get_linear_map_end_va(); 2492 } 2493 #endif 2494 2495 #ifdef CFG_NS_VIRTUALIZATION 2496 bool is_nexus(const void *va) 2497 { 2498 vaddr_t v = (vaddr_t)va; 2499 2500 return v >= VCORE_START_VA && v < VCORE_NEX_RW_PA + VCORE_NEX_RW_SZ; 2501 } 2502 #endif 2503 2504 void core_mmu_init_virtualization(void) 2505 { 2506 paddr_t b1 = 0; 2507 paddr_size_t s1 = 0; 2508 2509 static_assert(ARRAY_SIZE(secure_only) <= 2); 2510 if (ARRAY_SIZE(secure_only) == 2) { 2511 b1 = secure_only[1].paddr; 2512 s1 = secure_only[1].size; 2513 } 2514 virt_init_memory(&static_memory_map, secure_only[0].paddr, 2515 secure_only[0].size, b1, s1); 2516 } 2517 2518 vaddr_t io_pa_or_va(struct io_pa_va *p, size_t len) 2519 { 2520 assert(p->pa); 2521 if (cpu_mmu_enabled()) { 2522 if (!p->va) 2523 p->va = (vaddr_t)phys_to_virt_io(p->pa, len); 2524 assert(p->va); 2525 return p->va; 2526 } 2527 return p->pa; 2528 } 2529 2530 vaddr_t io_pa_or_va_secure(struct io_pa_va *p, size_t len) 2531 { 2532 assert(p->pa); 2533 if (cpu_mmu_enabled()) { 2534 if (!p->va) 2535 p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_SEC, 2536 len); 2537 assert(p->va); 2538 return p->va; 2539 } 2540 return p->pa; 2541 } 2542 2543 vaddr_t io_pa_or_va_nsec(struct io_pa_va *p, size_t len) 2544 { 2545 assert(p->pa); 2546 if (cpu_mmu_enabled()) { 2547 if (!p->va) 2548 p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_NSEC, 2549 len); 2550 assert(p->va); 2551 return p->va; 2552 } 2553 return p->pa; 2554 } 2555 2556 #ifdef CFG_CORE_RESERVED_SHM 2557 static TEE_Result teecore_init_pub_ram(void) 2558 { 2559 vaddr_t s = 0; 2560 vaddr_t e = 0; 2561 2562 /* get virtual addr/size of NSec shared mem allocated from teecore */ 2563 core_mmu_get_mem_by_type(MEM_AREA_NSEC_SHM, &s, &e); 2564 2565 if (s >= e || s & SMALL_PAGE_MASK || e & SMALL_PAGE_MASK) 2566 panic("invalid PUB RAM"); 2567 2568 /* extra check: we could rely on core_mmu_get_mem_by_type() */ 2569 if (!tee_vbuf_is_non_sec(s, e - s)) 2570 panic("PUB RAM is not non-secure"); 2571 2572 #ifdef CFG_PL310 2573 /* Allocate statically the l2cc mutex */ 2574 tee_l2cc_store_mutex_boot_pa(virt_to_phys((void *)s)); 2575 s += sizeof(uint32_t); /* size of a pl310 mutex */ 2576 s = ROUNDUP(s, SMALL_PAGE_SIZE); /* keep required alignment */ 2577 #endif 2578 2579 default_nsec_shm_paddr = virt_to_phys((void *)s); 2580 default_nsec_shm_size = e - s; 2581 2582 return TEE_SUCCESS; 2583 } 2584 early_init(teecore_init_pub_ram); 2585 #endif /*CFG_CORE_RESERVED_SHM*/ 2586 2587 void core_mmu_init_ta_ram(void) 2588 { 2589 vaddr_t s = 0; 2590 vaddr_t e = 0; 2591 paddr_t ps = 0; 2592 size_t size = 0; 2593 2594 /* 2595 * Get virtual addr/size of RAM where TA are loaded/executedNSec 2596 * shared mem allocated from teecore. 2597 */ 2598 if (IS_ENABLED(CFG_NS_VIRTUALIZATION)) 2599 virt_get_ta_ram(&s, &e); 2600 else 2601 core_mmu_get_mem_by_type(MEM_AREA_TA_RAM, &s, &e); 2602 2603 ps = virt_to_phys((void *)s); 2604 size = e - s; 2605 2606 if (!ps || (ps & CORE_MMU_USER_CODE_MASK) || 2607 !size || (size & CORE_MMU_USER_CODE_MASK)) 2608 panic("invalid TA RAM"); 2609 2610 /* extra check: we could rely on core_mmu_get_mem_by_type() */ 2611 if (!tee_pbuf_is_sec(ps, size)) 2612 panic("TA RAM is not secure"); 2613 2614 if (!tee_mm_is_empty(&tee_mm_sec_ddr)) 2615 panic("TA RAM pool is not empty"); 2616 2617 /* remove previous config and init TA ddr memory pool */ 2618 tee_mm_final(&tee_mm_sec_ddr); 2619 tee_mm_init(&tee_mm_sec_ddr, ps, size, CORE_MMU_USER_CODE_SHIFT, 2620 TEE_MM_POOL_NO_FLAGS); 2621 } 2622