1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2016, 2022 Linaro Limited 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * Copyright (c) 2022, Arm Limited and Contributors. All rights reserved. 6 */ 7 8 #include <assert.h> 9 #include <config.h> 10 #include <kernel/boot.h> 11 #include <kernel/linker.h> 12 #include <kernel/panic.h> 13 #include <kernel/spinlock.h> 14 #include <kernel/tee_l2cc_mutex.h> 15 #include <kernel/tee_misc.h> 16 #include <kernel/tlb_helpers.h> 17 #include <kernel/user_mode_ctx.h> 18 #include <kernel/virtualization.h> 19 #include <mm/core_memprot.h> 20 #include <mm/core_mmu.h> 21 #include <mm/mobj.h> 22 #include <mm/pgt_cache.h> 23 #include <mm/tee_pager.h> 24 #include <mm/vm.h> 25 #include <platform_config.h> 26 #include <string.h> 27 #include <trace.h> 28 #include <util.h> 29 30 #ifndef DEBUG_XLAT_TABLE 31 #define DEBUG_XLAT_TABLE 0 32 #endif 33 34 #define SHM_VASPACE_SIZE (1024 * 1024 * 32) 35 36 /* 37 * These variables are initialized before .bss is cleared. To avoid 38 * resetting them when .bss is cleared we're storing them in .data instead, 39 * even if they initially are zero. 40 */ 41 42 #ifdef CFG_CORE_RESERVED_SHM 43 /* Default NSec shared memory allocated from NSec world */ 44 unsigned long default_nsec_shm_size __nex_bss; 45 unsigned long default_nsec_shm_paddr __nex_bss; 46 #endif 47 48 static struct tee_mmap_region static_memory_map[CFG_MMAP_REGIONS 49 #ifdef CFG_CORE_ASLR 50 + 1 51 #endif 52 + 1] __nex_bss; 53 54 /* Define the platform's memory layout. */ 55 struct memaccess_area { 56 paddr_t paddr; 57 size_t size; 58 }; 59 60 #define MEMACCESS_AREA(a, s) { .paddr = a, .size = s } 61 62 static struct memaccess_area secure_only[] __nex_data = { 63 #ifdef TRUSTED_SRAM_BASE 64 MEMACCESS_AREA(TRUSTED_SRAM_BASE, TRUSTED_SRAM_SIZE), 65 #endif 66 MEMACCESS_AREA(TRUSTED_DRAM_BASE, TRUSTED_DRAM_SIZE), 67 }; 68 69 static struct memaccess_area nsec_shared[] __nex_data = { 70 #ifdef CFG_CORE_RESERVED_SHM 71 MEMACCESS_AREA(TEE_SHMEM_START, TEE_SHMEM_SIZE), 72 #endif 73 }; 74 75 #if defined(CFG_SECURE_DATA_PATH) 76 #ifdef CFG_TEE_SDP_MEM_BASE 77 register_sdp_mem(CFG_TEE_SDP_MEM_BASE, CFG_TEE_SDP_MEM_SIZE); 78 #endif 79 #ifdef TEE_SDP_TEST_MEM_BASE 80 register_sdp_mem(TEE_SDP_TEST_MEM_BASE, TEE_SDP_TEST_MEM_SIZE); 81 #endif 82 #endif 83 84 #ifdef CFG_CORE_RWDATA_NOEXEC 85 register_phys_mem_ul(MEM_AREA_TEE_RAM_RO, TEE_RAM_START, 86 VCORE_UNPG_RX_PA - TEE_RAM_START); 87 register_phys_mem_ul(MEM_AREA_TEE_RAM_RX, VCORE_UNPG_RX_PA, 88 VCORE_UNPG_RX_SZ_UNSAFE); 89 register_phys_mem_ul(MEM_AREA_TEE_RAM_RO, VCORE_UNPG_RO_PA, 90 VCORE_UNPG_RO_SZ_UNSAFE); 91 92 #ifdef CFG_VIRTUALIZATION 93 register_phys_mem_ul(MEM_AREA_NEX_RAM_RO, VCORE_UNPG_RW_PA, 94 VCORE_UNPG_RW_SZ_UNSAFE); 95 register_phys_mem_ul(MEM_AREA_NEX_RAM_RW, VCORE_NEX_RW_PA, 96 VCORE_NEX_RW_SZ_UNSAFE); 97 #else 98 register_phys_mem_ul(MEM_AREA_TEE_RAM_RW, VCORE_UNPG_RW_PA, 99 VCORE_UNPG_RW_SZ_UNSAFE); 100 #endif 101 102 #ifdef CFG_WITH_PAGER 103 register_phys_mem_ul(MEM_AREA_INIT_RAM_RX, VCORE_INIT_RX_PA, 104 VCORE_INIT_RX_SZ_UNSAFE); 105 register_phys_mem_ul(MEM_AREA_INIT_RAM_RO, VCORE_INIT_RO_PA, 106 VCORE_INIT_RO_SZ_UNSAFE); 107 #endif /*CFG_WITH_PAGER*/ 108 #else /*!CFG_CORE_RWDATA_NOEXEC*/ 109 register_phys_mem(MEM_AREA_TEE_RAM, TEE_RAM_START, TEE_RAM_PH_SIZE); 110 #endif /*!CFG_CORE_RWDATA_NOEXEC*/ 111 112 #ifdef CFG_VIRTUALIZATION 113 register_phys_mem(MEM_AREA_SEC_RAM_OVERALL, TRUSTED_DRAM_BASE, 114 TRUSTED_DRAM_SIZE); 115 #endif 116 117 #if defined(CFG_CORE_SANITIZE_KADDRESS) && defined(CFG_WITH_PAGER) 118 /* Asan ram is part of MEM_AREA_TEE_RAM_RW when pager is disabled */ 119 register_phys_mem_ul(MEM_AREA_TEE_ASAN, ASAN_MAP_PA, ASAN_MAP_SZ); 120 #endif 121 122 #ifndef CFG_VIRTUALIZATION 123 /* Every guest will have own TA RAM if virtualization support is enabled */ 124 register_phys_mem(MEM_AREA_TA_RAM, TA_RAM_START, TA_RAM_SIZE); 125 #endif 126 #ifdef CFG_CORE_RESERVED_SHM 127 register_phys_mem(MEM_AREA_NSEC_SHM, TEE_SHMEM_START, TEE_SHMEM_SIZE); 128 #endif 129 130 static unsigned int mmu_spinlock; 131 132 static uint32_t mmu_lock(void) 133 { 134 return cpu_spin_lock_xsave(&mmu_spinlock); 135 } 136 137 static void mmu_unlock(uint32_t exceptions) 138 { 139 cpu_spin_unlock_xrestore(&mmu_spinlock, exceptions); 140 } 141 142 static struct tee_mmap_region *get_memory_map(void) 143 { 144 if (IS_ENABLED(CFG_VIRTUALIZATION)) { 145 struct tee_mmap_region *map = virt_get_memory_map(); 146 147 if (map) 148 return map; 149 } 150 151 return static_memory_map; 152 } 153 154 static bool _pbuf_intersects(struct memaccess_area *a, size_t alen, 155 paddr_t pa, size_t size) 156 { 157 size_t n; 158 159 for (n = 0; n < alen; n++) 160 if (core_is_buffer_intersect(pa, size, a[n].paddr, a[n].size)) 161 return true; 162 return false; 163 } 164 165 #define pbuf_intersects(a, pa, size) \ 166 _pbuf_intersects((a), ARRAY_SIZE(a), (pa), (size)) 167 168 static bool _pbuf_is_inside(struct memaccess_area *a, size_t alen, 169 paddr_t pa, size_t size) 170 { 171 size_t n; 172 173 for (n = 0; n < alen; n++) 174 if (core_is_buffer_inside(pa, size, a[n].paddr, a[n].size)) 175 return true; 176 return false; 177 } 178 179 #define pbuf_is_inside(a, pa, size) \ 180 _pbuf_is_inside((a), ARRAY_SIZE(a), (pa), (size)) 181 182 static bool pa_is_in_map(struct tee_mmap_region *map, paddr_t pa, size_t len) 183 { 184 paddr_t end_pa = 0; 185 186 if (!map) 187 return false; 188 189 if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) 190 return false; 191 192 return (pa >= map->pa && end_pa <= map->pa + map->size - 1); 193 } 194 195 static bool va_is_in_map(struct tee_mmap_region *map, vaddr_t va) 196 { 197 if (!map) 198 return false; 199 return (va >= map->va && va <= (map->va + map->size - 1)); 200 } 201 202 /* check if target buffer fits in a core default map area */ 203 static bool pbuf_inside_map_area(unsigned long p, size_t l, 204 struct tee_mmap_region *map) 205 { 206 return core_is_buffer_inside(p, l, map->pa, map->size); 207 } 208 209 static struct tee_mmap_region *find_map_by_type(enum teecore_memtypes type) 210 { 211 struct tee_mmap_region *map; 212 213 for (map = get_memory_map(); !core_mmap_is_end_of_table(map); map++) 214 if (map->type == type) 215 return map; 216 return NULL; 217 } 218 219 static struct tee_mmap_region * 220 find_map_by_type_and_pa(enum teecore_memtypes type, paddr_t pa, size_t len) 221 { 222 struct tee_mmap_region *map; 223 224 for (map = get_memory_map(); !core_mmap_is_end_of_table(map); map++) { 225 if (map->type != type) 226 continue; 227 if (pa_is_in_map(map, pa, len)) 228 return map; 229 } 230 return NULL; 231 } 232 233 static struct tee_mmap_region *find_map_by_va(void *va) 234 { 235 struct tee_mmap_region *map = get_memory_map(); 236 unsigned long a = (unsigned long)va; 237 238 while (!core_mmap_is_end_of_table(map)) { 239 if (a >= map->va && a <= (map->va - 1 + map->size)) 240 return map; 241 map++; 242 } 243 return NULL; 244 } 245 246 static struct tee_mmap_region *find_map_by_pa(unsigned long pa) 247 { 248 struct tee_mmap_region *map = get_memory_map(); 249 250 while (!core_mmap_is_end_of_table(map)) { 251 if (pa >= map->pa && pa <= (map->pa + map->size - 1)) 252 return map; 253 map++; 254 } 255 return NULL; 256 } 257 258 #if defined(CFG_CORE_DYN_SHM) || defined(CFG_SECURE_DATA_PATH) 259 static bool pbuf_is_special_mem(paddr_t pbuf, size_t len, 260 const struct core_mmu_phys_mem *start, 261 const struct core_mmu_phys_mem *end) 262 { 263 const struct core_mmu_phys_mem *mem; 264 265 for (mem = start; mem < end; mem++) { 266 if (core_is_buffer_inside(pbuf, len, mem->addr, mem->size)) 267 return true; 268 } 269 270 return false; 271 } 272 #endif 273 274 #ifdef CFG_CORE_DYN_SHM 275 static void carve_out_phys_mem(struct core_mmu_phys_mem **mem, size_t *nelems, 276 paddr_t pa, size_t size) 277 { 278 struct core_mmu_phys_mem *m = *mem; 279 size_t n = 0; 280 281 while (true) { 282 if (n >= *nelems) { 283 DMSG("No need to carve out %#" PRIxPA " size %#zx", 284 pa, size); 285 return; 286 } 287 if (core_is_buffer_inside(pa, size, m[n].addr, m[n].size)) 288 break; 289 if (!core_is_buffer_outside(pa, size, m[n].addr, m[n].size)) 290 panic(); 291 n++; 292 } 293 294 if (pa == m[n].addr && size == m[n].size) { 295 /* Remove this entry */ 296 (*nelems)--; 297 memmove(m + n, m + n + 1, sizeof(*m) * (*nelems - n)); 298 m = nex_realloc(m, sizeof(*m) * *nelems); 299 if (!m) 300 panic(); 301 *mem = m; 302 } else if (pa == m[n].addr) { 303 m[n].addr += size; 304 m[n].size -= size; 305 } else if ((pa + size) == (m[n].addr + m[n].size)) { 306 m[n].size -= size; 307 } else { 308 /* Need to split the memory entry */ 309 m = nex_realloc(m, sizeof(*m) * (*nelems + 1)); 310 if (!m) 311 panic(); 312 *mem = m; 313 memmove(m + n + 1, m + n, sizeof(*m) * (*nelems - n)); 314 (*nelems)++; 315 m[n].size = pa - m[n].addr; 316 m[n + 1].size -= size + m[n].size; 317 m[n + 1].addr = pa + size; 318 } 319 } 320 321 static void check_phys_mem_is_outside(struct core_mmu_phys_mem *start, 322 size_t nelems, 323 struct tee_mmap_region *map) 324 { 325 size_t n; 326 327 for (n = 0; n < nelems; n++) { 328 if (!core_is_buffer_outside(start[n].addr, start[n].size, 329 map->pa, map->size)) { 330 EMSG("Non-sec mem (%#" PRIxPA ":%#" PRIxPASZ 331 ") overlaps map (type %d %#" PRIxPA ":%#zx)", 332 start[n].addr, start[n].size, 333 map->type, map->pa, map->size); 334 panic(); 335 } 336 } 337 } 338 339 static const struct core_mmu_phys_mem *discovered_nsec_ddr_start __nex_bss; 340 static size_t discovered_nsec_ddr_nelems __nex_bss; 341 342 static int cmp_pmem_by_addr(const void *a, const void *b) 343 { 344 const struct core_mmu_phys_mem *pmem_a = a; 345 const struct core_mmu_phys_mem *pmem_b = b; 346 347 return CMP_TRILEAN(pmem_a->addr, pmem_b->addr); 348 } 349 350 void core_mmu_set_discovered_nsec_ddr(struct core_mmu_phys_mem *start, 351 size_t nelems) 352 { 353 struct core_mmu_phys_mem *m = start; 354 size_t num_elems = nelems; 355 struct tee_mmap_region *map = static_memory_map; 356 const struct core_mmu_phys_mem __maybe_unused *pmem; 357 358 assert(!discovered_nsec_ddr_start); 359 assert(m && num_elems); 360 361 qsort(m, num_elems, sizeof(*m), cmp_pmem_by_addr); 362 363 /* 364 * Non-secure shared memory and also secure data 365 * path memory are supposed to reside inside 366 * non-secure memory. Since NSEC_SHM and SDP_MEM 367 * are used for a specific purpose make holes for 368 * those memory in the normal non-secure memory. 369 * 370 * This has to be done since for instance QEMU 371 * isn't aware of which memory range in the 372 * non-secure memory is used for NSEC_SHM. 373 */ 374 375 #ifdef CFG_SECURE_DATA_PATH 376 for (pmem = phys_sdp_mem_begin; pmem < phys_sdp_mem_end; pmem++) 377 carve_out_phys_mem(&m, &num_elems, pmem->addr, pmem->size); 378 #endif 379 380 carve_out_phys_mem(&m, &num_elems, TEE_RAM_START, TEE_RAM_PH_SIZE); 381 carve_out_phys_mem(&m, &num_elems, TA_RAM_START, TA_RAM_SIZE); 382 383 for (map = static_memory_map; !core_mmap_is_end_of_table(map); map++) { 384 switch (map->type) { 385 case MEM_AREA_NSEC_SHM: 386 carve_out_phys_mem(&m, &num_elems, map->pa, map->size); 387 break; 388 case MEM_AREA_EXT_DT: 389 case MEM_AREA_RES_VASPACE: 390 case MEM_AREA_SHM_VASPACE: 391 case MEM_AREA_TS_VASPACE: 392 case MEM_AREA_PAGER_VASPACE: 393 break; 394 default: 395 check_phys_mem_is_outside(m, num_elems, map); 396 } 397 } 398 399 discovered_nsec_ddr_start = m; 400 discovered_nsec_ddr_nelems = num_elems; 401 402 if (!core_mmu_check_end_pa(m[num_elems - 1].addr, 403 m[num_elems - 1].size)) 404 panic(); 405 } 406 407 static bool get_discovered_nsec_ddr(const struct core_mmu_phys_mem **start, 408 const struct core_mmu_phys_mem **end) 409 { 410 if (!discovered_nsec_ddr_start) 411 return false; 412 413 *start = discovered_nsec_ddr_start; 414 *end = discovered_nsec_ddr_start + discovered_nsec_ddr_nelems; 415 416 return true; 417 } 418 419 static bool pbuf_is_nsec_ddr(paddr_t pbuf, size_t len) 420 { 421 const struct core_mmu_phys_mem *start; 422 const struct core_mmu_phys_mem *end; 423 424 if (!get_discovered_nsec_ddr(&start, &end)) 425 return false; 426 427 return pbuf_is_special_mem(pbuf, len, start, end); 428 } 429 430 bool core_mmu_nsec_ddr_is_defined(void) 431 { 432 const struct core_mmu_phys_mem *start; 433 const struct core_mmu_phys_mem *end; 434 435 if (!get_discovered_nsec_ddr(&start, &end)) 436 return false; 437 438 return start != end; 439 } 440 #else 441 static bool pbuf_is_nsec_ddr(paddr_t pbuf __unused, size_t len __unused) 442 { 443 return false; 444 } 445 #endif /*CFG_CORE_DYN_SHM*/ 446 447 #define MSG_MEM_INSTERSECT(pa1, sz1, pa2, sz2) \ 448 EMSG("[%" PRIxPA " %" PRIx64 "] intersects [%" PRIxPA " %" PRIx64 "]", \ 449 pa1, (uint64_t)pa1 + (sz1), pa2, (uint64_t)pa2 + (sz2)) 450 451 #ifdef CFG_SECURE_DATA_PATH 452 static bool pbuf_is_sdp_mem(paddr_t pbuf, size_t len) 453 { 454 return pbuf_is_special_mem(pbuf, len, phys_sdp_mem_begin, 455 phys_sdp_mem_end); 456 } 457 458 struct mobj **core_sdp_mem_create_mobjs(void) 459 { 460 const struct core_mmu_phys_mem *mem; 461 struct mobj **mobj_base; 462 struct mobj **mobj; 463 int cnt = phys_sdp_mem_end - phys_sdp_mem_begin; 464 465 /* SDP mobjs table must end with a NULL entry */ 466 mobj_base = calloc(cnt + 1, sizeof(struct mobj *)); 467 if (!mobj_base) 468 panic("Out of memory"); 469 470 for (mem = phys_sdp_mem_begin, mobj = mobj_base; 471 mem < phys_sdp_mem_end; mem++, mobj++) { 472 *mobj = mobj_phys_alloc(mem->addr, mem->size, 473 TEE_MATTR_MEM_TYPE_CACHED, 474 CORE_MEM_SDP_MEM); 475 if (!*mobj) 476 panic("can't create SDP physical memory object"); 477 } 478 return mobj_base; 479 } 480 481 #else /* CFG_SECURE_DATA_PATH */ 482 static bool pbuf_is_sdp_mem(paddr_t pbuf __unused, size_t len __unused) 483 { 484 return false; 485 } 486 487 #endif /* CFG_SECURE_DATA_PATH */ 488 489 /* Check special memories comply with registered memories */ 490 static void verify_special_mem_areas(struct tee_mmap_region *mem_map, 491 size_t len, 492 const struct core_mmu_phys_mem *start, 493 const struct core_mmu_phys_mem *end, 494 const char *area_name __maybe_unused) 495 { 496 const struct core_mmu_phys_mem *mem; 497 const struct core_mmu_phys_mem *mem2; 498 struct tee_mmap_region *mmap; 499 size_t n; 500 501 if (start == end) { 502 DMSG("No %s memory area defined", area_name); 503 return; 504 } 505 506 for (mem = start; mem < end; mem++) 507 DMSG("%s memory [%" PRIxPA " %" PRIx64 "]", 508 area_name, mem->addr, (uint64_t)mem->addr + mem->size); 509 510 /* Check memories do not intersect each other */ 511 for (mem = start; mem + 1 < end; mem++) { 512 for (mem2 = mem + 1; mem2 < end; mem2++) { 513 if (core_is_buffer_intersect(mem2->addr, mem2->size, 514 mem->addr, mem->size)) { 515 MSG_MEM_INSTERSECT(mem2->addr, mem2->size, 516 mem->addr, mem->size); 517 panic("Special memory intersection"); 518 } 519 } 520 } 521 522 /* 523 * Check memories do not intersect any mapped memory. 524 * This is called before reserved VA space is loaded in mem_map. 525 */ 526 for (mem = start; mem < end; mem++) { 527 for (mmap = mem_map, n = 0; n < len; mmap++, n++) { 528 if (core_is_buffer_intersect(mem->addr, mem->size, 529 mmap->pa, mmap->size)) { 530 MSG_MEM_INSTERSECT(mem->addr, mem->size, 531 mmap->pa, mmap->size); 532 panic("Special memory intersection"); 533 } 534 } 535 } 536 } 537 538 static void add_phys_mem(struct tee_mmap_region *memory_map, size_t num_elems, 539 const struct core_mmu_phys_mem *mem, size_t *last) 540 { 541 size_t n = 0; 542 paddr_t pa; 543 paddr_size_t size; 544 545 /* 546 * If some ranges of memory of the same type do overlap 547 * each others they are coalesced into one entry. To help this 548 * added entries are sorted by increasing physical. 549 * 550 * Note that it's valid to have the same physical memory as several 551 * different memory types, for instance the same device memory 552 * mapped as both secure and non-secure. This will probably not 553 * happen often in practice. 554 */ 555 DMSG("%s type %s 0x%08" PRIxPA " size 0x%08" PRIxPASZ, 556 mem->name, teecore_memtype_name(mem->type), mem->addr, mem->size); 557 while (true) { 558 if (n >= (num_elems - 1)) { 559 EMSG("Out of entries (%zu) in memory_map", num_elems); 560 panic(); 561 } 562 if (n == *last) 563 break; 564 pa = memory_map[n].pa; 565 size = memory_map[n].size; 566 if (mem->type == memory_map[n].type && 567 ((pa <= (mem->addr + (mem->size - 1))) && 568 (mem->addr <= (pa + (size - 1))))) { 569 DMSG("Physical mem map overlaps 0x%" PRIxPA, mem->addr); 570 memory_map[n].pa = MIN(pa, mem->addr); 571 memory_map[n].size = MAX(size, mem->size) + 572 (pa - memory_map[n].pa); 573 return; 574 } 575 if (mem->type < memory_map[n].type || 576 (mem->type == memory_map[n].type && mem->addr < pa)) 577 break; /* found the spot where to insert this memory */ 578 n++; 579 } 580 581 memmove(memory_map + n + 1, memory_map + n, 582 sizeof(struct tee_mmap_region) * (*last - n)); 583 (*last)++; 584 memset(memory_map + n, 0, sizeof(memory_map[0])); 585 memory_map[n].type = mem->type; 586 memory_map[n].pa = mem->addr; 587 memory_map[n].size = mem->size; 588 } 589 590 static void add_va_space(struct tee_mmap_region *memory_map, size_t num_elems, 591 enum teecore_memtypes type, size_t size, size_t *last) 592 { 593 size_t n = 0; 594 595 DMSG("type %s size 0x%08zx", teecore_memtype_name(type), size); 596 while (true) { 597 if (n >= (num_elems - 1)) { 598 EMSG("Out of entries (%zu) in memory_map", num_elems); 599 panic(); 600 } 601 if (n == *last) 602 break; 603 if (type < memory_map[n].type) 604 break; 605 n++; 606 } 607 608 memmove(memory_map + n + 1, memory_map + n, 609 sizeof(struct tee_mmap_region) * (*last - n)); 610 (*last)++; 611 memset(memory_map + n, 0, sizeof(memory_map[0])); 612 memory_map[n].type = type; 613 memory_map[n].size = size; 614 } 615 616 uint32_t core_mmu_type_to_attr(enum teecore_memtypes t) 617 { 618 const uint32_t attr = TEE_MATTR_VALID_BLOCK; 619 const uint32_t cached = TEE_MATTR_MEM_TYPE_CACHED << 620 TEE_MATTR_MEM_TYPE_SHIFT; 621 const uint32_t noncache = TEE_MATTR_MEM_TYPE_DEV << 622 TEE_MATTR_MEM_TYPE_SHIFT; 623 624 switch (t) { 625 case MEM_AREA_TEE_RAM: 626 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | cached; 627 case MEM_AREA_TEE_RAM_RX: 628 case MEM_AREA_INIT_RAM_RX: 629 case MEM_AREA_IDENTITY_MAP_RX: 630 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRX | cached; 631 case MEM_AREA_TEE_RAM_RO: 632 case MEM_AREA_INIT_RAM_RO: 633 return attr | TEE_MATTR_SECURE | TEE_MATTR_PR | cached; 634 case MEM_AREA_TEE_RAM_RW: 635 case MEM_AREA_NEX_RAM_RO: /* This has to be r/w during init runtime */ 636 case MEM_AREA_NEX_RAM_RW: 637 case MEM_AREA_TEE_ASAN: 638 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 639 case MEM_AREA_TEE_COHERENT: 640 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRWX | noncache; 641 case MEM_AREA_TA_RAM: 642 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 643 case MEM_AREA_NSEC_SHM: 644 return attr | TEE_MATTR_PRW | cached; 645 case MEM_AREA_EXT_DT: 646 /* 647 * If CFG_MAP_EXT_DT_SECURE is enabled map the external device 648 * tree as secure non-cached memory, otherwise, fall back to 649 * non-secure mapping. 650 */ 651 if (IS_ENABLED(CFG_MAP_EXT_DT_SECURE)) 652 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | 653 noncache; 654 fallthrough; 655 case MEM_AREA_IO_NSEC: 656 return attr | TEE_MATTR_PRW | noncache; 657 case MEM_AREA_IO_SEC: 658 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | noncache; 659 case MEM_AREA_RAM_NSEC: 660 return attr | TEE_MATTR_PRW | cached; 661 case MEM_AREA_RAM_SEC: 662 case MEM_AREA_SEC_RAM_OVERALL: 663 return attr | TEE_MATTR_SECURE | TEE_MATTR_PRW | cached; 664 case MEM_AREA_RES_VASPACE: 665 case MEM_AREA_SHM_VASPACE: 666 return 0; 667 case MEM_AREA_PAGER_VASPACE: 668 return TEE_MATTR_SECURE; 669 default: 670 panic("invalid type"); 671 } 672 } 673 674 static bool __maybe_unused map_is_tee_ram(const struct tee_mmap_region *mm) 675 { 676 switch (mm->type) { 677 case MEM_AREA_TEE_RAM: 678 case MEM_AREA_TEE_RAM_RX: 679 case MEM_AREA_TEE_RAM_RO: 680 case MEM_AREA_TEE_RAM_RW: 681 case MEM_AREA_INIT_RAM_RX: 682 case MEM_AREA_INIT_RAM_RO: 683 case MEM_AREA_NEX_RAM_RW: 684 case MEM_AREA_NEX_RAM_RO: 685 case MEM_AREA_TEE_ASAN: 686 return true; 687 default: 688 return false; 689 } 690 } 691 692 static bool __maybe_unused map_is_secure(const struct tee_mmap_region *mm) 693 { 694 return !!(core_mmu_type_to_attr(mm->type) & TEE_MATTR_SECURE); 695 } 696 697 static bool __maybe_unused map_is_pgdir(const struct tee_mmap_region *mm) 698 { 699 return mm->region_size == CORE_MMU_PGDIR_SIZE; 700 } 701 702 static int cmp_mmap_by_lower_va(const void *a, const void *b) 703 { 704 const struct tee_mmap_region *mm_a = a; 705 const struct tee_mmap_region *mm_b = b; 706 707 return CMP_TRILEAN(mm_a->va, mm_b->va); 708 } 709 710 static void dump_mmap_table(struct tee_mmap_region *memory_map) 711 { 712 struct tee_mmap_region *map; 713 714 for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { 715 vaddr_t __maybe_unused vstart; 716 717 vstart = map->va + ((vaddr_t)map->pa & (map->region_size - 1)); 718 DMSG("type %-12s va 0x%08" PRIxVA "..0x%08" PRIxVA 719 " pa 0x%08" PRIxPA "..0x%08" PRIxPA " size 0x%08zx (%s)", 720 teecore_memtype_name(map->type), vstart, 721 vstart + map->size - 1, map->pa, 722 (paddr_t)(map->pa + map->size - 1), map->size, 723 map->region_size == SMALL_PAGE_SIZE ? "smallpg" : "pgdir"); 724 } 725 } 726 727 #if DEBUG_XLAT_TABLE 728 729 static void dump_xlat_table(vaddr_t va, unsigned int level) 730 { 731 struct core_mmu_table_info tbl_info; 732 unsigned int idx = 0; 733 paddr_t pa; 734 uint32_t attr; 735 736 core_mmu_find_table(NULL, va, level, &tbl_info); 737 va = tbl_info.va_base; 738 for (idx = 0; idx < tbl_info.num_entries; idx++) { 739 core_mmu_get_entry(&tbl_info, idx, &pa, &attr); 740 if (attr || level > CORE_MMU_BASE_TABLE_LEVEL) { 741 const char *security_bit = ""; 742 743 if (core_mmu_entry_have_security_bit(attr)) { 744 if (attr & TEE_MATTR_SECURE) 745 security_bit = "S"; 746 else 747 security_bit = "NS"; 748 } 749 750 if (attr & TEE_MATTR_TABLE) { 751 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 752 " TBL:0x%010" PRIxPA " %s", 753 level * 2, "", level, va, pa, 754 security_bit); 755 dump_xlat_table(va, level + 1); 756 } else if (attr) { 757 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 758 " PA:0x%010" PRIxPA " %s-%s-%s-%s", 759 level * 2, "", level, va, pa, 760 mattr_is_cached(attr) ? "MEM" : 761 "DEV", 762 attr & TEE_MATTR_PW ? "RW" : "RO", 763 attr & TEE_MATTR_PX ? "X " : "XN", 764 security_bit); 765 } else { 766 DMSG_RAW("%*s [LVL%d] VA:0x%010" PRIxVA 767 " INVALID\n", 768 level * 2, "", level, va); 769 } 770 } 771 va += BIT64(tbl_info.shift); 772 } 773 } 774 775 #else 776 777 static void dump_xlat_table(vaddr_t va __unused, int level __unused) 778 { 779 } 780 781 #endif 782 783 /* 784 * Reserves virtual memory space for pager usage. 785 * 786 * From the start of the first memory used by the link script + 787 * TEE_RAM_VA_SIZE should be covered, either with a direct mapping or empty 788 * mapping for pager usage. This adds translation tables as needed for the 789 * pager to operate. 790 */ 791 static void add_pager_vaspace(struct tee_mmap_region *mmap, size_t num_elems, 792 size_t *last) 793 { 794 paddr_t begin = 0; 795 paddr_t end = 0; 796 size_t size = 0; 797 size_t pos = 0; 798 size_t n = 0; 799 800 if (*last >= (num_elems - 1)) { 801 EMSG("Out of entries (%zu) in memory map", num_elems); 802 panic(); 803 } 804 805 for (n = 0; !core_mmap_is_end_of_table(mmap + n); n++) { 806 if (map_is_tee_ram(mmap + n)) { 807 if (!begin) 808 begin = mmap[n].pa; 809 pos = n + 1; 810 } 811 } 812 813 end = mmap[pos - 1].pa + mmap[pos - 1].size; 814 size = TEE_RAM_VA_SIZE - (end - begin); 815 if (!size) 816 return; 817 818 assert(pos <= *last); 819 memmove(mmap + pos + 1, mmap + pos, 820 sizeof(struct tee_mmap_region) * (*last - pos)); 821 (*last)++; 822 memset(mmap + pos, 0, sizeof(mmap[0])); 823 mmap[pos].type = MEM_AREA_PAGER_VASPACE; 824 mmap[pos].va = 0; 825 mmap[pos].size = size; 826 mmap[pos].region_size = SMALL_PAGE_SIZE; 827 mmap[pos].attr = core_mmu_type_to_attr(MEM_AREA_PAGER_VASPACE); 828 } 829 830 static void check_sec_nsec_mem_config(void) 831 { 832 size_t n = 0; 833 834 for (n = 0; n < ARRAY_SIZE(secure_only); n++) { 835 if (pbuf_intersects(nsec_shared, secure_only[n].paddr, 836 secure_only[n].size)) 837 panic("Invalid memory access config: sec/nsec"); 838 } 839 } 840 841 static size_t collect_mem_ranges(struct tee_mmap_region *memory_map, 842 size_t num_elems) 843 { 844 const struct core_mmu_phys_mem *mem = NULL; 845 size_t last = 0; 846 847 for (mem = phys_mem_map_begin; mem < phys_mem_map_end; mem++) { 848 struct core_mmu_phys_mem m = *mem; 849 850 /* Discard null size entries */ 851 if (!m.size) 852 continue; 853 854 /* Only unmapped virtual range may have a null phys addr */ 855 assert(m.addr || !core_mmu_type_to_attr(m.type)); 856 857 add_phys_mem(memory_map, num_elems, &m, &last); 858 } 859 860 if (IS_ENABLED(CFG_SECURE_DATA_PATH)) 861 verify_special_mem_areas(memory_map, num_elems, 862 phys_sdp_mem_begin, 863 phys_sdp_mem_end, "SDP"); 864 865 add_va_space(memory_map, num_elems, MEM_AREA_RES_VASPACE, 866 CFG_RESERVED_VASPACE_SIZE, &last); 867 868 add_va_space(memory_map, num_elems, MEM_AREA_SHM_VASPACE, 869 SHM_VASPACE_SIZE, &last); 870 871 memory_map[last].type = MEM_AREA_END; 872 873 return last; 874 } 875 876 static void assign_mem_granularity(struct tee_mmap_region *memory_map) 877 { 878 struct tee_mmap_region *map = NULL; 879 880 /* 881 * Assign region sizes, note that MEM_AREA_TEE_RAM always uses 882 * SMALL_PAGE_SIZE. 883 */ 884 for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { 885 paddr_t mask = map->pa | map->size; 886 887 if (!(mask & CORE_MMU_PGDIR_MASK)) 888 map->region_size = CORE_MMU_PGDIR_SIZE; 889 else if (!(mask & SMALL_PAGE_MASK)) 890 map->region_size = SMALL_PAGE_SIZE; 891 else 892 panic("Impossible memory alignment"); 893 894 if (map_is_tee_ram(map)) 895 map->region_size = SMALL_PAGE_SIZE; 896 } 897 } 898 899 static bool place_tee_ram_at_top(paddr_t paddr) 900 { 901 return paddr > BIT64(core_mmu_get_va_width()) / 2; 902 } 903 904 /* 905 * MMU arch driver shall override this function if it helps 906 * optimizing the memory footprint of the address translation tables. 907 */ 908 bool __weak core_mmu_prefer_tee_ram_at_top(paddr_t paddr) 909 { 910 return place_tee_ram_at_top(paddr); 911 } 912 913 static bool assign_mem_va_dir(vaddr_t tee_ram_va, 914 struct tee_mmap_region *memory_map, 915 bool tee_ram_at_top) 916 { 917 struct tee_mmap_region *map = NULL; 918 vaddr_t va = 0; 919 bool va_is_secure = true; 920 921 /* Clear eventual previous assignments */ 922 for (map = memory_map; !core_mmap_is_end_of_table(map); map++) 923 map->va = 0; 924 925 /* 926 * TEE RAM regions are always aligned with region_size. 927 * 928 * Note that MEM_AREA_PAGER_VASPACE also counts as TEE RAM here 929 * since it handles virtual memory which covers the part of the ELF 930 * that cannot fit directly into memory. 931 */ 932 va = tee_ram_va; 933 for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { 934 if (map_is_tee_ram(map) || 935 map->type == MEM_AREA_PAGER_VASPACE) { 936 assert(!(va & (map->region_size - 1))); 937 assert(!(map->size & (map->region_size - 1))); 938 map->va = va; 939 if (ADD_OVERFLOW(va, map->size, &va)) 940 return false; 941 if (va >= BIT64(core_mmu_get_va_width())) 942 return false; 943 } 944 } 945 946 if (tee_ram_at_top) { 947 /* 948 * Map non-tee ram regions at addresses lower than the tee 949 * ram region. 950 */ 951 va = tee_ram_va; 952 for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { 953 map->attr = core_mmu_type_to_attr(map->type); 954 if (map->va) 955 continue; 956 957 if (!IS_ENABLED(CFG_WITH_LPAE) && 958 va_is_secure != map_is_secure(map)) { 959 va_is_secure = !va_is_secure; 960 va = ROUNDDOWN(va, CORE_MMU_PGDIR_SIZE); 961 } 962 963 if (SUB_OVERFLOW(va, map->size, &va)) 964 return false; 965 va = ROUNDDOWN(va, map->region_size); 966 /* 967 * Make sure that va is aligned with pa for 968 * efficient pgdir mapping. Basically pa & 969 * pgdir_mask should be == va & pgdir_mask 970 */ 971 if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { 972 if (SUB_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, &va)) 973 return false; 974 va += (map->pa - va) & CORE_MMU_PGDIR_MASK; 975 } 976 map->va = va; 977 } 978 } else { 979 /* 980 * Map non-tee ram regions at addresses higher than the tee 981 * ram region. 982 */ 983 for (map = memory_map; !core_mmap_is_end_of_table(map); map++) { 984 map->attr = core_mmu_type_to_attr(map->type); 985 if (map->va) 986 continue; 987 988 if (!IS_ENABLED(CFG_WITH_LPAE) && 989 va_is_secure != map_is_secure(map)) { 990 va_is_secure = !va_is_secure; 991 if (ROUNDUP_OVERFLOW(va, CORE_MMU_PGDIR_SIZE, 992 &va)) 993 return false; 994 } 995 996 if (ROUNDUP_OVERFLOW(va, map->region_size, &va)) 997 return false; 998 /* 999 * Make sure that va is aligned with pa for 1000 * efficient pgdir mapping. Basically pa & 1001 * pgdir_mask should be == va & pgdir_mask 1002 */ 1003 if (map->size > 2 * CORE_MMU_PGDIR_SIZE) { 1004 vaddr_t offs = (map->pa - va) & 1005 CORE_MMU_PGDIR_MASK; 1006 1007 if (ADD_OVERFLOW(va, offs, &va)) 1008 return false; 1009 } 1010 1011 map->va = va; 1012 if (ADD_OVERFLOW(va, map->size, &va)) 1013 return false; 1014 if (va >= BIT64(core_mmu_get_va_width())) 1015 return false; 1016 } 1017 } 1018 1019 return true; 1020 } 1021 1022 static bool assign_mem_va(vaddr_t tee_ram_va, 1023 struct tee_mmap_region *memory_map) 1024 { 1025 bool tee_ram_at_top = place_tee_ram_at_top(tee_ram_va); 1026 1027 /* 1028 * Check that we're not overlapping with the user VA range. 1029 */ 1030 if (IS_ENABLED(CFG_WITH_LPAE)) { 1031 /* 1032 * User VA range is supposed to be defined after these 1033 * mappings have been established. 1034 */ 1035 assert(!core_mmu_user_va_range_is_defined()); 1036 } else { 1037 vaddr_t user_va_base = 0; 1038 size_t user_va_size = 0; 1039 1040 assert(core_mmu_user_va_range_is_defined()); 1041 core_mmu_get_user_va_range(&user_va_base, &user_va_size); 1042 if (tee_ram_va < (user_va_base + user_va_size)) 1043 return false; 1044 } 1045 1046 if (IS_ENABLED(CFG_WITH_PAGER)) { 1047 bool prefered_dir = core_mmu_prefer_tee_ram_at_top(tee_ram_va); 1048 1049 /* Try whole mapping covered by a single base xlat entry */ 1050 if (prefered_dir != tee_ram_at_top && 1051 assign_mem_va_dir(tee_ram_va, memory_map, prefered_dir)) 1052 return true; 1053 } 1054 1055 return assign_mem_va_dir(tee_ram_va, memory_map, tee_ram_at_top); 1056 } 1057 1058 static int cmp_init_mem_map(const void *a, const void *b) 1059 { 1060 const struct tee_mmap_region *mm_a = a; 1061 const struct tee_mmap_region *mm_b = b; 1062 int rc = 0; 1063 1064 rc = CMP_TRILEAN(mm_a->region_size, mm_b->region_size); 1065 if (!rc) 1066 rc = CMP_TRILEAN(mm_a->pa, mm_b->pa); 1067 /* 1068 * 32bit MMU descriptors cannot mix secure and non-secure mapping in 1069 * the same level2 table. Hence sort secure mapping from non-secure 1070 * mapping. 1071 */ 1072 if (!rc && !IS_ENABLED(CFG_WITH_LPAE)) 1073 rc = CMP_TRILEAN(map_is_secure(mm_a), map_is_secure(mm_b)); 1074 1075 return rc; 1076 } 1077 1078 static bool mem_map_add_id_map(struct tee_mmap_region *memory_map, 1079 size_t num_elems, size_t *last, 1080 vaddr_t id_map_start, vaddr_t id_map_end) 1081 { 1082 struct tee_mmap_region *map = NULL; 1083 vaddr_t start = ROUNDDOWN(id_map_start, SMALL_PAGE_SIZE); 1084 vaddr_t end = ROUNDUP(id_map_end, SMALL_PAGE_SIZE); 1085 size_t len = end - start; 1086 1087 if (*last >= num_elems - 1) { 1088 EMSG("Out of entries (%zu) in memory map", num_elems); 1089 panic(); 1090 } 1091 1092 for (map = memory_map; !core_mmap_is_end_of_table(map); map++) 1093 if (core_is_buffer_intersect(map->va, map->size, start, len)) 1094 return false; 1095 1096 *map = (struct tee_mmap_region){ 1097 .type = MEM_AREA_IDENTITY_MAP_RX, 1098 /* 1099 * Could use CORE_MMU_PGDIR_SIZE to potentially save a 1100 * translation table, at the increased risk of clashes with 1101 * the rest of the memory map. 1102 */ 1103 .region_size = SMALL_PAGE_SIZE, 1104 .pa = start, 1105 .va = start, 1106 .size = len, 1107 .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), 1108 }; 1109 1110 (*last)++; 1111 1112 return true; 1113 } 1114 1115 static unsigned long init_mem_map(struct tee_mmap_region *memory_map, 1116 size_t num_elems, unsigned long seed) 1117 { 1118 /* 1119 * @id_map_start and @id_map_end describes a physical memory range 1120 * that must be mapped Read-Only eXecutable at identical virtual 1121 * addresses. 1122 */ 1123 vaddr_t id_map_start = (vaddr_t)__identity_map_init_start; 1124 vaddr_t id_map_end = (vaddr_t)__identity_map_init_end; 1125 unsigned long offs = 0; 1126 size_t last = 0; 1127 1128 last = collect_mem_ranges(memory_map, num_elems); 1129 assign_mem_granularity(memory_map); 1130 1131 /* 1132 * To ease mapping and lower use of xlat tables, sort mapping 1133 * description moving small-page regions after the pgdir regions. 1134 */ 1135 qsort(memory_map, last, sizeof(struct tee_mmap_region), 1136 cmp_init_mem_map); 1137 1138 add_pager_vaspace(memory_map, num_elems, &last); 1139 if (IS_ENABLED(CFG_CORE_ASLR) && seed) { 1140 vaddr_t base_addr = TEE_RAM_START + seed; 1141 const unsigned int va_width = core_mmu_get_va_width(); 1142 const vaddr_t va_mask = GENMASK_64(va_width - 1, 1143 SMALL_PAGE_SHIFT); 1144 vaddr_t ba = base_addr; 1145 size_t n = 0; 1146 1147 for (n = 0; n < 3; n++) { 1148 if (n) 1149 ba = base_addr ^ BIT64(va_width - n); 1150 ba &= va_mask; 1151 if (assign_mem_va(ba, memory_map) && 1152 mem_map_add_id_map(memory_map, num_elems, &last, 1153 id_map_start, id_map_end)) { 1154 offs = ba - TEE_RAM_START; 1155 DMSG("Mapping core at %#"PRIxVA" offs %#lx", 1156 ba, offs); 1157 goto out; 1158 } else { 1159 DMSG("Failed to map core at %#"PRIxVA, ba); 1160 } 1161 } 1162 EMSG("Failed to map core with seed %#lx", seed); 1163 } 1164 1165 if (!assign_mem_va(TEE_RAM_START, memory_map)) 1166 panic(); 1167 1168 out: 1169 qsort(memory_map, last, sizeof(struct tee_mmap_region), 1170 cmp_mmap_by_lower_va); 1171 1172 dump_mmap_table(memory_map); 1173 1174 return offs; 1175 } 1176 1177 static void check_mem_map(struct tee_mmap_region *map) 1178 { 1179 struct tee_mmap_region *m = NULL; 1180 1181 for (m = map; !core_mmap_is_end_of_table(m); m++) { 1182 switch (m->type) { 1183 case MEM_AREA_TEE_RAM: 1184 case MEM_AREA_TEE_RAM_RX: 1185 case MEM_AREA_TEE_RAM_RO: 1186 case MEM_AREA_TEE_RAM_RW: 1187 case MEM_AREA_INIT_RAM_RX: 1188 case MEM_AREA_INIT_RAM_RO: 1189 case MEM_AREA_NEX_RAM_RW: 1190 case MEM_AREA_NEX_RAM_RO: 1191 case MEM_AREA_IDENTITY_MAP_RX: 1192 if (!pbuf_is_inside(secure_only, m->pa, m->size)) 1193 panic("TEE_RAM can't fit in secure_only"); 1194 break; 1195 case MEM_AREA_TA_RAM: 1196 if (!pbuf_is_inside(secure_only, m->pa, m->size)) 1197 panic("TA_RAM can't fit in secure_only"); 1198 break; 1199 case MEM_AREA_NSEC_SHM: 1200 if (!pbuf_is_inside(nsec_shared, m->pa, m->size)) 1201 panic("NS_SHM can't fit in nsec_shared"); 1202 break; 1203 case MEM_AREA_SEC_RAM_OVERALL: 1204 case MEM_AREA_TEE_COHERENT: 1205 case MEM_AREA_TEE_ASAN: 1206 case MEM_AREA_IO_SEC: 1207 case MEM_AREA_IO_NSEC: 1208 case MEM_AREA_EXT_DT: 1209 case MEM_AREA_RAM_SEC: 1210 case MEM_AREA_RAM_NSEC: 1211 case MEM_AREA_RES_VASPACE: 1212 case MEM_AREA_SHM_VASPACE: 1213 case MEM_AREA_PAGER_VASPACE: 1214 break; 1215 default: 1216 EMSG("Uhandled memtype %d", m->type); 1217 panic(); 1218 } 1219 } 1220 } 1221 1222 static struct tee_mmap_region *get_tmp_mmap(void) 1223 { 1224 struct tee_mmap_region *tmp_mmap = (void *)__heap1_start; 1225 1226 #ifdef CFG_WITH_PAGER 1227 if (__heap1_end - __heap1_start < (ptrdiff_t)sizeof(static_memory_map)) 1228 tmp_mmap = (void *)__heap2_start; 1229 #endif 1230 1231 memset(tmp_mmap, 0, sizeof(static_memory_map)); 1232 1233 return tmp_mmap; 1234 } 1235 1236 /* 1237 * core_init_mmu_map() - init tee core default memory mapping 1238 * 1239 * This routine sets the static default TEE core mapping. If @seed is > 0 1240 * and configured with CFG_CORE_ASLR it will map tee core at a location 1241 * based on the seed and return the offset from the link address. 1242 * 1243 * If an error happened: core_init_mmu_map is expected to panic. 1244 * 1245 * Note: this function is weak just to make it possible to exclude it from 1246 * the unpaged area. 1247 */ 1248 void __weak core_init_mmu_map(unsigned long seed, struct core_mmu_config *cfg) 1249 { 1250 #ifndef CFG_VIRTUALIZATION 1251 vaddr_t start = ROUNDDOWN((vaddr_t)__nozi_start, SMALL_PAGE_SIZE); 1252 #else 1253 vaddr_t start = ROUNDDOWN((vaddr_t)__vcore_nex_rw_start, 1254 SMALL_PAGE_SIZE); 1255 #endif 1256 vaddr_t len = ROUNDUP((vaddr_t)__nozi_end, SMALL_PAGE_SIZE) - start; 1257 struct tee_mmap_region *tmp_mmap = get_tmp_mmap(); 1258 unsigned long offs = 0; 1259 1260 check_sec_nsec_mem_config(); 1261 1262 /* 1263 * Add a entry covering the translation tables which will be 1264 * involved in some virt_to_phys() and phys_to_virt() conversions. 1265 */ 1266 static_memory_map[0] = (struct tee_mmap_region){ 1267 .type = MEM_AREA_TEE_RAM, 1268 .region_size = SMALL_PAGE_SIZE, 1269 .pa = start, 1270 .va = start, 1271 .size = len, 1272 .attr = core_mmu_type_to_attr(MEM_AREA_IDENTITY_MAP_RX), 1273 }; 1274 1275 COMPILE_TIME_ASSERT(CFG_MMAP_REGIONS >= 13); 1276 offs = init_mem_map(tmp_mmap, ARRAY_SIZE(static_memory_map), seed); 1277 1278 check_mem_map(tmp_mmap); 1279 core_init_mmu(tmp_mmap); 1280 dump_xlat_table(0x0, CORE_MMU_BASE_TABLE_LEVEL); 1281 core_init_mmu_regs(cfg); 1282 cfg->load_offset = offs; 1283 memcpy(static_memory_map, tmp_mmap, sizeof(static_memory_map)); 1284 } 1285 1286 bool core_mmu_mattr_is_ok(uint32_t mattr) 1287 { 1288 /* 1289 * Keep in sync with core_mmu_lpae.c:mattr_to_desc and 1290 * core_mmu_v7.c:mattr_to_texcb 1291 */ 1292 1293 switch ((mattr >> TEE_MATTR_MEM_TYPE_SHIFT) & TEE_MATTR_MEM_TYPE_MASK) { 1294 case TEE_MATTR_MEM_TYPE_DEV: 1295 case TEE_MATTR_MEM_TYPE_STRONGLY_O: 1296 case TEE_MATTR_MEM_TYPE_CACHED: 1297 return true; 1298 default: 1299 return false; 1300 } 1301 } 1302 1303 /* 1304 * test attributes of target physical buffer 1305 * 1306 * Flags: pbuf_is(SECURE, NOT_SECURE, RAM, IOMEM, KEYVAULT). 1307 * 1308 */ 1309 bool core_pbuf_is(uint32_t attr, paddr_t pbuf, size_t len) 1310 { 1311 struct tee_mmap_region *map; 1312 1313 /* Empty buffers complies with anything */ 1314 if (len == 0) 1315 return true; 1316 1317 switch (attr) { 1318 case CORE_MEM_SEC: 1319 return pbuf_is_inside(secure_only, pbuf, len); 1320 case CORE_MEM_NON_SEC: 1321 return pbuf_is_inside(nsec_shared, pbuf, len) || 1322 pbuf_is_nsec_ddr(pbuf, len); 1323 case CORE_MEM_TEE_RAM: 1324 return core_is_buffer_inside(pbuf, len, TEE_RAM_START, 1325 TEE_RAM_PH_SIZE); 1326 case CORE_MEM_TA_RAM: 1327 return core_is_buffer_inside(pbuf, len, TA_RAM_START, 1328 TA_RAM_SIZE); 1329 #ifdef CFG_CORE_RESERVED_SHM 1330 case CORE_MEM_NSEC_SHM: 1331 return core_is_buffer_inside(pbuf, len, TEE_SHMEM_START, 1332 TEE_SHMEM_SIZE); 1333 #endif 1334 case CORE_MEM_SDP_MEM: 1335 return pbuf_is_sdp_mem(pbuf, len); 1336 case CORE_MEM_CACHED: 1337 map = find_map_by_pa(pbuf); 1338 if (!map || !pbuf_inside_map_area(pbuf, len, map)) 1339 return false; 1340 return mattr_is_cached(map->attr); 1341 default: 1342 return false; 1343 } 1344 } 1345 1346 /* test attributes of target virtual buffer (in core mapping) */ 1347 bool core_vbuf_is(uint32_t attr, const void *vbuf, size_t len) 1348 { 1349 paddr_t p; 1350 1351 /* Empty buffers complies with anything */ 1352 if (len == 0) 1353 return true; 1354 1355 p = virt_to_phys((void *)vbuf); 1356 if (!p) 1357 return false; 1358 1359 return core_pbuf_is(attr, p, len); 1360 } 1361 1362 /* core_va2pa - teecore exported service */ 1363 static int __maybe_unused core_va2pa_helper(void *va, paddr_t *pa) 1364 { 1365 struct tee_mmap_region *map; 1366 1367 map = find_map_by_va(va); 1368 if (!va_is_in_map(map, (vaddr_t)va)) 1369 return -1; 1370 1371 /* 1372 * We can calculate PA for static map. Virtual address ranges 1373 * reserved to core dynamic mapping return a 'match' (return 0;) 1374 * together with an invalid null physical address. 1375 */ 1376 if (map->pa) 1377 *pa = map->pa + (vaddr_t)va - map->va; 1378 else 1379 *pa = 0; 1380 1381 return 0; 1382 } 1383 1384 static void *map_pa2va(struct tee_mmap_region *map, paddr_t pa, size_t len) 1385 { 1386 if (!pa_is_in_map(map, pa, len)) 1387 return NULL; 1388 1389 return (void *)(vaddr_t)(map->va + pa - map->pa); 1390 } 1391 1392 /* 1393 * teecore gets some memory area definitions 1394 */ 1395 void core_mmu_get_mem_by_type(unsigned int type, vaddr_t *s, vaddr_t *e) 1396 { 1397 struct tee_mmap_region *map = find_map_by_type(type); 1398 1399 if (map) { 1400 *s = map->va; 1401 *e = map->va + map->size; 1402 } else { 1403 *s = 0; 1404 *e = 0; 1405 } 1406 } 1407 1408 enum teecore_memtypes core_mmu_get_type_by_pa(paddr_t pa) 1409 { 1410 struct tee_mmap_region *map = find_map_by_pa(pa); 1411 1412 if (!map) 1413 return MEM_AREA_MAXTYPE; 1414 return map->type; 1415 } 1416 1417 void core_mmu_set_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, 1418 paddr_t pa, uint32_t attr) 1419 { 1420 assert(idx < tbl_info->num_entries); 1421 core_mmu_set_entry_primitive(tbl_info->table, tbl_info->level, 1422 idx, pa, attr); 1423 } 1424 1425 void core_mmu_get_entry(struct core_mmu_table_info *tbl_info, unsigned int idx, 1426 paddr_t *pa, uint32_t *attr) 1427 { 1428 assert(idx < tbl_info->num_entries); 1429 core_mmu_get_entry_primitive(tbl_info->table, tbl_info->level, 1430 idx, pa, attr); 1431 } 1432 1433 static void clear_region(struct core_mmu_table_info *tbl_info, 1434 struct tee_mmap_region *region) 1435 { 1436 unsigned int end = 0; 1437 unsigned int idx = 0; 1438 1439 /* va, len and pa should be block aligned */ 1440 assert(!core_mmu_get_block_offset(tbl_info, region->va)); 1441 assert(!core_mmu_get_block_offset(tbl_info, region->size)); 1442 assert(!core_mmu_get_block_offset(tbl_info, region->pa)); 1443 1444 idx = core_mmu_va2idx(tbl_info, region->va); 1445 end = core_mmu_va2idx(tbl_info, region->va + region->size); 1446 1447 while (idx < end) { 1448 core_mmu_set_entry(tbl_info, idx, 0, 0); 1449 idx++; 1450 } 1451 } 1452 1453 static void set_region(struct core_mmu_table_info *tbl_info, 1454 struct tee_mmap_region *region) 1455 { 1456 unsigned int end; 1457 unsigned int idx; 1458 paddr_t pa; 1459 1460 /* va, len and pa should be block aligned */ 1461 assert(!core_mmu_get_block_offset(tbl_info, region->va)); 1462 assert(!core_mmu_get_block_offset(tbl_info, region->size)); 1463 assert(!core_mmu_get_block_offset(tbl_info, region->pa)); 1464 1465 idx = core_mmu_va2idx(tbl_info, region->va); 1466 end = core_mmu_va2idx(tbl_info, region->va + region->size); 1467 pa = region->pa; 1468 1469 while (idx < end) { 1470 core_mmu_set_entry(tbl_info, idx, pa, region->attr); 1471 idx++; 1472 pa += BIT64(tbl_info->shift); 1473 } 1474 } 1475 1476 static void set_pg_region(struct core_mmu_table_info *dir_info, 1477 struct vm_region *region, struct pgt **pgt, 1478 struct core_mmu_table_info *pg_info) 1479 { 1480 struct tee_mmap_region r = { 1481 .va = region->va, 1482 .size = region->size, 1483 .attr = region->attr, 1484 }; 1485 vaddr_t end = r.va + r.size; 1486 uint32_t pgt_attr = (r.attr & TEE_MATTR_SECURE) | TEE_MATTR_TABLE; 1487 1488 while (r.va < end) { 1489 if (!pg_info->table || 1490 r.va >= (pg_info->va_base + CORE_MMU_PGDIR_SIZE)) { 1491 /* 1492 * We're assigning a new translation table. 1493 */ 1494 unsigned int idx; 1495 1496 /* Virtual addresses must grow */ 1497 assert(r.va > pg_info->va_base); 1498 1499 idx = core_mmu_va2idx(dir_info, r.va); 1500 pg_info->va_base = core_mmu_idx2va(dir_info, idx); 1501 1502 #ifdef CFG_PAGED_USER_TA 1503 /* 1504 * Advance pgt to va_base, note that we may need to 1505 * skip multiple page tables if there are large 1506 * holes in the vm map. 1507 */ 1508 while ((*pgt)->vabase < pg_info->va_base) { 1509 *pgt = SLIST_NEXT(*pgt, link); 1510 /* We should have allocated enough */ 1511 assert(*pgt); 1512 } 1513 assert((*pgt)->vabase == pg_info->va_base); 1514 pg_info->table = (*pgt)->tbl; 1515 #else 1516 assert(*pgt); /* We should have allocated enough */ 1517 pg_info->table = (*pgt)->tbl; 1518 *pgt = SLIST_NEXT(*pgt, link); 1519 #endif 1520 1521 core_mmu_set_entry(dir_info, idx, 1522 virt_to_phys(pg_info->table), 1523 pgt_attr); 1524 } 1525 1526 r.size = MIN(CORE_MMU_PGDIR_SIZE - (r.va - pg_info->va_base), 1527 end - r.va); 1528 1529 if (!mobj_is_paged(region->mobj)) { 1530 size_t granule = BIT(pg_info->shift); 1531 size_t offset = r.va - region->va + region->offset; 1532 1533 r.size = MIN(r.size, 1534 mobj_get_phys_granule(region->mobj)); 1535 r.size = ROUNDUP(r.size, SMALL_PAGE_SIZE); 1536 1537 if (mobj_get_pa(region->mobj, offset, granule, 1538 &r.pa) != TEE_SUCCESS) 1539 panic("Failed to get PA of unpaged mobj"); 1540 set_region(pg_info, &r); 1541 } 1542 r.va += r.size; 1543 } 1544 } 1545 1546 static bool can_map_at_level(paddr_t paddr, vaddr_t vaddr, 1547 size_t size_left, paddr_t block_size, 1548 struct tee_mmap_region *mm __maybe_unused) 1549 { 1550 /* VA and PA are aligned to block size at current level */ 1551 if ((vaddr | paddr) & (block_size - 1)) 1552 return false; 1553 1554 /* Remainder fits into block at current level */ 1555 if (size_left < block_size) 1556 return false; 1557 1558 #ifdef CFG_WITH_PAGER 1559 /* 1560 * If pager is enabled, we need to map tee ram 1561 * regions with small pages only 1562 */ 1563 if (map_is_tee_ram(mm) && block_size != SMALL_PAGE_SIZE) 1564 return false; 1565 #endif 1566 1567 return true; 1568 } 1569 1570 void core_mmu_map_region(struct mmu_partition *prtn, struct tee_mmap_region *mm) 1571 { 1572 struct core_mmu_table_info tbl_info; 1573 unsigned int idx; 1574 vaddr_t vaddr = mm->va; 1575 paddr_t paddr = mm->pa; 1576 ssize_t size_left = mm->size; 1577 unsigned int level; 1578 bool table_found; 1579 uint32_t old_attr; 1580 1581 assert(!((vaddr | paddr) & SMALL_PAGE_MASK)); 1582 1583 while (size_left > 0) { 1584 level = CORE_MMU_BASE_TABLE_LEVEL; 1585 1586 while (true) { 1587 paddr_t block_size = 0; 1588 1589 assert(level <= CORE_MMU_PGDIR_LEVEL); 1590 1591 table_found = core_mmu_find_table(prtn, vaddr, level, 1592 &tbl_info); 1593 if (!table_found) 1594 panic("can't find table for mapping"); 1595 1596 block_size = BIT64(tbl_info.shift); 1597 1598 idx = core_mmu_va2idx(&tbl_info, vaddr); 1599 if (!can_map_at_level(paddr, vaddr, size_left, 1600 block_size, mm)) { 1601 bool secure = mm->attr & TEE_MATTR_SECURE; 1602 1603 /* 1604 * This part of the region can't be mapped at 1605 * this level. Need to go deeper. 1606 */ 1607 if (!core_mmu_entry_to_finer_grained(&tbl_info, 1608 idx, 1609 secure)) 1610 panic("Can't divide MMU entry"); 1611 level++; 1612 continue; 1613 } 1614 1615 /* We can map part of the region at current level */ 1616 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 1617 if (old_attr) 1618 panic("Page is already mapped"); 1619 1620 core_mmu_set_entry(&tbl_info, idx, paddr, mm->attr); 1621 paddr += block_size; 1622 vaddr += block_size; 1623 size_left -= block_size; 1624 1625 break; 1626 } 1627 } 1628 } 1629 1630 TEE_Result core_mmu_map_pages(vaddr_t vstart, paddr_t *pages, size_t num_pages, 1631 enum teecore_memtypes memtype) 1632 { 1633 TEE_Result ret; 1634 struct core_mmu_table_info tbl_info; 1635 struct tee_mmap_region *mm; 1636 unsigned int idx; 1637 uint32_t old_attr; 1638 uint32_t exceptions; 1639 vaddr_t vaddr = vstart; 1640 size_t i; 1641 bool secure; 1642 1643 assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); 1644 1645 secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; 1646 1647 if (vaddr & SMALL_PAGE_MASK) 1648 return TEE_ERROR_BAD_PARAMETERS; 1649 1650 exceptions = mmu_lock(); 1651 1652 mm = find_map_by_va((void *)vaddr); 1653 if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) 1654 panic("VA does not belong to any known mm region"); 1655 1656 if (!core_mmu_is_dynamic_vaspace(mm)) 1657 panic("Trying to map into static region"); 1658 1659 for (i = 0; i < num_pages; i++) { 1660 if (pages[i] & SMALL_PAGE_MASK) { 1661 ret = TEE_ERROR_BAD_PARAMETERS; 1662 goto err; 1663 } 1664 1665 while (true) { 1666 if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, 1667 &tbl_info)) 1668 panic("Can't find pagetable for vaddr "); 1669 1670 idx = core_mmu_va2idx(&tbl_info, vaddr); 1671 if (tbl_info.shift == SMALL_PAGE_SHIFT) 1672 break; 1673 1674 /* This is supertable. Need to divide it. */ 1675 if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, 1676 secure)) 1677 panic("Failed to spread pgdir on small tables"); 1678 } 1679 1680 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 1681 if (old_attr) 1682 panic("Page is already mapped"); 1683 1684 core_mmu_set_entry(&tbl_info, idx, pages[i], 1685 core_mmu_type_to_attr(memtype)); 1686 vaddr += SMALL_PAGE_SIZE; 1687 } 1688 1689 /* 1690 * Make sure all the changes to translation tables are visible 1691 * before returning. TLB doesn't need to be invalidated as we are 1692 * guaranteed that there's no valid mapping in this range. 1693 */ 1694 core_mmu_table_write_barrier(); 1695 mmu_unlock(exceptions); 1696 1697 return TEE_SUCCESS; 1698 err: 1699 mmu_unlock(exceptions); 1700 1701 if (i) 1702 core_mmu_unmap_pages(vstart, i); 1703 1704 return ret; 1705 } 1706 1707 TEE_Result core_mmu_map_contiguous_pages(vaddr_t vstart, paddr_t pstart, 1708 size_t num_pages, 1709 enum teecore_memtypes memtype) 1710 { 1711 struct core_mmu_table_info tbl_info = { }; 1712 struct tee_mmap_region *mm = NULL; 1713 unsigned int idx = 0; 1714 uint32_t old_attr = 0; 1715 uint32_t exceptions = 0; 1716 vaddr_t vaddr = vstart; 1717 paddr_t paddr = pstart; 1718 size_t i = 0; 1719 bool secure = false; 1720 1721 assert(!(core_mmu_type_to_attr(memtype) & TEE_MATTR_PX)); 1722 1723 secure = core_mmu_type_to_attr(memtype) & TEE_MATTR_SECURE; 1724 1725 if ((vaddr | paddr) & SMALL_PAGE_MASK) 1726 return TEE_ERROR_BAD_PARAMETERS; 1727 1728 exceptions = mmu_lock(); 1729 1730 mm = find_map_by_va((void *)vaddr); 1731 if (!mm || !va_is_in_map(mm, vaddr + num_pages * SMALL_PAGE_SIZE - 1)) 1732 panic("VA does not belong to any known mm region"); 1733 1734 if (!core_mmu_is_dynamic_vaspace(mm)) 1735 panic("Trying to map into static region"); 1736 1737 for (i = 0; i < num_pages; i++) { 1738 while (true) { 1739 if (!core_mmu_find_table(NULL, vaddr, UINT_MAX, 1740 &tbl_info)) 1741 panic("Can't find pagetable for vaddr "); 1742 1743 idx = core_mmu_va2idx(&tbl_info, vaddr); 1744 if (tbl_info.shift == SMALL_PAGE_SHIFT) 1745 break; 1746 1747 /* This is supertable. Need to divide it. */ 1748 if (!core_mmu_entry_to_finer_grained(&tbl_info, idx, 1749 secure)) 1750 panic("Failed to spread pgdir on small tables"); 1751 } 1752 1753 core_mmu_get_entry(&tbl_info, idx, NULL, &old_attr); 1754 if (old_attr) 1755 panic("Page is already mapped"); 1756 1757 core_mmu_set_entry(&tbl_info, idx, paddr, 1758 core_mmu_type_to_attr(memtype)); 1759 paddr += SMALL_PAGE_SIZE; 1760 vaddr += SMALL_PAGE_SIZE; 1761 } 1762 1763 /* 1764 * Make sure all the changes to translation tables are visible 1765 * before returning. TLB doesn't need to be invalidated as we are 1766 * guaranteed that there's no valid mapping in this range. 1767 */ 1768 core_mmu_table_write_barrier(); 1769 mmu_unlock(exceptions); 1770 1771 return TEE_SUCCESS; 1772 } 1773 1774 void core_mmu_unmap_pages(vaddr_t vstart, size_t num_pages) 1775 { 1776 struct core_mmu_table_info tbl_info; 1777 struct tee_mmap_region *mm; 1778 size_t i; 1779 unsigned int idx; 1780 uint32_t exceptions; 1781 1782 exceptions = mmu_lock(); 1783 1784 mm = find_map_by_va((void *)vstart); 1785 if (!mm || !va_is_in_map(mm, vstart + num_pages * SMALL_PAGE_SIZE - 1)) 1786 panic("VA does not belong to any known mm region"); 1787 1788 if (!core_mmu_is_dynamic_vaspace(mm)) 1789 panic("Trying to unmap static region"); 1790 1791 for (i = 0; i < num_pages; i++, vstart += SMALL_PAGE_SIZE) { 1792 if (!core_mmu_find_table(NULL, vstart, UINT_MAX, &tbl_info)) 1793 panic("Can't find pagetable"); 1794 1795 if (tbl_info.shift != SMALL_PAGE_SHIFT) 1796 panic("Invalid pagetable level"); 1797 1798 idx = core_mmu_va2idx(&tbl_info, vstart); 1799 core_mmu_set_entry(&tbl_info, idx, 0, 0); 1800 } 1801 tlbi_all(); 1802 1803 mmu_unlock(exceptions); 1804 } 1805 1806 void core_mmu_populate_user_map(struct core_mmu_table_info *dir_info, 1807 struct user_mode_ctx *uctx) 1808 { 1809 struct core_mmu_table_info pg_info = { }; 1810 struct pgt_cache *pgt_cache = &thread_get_tsd()->pgt_cache; 1811 struct pgt *pgt = NULL; 1812 struct vm_region *r = NULL; 1813 struct vm_region *r_last = NULL; 1814 1815 /* Find the first and last valid entry */ 1816 r = TAILQ_FIRST(&uctx->vm_info.regions); 1817 if (!r) 1818 return; /* Nothing to map */ 1819 r_last = TAILQ_LAST(&uctx->vm_info.regions, vm_region_head); 1820 1821 /* 1822 * Allocate all page tables in advance. 1823 */ 1824 pgt_alloc(pgt_cache, uctx->ts_ctx, r->va, 1825 r_last->va + r_last->size - 1); 1826 pgt = SLIST_FIRST(pgt_cache); 1827 1828 core_mmu_set_info_table(&pg_info, dir_info->level + 1, 0, NULL); 1829 1830 TAILQ_FOREACH(r, &uctx->vm_info.regions, link) 1831 set_pg_region(dir_info, r, &pgt, &pg_info); 1832 } 1833 1834 TEE_Result core_mmu_remove_mapping(enum teecore_memtypes type, void *addr, 1835 size_t len) 1836 { 1837 struct core_mmu_table_info tbl_info = { }; 1838 struct tee_mmap_region *res_map = NULL; 1839 struct tee_mmap_region *map = NULL; 1840 paddr_t pa = virt_to_phys(addr); 1841 size_t granule = 0; 1842 ptrdiff_t i = 0; 1843 paddr_t p = 0; 1844 size_t l = 0; 1845 1846 map = find_map_by_type_and_pa(type, pa, len); 1847 if (!map) 1848 return TEE_ERROR_GENERIC; 1849 1850 res_map = find_map_by_type(MEM_AREA_RES_VASPACE); 1851 if (!res_map) 1852 return TEE_ERROR_GENERIC; 1853 if (!core_mmu_find_table(NULL, res_map->va, UINT_MAX, &tbl_info)) 1854 return TEE_ERROR_GENERIC; 1855 granule = BIT(tbl_info.shift); 1856 1857 if (map < static_memory_map || 1858 map >= static_memory_map + ARRAY_SIZE(static_memory_map)) 1859 return TEE_ERROR_GENERIC; 1860 i = map - static_memory_map; 1861 1862 /* Check that we have a full match */ 1863 p = ROUNDDOWN(pa, granule); 1864 l = ROUNDUP(len + pa - p, granule); 1865 if (map->pa != p || map->size != l) 1866 return TEE_ERROR_GENERIC; 1867 1868 clear_region(&tbl_info, map); 1869 tlbi_all(); 1870 1871 /* If possible remove the va range from res_map */ 1872 if (res_map->va - map->size == map->va) { 1873 res_map->va -= map->size; 1874 res_map->size += map->size; 1875 } 1876 1877 /* Remove the entry. */ 1878 memmove(map, map + 1, 1879 (ARRAY_SIZE(static_memory_map) - i - 1) * sizeof(*map)); 1880 1881 /* Clear the last new entry in case it was used */ 1882 memset(static_memory_map + ARRAY_SIZE(static_memory_map) - 1, 1883 0, sizeof(*map)); 1884 1885 return TEE_SUCCESS; 1886 } 1887 1888 struct tee_mmap_region * 1889 core_mmu_find_mapping_exclusive(enum teecore_memtypes type, size_t len) 1890 { 1891 struct tee_mmap_region *map = NULL; 1892 struct tee_mmap_region *map_found = NULL; 1893 1894 if (!len) 1895 return NULL; 1896 1897 for (map = get_memory_map(); !core_mmap_is_end_of_table(map); map++) { 1898 if (map->type != type) 1899 continue; 1900 1901 if (map_found) 1902 return NULL; 1903 1904 map_found = map; 1905 } 1906 1907 if (!map_found || map_found->size < len) 1908 return NULL; 1909 1910 return map_found; 1911 } 1912 1913 void *core_mmu_add_mapping(enum teecore_memtypes type, paddr_t addr, size_t len) 1914 { 1915 struct core_mmu_table_info tbl_info; 1916 struct tee_mmap_region *map; 1917 size_t n; 1918 size_t granule; 1919 paddr_t p; 1920 size_t l; 1921 1922 if (!len) 1923 return NULL; 1924 1925 if (!core_mmu_check_end_pa(addr, len)) 1926 return NULL; 1927 1928 /* Check if the memory is already mapped */ 1929 map = find_map_by_type_and_pa(type, addr, len); 1930 if (map && pbuf_inside_map_area(addr, len, map)) 1931 return (void *)(vaddr_t)(map->va + addr - map->pa); 1932 1933 /* Find the reserved va space used for late mappings */ 1934 map = find_map_by_type(MEM_AREA_RES_VASPACE); 1935 if (!map) 1936 return NULL; 1937 1938 if (!core_mmu_find_table(NULL, map->va, UINT_MAX, &tbl_info)) 1939 return NULL; 1940 1941 granule = BIT64(tbl_info.shift); 1942 p = ROUNDDOWN(addr, granule); 1943 l = ROUNDUP(len + addr - p, granule); 1944 1945 /* Ban overflowing virtual addresses */ 1946 if (map->size < l) 1947 return NULL; 1948 1949 /* 1950 * Something is wrong, we can't fit the va range into the selected 1951 * table. The reserved va range is possibly missaligned with 1952 * granule. 1953 */ 1954 if (core_mmu_va2idx(&tbl_info, map->va + len) >= tbl_info.num_entries) 1955 return NULL; 1956 1957 /* Find end of the memory map */ 1958 n = 0; 1959 while (!core_mmap_is_end_of_table(static_memory_map + n)) 1960 n++; 1961 1962 if (n < (ARRAY_SIZE(static_memory_map) - 1)) { 1963 /* There's room for another entry */ 1964 static_memory_map[n].va = map->va; 1965 static_memory_map[n].size = l; 1966 static_memory_map[n + 1].type = MEM_AREA_END; 1967 map->va += l; 1968 map->size -= l; 1969 map = static_memory_map + n; 1970 } else { 1971 /* 1972 * There isn't room for another entry, steal the reserved 1973 * entry as it's not useful for anything else any longer. 1974 */ 1975 map->size = l; 1976 } 1977 map->type = type; 1978 map->region_size = granule; 1979 map->attr = core_mmu_type_to_attr(type); 1980 map->pa = p; 1981 1982 set_region(&tbl_info, map); 1983 1984 /* Make sure the new entry is visible before continuing. */ 1985 core_mmu_table_write_barrier(); 1986 1987 return (void *)(vaddr_t)(map->va + addr - map->pa); 1988 } 1989 1990 #ifdef CFG_WITH_PAGER 1991 static vaddr_t get_linear_map_end(void) 1992 { 1993 /* this is synced with the generic linker file kern.ld.S */ 1994 return (vaddr_t)__heap2_end; 1995 } 1996 #endif 1997 1998 #if defined(CFG_TEE_CORE_DEBUG) 1999 static void check_pa_matches_va(void *va, paddr_t pa) 2000 { 2001 TEE_Result res = TEE_ERROR_GENERIC; 2002 vaddr_t v = (vaddr_t)va; 2003 paddr_t p = 0; 2004 struct core_mmu_table_info ti __maybe_unused = { }; 2005 2006 if (core_mmu_user_va_range_is_defined()) { 2007 vaddr_t user_va_base = 0; 2008 size_t user_va_size = 0; 2009 2010 core_mmu_get_user_va_range(&user_va_base, &user_va_size); 2011 if (v >= user_va_base && 2012 v <= (user_va_base - 1 + user_va_size)) { 2013 if (!core_mmu_user_mapping_is_active()) { 2014 if (pa) 2015 panic("issue in linear address space"); 2016 return; 2017 } 2018 2019 res = vm_va2pa(to_user_mode_ctx(thread_get_tsd()->ctx), 2020 va, &p); 2021 if (res == TEE_ERROR_NOT_SUPPORTED) 2022 return; 2023 if (res == TEE_SUCCESS && pa != p) 2024 panic("bad pa"); 2025 if (res != TEE_SUCCESS && pa) 2026 panic("false pa"); 2027 return; 2028 } 2029 } 2030 #ifdef CFG_WITH_PAGER 2031 if (is_unpaged(va)) { 2032 if (v - boot_mmu_config.load_offset != pa) 2033 panic("issue in linear address space"); 2034 return; 2035 } 2036 2037 if (tee_pager_get_table_info(v, &ti)) { 2038 uint32_t a; 2039 2040 /* 2041 * Lookups in the page table managed by the pager is 2042 * dangerous for addresses in the paged area as those pages 2043 * changes all the time. But some ranges are safe, 2044 * rw-locked areas when the page is populated for instance. 2045 */ 2046 core_mmu_get_entry(&ti, core_mmu_va2idx(&ti, v), &p, &a); 2047 if (a & TEE_MATTR_VALID_BLOCK) { 2048 paddr_t mask = BIT64(ti.shift) - 1; 2049 2050 p |= v & mask; 2051 if (pa != p) 2052 panic(); 2053 } else { 2054 if (pa) 2055 panic(); 2056 } 2057 return; 2058 } 2059 #endif 2060 2061 if (!core_va2pa_helper(va, &p)) { 2062 /* Verfiy only the static mapping (case non null phys addr) */ 2063 if (p && pa != p) { 2064 DMSG("va %p maps 0x%" PRIxPA ", expect 0x%" PRIxPA, 2065 va, p, pa); 2066 panic(); 2067 } 2068 } else { 2069 if (pa) { 2070 DMSG("va %p unmapped, expect 0x%" PRIxPA, va, pa); 2071 panic(); 2072 } 2073 } 2074 } 2075 #else 2076 static void check_pa_matches_va(void *va __unused, paddr_t pa __unused) 2077 { 2078 } 2079 #endif 2080 2081 paddr_t virt_to_phys(void *va) 2082 { 2083 paddr_t pa = 0; 2084 2085 if (!arch_va2pa_helper(va, &pa)) 2086 pa = 0; 2087 check_pa_matches_va(va, pa); 2088 return pa; 2089 } 2090 2091 #if defined(CFG_TEE_CORE_DEBUG) 2092 static void check_va_matches_pa(paddr_t pa, void *va) 2093 { 2094 paddr_t p = 0; 2095 2096 if (!va) 2097 return; 2098 2099 p = virt_to_phys(va); 2100 if (p != pa) { 2101 DMSG("va %p maps 0x%" PRIxPA " expect 0x%" PRIxPA, va, p, pa); 2102 panic(); 2103 } 2104 } 2105 #else 2106 static void check_va_matches_pa(paddr_t pa __unused, void *va __unused) 2107 { 2108 } 2109 #endif 2110 2111 static void *phys_to_virt_ts_vaspace(paddr_t pa, size_t len) 2112 { 2113 if (!core_mmu_user_mapping_is_active()) 2114 return NULL; 2115 2116 return vm_pa2va(to_user_mode_ctx(thread_get_tsd()->ctx), pa, len); 2117 } 2118 2119 #ifdef CFG_WITH_PAGER 2120 static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) 2121 { 2122 paddr_t end_pa = 0; 2123 2124 if (SUB_OVERFLOW(len, 1, &end_pa) || ADD_OVERFLOW(pa, end_pa, &end_pa)) 2125 return NULL; 2126 2127 if (pa >= TEE_LOAD_ADDR && pa < get_linear_map_end()) { 2128 if (end_pa > get_linear_map_end()) 2129 return NULL; 2130 return (void *)(vaddr_t)(pa + boot_mmu_config.load_offset); 2131 } 2132 2133 return tee_pager_phys_to_virt(pa, len); 2134 } 2135 #else 2136 static void *phys_to_virt_tee_ram(paddr_t pa, size_t len) 2137 { 2138 struct tee_mmap_region *mmap = NULL; 2139 2140 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM, pa, len); 2141 if (!mmap) 2142 mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RW, pa, len); 2143 if (!mmap) 2144 mmap = find_map_by_type_and_pa(MEM_AREA_NEX_RAM_RO, pa, len); 2145 if (!mmap) 2146 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RW, pa, len); 2147 if (!mmap) 2148 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RO, pa, len); 2149 if (!mmap) 2150 mmap = find_map_by_type_and_pa(MEM_AREA_TEE_RAM_RX, pa, len); 2151 /* 2152 * Note that MEM_AREA_INIT_RAM_RO and MEM_AREA_INIT_RAM_RX are only 2153 * used with pager and not needed here. 2154 */ 2155 return map_pa2va(mmap, pa, len); 2156 } 2157 #endif 2158 2159 void *phys_to_virt(paddr_t pa, enum teecore_memtypes m, size_t len) 2160 { 2161 void *va = NULL; 2162 2163 switch (m) { 2164 case MEM_AREA_TS_VASPACE: 2165 va = phys_to_virt_ts_vaspace(pa, len); 2166 break; 2167 case MEM_AREA_TEE_RAM: 2168 case MEM_AREA_TEE_RAM_RX: 2169 case MEM_AREA_TEE_RAM_RO: 2170 case MEM_AREA_TEE_RAM_RW: 2171 case MEM_AREA_NEX_RAM_RO: 2172 case MEM_AREA_NEX_RAM_RW: 2173 va = phys_to_virt_tee_ram(pa, len); 2174 break; 2175 case MEM_AREA_SHM_VASPACE: 2176 /* Find VA from PA in dynamic SHM is not yet supported */ 2177 va = NULL; 2178 break; 2179 default: 2180 va = map_pa2va(find_map_by_type_and_pa(m, pa, len), pa, len); 2181 } 2182 if (m != MEM_AREA_SEC_RAM_OVERALL) 2183 check_va_matches_pa(pa, va); 2184 return va; 2185 } 2186 2187 void *phys_to_virt_io(paddr_t pa, size_t len) 2188 { 2189 struct tee_mmap_region *map = NULL; 2190 void *va = NULL; 2191 2192 map = find_map_by_type_and_pa(MEM_AREA_IO_SEC, pa, len); 2193 if (!map) 2194 map = find_map_by_type_and_pa(MEM_AREA_IO_NSEC, pa, len); 2195 if (!map) 2196 return NULL; 2197 va = map_pa2va(map, pa, len); 2198 check_va_matches_pa(pa, va); 2199 return va; 2200 } 2201 2202 vaddr_t core_mmu_get_va(paddr_t pa, enum teecore_memtypes type, size_t len) 2203 { 2204 if (cpu_mmu_enabled()) 2205 return (vaddr_t)phys_to_virt(pa, type, len); 2206 2207 return (vaddr_t)pa; 2208 } 2209 2210 #ifdef CFG_WITH_PAGER 2211 bool is_unpaged(void *va) 2212 { 2213 vaddr_t v = (vaddr_t)va; 2214 2215 return v >= VCORE_START_VA && v < get_linear_map_end(); 2216 } 2217 #else 2218 bool is_unpaged(void *va __unused) 2219 { 2220 return true; 2221 } 2222 #endif 2223 2224 void core_mmu_init_virtualization(void) 2225 { 2226 virt_init_memory(static_memory_map); 2227 } 2228 2229 vaddr_t io_pa_or_va(struct io_pa_va *p, size_t len) 2230 { 2231 assert(p->pa); 2232 if (cpu_mmu_enabled()) { 2233 if (!p->va) 2234 p->va = (vaddr_t)phys_to_virt_io(p->pa, len); 2235 assert(p->va); 2236 return p->va; 2237 } 2238 return p->pa; 2239 } 2240 2241 vaddr_t io_pa_or_va_secure(struct io_pa_va *p, size_t len) 2242 { 2243 assert(p->pa); 2244 if (cpu_mmu_enabled()) { 2245 if (!p->va) 2246 p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_SEC, 2247 len); 2248 assert(p->va); 2249 return p->va; 2250 } 2251 return p->pa; 2252 } 2253 2254 vaddr_t io_pa_or_va_nsec(struct io_pa_va *p, size_t len) 2255 { 2256 assert(p->pa); 2257 if (cpu_mmu_enabled()) { 2258 if (!p->va) 2259 p->va = (vaddr_t)phys_to_virt(p->pa, MEM_AREA_IO_NSEC, 2260 len); 2261 assert(p->va); 2262 return p->va; 2263 } 2264 return p->pa; 2265 } 2266 2267 #ifdef CFG_CORE_RESERVED_SHM 2268 static TEE_Result teecore_init_pub_ram(void) 2269 { 2270 vaddr_t s = 0; 2271 vaddr_t e = 0; 2272 2273 /* get virtual addr/size of NSec shared mem allocated from teecore */ 2274 core_mmu_get_mem_by_type(MEM_AREA_NSEC_SHM, &s, &e); 2275 2276 if (s >= e || s & SMALL_PAGE_MASK || e & SMALL_PAGE_MASK) 2277 panic("invalid PUB RAM"); 2278 2279 /* extra check: we could rely on core_mmu_get_mem_by_type() */ 2280 if (!tee_vbuf_is_non_sec(s, e - s)) 2281 panic("PUB RAM is not non-secure"); 2282 2283 #ifdef CFG_PL310 2284 /* Allocate statically the l2cc mutex */ 2285 tee_l2cc_store_mutex_boot_pa(virt_to_phys((void *)s)); 2286 s += sizeof(uint32_t); /* size of a pl310 mutex */ 2287 s = ROUNDUP(s, SMALL_PAGE_SIZE); /* keep required alignment */ 2288 #endif 2289 2290 default_nsec_shm_paddr = virt_to_phys((void *)s); 2291 default_nsec_shm_size = e - s; 2292 2293 return TEE_SUCCESS; 2294 } 2295 early_init(teecore_init_pub_ram); 2296 #endif /*CFG_CORE_RESERVED_SHM*/ 2297 2298 void core_mmu_init_ta_ram(void) 2299 { 2300 vaddr_t s = 0; 2301 vaddr_t e = 0; 2302 paddr_t ps = 0; 2303 size_t size = 0; 2304 2305 /* 2306 * Get virtual addr/size of RAM where TA are loaded/executedNSec 2307 * shared mem allocated from teecore. 2308 */ 2309 if (IS_ENABLED(CFG_VIRTUALIZATION)) 2310 virt_get_ta_ram(&s, &e); 2311 else 2312 core_mmu_get_mem_by_type(MEM_AREA_TA_RAM, &s, &e); 2313 2314 ps = virt_to_phys((void *)s); 2315 size = e - s; 2316 2317 if (!ps || (ps & CORE_MMU_USER_CODE_MASK) || 2318 !size || (size & CORE_MMU_USER_CODE_MASK)) 2319 panic("invalid TA RAM"); 2320 2321 /* extra check: we could rely on core_mmu_get_mem_by_type() */ 2322 if (!tee_pbuf_is_sec(ps, size)) 2323 panic("TA RAM is not secure"); 2324 2325 if (!tee_mm_is_empty(&tee_mm_sec_ddr)) 2326 panic("TA RAM pool is not empty"); 2327 2328 /* remove previous config and init TA ddr memory pool */ 2329 tee_mm_final(&tee_mm_sec_ddr); 2330 tee_mm_init(&tee_mm_sec_ddr, ps, size, CORE_MMU_USER_CODE_SHIFT, 2331 TEE_MM_POOL_NO_FLAGS); 2332 } 2333