1 /* 2 * Copyright (c) 2016, Linaro Limited 3 * Copyright (c) 2014, STMicroelectronics International N.V. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, 10 * this list of conditions and the following disclaimer. 11 * 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <arm.h> 30 #include <assert.h> 31 #include <keep.h> 32 #include <sys/queue.h> 33 #include <kernel/abort.h> 34 #include <kernel/panic.h> 35 #include <kernel/tee_misc.h> 36 #include <kernel/tee_ta_manager.h> 37 #include <kernel/thread.h> 38 #include <kernel/tz_proc.h> 39 #include <mm/core_memprot.h> 40 #include <mm/tee_mm.h> 41 #include <mm/tee_mmu_defs.h> 42 #include <mm/tee_pager.h> 43 #include <types_ext.h> 44 #include <stdlib.h> 45 #include <tee_api_defines.h> 46 #include <tee/tee_cryp_provider.h> 47 #include <trace.h> 48 #include <utee_defines.h> 49 #include <util.h> 50 51 #include "pager_private.h" 52 53 #define PAGER_AE_KEY_BITS 256 54 55 struct pager_rw_pstate { 56 uint64_t iv; 57 uint8_t tag[PAGER_AES_GCM_TAG_LEN]; 58 }; 59 60 enum area_type { 61 AREA_TYPE_RO, 62 AREA_TYPE_RW, 63 AREA_TYPE_LOCK, 64 }; 65 66 struct tee_pager_area { 67 union { 68 const uint8_t *hashes; 69 struct pager_rw_pstate *rwp; 70 } u; 71 uint8_t *store; 72 enum area_type type; 73 uint32_t flags; 74 vaddr_t base; 75 size_t size; 76 struct pgt *pgt; 77 TAILQ_ENTRY(tee_pager_area) link; 78 }; 79 80 TAILQ_HEAD(tee_pager_area_head, tee_pager_area); 81 82 static struct tee_pager_area_head tee_pager_area_head = 83 TAILQ_HEAD_INITIALIZER(tee_pager_area_head); 84 85 #define INVALID_PGIDX UINT_MAX 86 87 /* 88 * struct tee_pager_pmem - Represents a physical page used for paging. 89 * 90 * @pgidx an index of the entry in area->ti. 91 * @va_alias Virtual address where the physical page always is aliased. 92 * Used during remapping of the page when the content need to 93 * be updated before it's available at the new location. 94 * @area a pointer to the pager area 95 */ 96 struct tee_pager_pmem { 97 unsigned pgidx; 98 void *va_alias; 99 struct tee_pager_area *area; 100 TAILQ_ENTRY(tee_pager_pmem) link; 101 }; 102 103 /* The list of physical pages. The first page in the list is the oldest */ 104 TAILQ_HEAD(tee_pager_pmem_head, tee_pager_pmem); 105 106 static struct tee_pager_pmem_head tee_pager_pmem_head = 107 TAILQ_HEAD_INITIALIZER(tee_pager_pmem_head); 108 109 static struct tee_pager_pmem_head tee_pager_lock_pmem_head = 110 TAILQ_HEAD_INITIALIZER(tee_pager_lock_pmem_head); 111 112 static uint8_t pager_ae_key[PAGER_AE_KEY_BITS / 8]; 113 114 /* number of pages hidden */ 115 #define TEE_PAGER_NHIDE (tee_pager_npages / 3) 116 117 /* Number of registered physical pages, used hiding pages. */ 118 static size_t tee_pager_npages; 119 120 #ifdef CFG_WITH_STATS 121 static struct tee_pager_stats pager_stats; 122 123 static inline void incr_ro_hits(void) 124 { 125 pager_stats.ro_hits++; 126 } 127 128 static inline void incr_rw_hits(void) 129 { 130 pager_stats.rw_hits++; 131 } 132 133 static inline void incr_hidden_hits(void) 134 { 135 pager_stats.hidden_hits++; 136 } 137 138 static inline void incr_zi_released(void) 139 { 140 pager_stats.zi_released++; 141 } 142 143 static inline void incr_npages_all(void) 144 { 145 pager_stats.npages_all++; 146 } 147 148 static inline void set_npages(void) 149 { 150 pager_stats.npages = tee_pager_npages; 151 } 152 153 void tee_pager_get_stats(struct tee_pager_stats *stats) 154 { 155 *stats = pager_stats; 156 157 pager_stats.hidden_hits = 0; 158 pager_stats.ro_hits = 0; 159 pager_stats.rw_hits = 0; 160 pager_stats.zi_released = 0; 161 } 162 163 #else /* CFG_WITH_STATS */ 164 static inline void incr_ro_hits(void) { } 165 static inline void incr_rw_hits(void) { } 166 static inline void incr_hidden_hits(void) { } 167 static inline void incr_zi_released(void) { } 168 static inline void incr_npages_all(void) { } 169 static inline void set_npages(void) { } 170 171 void tee_pager_get_stats(struct tee_pager_stats *stats) 172 { 173 memset(stats, 0, sizeof(struct tee_pager_stats)); 174 } 175 #endif /* CFG_WITH_STATS */ 176 177 static struct pgt pager_core_pgt; 178 struct core_mmu_table_info tee_pager_tbl_info; 179 static struct core_mmu_table_info pager_alias_tbl_info; 180 181 static unsigned pager_lock = SPINLOCK_UNLOCK; 182 183 /* Defines the range of the alias area */ 184 static tee_mm_entry_t *pager_alias_area; 185 /* 186 * Physical pages are added in a stack like fashion to the alias area, 187 * @pager_alias_next_free gives the address of next free entry if 188 * @pager_alias_next_free is != 0 189 */ 190 static uintptr_t pager_alias_next_free; 191 192 static void set_alias_area(tee_mm_entry_t *mm) 193 { 194 struct core_mmu_table_info *ti = &pager_alias_tbl_info; 195 size_t tbl_va_size; 196 unsigned idx; 197 unsigned last_idx; 198 vaddr_t smem = tee_mm_get_smem(mm); 199 size_t nbytes = tee_mm_get_bytes(mm); 200 201 DMSG("0x%" PRIxVA " - 0x%" PRIxVA, smem, smem + nbytes); 202 203 if (pager_alias_area) 204 panic("null pager_alias_area"); 205 206 if (!ti->num_entries && !core_mmu_find_table(smem, UINT_MAX, ti)) 207 panic("Can't find translation table"); 208 209 if ((1 << ti->shift) != SMALL_PAGE_SIZE) 210 panic("Unsupported page size in translation table"); 211 212 tbl_va_size = (1 << ti->shift) * ti->num_entries; 213 if (!core_is_buffer_inside(smem, nbytes, 214 ti->va_base, tbl_va_size)) { 215 EMSG("area 0x%" PRIxVA " len 0x%zx doesn't fit it translation table 0x%" PRIxVA " len 0x%zx", 216 smem, nbytes, ti->va_base, tbl_va_size); 217 panic(); 218 } 219 220 if (smem & SMALL_PAGE_MASK || nbytes & SMALL_PAGE_MASK) 221 panic("invalid area alignment"); 222 223 pager_alias_area = mm; 224 pager_alias_next_free = smem; 225 226 /* Clear all mapping in the alias area */ 227 idx = core_mmu_va2idx(ti, smem); 228 last_idx = core_mmu_va2idx(ti, smem + nbytes); 229 for (; idx < last_idx; idx++) 230 core_mmu_set_entry(ti, idx, 0, 0); 231 232 /* TODO only invalidate entries touched above */ 233 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 234 } 235 236 static void generate_ae_key(void) 237 { 238 if (rng_generate(pager_ae_key, sizeof(pager_ae_key)) != TEE_SUCCESS) 239 panic("failed to generate random"); 240 } 241 242 void tee_pager_init(tee_mm_entry_t *mm_alias) 243 { 244 set_alias_area(mm_alias); 245 generate_ae_key(); 246 } 247 248 static void *pager_add_alias_page(paddr_t pa) 249 { 250 unsigned idx; 251 struct core_mmu_table_info *ti = &pager_alias_tbl_info; 252 uint32_t attr = TEE_MATTR_VALID_BLOCK | TEE_MATTR_GLOBAL | 253 (TEE_MATTR_CACHE_CACHED << TEE_MATTR_CACHE_SHIFT) | 254 TEE_MATTR_SECURE | TEE_MATTR_PRW; 255 256 DMSG("0x%" PRIxPA, pa); 257 258 if (!pager_alias_next_free || !ti->num_entries) 259 panic("invalid alias entry"); 260 261 idx = core_mmu_va2idx(ti, pager_alias_next_free); 262 core_mmu_set_entry(ti, idx, pa, attr); 263 pgt_inc_used_entries(&pager_core_pgt); 264 pager_alias_next_free += SMALL_PAGE_SIZE; 265 if (pager_alias_next_free >= (tee_mm_get_smem(pager_alias_area) + 266 tee_mm_get_bytes(pager_alias_area))) 267 pager_alias_next_free = 0; 268 return (void *)core_mmu_idx2va(ti, idx); 269 } 270 271 static struct tee_pager_area *alloc_area(struct pgt *pgt, 272 vaddr_t base, size_t size, 273 uint32_t flags, const void *store, 274 const void *hashes) 275 { 276 struct tee_pager_area *area = calloc(1, sizeof(*area)); 277 enum area_type at; 278 tee_mm_entry_t *mm_store = NULL; 279 280 if (!area) 281 return NULL; 282 283 if (flags & (TEE_MATTR_PW | TEE_MATTR_UW)) { 284 if (flags & TEE_MATTR_LOCKED) { 285 at = AREA_TYPE_LOCK; 286 goto out; 287 } 288 mm_store = tee_mm_alloc(&tee_mm_sec_ddr, size); 289 if (!mm_store) 290 goto bad; 291 area->store = phys_to_virt(tee_mm_get_smem(mm_store), 292 MEM_AREA_TA_RAM); 293 if (!area->store) 294 goto bad; 295 area->u.rwp = calloc(size / SMALL_PAGE_SIZE, 296 sizeof(struct pager_rw_pstate)); 297 if (!area->u.rwp) 298 goto bad; 299 at = AREA_TYPE_RW; 300 } else { 301 area->store = (void *)store; 302 area->u.hashes = hashes; 303 at = AREA_TYPE_RO; 304 } 305 out: 306 area->pgt = pgt; 307 area->base = base; 308 area->size = size; 309 area->flags = flags; 310 area->type = at; 311 return area; 312 bad: 313 tee_mm_free(mm_store); 314 free(area->u.rwp); 315 free(area); 316 return NULL; 317 } 318 319 static void area_insert_tail(struct tee_pager_area *area) 320 { 321 uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); 322 323 cpu_spin_lock(&pager_lock); 324 325 TAILQ_INSERT_TAIL(&tee_pager_area_head, area, link); 326 327 cpu_spin_unlock(&pager_lock); 328 thread_set_exceptions(exceptions); 329 } 330 KEEP_PAGER(area_insert_tail); 331 332 static size_t tbl_usage_count(struct pgt *pgt) 333 { 334 size_t n; 335 paddr_t pa; 336 size_t usage = 0; 337 338 for (n = 0; n < tee_pager_tbl_info.num_entries; n++) { 339 core_mmu_get_entry_primitive(pgt->tbl, tee_pager_tbl_info.level, 340 n, &pa, NULL); 341 if (pa) 342 usage++; 343 } 344 return usage; 345 } 346 347 bool tee_pager_add_core_area(vaddr_t base, size_t size, uint32_t flags, 348 const void *store, const void *hashes) 349 { 350 struct tee_pager_area *area; 351 size_t tbl_va_size; 352 struct core_mmu_table_info *ti = &tee_pager_tbl_info; 353 354 DMSG("0x%" PRIxPTR " - 0x%" PRIxPTR " : flags 0x%x, store %p, hashes %p", 355 base, base + size, flags, store, hashes); 356 357 if (base & SMALL_PAGE_MASK || size & SMALL_PAGE_MASK || !size) { 358 EMSG("invalid pager area [%" PRIxVA " +0x%zx]", base, size); 359 panic(); 360 } 361 362 if (!(flags & TEE_MATTR_PW) && (!store || !hashes)) 363 panic("write pages cannot provide store or hashes"); 364 365 if ((flags & TEE_MATTR_PW) && (store || hashes)) 366 panic("non-write pages must provide store and hashes"); 367 368 if (!pager_core_pgt.tbl) { 369 pager_core_pgt.tbl = ti->table; 370 pgt_set_used_entries(&pager_core_pgt, 371 tbl_usage_count(&pager_core_pgt)); 372 } 373 374 tbl_va_size = (1 << ti->shift) * ti->num_entries; 375 if (!core_is_buffer_inside(base, size, ti->va_base, tbl_va_size)) { 376 DMSG("area 0x%" PRIxPTR " len 0x%zx doesn't fit it translation table 0x%" PRIxVA " len 0x%zx", 377 base, size, ti->va_base, tbl_va_size); 378 return false; 379 } 380 381 area = alloc_area(&pager_core_pgt, base, size, flags, store, hashes); 382 if (!area) 383 return false; 384 385 area_insert_tail(area); 386 return true; 387 } 388 389 #ifdef CFG_PAGED_USER_TA 390 bool tee_pager_add_uta_area(struct user_ta_ctx *utc, vaddr_t base, size_t size) 391 { 392 struct tee_pager_area *area; 393 uint32_t flags; 394 vaddr_t b = base; 395 size_t s = size; 396 397 if (!utc->areas) { 398 utc->areas = malloc(sizeof(*utc->areas)); 399 if (!utc->areas) 400 return false; 401 TAILQ_INIT(utc->areas); 402 } 403 404 flags = TEE_MATTR_PRW | TEE_MATTR_URWX; 405 406 while (s) { 407 size_t s2; 408 409 s2 = MIN(CORE_MMU_PGDIR_SIZE - (b & CORE_MMU_PGDIR_MASK), s); 410 411 /* Table info will be set when the context is activated. */ 412 area = alloc_area(NULL, b, s2, flags, NULL, NULL); 413 if (!area) 414 return false; 415 TAILQ_INSERT_TAIL(utc->areas, area, link); 416 b += s2; 417 s -= s2; 418 } 419 420 return true; 421 } 422 423 void tee_pager_rem_uta_areas(struct user_ta_ctx *utc) 424 { 425 struct tee_pager_area *area; 426 427 if (!utc->areas) 428 return; 429 430 while (true) { 431 area = TAILQ_FIRST(utc->areas); 432 if (!area) 433 break; 434 TAILQ_REMOVE(utc->areas, area, link); 435 tee_mm_free(tee_mm_find(&tee_mm_sec_ddr, 436 virt_to_phys(area->store))); 437 if (area->type == AREA_TYPE_RW) 438 free(area->u.rwp); 439 free(area); 440 } 441 442 free(utc->areas); 443 } 444 #endif /*CFG_PAGED_USER_TA*/ 445 446 static struct tee_pager_area *find_area(struct tee_pager_area_head *areas, 447 vaddr_t va) 448 { 449 struct tee_pager_area *area; 450 451 if (!areas) 452 return NULL; 453 454 TAILQ_FOREACH(area, areas, link) { 455 if (core_is_buffer_inside(va, 1, area->base, area->size)) 456 return area; 457 } 458 return NULL; 459 } 460 461 #ifdef CFG_PAGED_USER_TA 462 static struct tee_pager_area *find_uta_area(vaddr_t va) 463 { 464 struct tee_ta_ctx *ctx = thread_get_tsd()->ctx; 465 466 if (!ctx || !is_user_ta_ctx(ctx)) 467 return NULL; 468 return find_area(to_user_ta_ctx(ctx)->areas, va); 469 } 470 #else 471 static struct tee_pager_area *find_uta_area(vaddr_t va __unused) 472 { 473 return NULL; 474 } 475 #endif /*CFG_PAGED_USER_TA*/ 476 477 478 static uint32_t get_area_mattr(uint32_t area_flags) 479 { 480 uint32_t attr = TEE_MATTR_VALID_BLOCK | TEE_MATTR_SECURE | 481 TEE_MATTR_CACHE_CACHED << TEE_MATTR_CACHE_SHIFT | 482 (area_flags & (TEE_MATTR_PRWX | TEE_MATTR_URWX)); 483 484 if (!(area_flags & (TEE_MATTR_UR | TEE_MATTR_UX | TEE_MATTR_UW))) 485 attr |= TEE_MATTR_GLOBAL; 486 487 return attr; 488 } 489 490 static paddr_t get_pmem_pa(struct tee_pager_pmem *pmem) 491 { 492 paddr_t pa; 493 unsigned idx; 494 495 idx = core_mmu_va2idx(&pager_alias_tbl_info, (vaddr_t)pmem->va_alias); 496 core_mmu_get_entry(&pager_alias_tbl_info, idx, &pa, NULL); 497 return pa; 498 } 499 500 static bool decrypt_page(struct pager_rw_pstate *rwp, const void *src, 501 void *dst) 502 { 503 struct pager_aes_gcm_iv iv = { 504 { (vaddr_t)rwp, rwp->iv >> 32, rwp->iv } 505 }; 506 507 return pager_aes_gcm_decrypt(pager_ae_key, sizeof(pager_ae_key), 508 &iv, rwp->tag, src, dst, SMALL_PAGE_SIZE); 509 } 510 511 static void encrypt_page(struct pager_rw_pstate *rwp, void *src, void *dst) 512 { 513 struct pager_aes_gcm_iv iv; 514 515 assert((rwp->iv + 1) > rwp->iv); 516 rwp->iv++; 517 /* 518 * IV is constructed as recommended in section "8.2.1 Deterministic 519 * Construction" of "Recommendation for Block Cipher Modes of 520 * Operation: Galois/Counter Mode (GCM) and GMAC", 521 * http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf 522 */ 523 iv.iv[0] = (vaddr_t)rwp; 524 iv.iv[1] = rwp->iv >> 32; 525 iv.iv[2] = rwp->iv; 526 527 if (!pager_aes_gcm_encrypt(pager_ae_key, sizeof(pager_ae_key), 528 &iv, rwp->tag, 529 src, dst, SMALL_PAGE_SIZE)) 530 panic("gcm failed"); 531 } 532 533 static void tee_pager_load_page(struct tee_pager_area *area, vaddr_t page_va, 534 void *va_alias) 535 { 536 size_t idx = (page_va - area->base) >> SMALL_PAGE_SHIFT; 537 const void *stored_page = area->store + idx * SMALL_PAGE_SIZE; 538 539 switch (area->type) { 540 case AREA_TYPE_RO: 541 { 542 const void *hash = area->u.hashes + 543 idx * TEE_SHA256_HASH_SIZE; 544 545 memcpy(va_alias, stored_page, SMALL_PAGE_SIZE); 546 incr_ro_hits(); 547 548 if (hash_sha256_check(hash, va_alias, 549 SMALL_PAGE_SIZE) != TEE_SUCCESS) { 550 EMSG("PH 0x%" PRIxVA " failed", page_va); 551 panic(); 552 } 553 } 554 break; 555 case AREA_TYPE_RW: 556 FMSG("Restore %p %#" PRIxVA " iv %#" PRIx64, 557 va_alias, page_va, area->u.rwp[idx].iv); 558 if (!area->u.rwp[idx].iv) 559 memset(va_alias, 0, SMALL_PAGE_SIZE); 560 else if (!decrypt_page(&area->u.rwp[idx], stored_page, 561 va_alias)) { 562 EMSG("PH 0x%" PRIxVA " failed", page_va); 563 panic(); 564 } 565 incr_rw_hits(); 566 break; 567 case AREA_TYPE_LOCK: 568 FMSG("Zero init %p %#" PRIxVA, va_alias, page_va); 569 memset(va_alias, 0, SMALL_PAGE_SIZE); 570 break; 571 default: 572 panic(); 573 } 574 } 575 576 static void tee_pager_save_page(struct tee_pager_pmem *pmem, uint32_t attr) 577 { 578 const uint32_t dirty_bits = TEE_MATTR_PW | TEE_MATTR_UW | 579 TEE_MATTR_HIDDEN_DIRTY_BLOCK; 580 581 if (pmem->area->type == AREA_TYPE_RW && (attr & dirty_bits)) { 582 size_t offs = pmem->area->base & CORE_MMU_PGDIR_MASK; 583 size_t idx = pmem->pgidx - (offs >> SMALL_PAGE_SHIFT); 584 void *stored_page = pmem->area->store + idx * SMALL_PAGE_SIZE; 585 586 assert(pmem->area->flags & (TEE_MATTR_PW | TEE_MATTR_UW)); 587 encrypt_page(&pmem->area->u.rwp[idx], pmem->va_alias, 588 stored_page); 589 FMSG("Saved %#" PRIxVA " iv %#" PRIx64, 590 pmem->area->base + idx * SMALL_PAGE_SIZE, 591 pmem->area->u.rwp[idx].iv); 592 } 593 } 594 595 static void area_get_entry(struct tee_pager_area *area, size_t idx, 596 paddr_t *pa, uint32_t *attr) 597 { 598 assert(area->pgt); 599 assert(idx < tee_pager_tbl_info.num_entries); 600 core_mmu_get_entry_primitive(area->pgt->tbl, tee_pager_tbl_info.level, 601 idx, pa, attr); 602 } 603 604 static void area_set_entry(struct tee_pager_area *area, size_t idx, 605 paddr_t pa, uint32_t attr) 606 { 607 assert(area->pgt); 608 assert(idx < tee_pager_tbl_info.num_entries); 609 core_mmu_set_entry_primitive(area->pgt->tbl, tee_pager_tbl_info.level, 610 idx, pa, attr); 611 } 612 613 static size_t area_va2idx(struct tee_pager_area *area, vaddr_t va) 614 { 615 return (va - (area->base & ~CORE_MMU_PGDIR_MASK)) >> SMALL_PAGE_SHIFT; 616 } 617 618 #ifdef CFG_PAGED_USER_TA 619 bool tee_pager_set_uta_area(struct user_ta_ctx *utc, vaddr_t base, size_t size, 620 uint32_t flags) 621 { 622 bool ret; 623 vaddr_t b = base; 624 size_t s = size; 625 size_t s2; 626 struct tee_pager_area *area = find_area(utc->areas, b); 627 uint32_t exceptions; 628 struct tee_pager_pmem *pmem; 629 paddr_t pa; 630 uint32_t a; 631 uint32_t f; 632 633 f = (flags & TEE_MATTR_URWX) | TEE_MATTR_UR | TEE_MATTR_PR; 634 if (f & TEE_MATTR_UW) 635 f |= TEE_MATTR_PW; 636 f = get_area_mattr(f); 637 638 exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); 639 cpu_spin_lock(&pager_lock); 640 641 while (s) { 642 s2 = MIN(CORE_MMU_PGDIR_SIZE - (b & CORE_MMU_PGDIR_MASK), s); 643 if (!area || area->base != b || area->size != s2) { 644 ret = false; 645 goto out; 646 } 647 b += s2; 648 s -= s2; 649 650 TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) { 651 if (pmem->area != area) 652 continue; 653 area_get_entry(pmem->area, pmem->pgidx, &pa, &a); 654 if (a & TEE_MATTR_VALID_BLOCK) 655 assert(pa == get_pmem_pa(pmem)); 656 else 657 pa = get_pmem_pa(pmem); 658 if (a == f) 659 continue; 660 area_set_entry(pmem->area, pmem->pgidx, 0, 0); 661 /* TODO only invalidate entries touched above */ 662 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 663 if (!(flags & TEE_MATTR_UW)) 664 tee_pager_save_page(pmem, a); 665 area_set_entry(pmem->area, pmem->pgidx, pa, f); 666 } 667 668 area->flags = f; 669 area = TAILQ_NEXT(area, link); 670 } 671 672 ret = true; 673 out: 674 cpu_spin_unlock(&pager_lock); 675 thread_set_exceptions(exceptions); 676 return ret; 677 } 678 KEEP_PAGER(tee_pager_set_uta_area); 679 #endif /*CFG_PAGED_USER_TA*/ 680 681 static bool tee_pager_unhide_page(vaddr_t page_va) 682 { 683 struct tee_pager_pmem *pmem; 684 685 TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) { 686 paddr_t pa; 687 uint32_t attr; 688 689 if (pmem->pgidx == INVALID_PGIDX) 690 continue; 691 692 area_get_entry(pmem->area, pmem->pgidx, &pa, &attr); 693 694 if (!(attr & 695 (TEE_MATTR_HIDDEN_BLOCK | TEE_MATTR_HIDDEN_DIRTY_BLOCK))) 696 continue; 697 698 if (area_va2idx(pmem->area, page_va) == pmem->pgidx) { 699 uint32_t a = get_area_mattr(pmem->area->flags); 700 701 /* page is hidden, show and move to back */ 702 if (pa != get_pmem_pa(pmem)) 703 panic("unexpected pa"); 704 705 /* 706 * If it's not a dirty block, then it should be 707 * read only. 708 */ 709 if (!(attr & TEE_MATTR_HIDDEN_DIRTY_BLOCK)) 710 a &= ~(TEE_MATTR_PW | TEE_MATTR_UW); 711 else 712 FMSG("Unhide %#" PRIxVA, page_va); 713 714 if (page_va == 0x8000a000) 715 FMSG("unhide %#" PRIxVA " a %#" PRIX32, 716 page_va, a); 717 area_set_entry(pmem->area, pmem->pgidx, pa, a); 718 719 TAILQ_REMOVE(&tee_pager_pmem_head, pmem, link); 720 TAILQ_INSERT_TAIL(&tee_pager_pmem_head, pmem, link); 721 722 /* TODO only invalidate entry touched above */ 723 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 724 725 incr_hidden_hits(); 726 return true; 727 } 728 } 729 730 return false; 731 } 732 733 static void tee_pager_hide_pages(void) 734 { 735 struct tee_pager_pmem *pmem; 736 size_t n = 0; 737 738 TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) { 739 paddr_t pa; 740 uint32_t attr; 741 uint32_t a; 742 743 if (n >= TEE_PAGER_NHIDE) 744 break; 745 n++; 746 747 /* we cannot hide pages when pmem->area is not defined. */ 748 if (!pmem->area) 749 continue; 750 751 area_get_entry(pmem->area, pmem->pgidx, &pa, &attr); 752 if (!(attr & TEE_MATTR_VALID_BLOCK)) 753 continue; 754 755 assert(pa == get_pmem_pa(pmem)); 756 if (attr & (TEE_MATTR_PW | TEE_MATTR_UW)){ 757 a = TEE_MATTR_HIDDEN_DIRTY_BLOCK; 758 FMSG("Hide %#" PRIxVA, 759 pmem->area->ti->va_base + 760 pmem->pgidx * SMALL_PAGE_SIZE); 761 } else 762 a = TEE_MATTR_HIDDEN_BLOCK; 763 area_set_entry(pmem->area, pmem->pgidx, pa, a); 764 } 765 766 /* TODO only invalidate entries touched above */ 767 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 768 } 769 770 /* 771 * Find mapped pmem, hide and move to pageble pmem. 772 * Return false if page was not mapped, and true if page was mapped. 773 */ 774 static bool tee_pager_release_one_phys(struct tee_pager_area *area, 775 vaddr_t page_va) 776 { 777 struct tee_pager_pmem *pmem; 778 unsigned pgidx; 779 paddr_t pa; 780 uint32_t attr; 781 782 pgidx = area_va2idx(area, page_va); 783 area_get_entry(area, pgidx, &pa, &attr); 784 785 FMSG("%" PRIxVA " : %" PRIxPA "|%x", page_va, pa, attr); 786 787 TAILQ_FOREACH(pmem, &tee_pager_lock_pmem_head, link) { 788 if (pmem->area != area || pmem->pgidx != pgidx) 789 continue; 790 791 assert(pa == get_pmem_pa(pmem)); 792 area_set_entry(area, pgidx, 0, 0); 793 pgt_dec_used_entries(area->pgt); 794 TAILQ_REMOVE(&tee_pager_lock_pmem_head, pmem, link); 795 pmem->area = NULL; 796 pmem->pgidx = INVALID_PGIDX; 797 tee_pager_npages++; 798 set_npages(); 799 TAILQ_INSERT_HEAD(&tee_pager_pmem_head, pmem, link); 800 incr_zi_released(); 801 return true; 802 } 803 804 return false; 805 } 806 807 /* Finds the oldest page and unmats it from its old virtual address */ 808 static struct tee_pager_pmem *tee_pager_get_page(struct tee_pager_area *area) 809 { 810 struct tee_pager_pmem *pmem; 811 812 pmem = TAILQ_FIRST(&tee_pager_pmem_head); 813 if (!pmem) { 814 EMSG("No pmem entries"); 815 return NULL; 816 } 817 if (pmem->pgidx != INVALID_PGIDX) { 818 uint32_t a; 819 820 assert(pmem->area && pmem->area->pgt); 821 area_get_entry(pmem->area, pmem->pgidx, NULL, &a); 822 area_set_entry(pmem->area, pmem->pgidx, 0, 0); 823 pgt_dec_used_entries(pmem->area->pgt); 824 /* TODO only invalidate entries touched above */ 825 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 826 tee_pager_save_page(pmem, a); 827 } 828 829 TAILQ_REMOVE(&tee_pager_pmem_head, pmem, link); 830 pmem->pgidx = INVALID_PGIDX; 831 pmem->area = NULL; 832 if (area->type == AREA_TYPE_LOCK) { 833 /* Move page to lock list */ 834 if (tee_pager_npages <= 0) 835 panic("running out of page"); 836 tee_pager_npages--; 837 set_npages(); 838 TAILQ_INSERT_TAIL(&tee_pager_lock_pmem_head, pmem, link); 839 } else { 840 /* move page to back */ 841 TAILQ_INSERT_TAIL(&tee_pager_pmem_head, pmem, link); 842 } 843 844 return pmem; 845 } 846 847 static bool pager_update_permissions(struct tee_pager_area *area, 848 struct abort_info *ai, bool *handled) 849 { 850 unsigned int pgidx = area_va2idx(area, ai->va); 851 uint32_t attr; 852 paddr_t pa; 853 854 *handled = false; 855 856 area_get_entry(area, pgidx, &pa, &attr); 857 858 /* Not mapped */ 859 if (!(attr & TEE_MATTR_VALID_BLOCK)) 860 return false; 861 862 /* Not readable, should not happen */ 863 if (abort_is_user_exception(ai)) { 864 if (!(attr & TEE_MATTR_UR)) 865 return true; 866 } else { 867 if (!(attr & TEE_MATTR_PR)) { 868 abort_print_error(ai); 869 panic(); 870 } 871 } 872 873 switch (core_mmu_get_fault_type(ai->fault_descr)) { 874 case CORE_MMU_FAULT_TRANSLATION: 875 case CORE_MMU_FAULT_READ_PERMISSION: 876 if (ai->abort_type == ABORT_TYPE_PREFETCH) { 877 /* Check attempting to execute from an NOX page */ 878 if (abort_is_user_exception(ai)) { 879 if (!(attr & TEE_MATTR_UX)) 880 return true; 881 } else { 882 if (!(attr & TEE_MATTR_PX)) { 883 abort_print_error(ai); 884 panic(); 885 } 886 } 887 } 888 /* Since the page is mapped now it's OK */ 889 break; 890 case CORE_MMU_FAULT_WRITE_PERMISSION: 891 /* Check attempting to write to an RO page */ 892 if (abort_is_user_exception(ai)) { 893 if (!(area->flags & TEE_MATTR_UW)) 894 return true; 895 if (!(attr & TEE_MATTR_UW)) { 896 FMSG("Dirty %p", 897 (void *)(ai->va & ~SMALL_PAGE_MASK)); 898 area_set_entry(area, pgidx, pa, 899 get_area_mattr(area->flags)); 900 /* TODO only invalidate entry above */ 901 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 902 } 903 904 } else { 905 if (!(area->flags & TEE_MATTR_PW)) { 906 abort_print_error(ai); 907 panic(); 908 } 909 if (!(attr & TEE_MATTR_PW)) { 910 FMSG("Dirty %p", 911 (void *)(ai->va & ~SMALL_PAGE_MASK)); 912 area_set_entry(area, pgidx, pa, 913 get_area_mattr(area->flags)); 914 /* TODO only invalidate entry above */ 915 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 916 } 917 } 918 /* Since permissions has been updated now it's OK */ 919 break; 920 default: 921 /* Some fault we can't deal with */ 922 if (abort_is_user_exception(ai)) 923 return true; 924 abort_print_error(ai); 925 panic(); 926 } 927 *handled = true; 928 return true; 929 } 930 931 #ifdef CFG_TEE_CORE_DEBUG 932 static void stat_handle_fault(void) 933 { 934 static size_t num_faults; 935 static size_t min_npages = SIZE_MAX; 936 static size_t total_min_npages = SIZE_MAX; 937 938 num_faults++; 939 if ((num_faults % 1024) == 0 || tee_pager_npages < total_min_npages) { 940 DMSG("nfaults %zu npages %zu (min %zu)", 941 num_faults, tee_pager_npages, min_npages); 942 min_npages = tee_pager_npages; /* reset */ 943 } 944 if (tee_pager_npages < min_npages) 945 min_npages = tee_pager_npages; 946 if (tee_pager_npages < total_min_npages) 947 total_min_npages = tee_pager_npages; 948 } 949 #else 950 static void stat_handle_fault(void) 951 { 952 } 953 #endif 954 955 bool tee_pager_handle_fault(struct abort_info *ai) 956 { 957 struct tee_pager_area *area; 958 vaddr_t page_va = ai->va & ~SMALL_PAGE_MASK; 959 uint32_t exceptions; 960 bool ret; 961 962 #ifdef TEE_PAGER_DEBUG_PRINT 963 abort_print(ai); 964 #endif 965 966 /* 967 * We're updating pages that can affect several active CPUs at a 968 * time below. We end up here because a thread tries to access some 969 * memory that isn't available. We have to be careful when making 970 * that memory available as other threads may succeed in accessing 971 * that address the moment after we've made it available. 972 * 973 * That means that we can't just map the memory and populate the 974 * page, instead we use the aliased mapping to populate the page 975 * and once everything is ready we map it. 976 */ 977 exceptions = thread_mask_exceptions(THREAD_EXCP_IRQ); 978 cpu_spin_lock(&pager_lock); 979 980 stat_handle_fault(); 981 982 /* check if the access is valid */ 983 if (abort_is_user_exception(ai)) { 984 area = find_uta_area(ai->va); 985 986 } else { 987 area = find_area(&tee_pager_area_head, ai->va); 988 if (!area) 989 area = find_uta_area(ai->va); 990 } 991 if (!area) { 992 ret = false; 993 goto out; 994 } 995 996 if (!tee_pager_unhide_page(page_va)) { 997 struct tee_pager_pmem *pmem = NULL; 998 uint32_t attr; 999 1000 /* 1001 * The page wasn't hidden, but some other core may have 1002 * updated the table entry before we got here or we need 1003 * to make a read-only page read-write (dirty). 1004 */ 1005 if (pager_update_permissions(area, ai, &ret)) { 1006 /* 1007 * Nothing more to do with the abort. The problem 1008 * could already have been dealt with from another 1009 * core or if ret is false the TA will be paniced. 1010 */ 1011 goto out; 1012 } 1013 1014 pmem = tee_pager_get_page(area); 1015 if (!pmem) { 1016 abort_print(ai); 1017 panic(); 1018 } 1019 1020 /* load page code & data */ 1021 tee_pager_load_page(area, page_va, pmem->va_alias); 1022 1023 /* 1024 * We've updated the page using the aliased mapping and 1025 * some cache maintenence is now needed if it's an 1026 * executable page. 1027 * 1028 * Since the d-cache is a Physically-indexed, 1029 * physically-tagged (PIPT) cache we can clean the aliased 1030 * address instead of the real virtual address. 1031 * 1032 * The i-cache can also be PIPT, but may be something else 1033 * to, to keep it simple we invalidate the entire i-cache. 1034 * As a future optimization we may invalidate only the 1035 * aliased area if it a PIPT cache else the entire cache. 1036 */ 1037 if (area->flags & (TEE_MATTR_PX | TEE_MATTR_UX)) { 1038 /* 1039 * Doing these operations to LoUIS (Level of 1040 * unification, Inner Shareable) would be enough 1041 */ 1042 cache_maintenance_l1(DCACHE_AREA_CLEAN, 1043 pmem->va_alias, SMALL_PAGE_SIZE); 1044 1045 cache_maintenance_l1(ICACHE_INVALIDATE, NULL, 0); 1046 } 1047 1048 pmem->area = area; 1049 pmem->pgidx = area_va2idx(area, ai->va); 1050 attr = get_area_mattr(area->flags) & 1051 ~(TEE_MATTR_PW | TEE_MATTR_UW); 1052 area_set_entry(area, pmem->pgidx, get_pmem_pa(pmem), attr); 1053 pgt_inc_used_entries(area->pgt); 1054 1055 FMSG("Mapped 0x%" PRIxVA " -> 0x%" PRIxPA, 1056 area_idx2va(area, pmem->pgidx), get_pmem_pa(pmem)); 1057 1058 } 1059 1060 tee_pager_hide_pages(); 1061 ret = true; 1062 out: 1063 cpu_spin_unlock(&pager_lock); 1064 thread_unmask_exceptions(exceptions); 1065 return ret; 1066 } 1067 1068 void tee_pager_add_pages(vaddr_t vaddr, size_t npages, bool unmap) 1069 { 1070 struct core_mmu_table_info *ti = &tee_pager_tbl_info; 1071 size_t n; 1072 1073 DMSG("0x%" PRIxVA " - 0x%" PRIxVA " : %d", 1074 vaddr, vaddr + npages * SMALL_PAGE_SIZE, (int)unmap); 1075 1076 /* setup memory */ 1077 for (n = 0; n < npages; n++) { 1078 struct tee_pager_pmem *pmem; 1079 vaddr_t va = vaddr + n * SMALL_PAGE_SIZE; 1080 unsigned pgidx = core_mmu_va2idx(ti, va); 1081 paddr_t pa; 1082 uint32_t attr; 1083 1084 /* 1085 * Note that we can only support adding pages in the 1086 * valid range of this table info, currently not a problem. 1087 */ 1088 core_mmu_get_entry(ti, pgidx, &pa, &attr); 1089 1090 /* Ignore unmapped pages/blocks */ 1091 if (!(attr & TEE_MATTR_VALID_BLOCK)) 1092 continue; 1093 1094 pmem = malloc(sizeof(struct tee_pager_pmem)); 1095 if (!pmem) 1096 panic("out of mem"); 1097 1098 pmem->va_alias = pager_add_alias_page(pa); 1099 1100 if (unmap) { 1101 pmem->area = NULL; 1102 pmem->pgidx = INVALID_PGIDX; 1103 core_mmu_set_entry(ti, pgidx, 0, 0); 1104 pgt_dec_used_entries(&pager_core_pgt); 1105 } else { 1106 /* 1107 * The page is still mapped, let's assign the area 1108 * and update the protection bits accordingly. 1109 */ 1110 pmem->area = find_area(&tee_pager_area_head, va); 1111 assert(pmem->area->pgt == &pager_core_pgt); 1112 pmem->pgidx = pgidx; 1113 assert(pa == get_pmem_pa(pmem)); 1114 area_set_entry(pmem->area, pgidx, pa, 1115 get_area_mattr(pmem->area->flags)); 1116 } 1117 1118 tee_pager_npages++; 1119 incr_npages_all(); 1120 set_npages(); 1121 TAILQ_INSERT_TAIL(&tee_pager_pmem_head, pmem, link); 1122 } 1123 1124 /* Invalidate secure TLB */ 1125 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 1126 } 1127 1128 #ifdef CFG_PAGED_USER_TA 1129 static struct pgt *find_pgt(struct pgt *pgt, vaddr_t va) 1130 { 1131 struct pgt *p = pgt; 1132 1133 while (p && (va & ~CORE_MMU_PGDIR_MASK) != p->vabase) 1134 p = SLIST_NEXT(p, link); 1135 return p; 1136 } 1137 1138 void tee_pager_assign_uta_tables(struct user_ta_ctx *utc) 1139 { 1140 struct tee_pager_area *area; 1141 struct pgt *pgt = SLIST_FIRST(&thread_get_tsd()->pgt_cache); 1142 1143 TAILQ_FOREACH(area, utc->areas, link) { 1144 if (!area->pgt) 1145 area->pgt = find_pgt(pgt, area->base); 1146 else 1147 assert(area->pgt == find_pgt(pgt, area->base)); 1148 if (!area->pgt) 1149 panic(); 1150 } 1151 } 1152 1153 void tee_pager_pgt_save_and_release_entries(struct pgt *pgt) 1154 { 1155 struct tee_pager_pmem *pmem; 1156 struct tee_pager_area *area; 1157 uint32_t exceptions; 1158 uint32_t attr; 1159 1160 exceptions = thread_mask_exceptions(THREAD_EXCP_IRQ); 1161 cpu_spin_lock(&pager_lock); 1162 1163 if (!pgt->num_used_entries) 1164 goto out; 1165 1166 TAILQ_FOREACH(pmem, &tee_pager_pmem_head, link) { 1167 if (!pmem->area || pmem->pgidx == INVALID_PGIDX) 1168 continue; 1169 if (pmem->area->pgt == pgt) { 1170 area_get_entry(pmem->area, pmem->pgidx, NULL, &attr); 1171 area_set_entry(pmem->area, pmem->pgidx, 0, 0); 1172 tee_pager_save_page(pmem, attr); 1173 pmem->pgidx = INVALID_PGIDX; 1174 pmem->area = NULL; 1175 pgt->num_used_entries--; 1176 } 1177 } 1178 assert(!pgt->num_used_entries); 1179 1180 out: 1181 if (is_user_ta_ctx(pgt->ctx)) { 1182 TAILQ_FOREACH(area, to_user_ta_ctx(pgt->ctx)->areas, link) { 1183 if (area->pgt == pgt) 1184 area->pgt = NULL; 1185 } 1186 } 1187 1188 cpu_spin_unlock(&pager_lock); 1189 thread_unmask_exceptions(exceptions); 1190 } 1191 KEEP_PAGER(tee_pager_pgt_save_and_release_entries); 1192 #endif /*CFG_PAGED_USER_TA*/ 1193 1194 void tee_pager_release_phys(void *addr, size_t size) 1195 { 1196 bool unmaped = false; 1197 vaddr_t va = (vaddr_t)addr; 1198 vaddr_t begin = ROUNDUP(va, SMALL_PAGE_SIZE); 1199 vaddr_t end = ROUNDDOWN(va + size, SMALL_PAGE_SIZE); 1200 struct tee_pager_area *area; 1201 uint32_t exceptions; 1202 1203 if (!size) 1204 return; 1205 1206 area = find_area(&tee_pager_area_head, begin); 1207 if (!area || 1208 area != find_area(&tee_pager_area_head, end - SMALL_PAGE_SIZE)) 1209 panic(); 1210 1211 exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); 1212 cpu_spin_lock(&pager_lock); 1213 1214 for (va = begin; va < end; va += SMALL_PAGE_SIZE) 1215 unmaped |= tee_pager_release_one_phys(area, va); 1216 1217 /* Invalidate secure TLB */ 1218 if (unmaped) 1219 core_tlb_maintenance(TLBINV_UNIFIEDTLB, 0); 1220 1221 cpu_spin_unlock(&pager_lock); 1222 thread_set_exceptions(exceptions); 1223 } 1224 KEEP_PAGER(tee_pager_release_phys); 1225 1226 void *tee_pager_alloc(size_t size, uint32_t flags) 1227 { 1228 tee_mm_entry_t *mm; 1229 uint32_t f = TEE_MATTR_PW | TEE_MATTR_PR | (flags & TEE_MATTR_LOCKED); 1230 1231 if (!size) 1232 return NULL; 1233 1234 mm = tee_mm_alloc(&tee_mm_vcore, ROUNDUP(size, SMALL_PAGE_SIZE)); 1235 if (!mm) 1236 return NULL; 1237 1238 tee_pager_add_core_area(tee_mm_get_smem(mm), tee_mm_get_bytes(mm), 1239 f, NULL, NULL); 1240 1241 return (void *)tee_mm_get_smem(mm); 1242 } 1243