Lines Matching +full:- +full:p
1 // SPDX-License-Identifier: BSD-2-Clause
73 assert(pgt && pgt->parent); in free_pgt()
74 parent = pgt->parent; in free_pgt()
75 assert(parent->num_used <= PGT_PARENT_TBL_COUNT && in free_pgt()
76 parent->num_used > 0); in free_pgt()
77 if (parent->num_used == PGT_PARENT_TBL_COUNT) in free_pgt()
79 parent->num_used--; in free_pgt()
81 if (!parent->num_used && SLIST_NEXT(SLIST_FIRST(&parent_list), link)) { in free_pgt()
87 tee_mm_free(parent->mm); in free_pgt()
90 SLIST_INSERT_HEAD(&parent->pgt_cache, pgt, link); in free_pgt()
91 pgt->vabase = 0; in free_pgt()
92 pgt->populated = false; in free_pgt()
110 parent->mm = phys_mem_ta_alloc(PGT_PARENT_SIZE); in alloc_pgt_parent()
111 if (!parent->mm) { in alloc_pgt_parent()
115 tbl = phys_to_virt(tee_mm_get_smem(parent->mm), in alloc_pgt_parent()
120 SLIST_INIT(&parent->pgt_cache); in alloc_pgt_parent()
125 SLIST_INSERT_HEAD(&parent->pgt_cache, pgt + n, link); in alloc_pgt_parent()
148 pgt = SLIST_FIRST(&parent->pgt_cache); in alloc_pgt()
149 SLIST_REMOVE_HEAD(&parent->pgt_cache, link); in alloc_pgt()
150 parent->num_used++; in alloc_pgt()
151 assert(pgt && parent->num_used <= PGT_PARENT_TBL_COUNT); in alloc_pgt()
152 if (parent->num_used == PGT_PARENT_TBL_COUNT) in alloc_pgt()
155 pgt->vabase = vabase; in alloc_pgt()
161 static bool pgt_entry_matches(struct pgt *p, vaddr_t begin, vaddr_t last) in pgt_entry_matches() argument
163 if (!p) in pgt_entry_matches()
167 return core_is_buffer_inside(p->vabase, CORE_MMU_PGDIR_SIZE, begin, in pgt_entry_matches()
168 last - begin); in pgt_entry_matches()
173 struct pgt_cache *pgt_cache = &uctx->pgt_cache; in pgt_flush_range()
175 struct pgt *p = NULL; in pgt_flush_range() local
181 p = SLIST_FIRST(pgt_cache); in pgt_flush_range()
182 while (pgt_entry_matches(p, begin, last)) { in pgt_flush_range()
184 free_pgt(p); in pgt_flush_range()
185 p = SLIST_FIRST(pgt_cache); in pgt_flush_range()
189 * p either points to the first element in the list or it's NULL, in pgt_flush_range()
192 if (!p) in pgt_flush_range()
200 next_p = SLIST_NEXT(p, link); in pgt_flush_range()
204 SLIST_REMOVE_AFTER(p, link); in pgt_flush_range()
209 p = SLIST_NEXT(p, link); in pgt_flush_range()
215 struct pgt_cache *pgt_cache = &uctx->pgt_cache; in pgt_flush()
216 struct pgt *p = NULL; in pgt_flush() local
219 p = SLIST_FIRST(pgt_cache); in pgt_flush()
220 if (!p) in pgt_flush()
223 free_pgt(p); in pgt_flush()
229 struct pgt_cache *pgt_cache = &uctx->pgt_cache; in pgt_clear_range()
230 struct pgt *p = NULL; in pgt_clear_range() local
239 SLIST_FOREACH(p, pgt_cache, link) { in pgt_clear_range()
240 vaddr_t b = MAX(p->vabase, begin); in pgt_clear_range()
241 vaddr_t e = MIN(p->vabase + CORE_MMU_PGDIR_SIZE, end); in pgt_clear_range()
246 tbl = p->tbl; in pgt_clear_range()
247 idx = (b - p->vabase) / SMALL_PAGE_SIZE; in pgt_clear_range()
248 n = (e - b) / SMALL_PAGE_SIZE; in pgt_clear_range()
253 static struct pgt *prune_before_va(struct pgt_cache *pgt_cache, struct pgt *p, in prune_before_va() argument
256 while (p && p->vabase < va) { in prune_before_va()
258 assert(p == SLIST_NEXT(pp, link)); in prune_before_va()
260 free_pgt(p); in prune_before_va()
261 p = SLIST_NEXT(pp, link); in prune_before_va()
263 assert(p == SLIST_FIRST(pgt_cache)); in prune_before_va()
265 free_pgt(p); in prune_before_va()
266 p = SLIST_FIRST(pgt_cache); in prune_before_va()
270 return p; in prune_before_va()
275 struct pgt_cache *pgt_cache = &uctx->pgt_cache; in pgt_check_avail()
276 struct vm_info *vm_info = &uctx->vm_info; in pgt_check_avail()
277 struct pgt *p = SLIST_FIRST(pgt_cache); in pgt_check_avail() local
290 TAILQ_FOREACH(r, &vm_info->regions, link) { in pgt_check_avail()
291 for (va = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); in pgt_check_avail()
292 va < r->va + r->size; va += CORE_MMU_PGDIR_SIZE) { in pgt_check_avail()
294 p = prune_before_va(pgt_cache, p, pp, va); in pgt_check_avail()
295 if (!p) in pgt_check_avail()
298 if (p->vabase < va) { in pgt_check_avail()
299 pp = p; in pgt_check_avail()
300 p = SLIST_NEXT(pp, link); in pgt_check_avail()
301 if (!p) in pgt_check_avail()
306 if (p->vabase == va) in pgt_check_avail()
312 p = SLIST_FIRST(pgt_cache); in pgt_check_avail()
314 TAILQ_FOREACH(r, &vm_info->regions, link) { in pgt_check_avail()
315 for (va = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); in pgt_check_avail()
316 va < r->va + r->size; va += CORE_MMU_PGDIR_SIZE) { in pgt_check_avail()
317 if (p && p->vabase < va) { in pgt_check_avail()
318 pp = p; in pgt_check_avail()
319 p = SLIST_NEXT(pp, link); in pgt_check_avail()
322 if (p) { in pgt_check_avail()
323 if (p->vabase == va) in pgt_check_avail()
325 assert(p->vabase > va); in pgt_check_avail()
328 p = alloc_pgt(va); in pgt_check_avail()
329 if (!p) in pgt_check_avail()
333 SLIST_INSERT_AFTER(pp, p, link); in pgt_check_avail()
335 SLIST_INSERT_HEAD(pgt_cache, p, link); in pgt_check_avail()
375 struct pgt *p = pgt_entries + n; in pgt_init() local
377 p->tbl = tee_pager_alloc(PGT_SIZE); in pgt_init()
378 SLIST_INSERT_HEAD(&pgt_free_list, p, link); in pgt_init()
383 * Four translation tables per page -> need to keep track of the page
399 struct pgt *p = pgt_entries + in pgt_init() local
402 p->tbl = tbl + m * PGT_SIZE; in pgt_init()
403 p->parent = &pgt_parents[n]; in pgt_init()
404 SLIST_INSERT_HEAD(&pgt_parents[n].pgt_cache, p, link); in pgt_init()
422 struct pgt *p = pgt_entries + n; in pgt_init() local
424 p->tbl = pgt_tables[n]; in pgt_init()
425 SLIST_INSERT_HEAD(&pgt_free_list, p, link); in pgt_init()
434 struct pgt *p = SLIST_FIRST(&pgt_free_list); in pop_from_free_list() local
436 if (p) { in pop_from_free_list()
438 memset(p->tbl, 0, PGT_SIZE); in pop_from_free_list()
439 p->populated = false; in pop_from_free_list()
441 return p; in pop_from_free_list()
444 static void push_to_free_list(struct pgt *p) in push_to_free_list() argument
446 SLIST_INSERT_HEAD(&pgt_free_list, p, link); in push_to_free_list()
448 tee_pager_release_phys(p->tbl, PGT_SIZE); in push_to_free_list()
453 * Four translation tables per page -> need to keep track of the page
461 struct pgt *p = SLIST_FIRST(&pgt_parents[n].pgt_cache); in pop_from_free_list() local
463 if (p) { in pop_from_free_list()
466 memset(p->tbl, 0, PGT_SIZE); in pop_from_free_list()
467 p->populated = false; in pop_from_free_list()
468 return p; in pop_from_free_list()
474 static void push_to_free_list(struct pgt *p) in push_to_free_list() argument
476 SLIST_INSERT_HEAD(&p->parent->pgt_cache, p, link); in push_to_free_list()
477 assert(p->parent->num_used > 0); in push_to_free_list()
478 p->parent->num_used--; in push_to_free_list()
479 if (!p->parent->num_used) { in push_to_free_list()
480 vaddr_t va = (vaddr_t)p->tbl & ~SMALL_PAGE_MASK; in push_to_free_list()
494 return pgt->ctx == ctx && pgt->vabase == vabase; in match_pgt()
500 struct pgt *p; in pop_from_cache_list() local
511 p = SLIST_NEXT(pgt, link); in pop_from_cache_list()
512 if (!p) in pop_from_cache_list()
514 if (match_pgt(p, vabase, ctx)) { in pop_from_cache_list()
518 pgt = p; in pop_from_cache_list()
520 return p; in pop_from_cache_list()
526 return pgt->num_used_entries; in get_num_used_entries()
568 struct pgt *p = SLIST_FIRST(pgt_cache); in pgt_free_unlocked() local
577 if (IS_ENABLED(CFG_PAGED_USER_TA) && !get_num_used_entries(p)) { in pgt_free_unlocked()
578 tee_pager_pgt_save_and_release_entries(p); in pgt_free_unlocked()
579 p->ctx = NULL; in pgt_free_unlocked()
580 p->vabase = 0; in pgt_free_unlocked()
582 push_to_free_list(p); in pgt_free_unlocked()
586 push_to_cache_list(p); in pgt_free_unlocked()
592 struct pgt *p = pop_from_cache_list(vabase, ctx); in pop_from_some_list() local
594 if (p) in pop_from_some_list()
595 return p; in pop_from_some_list()
596 p = pop_from_free_list(); in pop_from_some_list()
597 if (!p) { in pop_from_some_list()
598 p = pop_least_used_from_cache_list(); in pop_from_some_list()
599 if (!p) in pop_from_some_list()
601 tee_pager_pgt_save_and_release_entries(p); in pop_from_some_list()
602 memset(p->tbl, 0, PGT_SIZE); in pop_from_some_list()
603 p->populated = false; in pop_from_some_list()
605 p->ctx = ctx; in pop_from_some_list()
606 p->vabase = vabase; in pop_from_some_list()
607 return p; in pop_from_some_list()
612 struct ts_ctx *ctx = uctx->ts_ctx; in pgt_flush()
614 struct pgt *p = NULL; in pgt_flush() local
619 p = SLIST_FIRST(&pgt_cache_list); in pgt_flush()
620 if (!p) in pgt_flush()
622 if (p->ctx != ctx) in pgt_flush()
625 tee_pager_pgt_save_and_release_entries(p); in pgt_flush()
626 p->ctx = NULL; in pgt_flush()
627 p->vabase = 0; in pgt_flush()
628 push_to_free_list(p); in pgt_flush()
631 pp = p; in pgt_flush()
633 p = SLIST_NEXT(pp, link); in pgt_flush()
634 if (!p) in pgt_flush()
636 if (p->ctx == ctx) { in pgt_flush()
638 tee_pager_pgt_save_and_release_entries(p); in pgt_flush()
639 p->ctx = NULL; in pgt_flush()
640 p->vabase = 0; in pgt_flush()
641 push_to_free_list(p); in pgt_flush()
643 pp = p; in pgt_flush()
651 static void flush_pgt_entry(struct pgt *p) in flush_pgt_entry() argument
653 tee_pager_pgt_save_and_release_entries(p); in flush_pgt_entry()
654 p->ctx = NULL; in flush_pgt_entry()
655 p->vabase = 0; in flush_pgt_entry()
658 static bool pgt_entry_matches(struct pgt *p, void *ctx, vaddr_t begin, in pgt_entry_matches() argument
661 if (!p) in pgt_entry_matches()
663 if (p->ctx != ctx) in pgt_entry_matches()
667 if (!core_is_buffer_inside(p->vabase, CORE_MMU_PGDIR_SIZE, begin, in pgt_entry_matches()
668 last - begin)) in pgt_entry_matches()
677 struct pgt *p; in flush_ctx_range_from_list() local
684 p = SLIST_FIRST(pgt_cache); in flush_ctx_range_from_list()
685 while (pgt_entry_matches(p, ctx, begin, last)) { in flush_ctx_range_from_list()
686 flush_pgt_entry(p); in flush_ctx_range_from_list()
688 push_to_free_list(p); in flush_ctx_range_from_list()
689 p = SLIST_FIRST(pgt_cache); in flush_ctx_range_from_list()
693 * p either points to the first element in the list or it's NULL, in flush_ctx_range_from_list()
696 if (!p) in flush_ctx_range_from_list()
704 next_p = SLIST_NEXT(p, link); in flush_ctx_range_from_list()
709 SLIST_REMOVE_AFTER(p, link); in flush_ctx_range_from_list()
714 p = SLIST_NEXT(p, link); in flush_ctx_range_from_list()
720 struct pgt_cache *pgt_cache = &uctx->pgt_cache; in pgt_flush_range()
721 struct ts_ctx *ctx = uctx->ts_ctx; in pgt_flush_range()
735 struct pgt *p = NULL; in clear_ctx_range_from_list() local
744 SLIST_FOREACH(p, pgt_cache, link) { in clear_ctx_range_from_list()
745 vaddr_t b = MAX(p->vabase, begin); in clear_ctx_range_from_list()
746 vaddr_t e = MIN(p->vabase + CORE_MMU_PGDIR_SIZE, end); in clear_ctx_range_from_list()
748 if (p->ctx != ctx) in clear_ctx_range_from_list()
753 tbl = p->tbl; in clear_ctx_range_from_list()
754 idx = (b - p->vabase) / SMALL_PAGE_SIZE; in clear_ctx_range_from_list()
755 n = (e - b) / SMALL_PAGE_SIZE; in clear_ctx_range_from_list()
762 struct pgt_cache *pgt_cache = &uctx->pgt_cache; in pgt_clear_range()
763 struct ts_ctx *ctx = uctx->ts_ctx; in pgt_clear_range()
778 struct pgt *p = NULL; in pgt_alloc_unlocked() local
781 TAILQ_FOREACH(r, &vm_info->regions, link) { in pgt_alloc_unlocked()
782 for (va = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); in pgt_alloc_unlocked()
783 va < r->va + r->size; va += CORE_MMU_PGDIR_SIZE) { in pgt_alloc_unlocked()
784 if (p && p->vabase == va) in pgt_alloc_unlocked()
786 p = pop_from_some_list(va, ctx); in pgt_alloc_unlocked()
787 if (!p) { in pgt_alloc_unlocked()
792 SLIST_INSERT_AFTER(pp, p, link); in pgt_alloc_unlocked()
794 SLIST_INSERT_HEAD(pgt_cache, p, link); in pgt_alloc_unlocked()
795 pp = p; in pgt_alloc_unlocked()
804 struct vm_info *vm_info = &uctx->vm_info; in pgt_check_avail()
810 TAILQ_FOREACH(r, &vm_info->regions, link) { in pgt_check_avail()
811 for (va = ROUNDDOWN(r->va, CORE_MMU_PGDIR_SIZE); in pgt_check_avail()
812 va < r->va + r->size; va += CORE_MMU_PGDIR_SIZE) { in pgt_check_avail()
825 struct pgt_cache *pgt_cache = &uctx->pgt_cache; in pgt_get_all()
826 struct vm_info *vm_info = &uctx->vm_info; in pgt_get_all()
828 if (TAILQ_EMPTY(&vm_info->regions)) in pgt_get_all()
834 while (!pgt_alloc_unlocked(pgt_cache, uctx->ts_ctx, vm_info)) { in pgt_get_all()
846 struct pgt_cache *pgt_cache = &uctx->pgt_cache; in pgt_put_all()