66 static void free_pgt(struct pgt *pgt)  in free_pgt()  argument
73 	assert(pgt && pgt->parent);  in free_pgt()
74 	parent = pgt->parent;  in free_pgt()
90 		SLIST_INSERT_HEAD(&parent->pgt_cache, pgt, link);  in free_pgt()
91 		pgt->vabase = 0;  in free_pgt()
92 		pgt->populated = false;  in free_pgt()
101 	struct pgt *pgt = NULL;  in alloc_pgt_parent()  local
106 	sz = sizeof(*parent) + sizeof(*pgt) * PGT_PARENT_TBL_COUNT;  in alloc_pgt_parent()
121 	pgt = (struct pgt *)(parent + 1);  in alloc_pgt_parent()
123 		pgt[n].parent = parent;  in alloc_pgt_parent()
124 		pgt[n].tbl = tbl + n * PGT_SIZE;  in alloc_pgt_parent()
125 		SLIST_INSERT_HEAD(&parent->pgt_cache, pgt + n, link);  in alloc_pgt_parent()
131 static struct pgt *alloc_pgt(vaddr_t vabase)  in alloc_pgt()
135 	struct pgt *pgt = NULL;  in alloc_pgt()  local
148 	pgt = SLIST_FIRST(&parent->pgt_cache);  in alloc_pgt()
151 	assert(pgt && parent->num_used <= PGT_PARENT_TBL_COUNT);  in alloc_pgt()
155 	pgt->vabase = vabase;  in alloc_pgt()
158 	return pgt;  in alloc_pgt()
161 static bool pgt_entry_matches(struct pgt *p, vaddr_t begin, vaddr_t last)  in pgt_entry_matches()
174 	struct pgt *next_p = NULL;  in pgt_flush_range()
175 	struct pgt *p = NULL;  in pgt_flush_range()
216 	struct pgt *p = NULL;  in pgt_flush()
230 	struct pgt *p = NULL;  in pgt_clear_range()
253 static struct pgt *prune_before_va(struct pgt_cache *pgt_cache, struct pgt *p,  in prune_before_va()
254 				   struct pgt *pp, vaddr_t va)  in prune_before_va()
277 	struct pgt *p = SLIST_FIRST(pgt_cache);  in pgt_check_avail()
279 	struct pgt *pp = NULL;  in pgt_check_avail()
360 static struct pgt pgt_entries[PGT_CACHE_SIZE];
375 		struct pgt *p = pgt_entries + n;  in pgt_init()
399 			struct pgt *p = pgt_entries +  in pgt_init()
422 		struct pgt *p = pgt_entries + n;  in pgt_init()
432 static struct pgt *pop_from_free_list(void)  in pop_from_free_list()
434 	struct pgt *p = SLIST_FIRST(&pgt_free_list);  in pop_from_free_list()
444 static void push_to_free_list(struct pgt *p)  in push_to_free_list()
456 static struct pgt *pop_from_free_list(void)  in pop_from_free_list()
461 		struct pgt *p = SLIST_FIRST(&pgt_parents[n].pgt_cache);  in pop_from_free_list()
474 static void push_to_free_list(struct pgt *p)  in push_to_free_list()
487 static void push_to_cache_list(struct pgt *pgt)  in push_to_cache_list()  argument
489 	SLIST_INSERT_HEAD(&pgt_cache_list, pgt, link);  in push_to_cache_list()
492 static bool match_pgt(struct pgt *pgt, vaddr_t vabase, void *ctx)  in match_pgt()  argument
494 	return pgt->ctx == ctx && pgt->vabase == vabase;  in match_pgt()
497 static struct pgt *pop_from_cache_list(vaddr_t vabase, void *ctx)  in pop_from_cache_list()
499 	struct pgt *pgt;  in pop_from_cache_list()  local
500 	struct pgt *p;  in pop_from_cache_list()
502 	pgt = SLIST_FIRST(&pgt_cache_list);  in pop_from_cache_list()
503 	if (!pgt)  in pop_from_cache_list()
505 	if (match_pgt(pgt, vabase, ctx)) {  in pop_from_cache_list()
507 		return pgt;  in pop_from_cache_list()
511 		p = SLIST_NEXT(pgt, link);  in pop_from_cache_list()
515 			SLIST_REMOVE_AFTER(pgt, link);  in pop_from_cache_list()
518 		pgt = p;  in pop_from_cache_list()
523 static uint16_t get_num_used_entries(struct pgt *pgt __maybe_unused)  in get_num_used_entries()
526 	return pgt->num_used_entries;  in get_num_used_entries()
532 static struct pgt *pop_least_used_from_cache_list(void)  in pop_least_used_from_cache_list()
534 	struct pgt *pgt = NULL;  in pop_least_used_from_cache_list()  local
535 	struct pgt *p_prev = NULL;  in pop_least_used_from_cache_list()
539 	pgt = SLIST_FIRST(&pgt_cache_list);  in pop_least_used_from_cache_list()
540 	if (!pgt)  in pop_least_used_from_cache_list()
542 	least_used = get_num_used_entries(pgt);  in pop_least_used_from_cache_list()
545 		if (!SLIST_NEXT(pgt, link))  in pop_least_used_from_cache_list()
547 		next_used = get_num_used_entries(SLIST_NEXT(pgt, link));  in pop_least_used_from_cache_list()
549 			p_prev = pgt;  in pop_least_used_from_cache_list()
552 		pgt = SLIST_NEXT(pgt, link);  in pop_least_used_from_cache_list()
556 		pgt = SLIST_NEXT(p_prev, link);  in pop_least_used_from_cache_list()
559 		pgt = SLIST_FIRST(&pgt_cache_list);  in pop_least_used_from_cache_list()
562 	return pgt;  in pop_least_used_from_cache_list()
568 		struct pgt *p = SLIST_FIRST(pgt_cache);  in pgt_free_unlocked()
590 static struct pgt *pop_from_some_list(vaddr_t vabase, void *ctx)  in pop_from_some_list()
592 	struct pgt *p = pop_from_cache_list(vabase, ctx);  in pop_from_some_list()
613 	struct pgt *pp = NULL;  in pgt_flush()
614 	struct pgt *p = NULL;  in pgt_flush()
651 static void flush_pgt_entry(struct pgt *p)  in flush_pgt_entry()
658 static bool pgt_entry_matches(struct pgt *p, void *ctx, vaddr_t begin,  in pgt_entry_matches()
677 	struct pgt *p;  in flush_ctx_range_from_list()
678 	struct pgt *next_p;  in flush_ctx_range_from_list()
735 	struct pgt *p = NULL;  in clear_ctx_range_from_list()
777 	struct pgt *pp = NULL;  in pgt_alloc_unlocked()
778 	struct pgt *p = NULL;  in pgt_alloc_unlocked()
859 struct pgt *pgt_pop_from_cache_list(vaddr_t vabase, struct ts_ctx *ctx)  in pgt_pop_from_cache_list()
861 	struct pgt *pgt = NULL;  in pgt_pop_from_cache_list()  local
864 	pgt = pop_from_cache_list(vabase, ctx);  in pgt_pop_from_cache_list()
867 	return pgt;  in pgt_pop_from_cache_list()
870 void pgt_push_to_cache_list(struct pgt *pgt)  in pgt_push_to_cache_list()  argument
873 	push_to_cache_list(pgt);  in pgt_push_to_cache_list()