57 	struct kvm_pgtable		*pgt;  member
115 static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)  in __kvm_pgd_page_idx()  argument
117 	u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */  in __kvm_pgd_page_idx()
118 	u64 mask = BIT(pgt->ia_bits) - 1;  in __kvm_pgd_page_idx()
125 	return __kvm_pgd_page_idx(data->pgt, data->addr);  in kvm_pgd_page_idx()
130 	struct kvm_pgtable pgt = {  in kvm_pgd_pages()  local
135 	return __kvm_pgd_page_idx(&pgt, -1ULL) + 1;  in kvm_pgd_pages()
255 	childp = kvm_pte_follow(pte, data->pgt->mm_ops);  in __kvm_pgtable_visit()
296 	struct kvm_pgtable *pgt = data->pgt;  in _kvm_pgtable_walk()  local
297 	u64 limit = BIT(pgt->ia_bits);  in _kvm_pgtable_walk()
302 	if (!pgt->pgd)  in _kvm_pgtable_walk()
306 		kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE];  in _kvm_pgtable_walk()
308 		ret = __kvm_pgtable_walk(data, ptep, pgt->start_level);  in _kvm_pgtable_walk()
316 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_walk()  argument
320 		.pgt	= pgt,  in kvm_pgtable_walk()
404 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,  in kvm_pgtable_hyp_map()  argument
410 		.mm_ops	= pgt->mm_ops,  in kvm_pgtable_hyp_map()
422 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_hyp_map()
428 int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,  in kvm_pgtable_hyp_init()  argument
433 	pgt->pgd = (kvm_pte_t *)mm_ops->zalloc_page(NULL);  in kvm_pgtable_hyp_init()
434 	if (!pgt->pgd)  in kvm_pgtable_hyp_init()
437 	pgt->ia_bits		= va_bits;  in kvm_pgtable_hyp_init()
438 	pgt->start_level	= KVM_PGTABLE_MAX_LEVELS - levels;  in kvm_pgtable_hyp_init()
439 	pgt->mm_ops		= mm_ops;  in kvm_pgtable_hyp_init()
440 	pgt->mmu		= NULL;  in kvm_pgtable_hyp_init()
453 void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)  in kvm_pgtable_hyp_destroy()  argument
458 		.arg	= pgt->mm_ops,  in kvm_pgtable_hyp_destroy()
461 	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));  in kvm_pgtable_hyp_destroy()
462 	pgt->mm_ops->put_page(pgt->pgd);  in kvm_pgtable_hyp_destroy()
463 	pgt->pgd = NULL;  in kvm_pgtable_hyp_destroy()
511 static bool stage2_has_fwb(struct kvm_pgtable *pgt)  in stage2_has_fwb()  argument
516 	return !(pgt->flags & KVM_PGTABLE_S2_NOFWB);  in stage2_has_fwb()
519 #define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt))  argument
521 static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,  in stage2_set_prot_attr()  argument
525 	kvm_pte_t attr = device ? KVM_S2_MEMATTR(pgt, DEVICE_nGnRE) :  in stage2_set_prot_attr()
526 			    KVM_S2_MEMATTR(pgt, NORMAL);  in stage2_set_prot_attr()
744 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_stage2_map()  argument
751 		.mmu		= pgt->mmu,  in kvm_pgtable_stage2_map()
753 		.mm_ops		= pgt->mm_ops,  in kvm_pgtable_stage2_map()
763 	if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys)))  in kvm_pgtable_stage2_map()
766 	ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);  in kvm_pgtable_stage2_map()
770 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_map()
775 int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_stage2_set_owner()  argument
781 		.mmu		= pgt->mmu,  in kvm_pgtable_stage2_set_owner()
783 		.mm_ops		= pgt->mm_ops,  in kvm_pgtable_stage2_set_owner()
797 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_set_owner()
801 static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)  in stage2_pte_cacheable()  argument
804 	return memattr == KVM_S2_MEMATTR(pgt, NORMAL);  in stage2_pte_cacheable()
811 	struct kvm_pgtable *pgt = arg;  in stage2_unmap_walker()  local
812 	struct kvm_s2_mmu *mmu = pgt->mmu;  in stage2_unmap_walker()
813 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in stage2_unmap_walker()
830 	} else if (stage2_pte_cacheable(pgt, pte)) {  in stage2_unmap_walker()
831 		need_flush = !stage2_has_fwb(pgt);  in stage2_unmap_walker()
852 int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_unmap()  argument
856 		.arg	= pgt,  in kvm_pgtable_stage2_unmap()
860 	return kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_unmap()
896 static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,  in stage2_update_leaf_attrs()  argument
913 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in stage2_update_leaf_attrs()
925 int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_wrprotect()  argument
927 	return stage2_update_leaf_attrs(pgt, addr, size, 0,  in kvm_pgtable_stage2_wrprotect()
932 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgtable_stage2_mkyoung()  argument
935 	stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,  in kvm_pgtable_stage2_mkyoung()
941 kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgtable_stage2_mkold()  argument
944 	stage2_update_leaf_attrs(pgt, addr, 1, 0, KVM_PTE_LEAF_ATTR_LO_S2_AF,  in kvm_pgtable_stage2_mkold()
955 bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgtable_stage2_is_young()  argument
958 	stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL);  in kvm_pgtable_stage2_is_young()
962 int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,  in kvm_pgtable_stage2_relax_perms()  argument
978 	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level);  in kvm_pgtable_stage2_relax_perms()
980 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level);  in kvm_pgtable_stage2_relax_perms()
988 	struct kvm_pgtable *pgt = arg;  in stage2_flush_walker()  local
989 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in stage2_flush_walker()
992 	if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte))  in stage2_flush_walker()
999 int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_flush()  argument
1004 		.arg	= pgt,  in kvm_pgtable_stage2_flush()
1007 	if (stage2_has_fwb(pgt))  in kvm_pgtable_stage2_flush()
1010 	return kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_flush()
1013 int kvm_pgtable_stage2_init_flags(struct kvm_pgtable *pgt, struct kvm_arch *arch,  in kvm_pgtable_stage2_init_flags()  argument
1024 	pgt->pgd = mm_ops->zalloc_pages_exact(pgd_sz);  in kvm_pgtable_stage2_init_flags()
1025 	if (!pgt->pgd)  in kvm_pgtable_stage2_init_flags()
1028 	pgt->ia_bits		= ia_bits;  in kvm_pgtable_stage2_init_flags()
1029 	pgt->start_level	= start_level;  in kvm_pgtable_stage2_init_flags()
1030 	pgt->mm_ops		= mm_ops;  in kvm_pgtable_stage2_init_flags()
1031 	pgt->mmu		= &arch->mmu;  in kvm_pgtable_stage2_init_flags()
1032 	pgt->flags		= flags;  in kvm_pgtable_stage2_init_flags()
1057 void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)  in kvm_pgtable_stage2_destroy()  argument
1064 		.arg	= pgt->mm_ops,  in kvm_pgtable_stage2_destroy()
1067 	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));  in kvm_pgtable_stage2_destroy()
1068 	pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;  in kvm_pgtable_stage2_destroy()
1069 	pgt->mm_ops->free_pages_exact(pgt->pgd, pgd_sz);  in kvm_pgtable_stage2_destroy()
1070 	pgt->pgd = NULL;  in kvm_pgtable_stage2_destroy()
1099 int kvm_pgtable_stage2_find_range(struct kvm_pgtable *pgt, u64 addr,  in kvm_pgtable_stage2_find_range()  argument
1113 	ret = stage2_set_prot_attr(pgt, prot, &attr);  in kvm_pgtable_stage2_find_range()
1118 	for (level = pgt->start_level; level < KVM_PGTABLE_MAX_LEVELS; level++) {  in kvm_pgtable_stage2_find_range()
1134 		ret = kvm_pgtable_walk(pgt, start, granule, &check_perm_walker);  in kvm_pgtable_stage2_find_range()