179 static void kvm_clear_pte(kvm_pte_t *ptep)  in kvm_clear_pte()  argument
181 	WRITE_ONCE(*ptep, 0);  in kvm_clear_pte()
184 static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp,  in kvm_set_table_pte()  argument
187 	kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));  in kvm_set_table_pte()
193 	smp_store_release(ptep, pte);  in kvm_set_table_pte()
215 				  u32 level, kvm_pte_t *ptep,  in kvm_pgtable_visitor_cb()  argument
219 	return walker->cb(addr, data->end, level, ptep, flag, walker->arg);  in kvm_pgtable_visitor_cb()
226 				      kvm_pte_t *ptep, u32 level)  in __kvm_pgtable_visit()  argument
230 	kvm_pte_t *childp, pte = *ptep;  in __kvm_pgtable_visit()
235 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
240 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
242 		pte = *ptep;  in __kvm_pgtable_visit()
261 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
279 		kvm_pte_t *ptep = &pgtable[idx];  in __kvm_pgtable_walk()  local
284 		ret = __kvm_pgtable_visit(data, ptep, level);  in __kvm_pgtable_walk()
306 		kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE];  in _kvm_pgtable_walk()  local
308 		ret = __kvm_pgtable_walk(data, ptep, pgt->start_level);  in _kvm_pgtable_walk()
335 static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)  in hyp_set_prot_attr()  argument
360 	*ptep = attr;  in hyp_set_prot_attr()
366 				    kvm_pte_t *ptep, struct hyp_map_data *data)  in hyp_map_walker_try_leaf()  argument
368 	kvm_pte_t new, old = *ptep;  in hyp_map_walker_try_leaf()
377 		smp_store_release(ptep, new);  in hyp_map_walker_try_leaf()
383 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_map_walker()  argument
390 	if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg))  in hyp_map_walker()
400 	kvm_set_table_pte(ptep, childp, mm_ops);  in hyp_map_walker()
444 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_free_walker()  argument
449 	mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops));  in hyp_free_walker()
522 				kvm_pte_t *ptep)  in stage2_set_prot_attr()  argument
542 	*ptep = attr;  in stage2_set_prot_attr()
565 static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr,  in stage2_put_pte()  argument
572 	if (kvm_pte_valid(*ptep)) {  in stage2_put_pte()
573 		kvm_clear_pte(ptep);  in stage2_put_pte()
577 	mm_ops->put_page(ptep);  in stage2_put_pte()
581 				      kvm_pte_t *ptep,  in stage2_map_walker_try_leaf()  argument
584 	kvm_pte_t new, old = *ptep;  in stage2_map_walker_try_leaf()
606 		stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);  in stage2_map_walker_try_leaf()
609 	smp_store_release(ptep, new);  in stage2_map_walker_try_leaf()
611 		mm_ops->get_page(ptep);  in stage2_map_walker_try_leaf()
618 				     kvm_pte_t *ptep,  in stage2_map_walk_table_pre()  argument
627 	data->childp = kvm_pte_follow(*ptep, data->mm_ops);  in stage2_map_walk_table_pre()
628 	kvm_clear_pte(ptep);  in stage2_map_walk_table_pre()
636 	data->anchor = ptep;  in stage2_map_walk_table_pre()
640 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walk_leaf()  argument
644 	kvm_pte_t *childp, pte = *ptep;  in stage2_map_walk_leaf()
649 			mm_ops->put_page(ptep);  in stage2_map_walk_leaf()
654 	ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data);  in stage2_map_walk_leaf()
674 		stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);  in stage2_map_walk_leaf()
676 	kvm_set_table_pte(ptep, childp, mm_ops);  in stage2_map_walk_leaf()
677 	mm_ops->get_page(ptep);  in stage2_map_walk_leaf()
683 				      kvm_pte_t *ptep,  in stage2_map_walk_table_post()  argument
693 	if (data->anchor == ptep) {  in stage2_map_walk_table_post()
697 		ret = stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walk_table_post()
699 		childp = kvm_pte_follow(*ptep, mm_ops);  in stage2_map_walk_table_post()
703 	mm_ops->put_page(ptep);  in stage2_map_walk_table_post()
727 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walker()  argument
734 		return stage2_map_walk_table_pre(addr, end, level, ptep, data);  in stage2_map_walker()
736 		return stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walker()
738 		return stage2_map_walk_table_post(addr, end, level, ptep, data);  in stage2_map_walker()
807 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_unmap_walker()  argument
814 	kvm_pte_t pte = *ptep, *childp = NULL;  in stage2_unmap_walker()
819 			kvm_clear_pte(ptep);  in stage2_unmap_walker()
820 			mm_ops->put_page(ptep);  in stage2_unmap_walker()
839 	stage2_put_pte(ptep, mmu, addr, level, mm_ops);  in stage2_unmap_walker()
870 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_attr_walker()  argument
874 	kvm_pte_t pte = *ptep;  in stage2_attr_walker()
891 		WRITE_ONCE(*ptep, pte);  in stage2_attr_walker()
984 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_flush_walker()  argument
990 	kvm_pte_t pte = *ptep;  in stage2_flush_walker()
1039 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_free_walker()  argument
1044 	kvm_pte_t pte = *ptep;  in stage2_free_walker()
1049 	mm_ops->put_page(ptep);  in stage2_free_walker()
1078 					  kvm_pte_t *ptep,  in stage2_check_permission_walker()  argument
1082 	kvm_pte_t old_attr, pte = *ptep, *new_attr = arg;  in stage2_check_permission_walker()