356 static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr)  in __gmap_unlink_by_vmaddr()  argument
363 	entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);  in __gmap_unlink_by_vmaddr()
381 	unsigned long vmaddr;  in __gmap_unmap_by_gaddr()  local
383 	vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host,  in __gmap_unmap_by_gaddr()
385 	return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0;  in __gmap_unmap_by_gaddr()
476 	unsigned long vmaddr;  in __gmap_translate()  local
478 	vmaddr = (unsigned long)  in __gmap_translate()
481 	return vmaddr ? (vmaddr | (gaddr & ~PMD_MASK)) : -EFAULT;  in __gmap_translate()
512 		 unsigned long vmaddr)  in gmap_unlink()  argument
519 		flush = __gmap_unlink_by_vmaddr(gmap, vmaddr);  in gmap_unlink()
540 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)  in __gmap_link()  argument
582 	pgd = pgd_offset(mm, vmaddr);  in __gmap_link()
584 	p4d = p4d_offset(pgd, vmaddr);  in __gmap_link()
586 	pud = pud_offset(p4d, vmaddr);  in __gmap_link()
591 	pmd = pmd_offset(pud, vmaddr);  in __gmap_link()
604 				       vmaddr >> PMD_SHIFT, table);  in __gmap_link()
639 	unsigned long vmaddr;  in gmap_fault()  local
647 	vmaddr = __gmap_translate(gmap, gaddr);  in gmap_fault()
648 	if (IS_ERR_VALUE(vmaddr)) {  in gmap_fault()
649 		rc = vmaddr;  in gmap_fault()
652 	if (fixup_user_fault(gmap->mm, vmaddr, fault_flags,  in gmap_fault()
664 	rc = __gmap_link(gmap, gaddr, vmaddr);  in gmap_fault()
676 	unsigned long vmaddr;  in __gmap_zap()  local
681 	vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host,  in __gmap_zap()
683 	if (vmaddr) {  in __gmap_zap()
684 		vmaddr |= gaddr & ~PMD_MASK;  in __gmap_zap()
686 		ptep = get_locked_pte(gmap->mm, vmaddr, &ptl);  in __gmap_zap()
688 			ptep_zap_unused(gmap->mm, vmaddr, ptep, 0);  in __gmap_zap()
697 	unsigned long gaddr, vmaddr, size;  in gmap_discard()  local
704 		vmaddr = (unsigned long)  in gmap_discard()
707 		if (!vmaddr)  in gmap_discard()
709 		vmaddr |= gaddr & ~PMD_MASK;  in gmap_discard()
711 		vma = find_vma(gmap->mm, vmaddr);  in gmap_discard()
721 		zap_page_range(vma, vmaddr, size);  in gmap_discard()
875 			     unsigned long vmaddr, int prot)  in gmap_pte_op_fixup()  argument
883 	if (fixup_user_fault(mm, vmaddr, fault_flags, &unlocked))  in gmap_pte_op_fixup()
889 	return __gmap_link(gmap, gaddr, vmaddr);  in gmap_pte_op_fixup()
1044 	unsigned long vmaddr, dist;  in gmap_protect_range()  local
1076 			vmaddr = __gmap_translate(gmap, gaddr);  in gmap_protect_range()
1077 			if (IS_ERR_VALUE(vmaddr))  in gmap_protect_range()
1078 				return vmaddr;  in gmap_protect_range()
1079 			rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);  in gmap_protect_range()
1132 	unsigned long address, vmaddr;  in gmap_read_table()  local
1157 		vmaddr = __gmap_translate(gmap, gaddr);  in gmap_read_table()
1158 		if (IS_ERR_VALUE(vmaddr)) {  in gmap_read_table()
1159 			rc = vmaddr;  in gmap_read_table()
1162 		rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ);  in gmap_read_table()
1178 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,  in gmap_insert_rmap()  argument
1184 	slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_insert_rmap()
1191 		radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT,  in gmap_insert_rmap()
1211 	unsigned long vmaddr;  in gmap_protect_rmap()  local
1219 		vmaddr = __gmap_translate(parent, paddr);  in gmap_protect_rmap()
1220 		if (IS_ERR_VALUE(vmaddr))  in gmap_protect_rmap()
1221 			return vmaddr;  in gmap_protect_rmap()
1238 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_protect_rmap()
1245 			rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ);  in gmap_protect_rmap()
2110 	unsigned long vmaddr, paddr;  in gmap_shadow_page()  local
2127 		vmaddr = __gmap_translate(parent, paddr);  in gmap_shadow_page()
2128 		if (IS_ERR_VALUE(vmaddr)) {  in gmap_shadow_page()
2129 			rc = vmaddr;  in gmap_shadow_page()
2150 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_shadow_page()
2160 		rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot);  in gmap_shadow_page()
2174 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,  in gmap_shadow_notify()  argument
2200 	head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_shadow_notify()
2236 void ptep_notify(struct mm_struct *mm, unsigned long vmaddr,  in ptep_notify()  argument
2249 					  vmaddr >> PMD_SHIFT);  in ptep_notify()
2260 				gmap_shadow_notify(sg, vmaddr, gaddr);  in ptep_notify()
2303 static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr,  in gmap_pmdp_clear()  argument
2314 						  vmaddr >> PMD_SHIFT);  in gmap_pmdp_clear()
2335 void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr)  in gmap_pmdp_invalidate()  argument
2337 	gmap_pmdp_clear(mm, vmaddr, 0);  in gmap_pmdp_invalidate()
2346 void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr)  in gmap_pmdp_csp()  argument
2348 	gmap_pmdp_clear(mm, vmaddr, 1);  in gmap_pmdp_csp()
2357 void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr)  in gmap_pmdp_idte_local()  argument
2367 					  vmaddr >> PMD_SHIFT);  in gmap_pmdp_idte_local()
2392 void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr)  in gmap_pmdp_idte_global()  argument
2402 					  vmaddr >> PMD_SHIFT);  in gmap_pmdp_idte_global()
2461 			     unsigned long gaddr, unsigned long vmaddr)  in gmap_sync_dirty_log_pmd()  argument
2476 		for (i = 0; i < _PAGE_ENTRIES; i++, vmaddr += PAGE_SIZE) {  in gmap_sync_dirty_log_pmd()
2477 			ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl);  in gmap_sync_dirty_log_pmd()
2480 			if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep))  in gmap_sync_dirty_log_pmd()