306 			    unsigned long init, unsigned long gaddr)  in gmap_alloc_table()  argument
322 		page->index = gaddr;  in gmap_alloc_table()
379 static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr)  in __gmap_unmap_by_gaddr()  argument
384 						   gaddr >> PMD_SHIFT);  in __gmap_unmap_by_gaddr()
474 unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr)  in __gmap_translate()  argument
479 		radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT);  in __gmap_translate()
481 	return vmaddr ? (vmaddr | (gaddr & ~PMD_MASK)) : -EFAULT;  in __gmap_translate()
494 unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr)  in gmap_translate()  argument
499 	rc = __gmap_translate(gmap, gaddr);  in gmap_translate()
527 			   unsigned long gaddr);
540 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)  in __gmap_link()  argument
556 		table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;  in __gmap_link()
559 				     gaddr & _REGION1_MASK))  in __gmap_link()
564 		table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;  in __gmap_link()
567 				     gaddr & _REGION2_MASK))  in __gmap_link()
572 		table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;  in __gmap_link()
575 				     gaddr & _REGION3_MASK))  in __gmap_link()
579 	table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;  in __gmap_link()
619 		gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr);  in __gmap_link()
636 int gmap_fault(struct gmap *gmap, unsigned long gaddr,  in gmap_fault()  argument
647 	vmaddr = __gmap_translate(gmap, gaddr);  in gmap_fault()
664 	rc = __gmap_link(gmap, gaddr, vmaddr);  in gmap_fault()
674 void __gmap_zap(struct gmap *gmap, unsigned long gaddr)  in __gmap_zap()  argument
682 						   gaddr >> PMD_SHIFT);  in __gmap_zap()
684 		vmaddr |= gaddr & ~PMD_MASK;  in __gmap_zap()
697 	unsigned long gaddr, vmaddr, size;  in gmap_discard()  local
701 	for (gaddr = from; gaddr < to;  in gmap_discard()
702 	     gaddr = (gaddr + PMD_SIZE) & PMD_MASK) {  in gmap_discard()
706 					  gaddr >> PMD_SHIFT);  in gmap_discard()
709 		vmaddr |= gaddr & ~PMD_MASK;  in gmap_discard()
720 		size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK));  in gmap_discard()
789 					     unsigned long gaddr, int level)  in gmap_table_walk()  argument
801 	    gaddr & (-1UL << (31 + (asce_type >> 2) * 11)))  in gmap_table_walk()
806 		table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;  in gmap_table_walk()
814 		table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;  in gmap_table_walk()
822 		table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;  in gmap_table_walk()
830 		table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;  in gmap_table_walk()
836 		table += (gaddr & _PAGE_INDEX) >> _PAGE_SHIFT;  in gmap_table_walk()
850 static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr,  in gmap_pte_op_walk()  argument
857 	table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */  in gmap_pte_op_walk()
860 	return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl);  in gmap_pte_op_walk()
874 static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr,  in gmap_pte_op_fixup()  argument
889 	return __gmap_link(gmap, gaddr, vmaddr);  in gmap_pte_op_fixup()
910 static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr)  in gmap_pmd_op_walk()  argument
915 	pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1);  in gmap_pmd_op_walk()
960 static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr,  in gmap_protect_pmd()  argument
973 		gmap_pmdp_xchg(gmap, pmdp, new, gaddr);  in gmap_protect_pmd()
979 		gmap_pmdp_xchg(gmap, pmdp, new, gaddr);  in gmap_protect_pmd()
1005 static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr,  in gmap_protect_pte()  argument
1016 	ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl);  in gmap_protect_pte()
1023 	rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits);  in gmap_protect_pte()
1041 static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,  in gmap_protect_range()  argument
1051 		pmdp = gmap_pmd_op_walk(gmap, gaddr);  in gmap_protect_range()
1054 				rc = gmap_protect_pte(gmap, gaddr, pmdp, prot,  in gmap_protect_range()
1058 					gaddr += PAGE_SIZE;  in gmap_protect_range()
1061 				rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot,  in gmap_protect_range()
1064 					dist = HPAGE_SIZE - (gaddr & ~HPAGE_MASK);  in gmap_protect_range()
1066 					gaddr = (gaddr & HPAGE_MASK) + HPAGE_SIZE;  in gmap_protect_range()
1076 			vmaddr = __gmap_translate(gmap, gaddr);  in gmap_protect_range()
1079 			rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);  in gmap_protect_range()
1101 int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr,  in gmap_mprotect_notify()  argument
1106 	if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap))  in gmap_mprotect_notify()
1111 	rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT);  in gmap_mprotect_notify()
1130 int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val)  in gmap_read_table()  argument
1142 		ptep = gmap_pte_op_walk(gmap, gaddr, &ptl);  in gmap_read_table()
1147 				address += gaddr & ~PAGE_MASK;  in gmap_read_table()
1157 		vmaddr = __gmap_translate(gmap, gaddr);  in gmap_read_table()
1162 		rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ);  in gmap_read_table()
2175 			       unsigned long gaddr)  in gmap_shadow_notify()  argument
2190 	if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start &&  in gmap_shadow_notify()
2191 	    gaddr < end) {  in gmap_shadow_notify()
2239 	unsigned long offset, gaddr = 0;  in ptep_notify()  local
2251 			gaddr = __gmap_segment_gaddr(table) + offset;  in ptep_notify()
2260 				gmap_shadow_notify(sg, vmaddr, gaddr);  in ptep_notify()
2264 			gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1);  in ptep_notify()
2271 			     unsigned long gaddr)  in pmdp_notify_gmap()  argument
2274 	gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1);  in pmdp_notify_gmap()
2288 			   unsigned long gaddr)  in gmap_pmdp_xchg()  argument
2290 	gaddr &= HPAGE_MASK;  in gmap_pmdp_xchg()
2291 	pmdp_notify_gmap(gmap, pmdp, gaddr);  in gmap_pmdp_xchg()
2294 		__pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce,  in gmap_pmdp_xchg()
2297 		__pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL);  in gmap_pmdp_xchg()
2308 	unsigned long gaddr;  in gmap_pmdp_clear()  local
2316 			gaddr = __gmap_segment_gaddr((unsigned long *)pmdp);  in gmap_pmdp_clear()
2317 			pmdp_notify_gmap(gmap, pmdp, gaddr);  in gmap_pmdp_clear()
2359 	unsigned long *entry, gaddr;  in gmap_pmdp_idte_local()  local
2370 			gaddr = __gmap_segment_gaddr(entry);  in gmap_pmdp_idte_local()
2371 			pmdp_notify_gmap(gmap, pmdp, gaddr);  in gmap_pmdp_idte_local()
2375 				__pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,  in gmap_pmdp_idte_local()
2378 				__pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL);  in gmap_pmdp_idte_local()
2394 	unsigned long *entry, gaddr;  in gmap_pmdp_idte_global()  local
2405 			gaddr = __gmap_segment_gaddr(entry);  in gmap_pmdp_idte_global()
2406 			pmdp_notify_gmap(gmap, pmdp, gaddr);  in gmap_pmdp_idte_global()
2410 				__pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,  in gmap_pmdp_idte_global()
2413 				__pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL);  in gmap_pmdp_idte_global()
2434 					  unsigned long gaddr)  in gmap_test_and_clear_dirty_pmd()  argument
2446 	gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0);  in gmap_test_and_clear_dirty_pmd()
2461 			     unsigned long gaddr, unsigned long vmaddr)  in gmap_sync_dirty_log_pmd()  argument
2468 	pmdp = gmap_pmd_op_walk(gmap, gaddr);  in gmap_sync_dirty_log_pmd()
2473 		if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr))  in gmap_sync_dirty_log_pmd()