48 				   pte_t *ptep, int nodat)  in ptep_ipte_local()  argument
61 		__ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);  in ptep_ipte_local()
63 		__ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);  in ptep_ipte_local()
68 				    pte_t *ptep, int nodat)  in ptep_ipte_global()  argument
81 		__ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);  in ptep_ipte_global()
83 		__ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);  in ptep_ipte_global()
88 				      unsigned long addr, pte_t *ptep,  in ptep_flush_direct()  argument
93 	old = *ptep;  in ptep_flush_direct()
99 		ptep_ipte_local(mm, addr, ptep, nodat);  in ptep_flush_direct()
101 		ptep_ipte_global(mm, addr, ptep, nodat);  in ptep_flush_direct()
107 				    unsigned long addr, pte_t *ptep,  in ptep_flush_lazy()  argument
112 	old = *ptep;  in ptep_flush_lazy()
118 		pte_val(*ptep) |= _PAGE_INVALID;  in ptep_flush_lazy()
121 		ptep_ipte_global(mm, addr, ptep, nodat);  in ptep_flush_lazy()
126 static inline pgste_t pgste_get_lock(pte_t *ptep)  in pgste_get_lock()  argument
139 		: "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])  in pgste_get_lock()
140 		: "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");  in pgste_get_lock()
145 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)  in pgste_set_unlock()  argument
151 		: "=Q" (ptep[PTRS_PER_PTE])  in pgste_set_unlock()
152 		: "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])  in pgste_set_unlock()
157 static inline pgste_t pgste_get(pte_t *ptep)  in pgste_get()  argument
161 	pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);  in pgste_get()
166 static inline void pgste_set(pte_t *ptep, pgste_t pgste)  in pgste_set()  argument
169 	*(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;  in pgste_set()
194 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,  in pgste_set_key()  argument
203 	VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));  in pgste_set_key()
216 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)  in pgste_set_pte()  argument
235 	*ptep = entry;  in pgste_set_pte()
241 				       pte_t *ptep, pgste_t pgste)  in pgste_pte_notify()  argument
249 		ptep_notify(mm, addr, ptep, bits);  in pgste_pte_notify()
256 				      unsigned long addr, pte_t *ptep)  in ptep_xchg_start()  argument
261 		pgste = pgste_get_lock(ptep);  in ptep_xchg_start()
262 		pgste = pgste_pte_notify(mm, addr, ptep, pgste);  in ptep_xchg_start()
268 				    unsigned long addr, pte_t *ptep,  in ptep_xchg_commit()  argument
273 			pgste_set_key(ptep, pgste, new, mm);  in ptep_xchg_commit()
280 		pgste = pgste_set_pte(ptep, pgste, new);  in ptep_xchg_commit()
281 		pgste_set_unlock(ptep, pgste);  in ptep_xchg_commit()
283 		*ptep = new;  in ptep_xchg_commit()
289 		       pte_t *ptep, pte_t new)  in ptep_xchg_direct()  argument
296 	pgste = ptep_xchg_start(mm, addr, ptep);  in ptep_xchg_direct()
298 	old = ptep_flush_direct(mm, addr, ptep, nodat);  in ptep_xchg_direct()
299 	old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);  in ptep_xchg_direct()
306 		     pte_t *ptep, pte_t new)  in ptep_xchg_lazy()  argument
313 	pgste = ptep_xchg_start(mm, addr, ptep);  in ptep_xchg_lazy()
315 	old = ptep_flush_lazy(mm, addr, ptep, nodat);  in ptep_xchg_lazy()
316 	old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);  in ptep_xchg_lazy()
323 			     pte_t *ptep)  in ptep_modify_prot_start()  argument
331 	pgste = ptep_xchg_start(mm, addr, ptep);  in ptep_modify_prot_start()
333 	old = ptep_flush_lazy(mm, addr, ptep, nodat);  in ptep_modify_prot_start()
336 		pgste_set(ptep, pgste);  in ptep_modify_prot_start()
342 			     pte_t *ptep, pte_t old_pte, pte_t pte)  in ptep_modify_prot_commit()  argument
350 		pgste = pgste_get(ptep);  in ptep_modify_prot_commit()
351 		pgste_set_key(ptep, pgste, pte, mm);  in ptep_modify_prot_commit()
352 		pgste = pgste_set_pte(ptep, pgste, pte);  in ptep_modify_prot_commit()
353 		pgste_set_unlock(ptep, pgste);  in ptep_modify_prot_commit()
355 		*ptep = pte;  in ptep_modify_prot_commit()
554 	pte_t *ptep;  in pgtable_trans_huge_withdraw()  local
567 	ptep = (pte_t *) pgtable;  in pgtable_trans_huge_withdraw()
568 	pte_val(*ptep) = _PAGE_INVALID;  in pgtable_trans_huge_withdraw()
569 	ptep++;  in pgtable_trans_huge_withdraw()
570 	pte_val(*ptep) = _PAGE_INVALID;  in pgtable_trans_huge_withdraw()
577 		     pte_t *ptep, pte_t entry)  in ptep_set_pte_at()  argument
583 	pgste = pgste_get_lock(ptep);  in ptep_set_pte_at()
585 	pgste_set_key(ptep, pgste, entry, mm);  in ptep_set_pte_at()
586 	pgste = pgste_set_pte(ptep, pgste, entry);  in ptep_set_pte_at()
587 	pgste_set_unlock(ptep, pgste);  in ptep_set_pte_at()
591 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)  in ptep_set_notify()  argument
596 	pgste = pgste_get_lock(ptep);  in ptep_set_notify()
598 	pgste_set_unlock(ptep, pgste);  in ptep_set_notify()
614 		    pte_t *ptep, int prot, unsigned long bit)  in ptep_force_prot()  argument
620 	pgste = pgste_get_lock(ptep);  in ptep_force_prot()
621 	entry = *ptep;  in ptep_force_prot()
627 		pgste_set_unlock(ptep, pgste);  in ptep_force_prot()
633 		ptep_flush_direct(mm, addr, ptep, nodat);  in ptep_force_prot()
638 		ptep_flush_direct(mm, addr, ptep, nodat);  in ptep_force_prot()
643 	pgste = pgste_set_pte(ptep, pgste, entry);  in ptep_force_prot()
644 	pgste_set_unlock(ptep, pgste);  in ptep_force_prot()
675 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)  in ptep_unshadow_pte()  argument
680 	pgste = pgste_get_lock(ptep);  in ptep_unshadow_pte()
683 	ptep_flush_direct(mm, saddr, ptep, nodat);  in ptep_unshadow_pte()
685 	pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));  in ptep_unshadow_pte()
686 	pgste_set_unlock(ptep, pgste);  in ptep_unshadow_pte()
702 		     pte_t *ptep, int reset)  in ptep_zap_unused()  argument
710 	pgste = pgste_get_lock(ptep);  in ptep_zap_unused()
712 	pte = *ptep;  in ptep_zap_unused()
717 		pte_clear(mm, addr, ptep);  in ptep_zap_unused()
721 	pgste_set_unlock(ptep, pgste);  in ptep_zap_unused()
725 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)  in ptep_zap_key()  argument
732 	pgste = pgste_get_lock(ptep);  in ptep_zap_key()
735 	ptev = pte_val(*ptep);  in ptep_zap_key()
738 	pgste_set_unlock(ptep, pgste);  in ptep_zap_key()
746 		       pte_t *ptep)  in ptep_test_and_clear_uc()  argument
753 	pgste = pgste_get_lock(ptep);  in ptep_test_and_clear_uc()
756 	pte = *ptep;  in ptep_test_and_clear_uc()
758 		pgste = pgste_pte_notify(mm, addr, ptep, pgste);  in ptep_test_and_clear_uc()
760 		ptep_ipte_global(mm, addr, ptep, nodat);  in ptep_test_and_clear_uc()
765 		*ptep = pte;  in ptep_test_and_clear_uc()
767 	pgste_set_unlock(ptep, pgste);  in ptep_test_and_clear_uc()
779 	pte_t *ptep;  in set_guest_storage_key()  local
804 	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);  in set_guest_storage_key()
805 	if (unlikely(!ptep))  in set_guest_storage_key()
808 	new = old = pgste_get_lock(ptep);  in set_guest_storage_key()
814 	if (!(pte_val(*ptep) & _PAGE_INVALID)) {  in set_guest_storage_key()
817 		paddr = pte_val(*ptep) & PAGE_MASK;  in set_guest_storage_key()
831 	pgste_set_unlock(ptep, new);  in set_guest_storage_key()
832 	pte_unmap_unlock(ptep, ptl);  in set_guest_storage_key()
881 	pte_t *ptep;  in reset_guest_reference_bit()  local
903 	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);  in reset_guest_reference_bit()
904 	if (unlikely(!ptep))  in reset_guest_reference_bit()
907 	new = old = pgste_get_lock(ptep);  in reset_guest_reference_bit()
911 	if (!(pte_val(*ptep) & _PAGE_INVALID)) {  in reset_guest_reference_bit()
912 		paddr = pte_val(*ptep) & PAGE_MASK;  in reset_guest_reference_bit()
923 	pgste_set_unlock(ptep, new);  in reset_guest_reference_bit()
924 	pte_unmap_unlock(ptep, ptl);  in reset_guest_reference_bit()
936 	pte_t *ptep;  in get_guest_storage_key()  local
959 	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);  in get_guest_storage_key()
960 	if (unlikely(!ptep))  in get_guest_storage_key()
963 	pgste = pgste_get_lock(ptep);  in get_guest_storage_key()
965 	paddr = pte_val(*ptep) & PAGE_MASK;  in get_guest_storage_key()
966 	if (!(pte_val(*ptep) & _PAGE_INVALID))  in get_guest_storage_key()
970 	pgste_set_unlock(ptep, pgste);  in get_guest_storage_key()
971 	pte_unmap_unlock(ptep, ptl);  in get_guest_storage_key()
995 	pte_t *ptep;  in pgste_perform_essa()  local
1005 	ptep = get_locked_pte(mm, hva, &ptl);  in pgste_perform_essa()
1006 	if (unlikely(!ptep))  in pgste_perform_essa()
1008 	pgste = pgste_get_lock(ptep);  in pgste_perform_essa()
1011 		*oldpte = pte_val(*ptep);  in pgste_perform_essa()
1025 		if (pte_val(*ptep) & _PAGE_INVALID)  in pgste_perform_essa()
1031 		if (pte_val(*ptep) & _PAGE_INVALID)  in pgste_perform_essa()
1036 		if (!(pte_val(*ptep) & _PAGE_INVALID)) {  in pgste_perform_essa()
1060 		if (!(pte_val(*ptep) & _PAGE_INVALID)) {  in pgste_perform_essa()
1078 	pgste_set_unlock(ptep, pgste);  in pgste_perform_essa()
1079 	pte_unmap_unlock(ptep, ptl);  in pgste_perform_essa()
1100 	pte_t *ptep;  in set_pgste_bits()  local
1105 	ptep = get_locked_pte(mm, hva, &ptl);  in set_pgste_bits()
1106 	if (unlikely(!ptep))  in set_pgste_bits()
1108 	new = pgste_get_lock(ptep);  in set_pgste_bits()
1113 	pgste_set_unlock(ptep, new);  in set_pgste_bits()
1114 	pte_unmap_unlock(ptep, ptl);  in set_pgste_bits()
1131 	pte_t *ptep;  in get_pgste()  local
1136 	ptep = get_locked_pte(mm, hva, &ptl);  in get_pgste()
1137 	if (unlikely(!ptep))  in get_pgste()
1139 	*pgstep = pgste_val(pgste_get(ptep));  in get_pgste()
1140 	pte_unmap_unlock(ptep, ptl);  in get_pgste()