389 		 * DAX PMD support.  in hugepage_init()
479 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)  in maybe_pmd_mkwrite()  argument
482 		pmd = pmd_mkwrite(pmd);  in maybe_pmd_mkwrite()
483 	return pmd;  in maybe_pmd_mkwrite()
613 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
614 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
639 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
640 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
696 		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,  in set_huge_zero_page()  argument
700 	if (!pmd_none(*pmd))  in set_huge_zero_page()
705 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in set_huge_zero_page()
706 	set_pmd_at(mm, haddr, pmd, entry);  in set_huge_zero_page()
739 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
741 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
753 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
773 		pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,  in insert_pfn_pmd()  argument
780 	ptl = pmd_lock(mm, pmd);  in insert_pfn_pmd()
781 	if (!pmd_none(*pmd)) {  in insert_pfn_pmd()
783 			if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {  in insert_pfn_pmd()
784 				WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));  in insert_pfn_pmd()
787 			entry = pmd_mkyoung(*pmd);  in insert_pfn_pmd()
789 			if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))  in insert_pfn_pmd()
790 				update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
805 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in insert_pfn_pmd()
810 	set_pmd_at(mm, addr, pmd, entry);  in insert_pfn_pmd()
811 	update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
820  * vmf_insert_pfn_pmd_prot - insert a pmd size pfn
826  * Insert a pmd size pfn. See vmf_insert_pfn() for additional info and
861 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd_prot()
952 		pmd_t *pmd, int flags)  in touch_pmd()  argument
956 	_pmd = pmd_mkyoung(*pmd);  in touch_pmd()
960 				pmd, _pmd, flags & FOLL_WRITE))  in touch_pmd()
961 		update_mmu_cache_pmd(vma, addr, pmd);  in touch_pmd()
965 		pmd_t *pmd, int flags, struct dev_pagemap **pgmap)  in follow_devmap_pmd()  argument
967 	unsigned long pfn = pmd_pfn(*pmd);  in follow_devmap_pmd()
971 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_devmap_pmd()
974 	 * When we COW a devmap PMD entry, we split it into PTEs, so we should  in follow_devmap_pmd()
984 	if (flags & FOLL_WRITE && !pmd_write(*pmd))  in follow_devmap_pmd()
987 	if (pmd_present(*pmd) && pmd_devmap(*pmd))  in follow_devmap_pmd()
993 		touch_pmd(vma, addr, pmd, flags);  in follow_devmap_pmd()
1019 	pmd_t pmd;  in copy_huge_pmd()  local
1036 	pmd = *src_pmd;  in copy_huge_pmd()
1039 	if (unlikely(is_swap_pmd(pmd))) {  in copy_huge_pmd()
1040 		swp_entry_t entry = pmd_to_swp_entry(pmd);  in copy_huge_pmd()
1042 		VM_BUG_ON(!is_pmd_migration_entry(pmd));  in copy_huge_pmd()
1045 			pmd = swp_entry_to_pmd(entry);  in copy_huge_pmd()
1047 				pmd = pmd_swp_mksoft_dirty(pmd);  in copy_huge_pmd()
1049 				pmd = pmd_swp_mkuffd_wp(pmd);  in copy_huge_pmd()
1050 			set_pmd_at(src_mm, addr, src_pmd, pmd);  in copy_huge_pmd()
1056 			pmd = pmd_swp_clear_uffd_wp(pmd);  in copy_huge_pmd()
1057 		set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1063 	if (unlikely(!pmd_trans_huge(pmd))) {  in copy_huge_pmd()
1068 	 * When page table lock is held, the huge zero pmd should not be  in copy_huge_pmd()
1069 	 * under splitting since we don't split the page itself, only pmd to  in copy_huge_pmd()
1072 	if (is_huge_zero_pmd(pmd)) {  in copy_huge_pmd()
1082 	src_page = pmd_page(pmd);  in copy_huge_pmd()
1110 		pmd = pmd_clear_uffd_wp(pmd);  in copy_huge_pmd()
1111 	pmd = pmd_mkold(pmd_wrprotect(pmd));  in copy_huge_pmd()
1112 	set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1256 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1257 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in huge_pmd_set_accessed()
1264 	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))  in huge_pmd_set_accessed()
1265 		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);  in huge_pmd_set_accessed()
1277 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1285 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1299 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1316 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))  in do_huge_pmd_wp_page()
1317 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1326 	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);  in do_huge_pmd_wp_page()
1331  * FOLL_FORCE can write to even unwritable pmd's, but only
1334 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)  in can_follow_write_pmd()  argument
1336 	return pmd_write(pmd) ||  in can_follow_write_pmd()
1337 	       ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));  in can_follow_write_pmd()
1342 				   pmd_t *pmd,  in follow_trans_huge_pmd()  argument
1348 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_trans_huge_pmd()
1350 	if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))  in follow_trans_huge_pmd()
1354 	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))  in follow_trans_huge_pmd()
1358 	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))  in follow_trans_huge_pmd()
1361 	page = pmd_page(*pmd);  in follow_trans_huge_pmd()
1368 		touch_pmd(vma, addr, pmd, flags);  in follow_trans_huge_pmd()
1377 		 * In most cases the pmd is the only mapping of the page as we  in follow_trans_huge_pmd()
1411 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)  in do_huge_pmd_numa_page()  argument
1424 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1425 	if (unlikely(!pmd_same(pmd, *vmf->pmd)))  in do_huge_pmd_numa_page()
1433 	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {  in do_huge_pmd_numa_page()
1434 		page = pmd_page(*vmf->pmd);  in do_huge_pmd_numa_page()
1442 	page = pmd_page(pmd);  in do_huge_pmd_numa_page()
1453 	if (!pmd_savedwrite(pmd))  in do_huge_pmd_numa_page()
1486 	/* Confirm the PMD did not change while page_table_lock was released */  in do_huge_pmd_numa_page()
1488 	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1516 		 * change_huge_pmd() released the pmd lock before  in do_huge_pmd_numa_page()
1535 				vmf->pmd, pmd, vmf->address, page, target_nid);  in do_huge_pmd_numa_page()
1545 	was_writable = pmd_savedwrite(pmd);  in do_huge_pmd_numa_page()
1546 	pmd = pmd_modify(pmd, vma->vm_page_prot);  in do_huge_pmd_numa_page()
1547 	pmd = pmd_mkyoung(pmd);  in do_huge_pmd_numa_page()
1549 		pmd = pmd_mkwrite(pmd);  in do_huge_pmd_numa_page()
1550 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1551 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1568  * Return true if we do MADV_FREE successfully on entire pmd page.
1572 		pmd_t *pmd, unsigned long addr, unsigned long next)  in madvise_free_huge_pmd()  argument
1582 	ptl = pmd_trans_huge_lock(pmd, vma);  in madvise_free_huge_pmd()
1586 	orig_pmd = *pmd;  in madvise_free_huge_pmd()
1625 		pmdp_invalidate(vma, addr, pmd);  in madvise_free_huge_pmd()
1629 		set_pmd_at(mm, addr, pmd, orig_pmd);  in madvise_free_huge_pmd()
1630 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in madvise_free_huge_pmd()
1641 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)  in zap_deposited_table()  argument
1645 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in zap_deposited_table()
1651 		 pmd_t *pmd, unsigned long addr)  in zap_huge_pmd()  argument
1658 	ptl = __pmd_trans_huge_lock(pmd, vma);  in zap_huge_pmd()
1667 	orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,  in zap_huge_pmd()
1669 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in zap_huge_pmd()
1672 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1677 		zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1697 			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");  in zap_huge_pmd()
1700 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1704 				zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1721 	 * With split pmd lock we also need to move preallocated  in pmd_move_must_withdraw()
1722 	 * PTE page table if new_pmd is on different PMD page table.  in pmd_move_must_withdraw()
1730 static pmd_t move_soft_dirty_pmd(pmd_t pmd)  in move_soft_dirty_pmd()  argument
1733 	if (unlikely(is_pmd_migration_entry(pmd)))  in move_soft_dirty_pmd()
1734 		pmd = pmd_swp_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1735 	else if (pmd_present(pmd))  in move_soft_dirty_pmd()
1736 		pmd = pmd_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1738 	return pmd;  in move_soft_dirty_pmd()
1745 	pmd_t pmd;  in move_huge_pmd()  local
1750 	 * The destination pmd shouldn't be established, free_pgtables()  in move_huge_pmd()
1767 		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);  in move_huge_pmd()
1768 		if (pmd_present(pmd))  in move_huge_pmd()
1777 		pmd = move_soft_dirty_pmd(pmd);  in move_huge_pmd()
1778 		set_pmd_at(mm, new_addr, new_pmd, pmd);  in move_huge_pmd()
1791  *  - 0 if PMD could not be locked
1792  *  - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1795 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in change_huge_pmd()  argument
1807 	ptl = __pmd_trans_huge_lock(pmd, vma);  in change_huge_pmd()
1811 	preserve_write = prot_numa && pmd_write(*pmd);  in change_huge_pmd()
1815 	if (is_swap_pmd(*pmd)) {  in change_huge_pmd()
1816 		swp_entry_t entry = pmd_to_swp_entry(*pmd);  in change_huge_pmd()
1818 		VM_BUG_ON(!is_pmd_migration_entry(*pmd));  in change_huge_pmd()
1827 			if (pmd_swp_soft_dirty(*pmd))  in change_huge_pmd()
1829 			if (pmd_swp_uffd_wp(*pmd))  in change_huge_pmd()
1831 			set_pmd_at(mm, addr, pmd, newpmd);  in change_huge_pmd()
1842 	if (prot_numa && is_huge_zero_pmd(*pmd))  in change_huge_pmd()
1845 	if (prot_numa && pmd_protnone(*pmd))  in change_huge_pmd()
1850 	 * to not clear pmd intermittently to avoid race with MADV_DONTNEED  in change_huge_pmd()
1858 	 *   pmd_trans_huge(*pmd) == 0 (without ptl)  in change_huge_pmd()
1859 	 *   // skip the pmd  in change_huge_pmd()
1861 	 *				 // pmd is re-established  in change_huge_pmd()
1863 	 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it  in change_huge_pmd()
1869 	entry = pmdp_invalidate(vma, addr, pmd);  in change_huge_pmd()
1886 	set_pmd_at(mm, addr, pmd, entry);  in change_huge_pmd()
1894  * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
1899 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)  in __pmd_trans_huge_lock()  argument
1902 	ptl = pmd_lock(vma->vm_mm, pmd);  in __pmd_trans_huge_lock()
1903 	if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||  in __pmd_trans_huge_lock()
1904 			pmd_devmap(*pmd)))  in __pmd_trans_huge_lock()
1993 		unsigned long haddr, pmd_t *pmd)  in __split_huge_zero_page_pmd()  argument
2001 	 * Leave pmd empty until pte is filled note that it is fine to delay  in __split_huge_zero_page_pmd()
2003 	 * replacing a zero pmd write protected page with a zero pte write  in __split_huge_zero_page_pmd()
2008 	pmdp_huge_clear_flush(vma, haddr, pmd);  in __split_huge_zero_page_pmd()
2010 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_zero_page_pmd()
2022 	smp_wmb(); /* make pte visible before pmd */  in __split_huge_zero_page_pmd()
2023 	pmd_populate(mm, pmd, pgtable);  in __split_huge_zero_page_pmd()
2026 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd_locked()  argument
2041 	VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)  in __split_huge_pmd_locked()
2042 				&& !pmd_devmap(*pmd));  in __split_huge_pmd_locked()
2047 		old_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in __split_huge_pmd_locked()
2053 			zap_deposited_table(mm, pmd);  in __split_huge_pmd_locked()
2074 	if (is_huge_zero_pmd(*pmd)) {  in __split_huge_pmd_locked()
2084 		return __split_huge_zero_page_pmd(vma, haddr, pmd);  in __split_huge_pmd_locked()
2088 	 * Up to this point the pmd is present and huge and userland has the  in __split_huge_pmd_locked()
2090 	 * place). If we overwrite the pmd with the not-huge version pointing  in __split_huge_pmd_locked()
2102 	 * current pmd notpresent (atomically because here the pmd_trans_huge  in __split_huge_pmd_locked()
2103 	 * must remain set at all times on the pmd until the split is complete  in __split_huge_pmd_locked()
2104 	 * for this pmd), then we flush the SMP TLB and finally we write the  in __split_huge_pmd_locked()
2105 	 * non-huge version of the pmd entry with pmd_populate.  in __split_huge_pmd_locked()
2107 	old_pmd = pmdp_invalidate(vma, haddr, pmd);  in __split_huge_pmd_locked()
2132 	 * Withdraw the table only after we mark the pmd entry invalid.  in __split_huge_pmd_locked()
2135 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_pmd_locked()
2209 	smp_wmb(); /* make pte visible before pmd */  in __split_huge_pmd_locked()
2210 	pmd_populate(mm, pmd, pgtable);  in __split_huge_pmd_locked()
2220 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd()  argument
2232 	ptl = pmd_lock(vma->vm_mm, pmd);  in __split_huge_pmd()
2236 	 * pmd against. Otherwise we can end up replacing wrong page.  in __split_huge_pmd()
2241 		if (page != pmd_page(*pmd))  in __split_huge_pmd()
2246 	if (pmd_trans_huge(*pmd)) {  in __split_huge_pmd()
2248 			page = pmd_page(*pmd);  in __split_huge_pmd()
2259 					_pmd = *pmd;  in __split_huge_pmd()
2263 					if (unlikely(!pmd_same(*pmd, _pmd))) {  in __split_huge_pmd()
2276 	} else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))  in __split_huge_pmd()
2278 	__split_huge_pmd_locked(vma, pmd, range.start, freeze);  in __split_huge_pmd()
2291 	 *  3) Split a huge pmd into pte pointing to the same page. No need  in __split_huge_pmd()
2305 	pmd_t *pmd;  in split_huge_pmd_address()  local
2319 	pmd = pmd_offset(pud, address);  in split_huge_pmd_address()
2321 	__split_huge_pmd(vma, pmd, address, freeze, page);  in split_huge_pmd_address()
2332 	 * an huge pmd.  in vma_adjust_trans_huge()
2342 	 * an huge pmd.  in vma_adjust_trans_huge()
2352 	 * contain an hugepage: check if we need to split an huge pmd.  in vma_adjust_trans_huge()
2975 	if (!(pvmw->pmd && !pvmw->pte))  in set_pmd_migration_entry()
2979 	pmdval = pmdp_invalidate(vma, address, pvmw->pmd);  in set_pmd_migration_entry()
2986 	set_pmd_at(mm, address, pvmw->pmd, pmdswp);  in set_pmd_migration_entry()
3000 	if (!(pvmw->pmd && !pvmw->pte))  in remove_migration_pmd()
3003 	entry = pmd_to_swp_entry(*pvmw->pmd);  in remove_migration_pmd()
3006 	if (pmd_swp_soft_dirty(*pvmw->pmd))  in remove_migration_pmd()
3010 	if (pmd_swp_uffd_wp(*pvmw->pmd))  in remove_migration_pmd()
3018 	set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);  in remove_migration_pmd()
3021 	update_mmu_cache_pmd(vma, address, pvmw->pmd);  in remove_migration_pmd()