750 		struct address_space *mapping, struct vm_fault *vmf,  in dax_insert_entry()  argument
774 		dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);  in dax_insert_entry()
1060 		struct vm_fault *vmf)  in dax_load_hole()  argument
1063 	unsigned long vaddr = vmf->address;  in dax_load_hole()
1067 	*entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,  in dax_load_hole()
1070 	ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);  in dax_load_hole()
1071 	trace_dax_load_hole(inode, vmf, ret);  in dax_load_hole()
1265 static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,  in dax_iomap_pte_fault()  argument
1268 	struct vm_area_struct *vma = vmf->vma;  in dax_iomap_pte_fault()
1270 	XA_STATE(xas, &mapping->i_pages, vmf->pgoff);  in dax_iomap_pte_fault()
1272 	unsigned long vaddr = vmf->address;  in dax_iomap_pte_fault()
1273 	loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;  in dax_iomap_pte_fault()
1278 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in dax_iomap_pte_fault()
1284 	trace_dax_pte_fault(inode, vmf, ret);  in dax_iomap_pte_fault()
1295 	if (write && !vmf->cow_page)  in dax_iomap_pte_fault()
1310 	if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {  in dax_iomap_pte_fault()
1332 	if (vmf->cow_page) {  in dax_iomap_pte_fault()
1338 			clear_user_highpage(vmf->cow_page, vaddr);  in dax_iomap_pte_fault()
1342 						  sector, vmf->cow_page, vaddr);  in dax_iomap_pte_fault()
1353 		__SetPageUptodate(vmf->cow_page);  in dax_iomap_pte_fault()
1354 		ret = finish_fault(vmf);  in dax_iomap_pte_fault()
1373 		entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,  in dax_iomap_pte_fault()
1391 		trace_dax_insert_mapping(inode, vmf, entry);  in dax_iomap_pte_fault()
1401 			ret = dax_load_hole(&xas, mapping, &entry, vmf);  in dax_iomap_pte_fault()
1430 	trace_dax_pte_fault_done(inode, vmf, ret);  in dax_iomap_pte_fault()
1435 static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,  in dax_pmd_load_hole()  argument
1438 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_pmd_load_hole()
1439 	unsigned long pmd_addr = vmf->address & PMD_MASK;  in dax_pmd_load_hole()
1440 	struct vm_area_struct *vma = vmf->vma;  in dax_pmd_load_hole()
1448 	zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);  in dax_pmd_load_hole()
1454 	*entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,  in dax_pmd_load_hole()
1463 	ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in dax_pmd_load_hole()
1464 	if (!pmd_none(*(vmf->pmd))) {  in dax_pmd_load_hole()
1470 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in dax_pmd_load_hole()
1473 	pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);  in dax_pmd_load_hole()
1475 	set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);  in dax_pmd_load_hole()
1477 	trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry);  in dax_pmd_load_hole()
1483 	trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry);  in dax_pmd_load_hole()
1487 static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,  in dax_iomap_pmd_fault()  argument
1490 	struct vm_area_struct *vma = vmf->vma;  in dax_iomap_pmd_fault()
1492 	XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);  in dax_iomap_pmd_fault()
1493 	unsigned long pmd_addr = vmf->address & PMD_MASK;  in dax_iomap_pmd_fault()
1494 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in dax_iomap_pmd_fault()
1514 	trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);  in dax_iomap_pmd_fault()
1522 	if ((vmf->pgoff & PG_PMD_COLOUR) !=  in dax_iomap_pmd_fault()
1523 	    ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))  in dax_iomap_pmd_fault()
1563 	if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&  in dax_iomap_pmd_fault()
1564 			!pmd_devmap(*vmf->pmd)) {  in dax_iomap_pmd_fault()
1591 		entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,  in dax_iomap_pmd_fault()
1608 		trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);  in dax_iomap_pmd_fault()
1609 		result = vmf_insert_pfn_pmd(vmf, pfn, write);  in dax_iomap_pmd_fault()
1615 		result = dax_pmd_load_hole(&xas, vmf, &iomap, &entry);  in dax_iomap_pmd_fault()
1641 		split_huge_pmd(vma, vmf->pmd, vmf->address);  in dax_iomap_pmd_fault()
1645 	trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);  in dax_iomap_pmd_fault()
1649 static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,  in dax_iomap_pmd_fault()  argument
1669 vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,  in dax_iomap_fault()  argument
1674 		return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);  in dax_iomap_fault()
1676 		return dax_iomap_pmd_fault(vmf, pfnp, ops);  in dax_iomap_fault()
1693 dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order)  in dax_insert_pfn_mkwrite()  argument
1695 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_insert_pfn_mkwrite()
1696 	XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);  in dax_insert_pfn_mkwrite()
1707 		trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,  in dax_insert_pfn_mkwrite()
1715 		ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);  in dax_insert_pfn_mkwrite()
1718 		ret = vmf_insert_pfn_pmd(vmf, pfn, FAULT_FLAG_WRITE);  in dax_insert_pfn_mkwrite()
1723 	trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);  in dax_insert_pfn_mkwrite()
1737 vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,  in dax_finish_sync_fault()  argument
1741 	loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;  in dax_finish_sync_fault()
1745 	err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);  in dax_finish_sync_fault()
1748 	return dax_insert_pfn_mkwrite(vmf, pfn, order);  in dax_finish_sync_fault()