Lines Matching refs:pvmw
10 static inline bool not_found(struct page_vma_mapped_walk *pvmw) in not_found() argument
12 page_vma_mapped_walk_done(pvmw); in not_found()
16 static bool map_pte(struct page_vma_mapped_walk *pvmw) in map_pte() argument
18 pvmw->pte = pte_offset_map(pvmw->pmd, pvmw->address); in map_pte()
19 if (!(pvmw->flags & PVMW_SYNC)) { in map_pte()
20 if (pvmw->flags & PVMW_MIGRATION) { in map_pte()
21 if (!is_swap_pte(*pvmw->pte)) in map_pte()
39 if (is_swap_pte(*pvmw->pte)) { in map_pte()
43 entry = pte_to_swp_entry(*pvmw->pte); in map_pte()
46 } else if (!pte_present(*pvmw->pte)) in map_pte()
50 pvmw->ptl = pte_lockptr(pvmw->vma->vm_mm, pvmw->pmd); in map_pte()
51 spin_lock(pvmw->ptl); in map_pte()
85 static bool check_pte(struct page_vma_mapped_walk *pvmw) in check_pte() argument
89 if (pvmw->flags & PVMW_MIGRATION) { in check_pte()
91 if (!is_swap_pte(*pvmw->pte)) in check_pte()
93 entry = pte_to_swp_entry(*pvmw->pte); in check_pte()
99 } else if (is_swap_pte(*pvmw->pte)) { in check_pte()
103 entry = pte_to_swp_entry(*pvmw->pte); in check_pte()
109 if (!pte_present(*pvmw->pte)) in check_pte()
112 pfn = pte_pfn(*pvmw->pte); in check_pte()
115 return pfn_is_match(pvmw->page, pfn); in check_pte()
118 static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size) in step_forward() argument
120 pvmw->address = (pvmw->address + size) & ~(size - 1); in step_forward()
121 if (!pvmw->address) in step_forward()
122 pvmw->address = ULONG_MAX; in step_forward()
149 bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) in page_vma_mapped_walk() argument
151 struct mm_struct *mm = pvmw->vma->vm_mm; in page_vma_mapped_walk()
152 struct page *page = pvmw->page; in page_vma_mapped_walk()
160 if (pvmw->pmd && !pvmw->pte) in page_vma_mapped_walk()
161 return not_found(pvmw); in page_vma_mapped_walk()
165 if (pvmw->pte) in page_vma_mapped_walk()
166 return not_found(pvmw); in page_vma_mapped_walk()
169 pvmw->pte = huge_pte_offset(mm, pvmw->address, page_size(page)); in page_vma_mapped_walk()
170 if (!pvmw->pte) in page_vma_mapped_walk()
173 pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte); in page_vma_mapped_walk()
174 spin_lock(pvmw->ptl); in page_vma_mapped_walk()
175 if (!check_pte(pvmw)) in page_vma_mapped_walk()
176 return not_found(pvmw); in page_vma_mapped_walk()
187 vma_address_end(page, pvmw->vma) : in page_vma_mapped_walk()
188 pvmw->address + PAGE_SIZE; in page_vma_mapped_walk()
189 if (pvmw->pte) in page_vma_mapped_walk()
193 pgd = pgd_offset(mm, pvmw->address); in page_vma_mapped_walk()
195 step_forward(pvmw, PGDIR_SIZE); in page_vma_mapped_walk()
198 p4d = p4d_offset(pgd, pvmw->address); in page_vma_mapped_walk()
200 step_forward(pvmw, P4D_SIZE); in page_vma_mapped_walk()
203 pud = pud_offset(p4d, pvmw->address); in page_vma_mapped_walk()
205 step_forward(pvmw, PUD_SIZE); in page_vma_mapped_walk()
209 pvmw->pmd = pmd_offset(pud, pvmw->address); in page_vma_mapped_walk()
215 pmde = READ_ONCE(*pvmw->pmd); in page_vma_mapped_walk()
218 pvmw->ptl = pmd_lock(mm, pvmw->pmd); in page_vma_mapped_walk()
219 pmde = *pvmw->pmd; in page_vma_mapped_walk()
221 if (pvmw->flags & PVMW_MIGRATION) in page_vma_mapped_walk()
222 return not_found(pvmw); in page_vma_mapped_walk()
224 return not_found(pvmw); in page_vma_mapped_walk()
231 !(pvmw->flags & PVMW_MIGRATION)) in page_vma_mapped_walk()
232 return not_found(pvmw); in page_vma_mapped_walk()
236 return not_found(pvmw); in page_vma_mapped_walk()
240 spin_unlock(pvmw->ptl); in page_vma_mapped_walk()
241 pvmw->ptl = NULL; in page_vma_mapped_walk()
248 if ((pvmw->flags & PVMW_SYNC) && in page_vma_mapped_walk()
250 spinlock_t *ptl = pmd_lock(mm, pvmw->pmd); in page_vma_mapped_walk()
254 step_forward(pvmw, PMD_SIZE); in page_vma_mapped_walk()
257 if (!map_pte(pvmw)) in page_vma_mapped_walk()
260 if (check_pte(pvmw)) in page_vma_mapped_walk()
264 pvmw->address += PAGE_SIZE; in page_vma_mapped_walk()
265 if (pvmw->address >= end) in page_vma_mapped_walk()
266 return not_found(pvmw); in page_vma_mapped_walk()
268 if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) { in page_vma_mapped_walk()
269 if (pvmw->ptl) { in page_vma_mapped_walk()
270 spin_unlock(pvmw->ptl); in page_vma_mapped_walk()
271 pvmw->ptl = NULL; in page_vma_mapped_walk()
273 pte_unmap(pvmw->pte); in page_vma_mapped_walk()
274 pvmw->pte = NULL; in page_vma_mapped_walk()
277 pvmw->pte++; in page_vma_mapped_walk()
278 if ((pvmw->flags & PVMW_SYNC) && !pvmw->ptl) { in page_vma_mapped_walk()
279 pvmw->ptl = pte_lockptr(mm, pvmw->pmd); in page_vma_mapped_walk()
280 spin_lock(pvmw->ptl); in page_vma_mapped_walk()
282 } while (pte_none(*pvmw->pte)); in page_vma_mapped_walk()
284 if (!pvmw->ptl) { in page_vma_mapped_walk()
285 pvmw->ptl = pte_lockptr(mm, pvmw->pmd); in page_vma_mapped_walk()
286 spin_lock(pvmw->ptl); in page_vma_mapped_walk()
289 } while (pvmw->address < end); in page_vma_mapped_walk()
305 struct page_vma_mapped_walk pvmw = { in page_mapped_in_vma() local
311 pvmw.address = vma_address(page, vma); in page_mapped_in_vma()
312 if (pvmw.address == -EFAULT) in page_mapped_in_vma()
314 if (!page_vma_mapped_walk(&pvmw)) in page_mapped_in_vma()
316 page_vma_mapped_walk_done(&pvmw); in page_mapped_in_vma()