Lines Matching refs:walk
24 unsigned long end, struct mm_walk *walk) in walk_pte_range_inner() argument
26 const struct mm_walk_ops *ops = walk->ops; in walk_pte_range_inner()
30 err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk); in walk_pte_range_inner()
42 struct mm_walk *walk) in walk_pte_range() argument
48 if (walk->no_vma) { in walk_pte_range()
50 err = walk_pte_range_inner(pte, addr, end, walk); in walk_pte_range()
53 pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); in walk_pte_range()
54 err = walk_pte_range_inner(pte, addr, end, walk); in walk_pte_range()
62 struct mm_walk *walk) in walk_pmd_range() argument
66 const struct mm_walk_ops *ops = walk->ops; in walk_pmd_range()
76 err = ops->pte_hole(addr, next, depth, walk); in walk_pmd_range()
82 walk->action = ACTION_SUBTREE; in walk_pmd_range()
89 err = ops->pmd_entry(pmd, addr, next, walk); in walk_pmd_range()
93 if (walk->action == ACTION_AGAIN) in walk_pmd_range()
100 if ((!walk->vma && (pmd_leaf(*pmd) || !pmd_present(*pmd))) || in walk_pmd_range()
101 walk->action == ACTION_CONTINUE || in walk_pmd_range()
105 if (walk->vma) { in walk_pmd_range()
106 split_huge_pmd(walk->vma, pmd, addr); in walk_pmd_range()
111 err = walk_pte_range(pmd, addr, next, walk); in walk_pmd_range()
120 struct mm_walk *walk) in walk_pud_range() argument
124 const struct mm_walk_ops *ops = walk->ops; in walk_pud_range()
134 err = ops->pte_hole(addr, next, depth, walk); in walk_pud_range()
140 walk->action = ACTION_SUBTREE; in walk_pud_range()
143 err = ops->pud_entry(pud, addr, next, walk); in walk_pud_range()
147 if (walk->action == ACTION_AGAIN) in walk_pud_range()
150 if ((!walk->vma && (pud_leaf(*pud) || !pud_present(*pud))) || in walk_pud_range()
151 walk->action == ACTION_CONTINUE || in walk_pud_range()
155 if (walk->vma) in walk_pud_range()
156 split_huge_pud(walk->vma, pud, addr); in walk_pud_range()
160 err = walk_pmd_range(pud, addr, next, walk); in walk_pud_range()
169 struct mm_walk *walk) in walk_p4d_range() argument
173 const struct mm_walk_ops *ops = walk->ops; in walk_p4d_range()
182 err = ops->pte_hole(addr, next, depth, walk); in walk_p4d_range()
188 err = ops->p4d_entry(p4d, addr, next, walk); in walk_p4d_range()
193 err = walk_pud_range(p4d, addr, next, walk); in walk_p4d_range()
202 struct mm_walk *walk) in walk_pgd_range() argument
206 const struct mm_walk_ops *ops = walk->ops; in walk_pgd_range()
209 if (walk->pgd) in walk_pgd_range()
210 pgd = walk->pgd + pgd_index(addr); in walk_pgd_range()
212 pgd = pgd_offset(walk->mm, addr); in walk_pgd_range()
217 err = ops->pte_hole(addr, next, 0, walk); in walk_pgd_range()
223 err = ops->pgd_entry(pgd, addr, next, walk); in walk_pgd_range()
229 err = walk_p4d_range(pgd, addr, next, walk); in walk_pgd_range()
246 struct mm_walk *walk) in walk_hugetlb_range() argument
248 struct vm_area_struct *vma = walk->vma; in walk_hugetlb_range()
254 const struct mm_walk_ops *ops = walk->ops; in walk_hugetlb_range()
259 pte = huge_pte_offset(walk->mm, addr & hmask, sz); in walk_hugetlb_range()
262 err = ops->hugetlb_entry(pte, hmask, addr, next, walk); in walk_hugetlb_range()
264 err = ops->pte_hole(addr, next, -1, walk); in walk_hugetlb_range()
275 struct mm_walk *walk) in walk_hugetlb_range() argument
289 struct mm_walk *walk) in walk_page_test() argument
291 struct vm_area_struct *vma = walk->vma; in walk_page_test()
292 const struct mm_walk_ops *ops = walk->ops; in walk_page_test()
295 return ops->test_walk(start, end, walk); in walk_page_test()
308 err = ops->pte_hole(start, end, -1, walk); in walk_page_test()
315 struct mm_walk *walk) in __walk_page_range() argument
318 struct vm_area_struct *vma = walk->vma; in __walk_page_range()
319 const struct mm_walk_ops *ops = walk->ops; in __walk_page_range()
322 err = ops->pre_vma(start, end, walk); in __walk_page_range()
329 err = walk_hugetlb_range(start, end, walk); in __walk_page_range()
331 err = walk_pgd_range(start, end, walk); in __walk_page_range()
334 ops->post_vma(walk); in __walk_page_range()
386 struct mm_walk walk = { in walk_page_range() local
395 if (!walk.mm) in walk_page_range()
398 mmap_assert_locked(walk.mm); in walk_page_range()
400 vma = find_vma(walk.mm, start); in walk_page_range()
403 walk.vma = NULL; in walk_page_range()
406 err = ops->pte_hole(start, next, -1, &walk); in walk_page_range()
408 walk.vma = NULL; in walk_page_range()
411 err = ops->pte_hole(start, next, -1, &walk); in walk_page_range()
413 walk.vma = vma; in walk_page_range()
417 err = walk_page_test(start, next, &walk); in walk_page_range()
429 err = __walk_page_range(start, next, &walk); in walk_page_range()
449 struct mm_walk walk = { in walk_page_range_novma() local
457 if (start >= end || !walk.mm) in walk_page_range_novma()
460 mmap_assert_write_locked(walk.mm); in walk_page_range_novma()
462 return walk_pgd_range(start, end, &walk); in walk_page_range_novma()
468 struct mm_walk walk = { in walk_page_vma() local
476 if (!walk.mm) in walk_page_vma()
479 mmap_assert_locked(walk.mm); in walk_page_vma()
481 err = walk_page_test(vma->vm_start, vma->vm_end, &walk); in walk_page_vma()
486 return __walk_page_range(vma->vm_start, vma->vm_end, &walk); in walk_page_vma()
523 struct mm_walk walk = { in walk_page_mapping() local
548 walk.vma = vma; in walk_page_mapping()
549 walk.mm = vma->vm_mm; in walk_page_mapping()
551 err = walk_page_test(vma->vm_start, vma->vm_end, &walk); in walk_page_mapping()
558 err = __walk_page_range(start_addr, end_addr, &walk); in walk_page_mapping()