Lines Matching refs:vm_mm

538 	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);  in print_bad_pte()
841 struct mm_struct *src_mm = src_vma->vm_mm; in copy_present_page()
897 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); in copy_present_page()
910 struct mm_struct *src_mm = src_vma->vm_mm; in copy_present_pte()
949 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); in copy_present_pte()
977 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_pte_range()
978 struct mm_struct *src_mm = src_vma->vm_mm; in copy_pte_range()
1084 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_pmd_range()
1085 struct mm_struct *src_mm = src_vma->vm_mm; in copy_pmd_range()
1121 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_pud_range()
1122 struct mm_struct *src_mm = src_vma->vm_mm; in copy_pud_range()
1158 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_p4d_range()
1184 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_page_range()
1185 struct mm_struct *src_mm = src_vma->vm_mm; in copy_page_range()
1510 pgd = pgd_offset(vma->vm_mm, addr); in unmap_page_range()
1588 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm, in unmap_vmas()
1611 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, in zap_page_range()
1613 tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end); in zap_page_range()
1614 update_hiwater_rss(vma->vm_mm); in zap_page_range()
1638 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, in zap_page_range_single()
1640 tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end); in zap_page_range_single()
1641 update_hiwater_rss(vma->vm_mm); in zap_page_range_single()
1733 struct mm_struct *mm = vma->vm_mm; in insert_page()
1774 struct mm_struct *const mm = vma->vm_mm; in insert_pages()
1847 BUG_ON(mmap_read_trylock(vma->vm_mm)); in vm_insert_pages()
1905 BUG_ON(mmap_read_trylock(vma->vm_mm)); in vm_insert_page()
1997 struct mm_struct *mm = vma->vm_mm; in insert_pfn()
2352 struct mm_struct *mm = vma->vm_mm; in remap_pfn_range()
2636 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); in pte_spinlock()
2659 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); in pte_spinlock()
2718 ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); in __pte_map_lock_speculative()
2743 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in pte_map_lock()
2754 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in pte_map_lock_addr()
2818 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); in pte_spinlock()
2825 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in pte_map_lock()
2832 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in pte_map_lock_addr()
2897 struct mm_struct *mm = vma->vm_mm; in cow_user_page()
3133 struct mm_struct *mm = vma->vm_mm; in wp_page_copy()
3409 mm_tlb_flush_pending(vmf->vma->vm_mm))) in do_wp_page()
3633 migration_entry_wait(vma->vm_mm, vmf->pmd, in do_swap_page()
3671 err = mem_cgroup_charge(page, vma->vm_mm, in do_swap_page()
3724 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT); in do_swap_page()
3735 locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags); in do_swap_page()
3788 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); in do_swap_page()
3789 dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS); in do_swap_page()
3804 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); in do_swap_page()
3805 arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte); in do_swap_page()
3891 if (pte_alloc(vma->vm_mm, vmf->pmd)) in do_anonymous_page()
3901 !mm_forbids_zeropage(vma->vm_mm)) { in do_anonymous_page()
3910 ret = check_stable_address_space(vma->vm_mm); in do_anonymous_page()
3936 if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL)) in do_anonymous_page()
3962 ret = check_stable_address_space(vma->vm_mm); in do_anonymous_page()
3974 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); in do_anonymous_page()
3978 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); in do_anonymous_page()
4026 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); in __do_fault()
4068 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); in deposit_prealloc_pte()
4073 mm_inc_nr_ptes(vma->vm_mm); in deposit_prealloc_pte()
4098 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); in do_set_pmd()
4104 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_set_pmd()
4115 add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR); in do_set_pmd()
4123 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in do_set_pmd()
4160 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); in do_set_pte()
4164 inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page)); in do_set_pte()
4167 set_pte_at(vma->vm_mm, addr, vmf->pte, entry); in do_set_pte()
4203 ret = check_stable_address_space(vma->vm_mm); in finish_fault()
4220 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in finish_fault()
4222 mm_inc_nr_ptes(vma->vm_mm); in finish_fault()
4223 pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); in finish_fault()
4227 } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) { in finish_fault()
4341 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); in do_fault_around()
4391 if (mem_cgroup_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL)) { in do_cow_fault()
4463 struct mm_struct *vm_mm = vma->vm_mm; in do_fault() local
4477 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, in do_fault()
4504 pte_free(vm_mm, vmf->prealloc_pte); in do_fault()
4781 if (!mmu_notifier_trylock(vmf->vma->vm_mm)) { in handle_pte_fault()
4787 mmu_notifier_unlock(vmf->vma->vm_mm); in handle_pte_fault()
4837 struct mm_struct *mm = vma->vm_mm; in __handle_mm_fault()
5260 count_memcg_event_mm(vma->vm_mm, PGFAULT); in handle_mm_fault()
5278 ret = hugetlb_fault(vma->vm_mm, vma, address, flags); in handle_mm_fault()
5490 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); in follow_pfn()
5511 if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) in follow_phys()