Lines Matching refs:gup_flags
840 unsigned int gup_flags, struct vm_area_struct **vma, in get_gate_page() argument
851 if (gup_flags & FOLL_WRITE) in get_gate_page()
877 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte))) in get_gate_page()
881 if (unlikely(!try_grab_page(*page, gup_flags))) { in get_gate_page()
951 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) in check_vma_flags() argument
954 int write = (gup_flags & FOLL_WRITE); in check_vma_flags()
955 int foreign = (gup_flags & FOLL_REMOTE); in check_vma_flags()
960 if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma)) in check_vma_flags()
963 if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma)) in check_vma_flags()
968 if (!(gup_flags & FOLL_FORCE)) in check_vma_flags()
983 if (!(gup_flags & FOLL_FORCE)) in check_vma_flags()
1063 unsigned int gup_flags, struct page **pages, in __get_user_pages() argument
1075 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); in __get_user_pages()
1082 if (!(gup_flags & FOLL_FORCE)) in __get_user_pages()
1083 gup_flags |= FOLL_NUMA; in __get_user_pages()
1087 unsigned int foll_flags = gup_flags; in __get_user_pages()
1095 gup_flags, &vma, in __get_user_pages()
1107 ret = check_vma_flags(vma, gup_flags); in __get_user_pages()
1114 gup_flags, locked); in __get_user_pages()
1121 BUG_ON(gup_flags & FOLL_NOWAIT); in __get_user_pages()
1444 int gup_flags; in populate_vma_page_range() local
1452 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK; in populate_vma_page_range()
1454 gup_flags &= ~FOLL_POPULATE; in populate_vma_page_range()
1461 gup_flags |= FOLL_WRITE; in populate_vma_page_range()
1468 gup_flags |= FOLL_FORCE; in populate_vma_page_range()
1474 return __get_user_pages(mm, start, nr_pages, gup_flags, in populate_vma_page_range()
1620 unsigned int gup_flags) in check_and_migrate_cma_pages() argument
1680 if (gup_flags & FOLL_PIN) in check_and_migrate_cma_pages()
1697 NULL, gup_flags); in check_and_migrate_cma_pages()
1715 unsigned int gup_flags) in check_and_migrate_cma_pages() argument
1730 unsigned int gup_flags) in __gup_longterm_locked() argument
1735 if (gup_flags & FOLL_LONGTERM) in __gup_longterm_locked()
1739 gup_flags); in __gup_longterm_locked()
1741 if (gup_flags & FOLL_LONGTERM) { in __gup_longterm_locked()
1744 vmas, gup_flags); in __gup_longterm_locked()
1750 static bool is_valid_gup_flags(unsigned int gup_flags) in is_valid_gup_flags() argument
1756 if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) in is_valid_gup_flags()
1763 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) in is_valid_gup_flags()
1772 unsigned int gup_flags, struct page **pages, in __get_user_pages_remote() argument
1782 if (gup_flags & FOLL_LONGTERM) { in __get_user_pages_remote()
1790 vmas, gup_flags | FOLL_TOUCH | in __get_user_pages_remote()
1796 gup_flags | FOLL_TOUCH | FOLL_REMOTE); in __get_user_pages_remote()
1861 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
1864 if (!is_valid_gup_flags(gup_flags)) in get_user_pages_remote()
1867 return __get_user_pages_remote(mm, start, nr_pages, gup_flags, in get_user_pages_remote()
1875 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
1883 unsigned int gup_flags, struct page **pages, in __get_user_pages_remote() argument
1907 unsigned int gup_flags, struct page **pages, in get_user_pages() argument
1910 if (!is_valid_gup_flags(gup_flags)) in get_user_pages()
1914 pages, vmas, gup_flags | FOLL_TOUCH); in get_user_pages()
1951 unsigned int gup_flags, struct page **pages, in get_user_pages_locked() argument
1960 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) in get_user_pages_locked()
1966 if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) in get_user_pages_locked()
1971 gup_flags | FOLL_TOUCH); in get_user_pages_locked()
1991 struct page **pages, unsigned int gup_flags) in get_user_pages_unlocked() argument
2003 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) in get_user_pages_unlocked()
2008 &locked, gup_flags | FOLL_TOUCH); in get_user_pages_unlocked()
2641 unsigned int gup_flags, struct page **pages) in __gup_longterm_unlocked() argument
2649 if (gup_flags & FOLL_LONGTERM) { in __gup_longterm_unlocked()
2653 pages, NULL, gup_flags); in __gup_longterm_unlocked()
2657 pages, gup_flags); in __gup_longterm_unlocked()
2665 unsigned int gup_flags, in lockless_pages_from_mm() argument
2676 if (gup_flags & FOLL_PIN) { in lockless_pages_from_mm()
2694 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned); in lockless_pages_from_mm()
2701 if (gup_flags & FOLL_PIN) { in lockless_pages_from_mm()
2712 unsigned int gup_flags, in internal_get_user_pages_fast() argument
2719 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | in internal_get_user_pages_fast()
2724 if (gup_flags & FOLL_PIN) in internal_get_user_pages_fast()
2727 if (!(gup_flags & FOLL_FAST_ONLY)) in internal_get_user_pages_fast()
2737 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages); in internal_get_user_pages_fast()
2738 if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY) in internal_get_user_pages_fast()
2744 ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags, in internal_get_user_pages_fast()
2779 unsigned int gup_flags, struct page **pages) in get_user_pages_fast_only() argument
2789 gup_flags |= FOLL_GET | FOLL_FAST_ONLY; in get_user_pages_fast_only()
2791 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags, in get_user_pages_fast_only()
2824 unsigned int gup_flags, struct page **pages) in get_user_pages_fast() argument
2826 if (!is_valid_gup_flags(gup_flags)) in get_user_pages_fast()
2835 gup_flags |= FOLL_GET; in get_user_pages_fast()
2836 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in get_user_pages_fast()
2857 unsigned int gup_flags, struct page **pages) in pin_user_pages_fast() argument
2860 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_fast()
2863 gup_flags |= FOLL_PIN; in pin_user_pages_fast()
2864 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in pin_user_pages_fast()
2875 unsigned int gup_flags, struct page **pages) in pin_user_pages_fast_only() argument
2883 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_fast_only()
2889 gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY); in pin_user_pages_fast_only()
2890 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags, in pin_user_pages_fast_only()
2929 unsigned int gup_flags, struct page **pages, in pin_user_pages_remote() argument
2933 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_remote()
2936 gup_flags |= FOLL_PIN; in pin_user_pages_remote()
2937 return __get_user_pages_remote(mm, start, nr_pages, gup_flags, in pin_user_pages_remote()
2961 unsigned int gup_flags, struct page **pages, in pin_user_pages() argument
2965 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages()
2968 gup_flags |= FOLL_PIN; in pin_user_pages()
2970 pages, vmas, gup_flags); in pin_user_pages()
2980 struct page **pages, unsigned int gup_flags) in pin_user_pages_unlocked() argument
2983 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_unlocked()
2986 gup_flags |= FOLL_PIN; in pin_user_pages_unlocked()
2987 return get_user_pages_unlocked(start, nr_pages, pages, gup_flags); in pin_user_pages_unlocked()
2997 unsigned int gup_flags, struct page **pages, in pin_user_pages_locked() argument
3006 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) in pin_user_pages_locked()
3010 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_locked()
3013 gup_flags |= FOLL_PIN; in pin_user_pages_locked()
3016 gup_flags | FOLL_TOUCH); in pin_user_pages_locked()