Lines Matching full:page
27 #include <linux/page-flags.h>
103 #include <asm/page.h>
139 * a zero page mapping on a read fault.
142 * related to the physical page in case of virtualization.
155 /* This function must be updated when the size of struct page grows above 80
162 static inline void __mm_zero_struct_page(struct page *page) in __mm_zero_struct_page() argument
164 unsigned long *_pp = (void *)page; in __mm_zero_struct_page()
166 /* Check that struct page is either 56, 64, 72, or 80 bytes */ in __mm_zero_struct_page()
167 BUILD_BUG_ON(sizeof(struct page) & 7); in __mm_zero_struct_page()
168 BUILD_BUG_ON(sizeof(struct page) < 56); in __mm_zero_struct_page()
169 BUILD_BUG_ON(sizeof(struct page) > 80); in __mm_zero_struct_page()
171 switch (sizeof(struct page)) { in __mm_zero_struct_page()
192 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))
230 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) argument
232 /* to align the pointer to the (next) page boundary */
238 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
279 #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
295 #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
296 #define VM_SYNC 0x00800000 /* Synchronous page faults */
307 #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
350 # define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */
430 * low four bits) to a page protection mask..
449 * whether we would allow page faults to retry by specifying these two
452 * (a) ALLOW_RETRY and !TRIED: this means the page fault allows retry, and
455 * (b) ALLOW_RETRY and TRIED: this means the page fault allows retry, and
458 * (c) !ALLOW_RETRY and !TRIED: this means the page fault does not allow retry
461 * be used. Note that page faults can be allowed to retry for multiple times,
464 * signals before a retry to make sure the continuous page faults can still be
482 * arch-specific page fault handlers.
497 * Return: true if the page fault allows retry and this is the first
523 * MM layer fills up gfp_mask for page allocations but fault handler might
536 pgoff_t pgoff; /* Logical page offset based on vma */
548 struct page *cow_page; /* Page handler may use for COW fault */
549 struct page *page; /* ->fault handlers should return a member
550 * page here, unless VM_FAULT_NOPAGE
556 * the 'address'. NULL if the page
559 spinlock_t *ptl; /* Page table lock.
560 * Protects pte page table if 'pte'
563 pgtable_t prealloc_pte; /* Pre-allocated pte page table.
564 * vm_ops->map_pages() sets up a page
567 * page table to avoid allocation from
571 * These entries are required when handling speculative page fault.
572 * This way the page handling is done using consistent field values.
579 /* page entry size for vm->huge_fault() */
589 * to the functions called when a no-page or a wp-page exception occurs.
603 /* notification that a previously read-only page is about to become
646 * page for @addr. This is useful if the default behavior
647 * (using pte_page()) would not find the correct page.
649 struct page *(*find_special_page)(struct vm_area_struct *vma,
742 * Methods to modify the page usage count.
744 * What counts for a page usage:
745 * - cache mapping (page->mapping)
746 * - private data (page->private)
747 * - page mapped in a task's page tables, each mapping
750 * Also, many kernel routines increase the page count before a critical
751 * routine so they can be sure the page doesn't go away from under them.
755 * Drop a ref, return true if the refcount fell to zero (the page has no users)
757 static inline int put_page_testzero(struct page *page) in put_page_testzero() argument
761 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); in put_page_testzero()
762 ret = page_ref_dec_and_test(page); in put_page_testzero()
763 page_pinner_put_page(page); in put_page_testzero()
769 * Try to grab a ref unless the page has a refcount of zero, return false if
774 static inline int get_page_unless_zero(struct page *page) in get_page_unless_zero() argument
776 return page_ref_add_unless(page, 1, 0); in get_page_unless_zero()
791 struct page *vmalloc_to_page(const void *addr);
853 static inline int head_compound_mapcount(struct page *head) in head_compound_mapcount()
859 * Mapcount of compound page as a whole, does not include mapped sub-pages.
863 static inline int compound_mapcount(struct page *page) in compound_mapcount() argument
865 VM_BUG_ON_PAGE(!PageCompound(page), page); in compound_mapcount()
866 page = compound_head(page); in compound_mapcount()
867 return head_compound_mapcount(page); in compound_mapcount()
871 * The atomic page->_mapcount, starts from -1: so that transitions
875 static inline void page_mapcount_reset(struct page *page) in page_mapcount_reset() argument
877 atomic_set(&(page)->_mapcount, -1); in page_mapcount_reset()
880 int __page_mapcount(struct page *page);
883 * Mapcount of 0-order page; when compound sub-page, includes
888 * They use this place in struct page differently.
890 static inline int page_mapcount(struct page *page) in page_mapcount() argument
892 if (unlikely(PageCompound(page))) in page_mapcount()
893 return __page_mapcount(page); in page_mapcount()
894 return atomic_read(&page->_mapcount) + 1; in page_mapcount()
898 int total_mapcount(struct page *page);
899 int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
901 static inline int total_mapcount(struct page *page) in total_mapcount() argument
903 return page_mapcount(page); in total_mapcount()
905 static inline int page_trans_huge_mapcount(struct page *page, in page_trans_huge_mapcount() argument
908 int mapcount = page_mapcount(page); in page_trans_huge_mapcount()
915 static inline struct page *virt_to_head_page(const void *x) in virt_to_head_page()
917 struct page *page = virt_to_page(x); in virt_to_head_page() local
919 return compound_head(page); in virt_to_head_page()
922 void __put_page(struct page *page);
926 void split_page(struct page *page, unsigned int order);
931 * These are _only_ valid on the head of a compound page.
933 typedef void compound_page_dtor(struct page *);
949 static inline void set_compound_page_dtor(struct page *page, in set_compound_page_dtor() argument
952 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page); in set_compound_page_dtor()
953 page[1].compound_dtor = compound_dtor; in set_compound_page_dtor()
956 static inline void destroy_compound_page(struct page *page) in destroy_compound_page() argument
958 VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page); in destroy_compound_page()
959 compound_page_dtors[page[1].compound_dtor](page); in destroy_compound_page()
962 static inline unsigned int compound_order(struct page *page) in compound_order() argument
964 if (!PageHead(page)) in compound_order()
966 return page[1].compound_order; in compound_order()
969 static inline bool hpage_pincount_available(struct page *page) in hpage_pincount_available() argument
972 * Can the page->hpage_pinned_refcount field be used? That field is in in hpage_pincount_available()
973 * the 3rd page of the compound page, so the smallest (2-page) compound in hpage_pincount_available()
976 page = compound_head(page); in hpage_pincount_available()
977 return PageCompound(page) && compound_order(page) > 1; in hpage_pincount_available()
980 static inline int head_compound_pincount(struct page *head) in head_compound_pincount()
985 static inline int compound_pincount(struct page *page) in compound_pincount() argument
987 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page); in compound_pincount()
988 page = compound_head(page); in compound_pincount()
989 return head_compound_pincount(page); in compound_pincount()
992 static inline void set_compound_order(struct page *page, unsigned int order) in set_compound_order() argument
994 page[1].compound_order = order; in set_compound_order()
995 page[1].compound_nr = 1U << order; in set_compound_order()
998 /* Returns the number of pages in this potentially compound page. */
999 static inline unsigned long compound_nr(struct page *page) in compound_nr() argument
1001 if (!PageHead(page)) in compound_nr()
1003 return page[1].compound_nr; in compound_nr()
1006 /* Returns the number of bytes in this potentially compound page. */
1007 static inline unsigned long page_size(struct page *page) in page_size() argument
1009 return PAGE_SIZE << compound_order(page); in page_size()
1012 /* Returns the number of bits needed for the number of bytes in a page */
1013 static inline unsigned int page_shift(struct page *page) in page_shift() argument
1015 return PAGE_SHIFT + compound_order(page); in page_shift()
1018 void free_compound_page(struct page *page);
1034 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page);
1035 void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr);
1042 * Multiple processes may "see" the same page. E.g. for untouched
1043 * mappings of /dev/null, all processes see the same page full of
1047 * For the non-reserved pages, page_count(page) denotes a reference count.
1048 * page_count() == 0 means the page is free. page->lru is then used for
1050 * page_count() > 0 means the page has been allocated.
1054 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
1058 * A page may be used by anyone else who does a __get_free_page().
1060 * be used through the normal accessor functions. The top bits of page->flags
1061 * and page->virtual store page management information, but all other fields
1063 * page is the responsibility of the one who allocated it, and those who have
1070 * A pagecache page contains an opaque `private' member, which belongs to the
1071 * page's address_space. Usually, this is the address of a circular list of
1072 * the page's disk buffers. PG_private must be set to tell the VM to call
1075 * A page may belong to an inode's memory mapping. In this case, page->mapping
1076 * is the pointer to the inode, and page->index is the file offset of the page,
1081 * case PG_swapcache is set, and page->private is an offset into the swapcache.
1084 * reference to the page. Setting PG_private should also increment the
1085 * refcount. The each user mapping also has a reference to the page.
1106 /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
1144 static inline enum zone_type page_zonenum(const struct page *page) in page_zonenum() argument
1146 ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT); in page_zonenum()
1147 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; in page_zonenum()
1151 static inline bool is_zone_device_page(const struct page *page) in is_zone_device_page() argument
1153 return page_zonenum(page) == ZONE_DEVICE; in is_zone_device_page()
1158 static inline bool is_zone_device_page(const struct page *page) in is_zone_device_page() argument
1165 void free_devmap_managed_page(struct page *page);
1168 static inline bool page_is_devmap_managed(struct page *page) in page_is_devmap_managed() argument
1172 if (!is_zone_device_page(page)) in page_is_devmap_managed()
1174 switch (page->pgmap->type) { in page_is_devmap_managed()
1184 void put_devmap_managed_page(struct page *page);
1187 static inline bool page_is_devmap_managed(struct page *page) in page_is_devmap_managed() argument
1192 static inline void put_devmap_managed_page(struct page *page) in put_devmap_managed_page() argument
1197 static inline bool is_device_private_page(const struct page *page) in is_device_private_page() argument
1201 is_zone_device_page(page) && in is_device_private_page()
1202 page->pgmap->type == MEMORY_DEVICE_PRIVATE; in is_device_private_page()
1205 static inline bool is_pci_p2pdma_page(const struct page *page) in is_pci_p2pdma_page() argument
1209 is_zone_device_page(page) && in is_pci_p2pdma_page()
1210 page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA; in is_pci_p2pdma_page()
1214 #define page_ref_zero_or_close_to_overflow(page) \ argument
1215 ((unsigned int) page_ref_count(page) + 127u <= 127u)
1217 static inline void get_page(struct page *page) in get_page() argument
1219 page = compound_head(page); in get_page()
1221 * Getting a normal page or the head of a compound page in get_page()
1222 * requires to already have an elevated page->_refcount. in get_page()
1224 VM_BUG_ON_PAGE(page_ref_zero_or_close_to_overflow(page), page); in get_page()
1225 page_ref_inc(page); in get_page()
1228 bool __must_check try_grab_page(struct page *page, unsigned int flags);
1230 static inline __must_check bool try_get_page(struct page *page) in try_get_page() argument
1232 page = compound_head(page); in try_get_page()
1233 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) in try_get_page()
1235 page_ref_inc(page); in try_get_page()
1239 static inline void put_page(struct page *page) in put_page() argument
1241 page = compound_head(page); in put_page()
1245 * 2 to 1, when refcount reach one it means the page is free and we in put_page()
1249 if (page_is_devmap_managed(page)) { in put_page()
1250 put_devmap_managed_page(page); in put_page()
1254 if (put_page_testzero(page)) in put_page()
1255 __put_page(page); in put_page()
1260 * the page's refcount so that two separate items are tracked: the original page
1262 * made against the page. ("gup-pinned" is another term for the latter).
1270 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference
1272 * simpler, due to the fact that adding an even power of two to the page
1281 * applications that don't have huge page reference counts, this won't be an
1286 * get_user_pages and page_mkclean and other calls that race to set up page
1291 void put_user_page(struct page *page);
1292 void unpin_user_page(struct page *page);
1293 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1295 void unpin_user_pages(struct page **pages, unsigned long npages);
1298 * page_maybe_dma_pinned() - report if a page is pinned for DMA.
1300 * This function checks if a page has been pinned via a call to
1306 * GUP_PIN_COUNTING_BIAS worth of normal page references".
1308 * False positives are OK, because: a) it's unlikely for a page to get that many
1313 * more tracking data available: the 3rd struct page in the compound page is
1319 * @page: pointer to page to be queried.
1320 * @Return: True, if it is likely that the page has been "dma-pinned".
1321 * False, if the page is definitely not dma-pinned.
1323 static inline bool page_maybe_dma_pinned(struct page *page) in page_maybe_dma_pinned() argument
1325 if (hpage_pincount_available(page)) in page_maybe_dma_pinned()
1326 return compound_pincount(page) > 0; in page_maybe_dma_pinned()
1336 return ((unsigned int)page_ref_count(compound_head(page))) >= in page_maybe_dma_pinned()
1348 * node id available in page flags.
1352 static inline int page_zone_id(struct page *page) in page_zone_id() argument
1354 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK; in page_zone_id()
1358 extern int page_to_nid(const struct page *page);
1360 static inline int page_to_nid(const struct page *page) in page_to_nid() argument
1362 struct page *p = (struct page *)page; in page_to_nid()
1406 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) in page_cpupid_xchg_last() argument
1408 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK); in page_cpupid_xchg_last()
1411 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1413 return page->_last_cpupid; in page_cpupid_last()
1415 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1417 page->_last_cpupid = -1 & LAST_CPUPID_MASK; in page_cpupid_reset_last()
1420 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1422 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; in page_cpupid_last()
1425 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
1427 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1429 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; in page_cpupid_reset_last()
1433 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) in page_cpupid_xchg_last() argument
1435 return page_to_nid(page); /* XXX */ in page_cpupid_xchg_last()
1438 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1440 return page_to_nid(page); /* XXX */ in page_cpupid_last()
1468 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1481 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid
1486 static inline u8 page_kasan_tag(const struct page *page) in page_kasan_tag() argument
1491 tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; in page_kasan_tag()
1498 static inline void page_kasan_tag_set(struct page *page, u8 tag) in page_kasan_tag_set() argument
1502 page->flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); in page_kasan_tag_set()
1503 page->flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; in page_kasan_tag_set()
1507 static inline void page_kasan_tag_reset(struct page *page) in page_kasan_tag_reset() argument
1510 page_kasan_tag_set(page, 0xff); in page_kasan_tag_reset()
1515 static inline u8 page_kasan_tag(const struct page *page) in page_kasan_tag() argument
1520 static inline void page_kasan_tag_set(struct page *page, u8 tag) { } in page_kasan_tag_set() argument
1521 static inline void page_kasan_tag_reset(struct page *page) { } in page_kasan_tag_reset() argument
1525 static inline struct zone *page_zone(const struct page *page) in page_zone() argument
1527 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; in page_zone()
1530 static inline pg_data_t *page_pgdat(const struct page *page) in page_pgdat() argument
1532 return NODE_DATA(page_to_nid(page)); in page_pgdat()
1536 static inline void set_page_section(struct page *page, unsigned long section) in set_page_section() argument
1538 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); in set_page_section()
1539 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; in set_page_section()
1542 static inline unsigned long page_to_section(const struct page *page) in page_to_section() argument
1544 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; in page_to_section()
1548 static inline void set_page_zone(struct page *page, enum zone_type zone) in set_page_zone() argument
1550 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); in set_page_zone()
1551 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; in set_page_zone()
1554 static inline void set_page_node(struct page *page, unsigned long node) in set_page_node() argument
1556 page->flags &= ~(NODES_MASK << NODES_PGSHIFT); in set_page_node()
1557 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; in set_page_node()
1560 static inline void set_page_links(struct page *page, enum zone_type zone, in set_page_links() argument
1563 set_page_zone(page, zone); in set_page_links()
1564 set_page_node(page, node); in set_page_links()
1566 set_page_section(page, pfn_to_section_nr(pfn)); in set_page_links()
1571 static inline struct mem_cgroup *page_memcg(struct page *page) in page_memcg() argument
1573 return page->mem_cgroup; in page_memcg()
1575 static inline struct mem_cgroup *page_memcg_rcu(struct page *page) in page_memcg_rcu() argument
1578 return READ_ONCE(page->mem_cgroup); in page_memcg_rcu()
1581 static inline struct mem_cgroup *page_memcg(struct page *page) in page_memcg() argument
1585 static inline struct mem_cgroup *page_memcg_rcu(struct page *page) in page_memcg_rcu() argument
1597 static __always_inline void *lowmem_page_address(const struct page *page) in lowmem_page_address() argument
1599 return page_to_virt(page); in lowmem_page_address()
1607 static inline void *page_address(const struct page *page) in page_address() argument
1609 return page->virtual; in page_address()
1611 static inline void set_page_address(struct page *page, void *address) in set_page_address() argument
1613 page->virtual = address; in set_page_address()
1619 void *page_address(const struct page *page);
1620 void set_page_address(struct page *page, void *virtual);
1625 #define page_address(page) lowmem_page_address(page) argument
1626 #define set_page_address(page, address) do { } while(0) argument
1630 extern void *page_rmapping(struct page *page);
1631 extern struct anon_vma *page_anon_vma(struct page *page);
1632 extern struct address_space *page_mapping(struct page *page);
1634 extern struct address_space *__page_file_mapping(struct page *);
1637 struct address_space *page_file_mapping(struct page *page) in page_file_mapping() argument
1639 if (unlikely(PageSwapCache(page))) in page_file_mapping()
1640 return __page_file_mapping(page); in page_file_mapping()
1642 return page->mapping; in page_file_mapping()
1645 extern pgoff_t __page_file_index(struct page *page);
1648 * Return the pagecache index of the passed page. Regular pagecache pages
1651 static inline pgoff_t page_index(struct page *page) in page_index() argument
1653 if (unlikely(PageSwapCache(page))) in page_index()
1654 return __page_file_index(page); in page_index()
1655 return page->index; in page_index()
1658 bool page_mapped(struct page *page);
1659 struct address_space *page_mapping(struct page *page);
1660 struct address_space *page_mapping_file(struct page *page);
1663 * Return true only if the page has been allocated with
1667 static inline bool page_is_pfmemalloc(struct page *page) in page_is_pfmemalloc() argument
1670 * Page index cannot be this large so this must be in page_is_pfmemalloc()
1671 * a pfmemalloc page. in page_is_pfmemalloc()
1673 return page->index == -1UL; in page_is_pfmemalloc()
1677 * Only to be called by the page allocator on a freshly allocated
1678 * page.
1680 static inline void set_page_pfmemalloc(struct page *page) in set_page_pfmemalloc() argument
1682 page->index = -1UL; in set_page_pfmemalloc()
1685 static inline void clear_page_pfmemalloc(struct page *page) in clear_page_pfmemalloc() argument
1687 page->index = 0; in clear_page_pfmemalloc()
1696 #define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1)) argument
1718 struct address_space *check_mapping; /* Check page->mapping if set */
1719 pgoff_t first_index; /* Lowest page->index to unmap */
1720 pgoff_t last_index; /* Highest page->index to unmap */
1721 struct page *single_page; /* Locked page to be unmapped */
1724 struct page *_vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1726 static inline struct page *vm_normal_page(struct vm_area_struct *vma, in vm_normal_page()
1732 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1765 * speculative page fault handler still needs to know it was changed. in vm_write_begin()
1792 int truncate_inode_page(struct address_space *mapping, struct page *page);
1793 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1794 int invalidate_inode_page(struct page *page);
1817 * Try speculative page fault for multithreaded user space task only. in handle_speculative_fault()
1843 void unmap_mapping_page(struct page *page);
1864 static inline void unmap_mapping_page(struct page *page) { } in unmap_mapping_page() argument
1886 unsigned int gup_flags, struct page **pages,
1890 unsigned int gup_flags, struct page **pages,
1893 unsigned int gup_flags, struct page **pages,
1896 unsigned int gup_flags, struct page **pages,
1899 unsigned int gup_flags, struct page **pages, int *locked);
1901 unsigned int gup_flags, struct page **pages, int *locked);
1903 struct page **pages, unsigned int gup_flags);
1905 struct page **pages, unsigned int gup_flags);
1908 unsigned int gup_flags, struct page **pages);
1910 unsigned int gup_flags, struct page **pages);
1920 bool got_ref; /* Did we pin pages by getting page ref? */
1940 static inline struct page **frame_vector_pages(struct frame_vector *vec) in frame_vector_pages()
1948 return (struct page **)(vec->ptrs); in frame_vector_pages()
1960 struct page **pages);
1961 int get_kernel_page(unsigned long start, int write, struct page **pages);
1962 struct page *get_dump_page(unsigned long addr);
1964 extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1965 extern void do_invalidatepage(struct page *page, unsigned int offset,
1968 void __set_page_dirty(struct page *, struct address_space *, int warn);
1969 int __set_page_dirty_nobuffers(struct page *page);
1970 int __set_page_dirty_no_writeback(struct page *page);
1972 struct page *page);
1973 void account_page_dirtied(struct page *page, struct address_space *mapping);
1974 void account_page_cleaned(struct page *page, struct address_space *mapping,
1976 int set_page_dirty(struct page *page);
1977 int set_page_dirty_lock(struct page *page);
1978 void __cancel_dirty_page(struct page *page);
1979 static inline void cancel_dirty_page(struct page *page) in cancel_dirty_page() argument
1982 if (PageDirty(page)) in cancel_dirty_page()
1983 __cancel_dirty_page(page); in cancel_dirty_page()
1985 int clear_page_dirty_for_io(struct page *page);
2021 unsigned int gup_flags, struct page **pages);
2023 unsigned int gup_flags, struct page **pages);
2026 unsigned int gup_flags, struct page **pagep) in get_user_page_fast_only()
2072 /* Optimized variant when page is already known not to be PageAnon */
2073 static inline int mm_counter_file(struct page *page) in mm_counter_file() argument
2075 if (PageSwapBacked(page)) in mm_counter_file()
2080 static inline int mm_counter(struct page *page) in mm_counter() argument
2082 if (PageAnon(page)) in mm_counter()
2084 return mm_counter_file(page); in mm_counter()
2297 extern bool ptlock_alloc(struct page *page);
2298 extern void ptlock_free(struct page *page);
2300 static inline spinlock_t *ptlock_ptr(struct page *page) in ptlock_ptr() argument
2302 return page->ptl; in ptlock_ptr()
2309 static inline bool ptlock_alloc(struct page *page) in ptlock_alloc() argument
2314 static inline void ptlock_free(struct page *page) in ptlock_free() argument
2318 static inline spinlock_t *ptlock_ptr(struct page *page) in ptlock_ptr() argument
2320 return &page->ptl; in ptlock_ptr()
2329 static inline bool ptlock_init(struct page *page) in ptlock_init() argument
2332 * prep_new_page() initialize page->private (and therefore page->ptl) in ptlock_init()
2335 * It can happen if arch try to use slab for page table allocation: in ptlock_init()
2336 * slab code uses page->slab_cache, which share storage with page->ptl. in ptlock_init()
2338 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page); in ptlock_init()
2339 if (!ptlock_alloc(page)) in ptlock_init()
2341 spin_lock_init(ptlock_ptr(page)); in ptlock_init()
2354 static inline bool ptlock_init(struct page *page) { return true; } in ptlock_init() argument
2355 static inline void ptlock_free(struct page *page) {} in ptlock_free() argument
2364 static inline bool pgtable_pte_page_ctor(struct page *page) in pgtable_pte_page_ctor() argument
2366 if (!ptlock_init(page)) in pgtable_pte_page_ctor()
2368 __SetPageTable(page); in pgtable_pte_page_ctor()
2369 inc_zone_page_state(page, NR_PAGETABLE); in pgtable_pte_page_ctor()
2373 static inline void pgtable_pte_page_dtor(struct page *page) in pgtable_pte_page_dtor() argument
2375 ptlock_free(page); in pgtable_pte_page_dtor()
2376 __ClearPageTable(page); in pgtable_pte_page_dtor()
2377 dec_zone_page_state(page, NR_PAGETABLE); in pgtable_pte_page_dtor()
2409 static struct page *pmd_to_page(pmd_t *pmd) in pmd_to_page()
2420 static inline bool pmd_ptlock_init(struct page *page) in pmd_ptlock_init() argument
2423 page->pmd_huge_pte = NULL; in pmd_ptlock_init()
2425 return ptlock_init(page); in pmd_ptlock_init()
2428 static inline void pmd_ptlock_free(struct page *page) in pmd_ptlock_free() argument
2431 VM_BUG_ON_PAGE(page->pmd_huge_pte, page); in pmd_ptlock_free()
2433 ptlock_free(page); in pmd_ptlock_free()
2445 static inline bool pmd_ptlock_init(struct page *page) { return true; } in pmd_ptlock_init() argument
2446 static inline void pmd_ptlock_free(struct page *page) {} in pmd_ptlock_free() argument
2459 static inline bool pgtable_pmd_page_ctor(struct page *page) in pgtable_pmd_page_ctor() argument
2461 if (!pmd_ptlock_init(page)) in pgtable_pmd_page_ctor()
2463 __SetPageTable(page); in pgtable_pmd_page_ctor()
2464 inc_zone_page_state(page, NR_PAGETABLE); in pgtable_pmd_page_ctor()
2468 static inline void pgtable_pmd_page_dtor(struct page *page) in pgtable_pmd_page_dtor() argument
2470 pmd_ptlock_free(page); in pgtable_pmd_page_dtor()
2471 __ClearPageTable(page); in pgtable_pmd_page_dtor()
2472 dec_zone_page_state(page, NR_PAGETABLE); in pgtable_pmd_page_dtor()
2509 * Free a highmem page into the buddy system, adjusting totalhigh_pages
2512 extern void free_highmem_page(struct page *page);
2515 extern void adjust_managed_page_count(struct page *page, long count);
2520 /* Free the reserved page into the buddy system, so it gets managed. */
2521 static inline void __free_reserved_page(struct page *page) in __free_reserved_page() argument
2523 ClearPageReserved(page); in __free_reserved_page()
2524 init_page_count(page); in __free_reserved_page()
2525 __free_page(page); in __free_reserved_page()
2528 static inline void free_reserved_page(struct page *page) in free_reserved_page() argument
2530 __free_reserved_page(page); in free_reserved_page()
2531 adjust_managed_page_count(page, 1); in free_reserved_page()
2534 static inline void mark_page_reserved(struct page *page) in mark_page_reserved() argument
2536 SetPageReserved(page); in mark_page_reserved()
2537 adjust_managed_page_count(page, -1); in mark_page_reserved()
2570 * An architecture is expected to register range of page frames backed by
2743 unsigned long flags, struct page **pages);
2809 /* mm/page-writeback.c */
2810 int __must_check write_one_page(struct page *page);
2911 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2913 struct page **pages, unsigned long *num);
2914 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2916 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2931 unsigned long addr, struct page *page) in vmf_insert_page() argument
2933 int err = vm_insert_page(vma, addr, page); in vmf_insert_page()
2959 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
2963 #define FOLL_TOUCH 0x02 /* mark page accessed */
2964 #define FOLL_GET 0x04 /* do get_page on page */
2969 #define FOLL_POPULATE 0x40 /* fault in page */
2971 #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2972 #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2973 #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2988 * FOLL_LONGTERM indicates that the page will be held for an indefinite time
2996 * pin, delaying writeback, bounce buffer page writeback, etc. As FS DAX was
3008 * that region. And so, CMA attempts to migrate the page before pinning, when
3011 * FOLL_PIN indicates that a special kind of tracking (not just page->_refcount,
3013 * anything that gets a page reference and then touches page data (for example,
3060 extern void __kernel_poison_pages(struct page *page, int numpages);
3061 extern void __kernel_unpoison_pages(struct page *page, int numpages);
3076 static inline void kernel_poison_pages(struct page *page, int numpages) in kernel_poison_pages() argument
3079 __kernel_poison_pages(page, numpages); in kernel_poison_pages()
3081 static inline void kernel_unpoison_pages(struct page *page, int numpages) in kernel_unpoison_pages() argument
3084 __kernel_unpoison_pages(page, numpages); in kernel_unpoison_pages()
3089 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { } in __kernel_poison_pages() argument
3090 static inline void kernel_poison_pages(struct page *page, int numpages) { } in kernel_poison_pages() argument
3091 static inline void kernel_unpoison_pages(struct page *page, int numpages) { } in kernel_unpoison_pages() argument
3130 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
3137 kernel_map_pages(struct page *page, int numpages, int enable) in kernel_map_pages() argument
3139 __kernel_map_pages(page, numpages, enable); in kernel_map_pages()
3142 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) in debug_pagealloc_map_pages() argument
3145 __kernel_map_pages(page, numpages, 1); in debug_pagealloc_map_pages()
3148 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) in debug_pagealloc_unmap_pages() argument
3151 __kernel_map_pages(page, numpages, 0); in debug_pagealloc_unmap_pages()
3155 extern bool kernel_page_present(struct page *page);
3159 kernel_map_pages(struct page *page, int numpages, int enable) {} in kernel_map_pages() argument
3160 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} in debug_pagealloc_map_pages() argument
3161 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} in debug_pagealloc_unmap_pages() argument
3163 static inline bool kernel_page_present(struct page *page) { return true; } in kernel_page_present() argument
3210 struct page * __populate_section_memmap(unsigned long pfn,
3232 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
3247 extern void shake_page(struct page *p, int access);
3289 extern void clear_huge_page(struct page *page,
3292 extern void copy_user_huge_page(struct page *dst, struct page *src,
3296 extern long copy_huge_page_from_user(struct page *dst_page,
3302 * vma_is_special_huge - Are transhuge page-table entries considered special?
3305 * Whether transhuge page-table entries are considered "special" following
3308 * Return: true if transhuge page-table entries should be considered special,
3333 static inline bool page_is_guard(struct page *page) in page_is_guard() argument
3338 return PageGuard(page); in page_is_guard()
3343 static inline bool page_is_guard(struct page *page) { return false; } in page_is_guard() argument
3352 extern int memcmp_pages(struct page *page1, struct page *page2);
3354 static inline int pages_identical(struct page *page1, struct page *page2) in pages_identical()