Lines Matching full:page
3 * Macros for manipulating and testing page->flags
18 * Various page->flags bits:
20 * PG_reserved is set for special pages. The "struct page" of such a page
25 * - Pages reserved or allocated early during boot (before the page allocator
27 * initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
29 * be given to the page allocator.
32 * - The zero page(s)
33 * - Pages not added to the page allocator when onlining a section because
49 * Consequently, PG_reserved for a page mapped into user space can indicate
50 * the zero page, the vDSO, MMIO pages or device memory.
53 * specific data (which is normally at page->private). It can be used by
60 * PG_locked also pins a page in pagecache, and blocks truncation of the file
63 * page_waitqueue(page) is a wait queue of all tasks waiting for the page
66 * PG_swapbacked is set when a page uses swap as a backing storage. This are
71 * PG_uptodate tells whether the page's contents is valid. When a read
72 * completes, the page becomes uptodate, unless a disk I/O error happened.
74 * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
77 * PG_error is set to indicate that an I/O error occurred on this page.
79 * PG_arch_1 is an architecture specific page state bit. The generic code
80 * guarantees that this bit is cleared for a page when it first is entered into
81 * the page cache.
83 * PG_hwpoison indicates that a page got corrupted in hardware and contains
90 * locked- and dirty-page accounting.
92 * The page flags field is split into two parts, the main flags area
105 PG_locked, /* Page is locked. Don't touch. */
112 …PG_waiters, /* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_…
120 PG_writeback, /* Page is under writeback */
121 PG_head, /* A head page */
124 PG_swapbacked, /* Page is backed by RAM/swap */
125 PG_unevictable, /* Page is "unevictable" */
127 PG_mlocked, /* Page is vma mlocked */
130 PG_uncached, /* Page has been mapped as uncached */
133 PG_hwpoison, /* hardware poisoned page. Don't touch */
151 PG_swapcache = PG_owner_priv_1, /* Swap page: swp_entry_t in private */
153 /* Two page bits are conscripted by FS-Cache to maintain local caching
157 PG_fscache = PG_private_2, /* page backed by cache */
160 /* Pinned in Xen as a read-only pagetable page. */
164 /* Has a grant mapping of another (foreign) domain's page. */
172 /* Compound pages. Stored in first tail page's flags */
175 /* non-lru isolated movable page */
184 struct page; /* forward declaration */
186 static inline struct page *compound_head(struct page *page) in compound_head() argument
188 unsigned long head = READ_ONCE(page->compound_head); in compound_head()
191 return (struct page *) (head - 1); in compound_head()
192 return page; in compound_head()
195 static __always_inline int PageTail(struct page *page) in PageTail() argument
197 return READ_ONCE(page->compound_head) & 1; in PageTail()
200 static __always_inline int PageCompound(struct page *page) in PageCompound() argument
202 return test_bit(PG_head, &page->flags) || PageTail(page); in PageCompound()
206 static inline int PagePoisoned(const struct page *page) in PagePoisoned() argument
208 return page->flags == PAGE_POISON_PATTERN; in PagePoisoned()
212 void page_init_poison(struct page *page, size_t size);
214 static inline void page_init_poison(struct page *page, size_t size) in page_init_poison() argument
220 * Page flags policies wrt compound pages
223 * check if this struct page poisoned/uninitialized
226 * the page flag is relevant for small, head and tail pages.
229 * for compound page all operations related to the page flag applied to
230 * head page.
233 * for compound page, callers only ever operate on the head page.
236 * modifications of the page flag must be done on small or head pages,
240 * the page flag is not relevant for compound pages.
243 * the page flag is stored in the first tail page.
245 #define PF_POISONED_CHECK(page) ({ \ argument
246 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
247 page; })
248 #define PF_ANY(page, enforce) PF_POISONED_CHECK(page) argument
249 #define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page)) argument
250 #define PF_ONLY_HEAD(page, enforce) ({ \ argument
251 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
252 PF_POISONED_CHECK(page); })
253 #define PF_NO_TAIL(page, enforce) ({ \ argument
254 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
255 PF_POISONED_CHECK(compound_head(page)); })
256 #define PF_NO_COMPOUND(page, enforce) ({ \ argument
257 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
258 PF_POISONED_CHECK(page); })
259 #define PF_SECOND(page, enforce) ({ \ argument
260 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
261 PF_POISONED_CHECK(&page[1]); })
264 * Macros to create function definitions for page flags
267 static __always_inline int Page##uname(struct page *page) \
268 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
271 static __always_inline void SetPage##uname(struct page *page) \
272 { set_bit(PG_##lname, &policy(page, 1)->flags); }
275 static __always_inline void ClearPage##uname(struct page *page) \
276 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
279 static __always_inline void __SetPage##uname(struct page *page) \
280 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
283 static __always_inline void __ClearPage##uname(struct page *page) \
284 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
287 static __always_inline int TestSetPage##uname(struct page *page) \
288 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
291 static __always_inline int TestClearPage##uname(struct page *page) \
292 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
309 static inline int Page##uname(const struct page *page) { return 0; }
312 static inline void SetPage##uname(struct page *page) { }
315 static inline void ClearPage##uname(struct page *page) { }
318 static inline void __ClearPage##uname(struct page *page) { }
321 static inline int TestSetPage##uname(struct page *page) { return 0; }
324 static inline int TestClearPage##uname(struct page *page) { return 0; }
365 * Private page markings that may be used by the filesystem that owns the page in PAGEFLAG()
377 * risky: they bypass page accounting. in PAGEFLAG()
400 static __always_inline int PageSwapCache(struct page *page) in PAGEFLAG()
403 page = compound_head(page); in PAGEFLAG()
405 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags); in PAGEFLAG()
437 extern bool take_page_off_buddy(struct page *page);
465 * On an anonymous page mapped into a user virtual memory area, in TESTPAGEFLAG()
466 * page->mapping points to its anon_vma, not to a struct address_space; in TESTPAGEFLAG()
469 * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled, in TESTPAGEFLAG()
471 * bit; and then page->mapping points, not to an anon_vma, but to a private in TESTPAGEFLAG()
472 * structure which KSM associates with that merged page. See ksm.h. in TESTPAGEFLAG()
475 * page and then page->mapping points a struct address_space. in TESTPAGEFLAG()
478 * address_space which maps the page from disk; whereas "page_mapped" in TESTPAGEFLAG()
479 * refers to user virtual address space into which the page is mapped. in TESTPAGEFLAG()
486 static __always_inline int PageMappingFlags(struct page *page) in TESTPAGEFLAG()
488 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0; in TESTPAGEFLAG()
491 static __always_inline int PageAnon(struct page *page) in PageAnon() argument
493 page = compound_head(page); in PageAnon()
494 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0; in PageAnon()
497 static __always_inline int __PageMovable(struct page *page) in __PageMovable() argument
499 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in __PageMovable()
505 * A KSM page is one of those write-protected "shared pages" or "merged pages"
506 * which KSM maps into multiple mms, wherever identical anonymous page content
507 * is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
508 * anon_vma, but to that page's node of the stable tree.
510 static __always_inline int PageKsm(struct page *page) in PageKsm() argument
512 page = compound_head(page); in PageKsm()
513 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in PageKsm()
520 u64 stable_page_flags(struct page *page);
522 static inline int PageUptodate(struct page *page) in PageUptodate() argument
525 page = compound_head(page); in PageUptodate()
526 ret = test_bit(PG_uptodate, &(page)->flags); in PageUptodate()
528 * Must ensure that the data we read out of the page is loaded in PageUptodate()
529 * _after_ we've loaded page->flags to check for PageUptodate. in PageUptodate()
530 * We can skip the barrier if the page is not uptodate, because in PageUptodate()
541 static __always_inline void __SetPageUptodate(struct page *page) in __SetPageUptodate() argument
543 VM_BUG_ON_PAGE(PageTail(page), page); in __SetPageUptodate()
545 __set_bit(PG_uptodate, &page->flags); in __SetPageUptodate()
548 static __always_inline void SetPageUptodate(struct page *page) in SetPageUptodate() argument
550 VM_BUG_ON_PAGE(PageTail(page), page); in SetPageUptodate()
553 * so that all previous stores issued in order to bring the page in SetPageUptodate()
557 set_bit(PG_uptodate, &page->flags); in SetPageUptodate()
562 int test_clear_page_writeback(struct page *page);
563 int __test_set_page_writeback(struct page *page, bool keep_write);
565 #define test_set_page_writeback(page) \ argument
566 __test_set_page_writeback(page, false)
567 #define test_set_page_writeback_keepwrite(page) \ argument
568 __test_set_page_writeback(page, true)
570 static inline void set_page_writeback(struct page *page) in set_page_writeback() argument
572 test_set_page_writeback(page); in set_page_writeback()
575 static inline void set_page_writeback_keepwrite(struct page *page) in set_page_writeback_keepwrite() argument
577 test_set_page_writeback_keepwrite(page); in set_page_writeback_keepwrite()
582 static __always_inline void set_compound_head(struct page *page, struct page *head) in __PAGEFLAG()
584 WRITE_ONCE(page->compound_head, (unsigned long)head + 1); in __PAGEFLAG()
587 static __always_inline void clear_compound_head(struct page *page) in clear_compound_head() argument
589 WRITE_ONCE(page->compound_head, 0); in clear_compound_head()
593 static inline void ClearPageCompound(struct page *page) in ClearPageCompound() argument
595 BUG_ON(!PageHead(page)); in ClearPageCompound()
596 ClearPageHead(page); in ClearPageCompound()
603 int PageHuge(struct page *page);
604 int PageHeadHuge(struct page *page);
605 bool page_huge_active(struct page *page);
610 static inline bool page_huge_active(struct page *page) in TESTPAGEFLAG_FALSE()
626 static inline int PageTransHuge(struct page *page) in PageTransHuge() argument
628 VM_BUG_ON_PAGE(PageTail(page), page); in PageTransHuge()
629 return PageHead(page); in PageTransHuge()
637 static inline int PageTransCompound(struct page *page) in PageTransCompound() argument
639 return PageCompound(page); in PageTransCompound()
644 * guarantees the primary MMU has the entire compound page mapped
646 * can also map the entire compound page. This allows the secondary
647 * MMUs to call get_user_pages() only once for each compound page and
648 * to immediately map the entire compound page with a single secondary
655 * MMU notifier, otherwise it may result in page->_mapcount check false
658 * We have to treat page cache THP differently since every subpage of it
663 static inline int PageTransCompoundMap(struct page *page) in PageTransCompoundMap() argument
665 struct page *head; in PageTransCompoundMap()
667 if (!PageTransCompound(page)) in PageTransCompoundMap()
670 if (PageAnon(page)) in PageTransCompoundMap()
671 return atomic_read(&page->_mapcount) < 0; in PageTransCompoundMap()
673 head = compound_head(page); in PageTransCompoundMap()
675 return atomic_read(&page->_mapcount) == in PageTransCompoundMap()
684 static inline int PageTransTail(struct page *page) in PageTransTail() argument
686 return PageTail(page); in PageTransTail()
690 * PageDoubleMap indicates that the compound page is mapped with PTEs as well
694 * per small page mapcount accounting (and its overhead from atomic operations)
697 * For the page PageDoubleMap means ->_mapcount in all sub-pages is offset up
719 * mistaken for a page type value. in PAGEFLAG()
731 #define PageType(page, flag) \ in PAGEFLAG() argument
732 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE) in PAGEFLAG()
734 static inline int page_has_type(struct page *page) in PAGEFLAG()
736 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE; in PAGEFLAG()
740 static __always_inline int Page##uname(struct page *page) \
742 return PageType(page, PG_##lname); \
744 static __always_inline void __SetPage##uname(struct page *page) \
746 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
747 page->page_type &= ~PG_##lname; \
749 static __always_inline void __ClearPage##uname(struct page *page) \
751 VM_BUG_ON_PAGE(!Page##uname(page), page); \
752 page->page_type |= PG_##lname; \
756 * PageBuddy() indicates that the page is free and in the buddy system
762 * PageOffline() indicates that the page is logically offline although the
782 * pages allocated with __GFP_ACCOUNT. It gets cleared on page free.
787 * Marks pages in use as page tables.
796 extern bool is_free_buddy_page(struct page *page);
804 static inline int PageSlabPfmemalloc(struct page *page) in PageSlabPfmemalloc() argument
806 VM_BUG_ON_PAGE(!PageSlab(page), page); in PageSlabPfmemalloc()
807 return PageActive(page); in PageSlabPfmemalloc()
810 static inline void SetPageSlabPfmemalloc(struct page *page) in SetPageSlabPfmemalloc() argument
812 VM_BUG_ON_PAGE(!PageSlab(page), page); in SetPageSlabPfmemalloc()
813 SetPageActive(page); in SetPageSlabPfmemalloc()
816 static inline void __ClearPageSlabPfmemalloc(struct page *page) in __ClearPageSlabPfmemalloc() argument
818 VM_BUG_ON_PAGE(!PageSlab(page), page); in __ClearPageSlabPfmemalloc()
819 __ClearPageActive(page); in __ClearPageSlabPfmemalloc()
822 static inline void ClearPageSlabPfmemalloc(struct page *page) in ClearPageSlabPfmemalloc() argument
824 VM_BUG_ON_PAGE(!PageSlab(page), page); in ClearPageSlabPfmemalloc()
825 ClearPageActive(page); in ClearPageSlabPfmemalloc()
835 * Flags checked when a page is freed. Pages being freed should not have
846 * Flags checked when a page is prepped for return by the page allocator.
848 * there has been a kernel bug or struct page corruption.
850 * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
851 * alloc-free cycle to prevent from reusing the page.
859 * page_has_private - Determine if page has private stuff
860 * @page: The page to be checked
862 * Determine if a page has private stuff, indicating that release routines
865 static inline int page_has_private(struct page *page) in page_has_private() argument
867 return !!(page->flags & PAGE_FLAGS_PRIVATE); in page_has_private()