1 /* SPDX-License-Identifier: GPL-2.0 */
23  * Bits in mapping->flags.
37  * mapping_set_error - record a writeback error in the address_space
38  * @mapping: the mapping in which an error should be set
39  * @error: the error to set in the mapping
47  * mapping_set_error to record the error in the mapping so that it can be
50 static inline void mapping_set_error(struct address_space *mapping, int error)  in mapping_set_error()  argument
56 	__filemap_set_wb_err(mapping, error);  in mapping_set_error()
59 	if (mapping->host)  in mapping_set_error()
60 		errseq_set(&mapping->host->i_sb->s_wb_err, error);  in mapping_set_error()
63 	if (error == -ENOSPC)  in mapping_set_error()
64 		set_bit(AS_ENOSPC, &mapping->flags);  in mapping_set_error()
66 		set_bit(AS_EIO, &mapping->flags);  in mapping_set_error()
69 static inline void mapping_set_unevictable(struct address_space *mapping)  in mapping_set_unevictable()  argument
71 	set_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_set_unevictable()
74 static inline void mapping_clear_unevictable(struct address_space *mapping)  in mapping_clear_unevictable()  argument
76 	clear_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_clear_unevictable()
79 static inline bool mapping_unevictable(struct address_space *mapping)  in mapping_unevictable()  argument
81 	return mapping && test_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_unevictable()
84 static inline void mapping_set_exiting(struct address_space *mapping)  in mapping_set_exiting()  argument
86 	set_bit(AS_EXITING, &mapping->flags);  in mapping_set_exiting()
89 static inline int mapping_exiting(struct address_space *mapping)  in mapping_exiting()  argument
91 	return test_bit(AS_EXITING, &mapping->flags);  in mapping_exiting()
94 static inline void mapping_set_no_writeback_tags(struct address_space *mapping)  in mapping_set_no_writeback_tags()  argument
96 	set_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);  in mapping_set_no_writeback_tags()
99 static inline int mapping_use_writeback_tags(struct address_space *mapping)  in mapping_use_writeback_tags()  argument
101 	return !test_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);  in mapping_use_writeback_tags()
104 static inline gfp_t mapping_gfp_mask(struct address_space * mapping)  in mapping_gfp_mask()  argument
106 	return mapping->gfp_mask;  in mapping_gfp_mask()
109 /* Restricts the given gfp_mask to what the mapping allows. */
110 static inline gfp_t mapping_gfp_constraint(struct address_space *mapping,  in mapping_gfp_constraint()  argument
113 	return mapping_gfp_mask(mapping) & gfp_mask;  in mapping_gfp_constraint()
117  * This is non-atomic.  Only to be used before the mapping is activated.
122 	m->gfp_mask = mask;  in mapping_set_gfp_mask()
125 static inline bool mapping_thp_support(struct address_space *mapping)  in mapping_thp_support()  argument
127 	return test_bit(AS_THP_SUPPORT, &mapping->flags);  in mapping_thp_support()
130 static inline int filemap_nr_thps(struct address_space *mapping)  in filemap_nr_thps()  argument
133 	return atomic_read(&mapping->nr_thps);  in filemap_nr_thps()
139 static inline void filemap_nr_thps_inc(struct address_space *mapping)  in filemap_nr_thps_inc()  argument
142 	if (!mapping_thp_support(mapping))  in filemap_nr_thps_inc()
143 		atomic_inc(&mapping->nr_thps);  in filemap_nr_thps_inc()
149 static inline void filemap_nr_thps_dec(struct address_space *mapping)  in filemap_nr_thps_dec()  argument
152 	if (!mapping_thp_support(mapping))  in filemap_nr_thps_dec()
153 		atomic_dec(&mapping->nr_thps);  in filemap_nr_thps_dec()
167  * been used to lookup the page in the pagecache radix-tree (or page table):
178  * get_user_pages) locking protocol, where the lookup-side (eg. find_get_page)
184  * Remove-side that cares about stability of _refcount (eg. reclaim) has the
191  * - 2 runs before A: in this case, A sees elevated refcount and bails out
192  * - A runs before 2: in this case, 2 sees zero refcount and retries;
199  * such a re-insertion, depending on order that locks are granted.
212 	 * Preempt must be disabled here - we rely on rcu_read_lock doing  in __page_cache_add_speculative()
249  * attach_page_private - Attach private data to a page.
250  * @page: Page to attach data to.
251  * @data: Data to attach to page.
253  * Attaching private data to a page increments the page's reference count.
254  * The data must be detached before the page will be freed.
256 static inline void attach_page_private(struct page *page, void *data)  in attach_page_private()  argument
259 	set_page_private(page, (unsigned long)data);  in attach_page_private()
264  * detach_page_private - Detach private data from a page.
265  * @page: Page to detach data from.
267  * Removes the data that was previously attached to the page and decrements
270  * Return: Data that was attached to the page.
274 	void *data = (void *)page_private(page);  in detach_page_private()  local
282 	return data;  in detach_page_private()
303 pgoff_t page_cache_next_miss(struct address_space *mapping,
305 pgoff_t page_cache_prev_miss(struct address_space *mapping,
317 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
321  * find_get_page - find and get a page reference
322  * @mapping: the address_space to search
325  * Looks up the page cache slot at @mapping & @offset.  If there is a
330 static inline struct page *find_get_page(struct address_space *mapping,  in find_get_page()  argument
333 	return pagecache_get_page(mapping, offset, 0, 0);  in find_get_page()
336 static inline struct page *find_get_page_flags(struct address_space *mapping,  in find_get_page_flags()  argument
339 	return pagecache_get_page(mapping, offset, fgp_flags, 0);  in find_get_page_flags()
343  * find_lock_page - locate, pin and lock a pagecache page
344  * @mapping: the address_space to search
347  * Looks up the page cache entry at @mapping & @index.  If there is a
355 static inline struct page *find_lock_page(struct address_space *mapping,  in find_lock_page()  argument
358 	return pagecache_get_page(mapping, index, FGP_LOCK, 0);  in find_lock_page()
362  * find_lock_head - Locate, pin and lock a pagecache page.
363  * @mapping: The address_space to search.
366  * Looks up the page cache entry at @mapping & @index.  If there is a
374 static inline struct page *find_lock_head(struct address_space *mapping,  in find_lock_head()  argument
377 	return pagecache_get_page(mapping, index, FGP_LOCK | FGP_HEAD, 0);  in find_lock_head()
381  * find_or_create_page - locate or add a pagecache page
382  * @mapping: the page's address_space
383  * @index: the page's index into the mapping
386  * Looks up the page cache slot at @mapping & @offset.  If there is a
399 static inline struct page *find_or_create_page(struct address_space *mapping,  in find_or_create_page()  argument
402 	return pagecache_get_page(mapping, index,  in find_or_create_page()
408  * grab_cache_page_nowait - returns locked page at given index in given cache
409  * @mapping: target address_space
413  * This is intended for speculative data generators, where the data can
420 static inline struct page *grab_cache_page_nowait(struct address_space *mapping,  in grab_cache_page_nowait()  argument
423 	return pagecache_get_page(mapping, index,  in grab_cache_page_nowait()
425 			mapping_gfp_mask(mapping));  in grab_cache_page_nowait()
433 		return head->index == index;  in thp_contains()
434 	return page_index(head) == (index & ~(thp_nr_pages(head) - 1UL));  in thp_contains()
447 	return head + (index & (thp_nr_pages(head) - 1));  in find_subpage()
450 unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
453 unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
456 static inline unsigned find_get_pages(struct address_space *mapping,  in find_get_pages()  argument
460 	return find_get_pages_range(mapping, start, (pgoff_t)-1, nr_pages,  in find_get_pages()
463 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start,
465 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
468 static inline unsigned find_get_pages_tag(struct address_space *mapping,  in find_get_pages_tag()  argument
472 	return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag,  in find_get_pages_tag()
476 struct page *grab_cache_page_write_begin(struct address_space *mapping,
482 static inline struct page *grab_cache_page(struct address_space *mapping,  in grab_cache_page()  argument
485 	return find_or_create_page(mapping, index, mapping_gfp_mask(mapping));  in grab_cache_page()
488 extern struct page * read_cache_page(struct address_space *mapping,
489 				pgoff_t index, filler_t *filler, void *data);
490 extern struct page * read_cache_page_gfp(struct address_space *mapping,
492 extern int read_cache_pages(struct address_space *mapping,
493 		struct list_head *pages, filler_t *filler, void *data);
495 static inline struct page *read_mapping_page(struct address_space *mapping,  in read_mapping_page()  argument
496 				pgoff_t index, void *data)  in read_mapping_page()  argument
498 	return read_cache_page(mapping, index, NULL, data);  in read_mapping_page()
502  * Get index of the page within radix-tree (but not for hugetlb pages).
503  * (TODO: remove once hugetlb pages will have ->index in PAGE_SIZE)
510 		return page->index;  in page_to_index()
513 	 *  We don't initialize ->index for tail pages: calculate based on  in page_to_index()
516 	pgoff = compound_head(page)->index;  in page_to_index()
517 	pgoff += page - compound_head(page);  in page_to_index()
525  * (TODO: hugetlb pages should have ->index in PAGE_SIZE)
535  * Return byte-offset into filesystem object for page.
539 	return ((loff_t)page->index) << PAGE_SHIFT;  in page_offset()
556 	pgoff = (address - READ_ONCE(vma->vm_start)) >> PAGE_SHIFT;  in linear_page_index()
557 	pgoff += READ_ONCE(vma->vm_pgoff);  in linear_page_index()
576 	if (wait_page->page != key->page)  in wake_page_match()
578 	key->page_match = 1;  in wake_page_match()
580 	if (wait_page->bit_nr != key->bit_nr)  in wake_page_match()
599 	return (likely(!test_and_set_bit_lock(PG_locked, &page->flags)));  in trylock_page()
614  * signals.  It returns 0 if it locked the page and -EINTR if it was
626  * lock_page_async - Lock the page, unless this would block. If the page
630  * Returns 0 if the page is locked successfully, or -EIOCBQUEUED if the page
642  * lock_page_or_retry - Lock the page, unless this would block and the
666  * ie with increased "page->count" so that the page won't
700 	char __user *end = uaddr + size - 1;  in fault_in_pages_writeable()
706 		return -EFAULT;  in fault_in_pages_writeable()
713 			return -EFAULT;  in fault_in_pages_writeable()
728 	const char __user *end = uaddr + size - 1;  in fault_in_pages_readable()
734 		return -EFAULT;  in fault_in_pages_readable()
738 			return -EFAULT;  in fault_in_pages_readable()
752 int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
754 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
759 void delete_from_page_cache_batch(struct address_space *mapping,
767 		struct address_space *mapping, pgoff_t offset, gfp_t gfp_mask)  in add_to_page_cache()  argument
772 	error = add_to_page_cache_locked(page, mapping, offset, gfp_mask);  in add_to_page_cache()
779  * struct readahead_control - Describes a readahead request.
782  * implement the ->readahead method should call readahead_page() or
791  * @mapping: Readahead this filesystem object.
795 	struct address_space *mapping;  member
805 		.mapping = m,						\
819  * page_cache_sync_readahead - generic file readahead
820  * @mapping: address_space which holds the pagecache and I/O vectors
832 void page_cache_sync_readahead(struct address_space *mapping,  in page_cache_sync_readahead()  argument
836 	DEFINE_READAHEAD(ractl, file, mapping, index);  in page_cache_sync_readahead()
841  * page_cache_async_readahead - file readahead for marked pages
842  * @mapping: address_space which holds the pagecache and I/O vectors
855 void page_cache_async_readahead(struct address_space *mapping,  in page_cache_async_readahead()  argument
859 	DEFINE_READAHEAD(ractl, file, mapping, index);  in page_cache_async_readahead()
864  * readahead_page - Get the next page to read.
876 	BUG_ON(rac->_batch_count > rac->_nr_pages);  in readahead_page()
877 	rac->_nr_pages -= rac->_batch_count;  in readahead_page()
878 	rac->_index += rac->_batch_count;  in readahead_page()
880 	if (!rac->_nr_pages) {  in readahead_page()
881 		rac->_batch_count = 0;  in readahead_page()
885 	page = xa_load(&rac->mapping->i_pages, rac->_index);  in readahead_page()
887 	rac->_batch_count = thp_nr_pages(page);  in readahead_page()
896 	XA_STATE(xas, &rac->mapping->i_pages, 0);  in __readahead_batch()
899 	BUG_ON(rac->_batch_count > rac->_nr_pages);  in __readahead_batch()
900 	rac->_nr_pages -= rac->_batch_count;  in __readahead_batch()
901 	rac->_index += rac->_batch_count;  in __readahead_batch()
902 	rac->_batch_count = 0;  in __readahead_batch()
904 	xas_set(&xas, rac->_index);  in __readahead_batch()
906 	xas_for_each(&xas, page, rac->_index + rac->_nr_pages - 1) {  in __readahead_batch()
912 		rac->_batch_count += thp_nr_pages(page);  in __readahead_batch()
915 		 * The page cache isn't using multi-index entries yet,  in __readahead_batch()
921 			xas_set(&xas, rac->_index + rac->_batch_count);  in __readahead_batch()
932  * readahead_page_batch - Get a batch of pages to read.
946  * readahead_pos - The byte offset into the file of this readahead request.
951 	return (loff_t)rac->_index * PAGE_SIZE;  in readahead_pos()
955  * readahead_length - The number of bytes in this readahead request.
960 	return (loff_t)rac->_nr_pages * PAGE_SIZE;  in readahead_length()
964  * readahead_index - The index of the first page in this readahead request.
969 	return rac->_index;  in readahead_index()
973  * readahead_count - The number of pages in this readahead request.
978 	return rac->_nr_pages;  in readahead_count()
983 	return (unsigned long)(inode->i_size + PAGE_SIZE - 1) >>  in dir_pages()
988  * page_mkwrite_check_truncate - check if page was truncated
993  * or -EFAULT if the page was truncated.
1002 	if (page->mapping != inode->i_mapping)  in page_mkwrite_check_truncate()
1003 		return -EFAULT;  in page_mkwrite_check_truncate()
1006 	if (page->index < index)  in page_mkwrite_check_truncate()
1009 	if (page->index > index || !offset)  in page_mkwrite_check_truncate()
1010 		return -EFAULT;  in page_mkwrite_check_truncate()
1016  * i_blocks_per_page - How many blocks fit in this page.
1029 	return thp_size(page) >> inode->i_blkbits;  in i_blocks_per_page()