125 static void page_cache_delete(struct address_space *mapping,  in page_cache_delete()  argument
128 	XA_STATE(xas, &mapping->i_pages, page->index);  in page_cache_delete()
131 	mapping_set_update(&xas, mapping);  in page_cache_delete()
146 	page->mapping = NULL;  in page_cache_delete()
150 		mapping->nrexceptional += nr;  in page_cache_delete()
159 	mapping->nrpages -= nr;  in page_cache_delete()
162 static void unaccount_page_cache_page(struct address_space *mapping,  in unaccount_page_cache_page()  argument
175 		cleancache_invalidate_page(mapping, page);  in unaccount_page_cache_page()
189 		if (mapping_exiting(mapping) &&  in unaccount_page_cache_page()
215 		filemap_nr_thps_dec(mapping);  in unaccount_page_cache_page()
229 		account_page_cleaned(page, mapping, inode_to_wb(mapping->host));  in unaccount_page_cache_page()
239 	struct address_space *mapping = page->mapping;  in __delete_from_page_cache()  local
243 	unaccount_page_cache_page(mapping, page);  in __delete_from_page_cache()
244 	page_cache_delete(mapping, page, shadow);  in __delete_from_page_cache()
247 static void page_cache_free_page(struct address_space *mapping,  in page_cache_free_page()  argument
252 	freepage = mapping->a_ops->freepage;  in page_cache_free_page()
274 	struct address_space *mapping = page_mapping(page);  in delete_from_page_cache()  local
278 	xa_lock_irqsave(&mapping->i_pages, flags);  in delete_from_page_cache()
280 	xa_unlock_irqrestore(&mapping->i_pages, flags);  in delete_from_page_cache()
282 	page_cache_free_page(mapping, page);  in delete_from_page_cache()
300 static void page_cache_delete_batch(struct address_space *mapping,  in page_cache_delete_batch()  argument
303 	XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index);  in page_cache_delete_batch()
308 	mapping_set_update(&xas, mapping);  in page_cache_delete_batch()
332 			page->mapping = NULL;  in page_cache_delete_batch()
345 	mapping->nrpages -= total_pages;  in page_cache_delete_batch()
348 void delete_from_page_cache_batch(struct address_space *mapping,  in delete_from_page_cache_batch()  argument
357 	xa_lock_irqsave(&mapping->i_pages, flags);  in delete_from_page_cache_batch()
361 		unaccount_page_cache_page(mapping, pvec->pages[i]);  in delete_from_page_cache_batch()
363 	page_cache_delete_batch(mapping, pvec);  in delete_from_page_cache_batch()
364 	xa_unlock_irqrestore(&mapping->i_pages, flags);  in delete_from_page_cache_batch()
367 		page_cache_free_page(mapping, pvec->pages[i]);  in delete_from_page_cache_batch()
370 int filemap_check_errors(struct address_space *mapping)  in filemap_check_errors()  argument
374 	if (test_bit(AS_ENOSPC, &mapping->flags) &&  in filemap_check_errors()
375 	    test_and_clear_bit(AS_ENOSPC, &mapping->flags))  in filemap_check_errors()
377 	if (test_bit(AS_EIO, &mapping->flags) &&  in filemap_check_errors()
378 	    test_and_clear_bit(AS_EIO, &mapping->flags))  in filemap_check_errors()
384 static int filemap_check_and_keep_errors(struct address_space *mapping)  in filemap_check_and_keep_errors()  argument
387 	if (test_bit(AS_EIO, &mapping->flags))  in filemap_check_and_keep_errors()
389 	if (test_bit(AS_ENOSPC, &mapping->flags))  in filemap_check_and_keep_errors()
411 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,  in __filemap_fdatawrite_range()  argument
422 	if (!mapping_can_writeback(mapping) ||  in __filemap_fdatawrite_range()
423 	    !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))  in __filemap_fdatawrite_range()
426 	wbc_attach_fdatawrite_inode(&wbc, mapping->host);  in __filemap_fdatawrite_range()
427 	ret = do_writepages(mapping, &wbc);  in __filemap_fdatawrite_range()
432 static inline int __filemap_fdatawrite(struct address_space *mapping,  in __filemap_fdatawrite()  argument
435 	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);  in __filemap_fdatawrite()
438 int filemap_fdatawrite(struct address_space *mapping)  in filemap_fdatawrite()  argument
440 	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);  in filemap_fdatawrite()
444 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,  in filemap_fdatawrite_range()  argument
447 	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);  in filemap_fdatawrite_range()
460 int filemap_flush(struct address_space *mapping)  in filemap_flush()  argument
462 	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);  in filemap_flush()
478 bool filemap_range_has_page(struct address_space *mapping,  in filemap_range_has_page()  argument
482 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);  in filemap_range_has_page()
509 static void __filemap_fdatawait_range(struct address_space *mapping,  in __filemap_fdatawait_range()  argument
524 		nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index,  in __filemap_fdatawait_range()
556 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,  in filemap_fdatawait_range()  argument
559 	__filemap_fdatawait_range(mapping, start_byte, end_byte);  in filemap_fdatawait_range()
560 	return filemap_check_errors(mapping);  in filemap_fdatawait_range()
578 int filemap_fdatawait_range_keep_errors(struct address_space *mapping,  in filemap_fdatawait_range_keep_errors()  argument
581 	__filemap_fdatawait_range(mapping, start_byte, end_byte);  in filemap_fdatawait_range_keep_errors()
582 	return filemap_check_and_keep_errors(mapping);  in filemap_fdatawait_range_keep_errors()
604 	struct address_space *mapping = file->f_mapping;  in file_fdatawait_range()  local
606 	__filemap_fdatawait_range(mapping, start_byte, end_byte);  in file_fdatawait_range()
625 int filemap_fdatawait_keep_errors(struct address_space *mapping)  in filemap_fdatawait_keep_errors()  argument
627 	__filemap_fdatawait_range(mapping, 0, LLONG_MAX);  in filemap_fdatawait_keep_errors()
628 	return filemap_check_and_keep_errors(mapping);  in filemap_fdatawait_keep_errors()
633 static bool mapping_needs_writeback(struct address_space *mapping)  in mapping_needs_writeback()  argument
635 	if (dax_mapping(mapping))  in mapping_needs_writeback()
636 		return mapping->nrexceptional;  in mapping_needs_writeback()
638 	return mapping->nrpages;  in mapping_needs_writeback()
654 int filemap_write_and_wait_range(struct address_space *mapping,  in filemap_write_and_wait_range()  argument
659 	if (mapping_needs_writeback(mapping)) {  in filemap_write_and_wait_range()
660 		err = __filemap_fdatawrite_range(mapping, lstart, lend,  in filemap_write_and_wait_range()
669 			int err2 = filemap_fdatawait_range(mapping,  in filemap_write_and_wait_range()
675 			filemap_check_errors(mapping);  in filemap_write_and_wait_range()
678 		err = filemap_check_errors(mapping);  in filemap_write_and_wait_range()
684 void __filemap_set_wb_err(struct address_space *mapping, int err)  in __filemap_set_wb_err()  argument
686 	errseq_t eseq = errseq_set(&mapping->wb_err, err);  in __filemap_set_wb_err()
688 	trace_filemap_set_wb_err(mapping, eseq);  in __filemap_set_wb_err()
720 	struct address_space *mapping = file->f_mapping;  in file_check_and_advance_wb_err()  local
723 	if (errseq_check(&mapping->wb_err, old)) {  in file_check_and_advance_wb_err()
727 		err = errseq_check_and_advance(&mapping->wb_err,  in file_check_and_advance_wb_err()
738 	clear_bit(AS_EIO, &mapping->flags);  in file_check_and_advance_wb_err()
739 	clear_bit(AS_ENOSPC, &mapping->flags);  in file_check_and_advance_wb_err()
763 	struct address_space *mapping = file->f_mapping;  in file_write_and_wait_range()  local
765 	if (mapping_needs_writeback(mapping)) {  in file_write_and_wait_range()
766 		err = __filemap_fdatawrite_range(mapping, lstart, lend,  in file_write_and_wait_range()
770 			__filemap_fdatawait_range(mapping, lstart, lend);  in file_write_and_wait_range()
797 	struct address_space *mapping = old->mapping;  in replace_page_cache_page()  local
798 	void (*freepage)(struct page *) = mapping->a_ops->freepage;  in replace_page_cache_page()
800 	XA_STATE(xas, &mapping->i_pages, offset);  in replace_page_cache_page()
805 	VM_BUG_ON_PAGE(new->mapping, new);  in replace_page_cache_page()
808 	new->mapping = mapping;  in replace_page_cache_page()
816 	old->mapping = NULL;  in replace_page_cache_page()
836 					struct address_space *mapping,  in __add_to_page_cache_locked()  argument
840 	XA_STATE(xas, &mapping->i_pages, offset);  in __add_to_page_cache_locked()
847 	mapping_set_update(&xas, mapping);  in __add_to_page_cache_locked()
850 	page->mapping = mapping;  in __add_to_page_cache_locked()
894 			mapping->nrexceptional--;  in __add_to_page_cache_locked()
895 		mapping->nrpages++;  in __add_to_page_cache_locked()
914 	page->mapping = NULL;  in __add_to_page_cache_locked()
933 int add_to_page_cache_locked(struct page *page, struct address_space *mapping,  in add_to_page_cache_locked()  argument
936 	return __add_to_page_cache_locked(page, mapping, offset,  in add_to_page_cache_locked()
941 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,  in add_to_page_cache_lru()  argument
948 	ret = __add_to_page_cache_locked(page, mapping, offset,  in add_to_page_cache_lru()
1528 			struct address_space *mapping;  in page_endio()  local
1531 			mapping = page_mapping(page);  in page_endio()
1532 			if (mapping)  in page_endio()
1533 				mapping_set_error(mapping, err);  in page_endio()
1629 pgoff_t page_cache_next_miss(struct address_space *mapping,  in page_cache_next_miss()  argument
1632 	XA_STATE(xas, &mapping->i_pages, index);  in page_cache_next_miss()
1665 pgoff_t page_cache_prev_miss(struct address_space *mapping,  in page_cache_prev_miss()  argument
1668 	XA_STATE(xas, &mapping->i_pages, index);  in page_cache_prev_miss()
1695 struct page *find_get_entry(struct address_space *mapping, pgoff_t index)  in find_get_entry()  argument
1697 	XA_STATE(xas, &mapping->i_pages, index);  in find_get_entry()
1745 struct page *find_lock_entry(struct address_space *mapping, pgoff_t index)  in find_lock_entry()  argument
1750 	page = find_get_entry(mapping, index);  in find_lock_entry()
1754 		if (unlikely(page->mapping != mapping)) {  in find_lock_entry()
1796 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,  in pagecache_get_page()  argument
1802 	page = find_get_entry(mapping, index);  in pagecache_get_page()
1806 	trace_android_vh_pagecache_get_page(mapping, index, fgp_flags,  in pagecache_get_page()
1822 		if (unlikely(page->mapping != mapping)) {  in pagecache_get_page()
1843 		if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))  in pagecache_get_page()
1859 		err = add_to_page_cache_lru(page, mapping, index, gfp_mask);  in pagecache_get_page()
1906 unsigned find_get_entries(struct address_space *mapping,  in find_get_entries()  argument
1910 	XA_STATE(xas, &mapping->i_pages, start);  in find_get_entries()
1980 unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,  in find_get_pages_range()  argument
1984 	XA_STATE(xas, &mapping->i_pages, *start);  in find_get_pages_range()
2046 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,  in find_get_pages_contig()  argument
2049 	XA_STATE(xas, &mapping->i_pages, index);  in find_get_pages_contig()
2102 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,  in find_get_pages_range_tag()  argument
2106 	XA_STATE(xas, &mapping->i_pages, *index);  in find_get_pages_range_tag()
2201 	struct address_space *mapping = filp->f_mapping;  in generic_file_buffered_read()  local
2202 	struct inode *inode = mapping->host;  in generic_file_buffered_read()
2243 		page = find_get_page(mapping, index);  in generic_file_buffered_read()
2247 			page_cache_sync_readahead(mapping,  in generic_file_buffered_read()
2250 			page = find_get_page(mapping, index);  in generic_file_buffered_read()
2259 			page_cache_async_readahead(mapping,  in generic_file_buffered_read()
2289 					!mapping->a_ops->is_partially_uptodate)  in generic_file_buffered_read()
2297 			if (!page->mapping)  in generic_file_buffered_read()
2299 			if (!mapping->a_ops->is_partially_uptodate(page,  in generic_file_buffered_read()
2336 		if (mapping_writably_mapped(mapping))  in generic_file_buffered_read()
2384 		if (!page->mapping) {  in generic_file_buffered_read()
2409 		error = mapping->a_ops->readpage(filp, page);  in generic_file_buffered_read()
2434 				if (page->mapping == NULL) {  in generic_file_buffered_read()
2462 		page = page_cache_alloc(mapping);  in generic_file_buffered_read()
2467 		error = add_to_page_cache_lru(page, mapping, index,  in generic_file_buffered_read()
2468 				mapping_gfp_constraint(mapping, GFP_KERNEL));  in generic_file_buffered_read()
2525 		struct address_space *mapping = file->f_mapping;  in generic_file_read_iter()  local
2526 		struct inode *inode = mapping->host;  in generic_file_read_iter()
2531 			if (filemap_range_has_page(mapping, iocb->ki_pos,  in generic_file_read_iter()
2535 			retval = filemap_write_and_wait_range(mapping,  in generic_file_read_iter()
2544 		retval = mapping->a_ops->direct_IO(iocb, iter);  in generic_file_read_iter()
2628 	struct address_space *mapping = file->f_mapping;  in do_sync_mmap_readahead()  local
2629 	DEFINE_READAHEAD(ractl, file, mapping, vmf->pgoff);  in do_sync_mmap_readahead()
2679 	struct address_space *mapping = file->f_mapping;  in do_async_mmap_readahead()  local
2692 		page_cache_async_readahead(mapping, ra, file,  in do_async_mmap_readahead()
2728 	struct address_space *mapping = file->f_mapping;  in filemap_fault()  local
2730 	struct inode *inode = mapping->host;  in filemap_fault()
2738 		page = find_get_page(mapping, offset);  in filemap_fault()
2745 		if (unlikely(compound_head(page)->mapping != mapping))  in filemap_fault()
2791 	page = find_get_page(mapping, offset);  in filemap_fault()
2805 		page = pagecache_get_page(mapping, offset,  in filemap_fault()
2819 	if (unlikely(compound_head(page)->mapping != mapping)) {  in filemap_fault()
2867 	error = mapping->a_ops->readpage(file, page);  in filemap_fault()
2940 				       struct address_space *mapping,  in next_uptodate_page()  argument
2965 		if (page->mapping != mapping)  in next_uptodate_page()
2969 		max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);  in next_uptodate_page()
2982 static inline struct page *first_map_page(struct address_space *mapping,  in first_map_page()  argument
2987 				  mapping, xas, end_pgoff);  in first_map_page()
2990 static inline struct page *next_map_page(struct address_space *mapping,  in next_map_page()  argument
2995 				  mapping, xas, end_pgoff);  in next_map_page()
3011 	struct address_space *mapping = file->f_mapping;  in filemap_map_pages()  local
3014 	XA_STATE(xas, &mapping->i_pages, start_pgoff);  in filemap_map_pages()
3021 	head = first_map_page(mapping, &xas, end_pgoff);  in filemap_map_pages()
3065 	} while ((head = next_map_page(mapping, &xas, end_pgoff)) != NULL);  in filemap_map_pages()
3083 	if (page->mapping != inode->i_mapping) {  in filemap_page_mkwrite()
3113 	struct address_space *mapping = file->f_mapping;  in generic_file_mmap()  local
3115 	if (!mapping->a_ops->readpage)  in generic_file_mmap()
3162 static struct page *do_read_cache_page(struct address_space *mapping,  in do_read_cache_page()  argument
3171 	page = find_get_page(mapping, index);  in do_read_cache_page()
3176 		err = add_to_page_cache_lru(page, mapping, index, gfp);  in do_read_cache_page()
3189 			err = mapping->a_ops->readpage(data, page);  in do_read_cache_page()
3243 	if (!page->mapping) {  in do_read_cache_page()
3283 struct page *read_cache_page(struct address_space *mapping,  in read_cache_page()  argument
3288 	return do_read_cache_page(mapping, index, filler, data,  in read_cache_page()
3289 			mapping_gfp_mask(mapping));  in read_cache_page()
3306 struct page *read_cache_page_gfp(struct address_space *mapping,  in read_cache_page_gfp()  argument
3310 	return do_read_cache_page(mapping, index, NULL, NULL, gfp);  in read_cache_page_gfp()
3314 int pagecache_write_begin(struct file *file, struct address_space *mapping,  in pagecache_write_begin()  argument
3318 	const struct address_space_operations *aops = mapping->a_ops;  in pagecache_write_begin()
3320 	return aops->write_begin(file, mapping, pos, len, flags,  in pagecache_write_begin()
3325 int pagecache_write_end(struct file *file, struct address_space *mapping,  in pagecache_write_end()  argument
3329 	const struct address_space_operations *aops = mapping->a_ops;  in pagecache_write_end()
3331 	return aops->write_end(file, mapping, pos, len, copied, page, fsdata);  in pagecache_write_end()
3360 	struct address_space *mapping = file->f_mapping;  in generic_file_direct_write()  local
3361 	struct inode	*inode = mapping->host;  in generic_file_direct_write()
3376 		written = filemap_write_and_wait_range(mapping, pos,  in generic_file_direct_write()
3388 	written = invalidate_inode_pages2_range(mapping,  in generic_file_direct_write()
3400 	written = mapping->a_ops->direct_IO(iocb, from);  in generic_file_direct_write()
3419 	if (written > 0 && mapping->nrpages &&  in generic_file_direct_write()
3420 	    invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end))  in generic_file_direct_write()
3442 struct page *grab_cache_page_write_begin(struct address_space *mapping,  in grab_cache_page_write_begin()  argument
3451 	page = pagecache_get_page(mapping, index, fgp_flags,  in grab_cache_page_write_begin()
3452 			mapping_gfp_mask(mapping));  in grab_cache_page_write_begin()
3463 	struct address_space *mapping = file->f_mapping;  in generic_perform_write()  local
3464 	const struct address_space_operations *a_ops = mapping->a_ops;  in generic_perform_write()
3501 		status = a_ops->write_begin(file, mapping, pos, bytes, flags,  in generic_perform_write()
3506 		if (mapping_writably_mapped(mapping))  in generic_perform_write()
3512 		status = a_ops->write_end(file, mapping, pos, bytes, copied,  in generic_perform_write()
3537 		balance_dirty_pages_ratelimited(mapping);  in generic_perform_write()
3568 	struct address_space * mapping = file->f_mapping;  in __generic_file_write_iter()  local
3569 	struct inode 	*inode = mapping->host;  in __generic_file_write_iter()
3616 		err = filemap_write_and_wait_range(mapping, pos, endbyte);  in __generic_file_write_iter()
3620 			invalidate_mapping_pages(mapping,  in __generic_file_write_iter()
3690 	struct address_space * const mapping = page->mapping;  in try_to_release_page()  local
3696 	if (mapping && mapping->a_ops->releasepage)  in try_to_release_page()
3697 		return mapping->a_ops->releasepage(page, gfp_mask);  in try_to_release_page()