Lines Matching +full:buffered +full:- +full:negative
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1994-1999 Linus Torvalds
28 #include <linux/error-injection.h>
31 #include <linux/backing-dev.h>
68 * finished 'unifying' the page and buffer cache and SMP-threaded the
69 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
71 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
77 * ->i_mmap_rwsem (truncate_pagecache)
78 * ->private_lock (__free_pte->__set_page_dirty_buffers)
79 * ->swap_lock (exclusive_swap_page, others)
80 * ->i_pages lock
82 * ->i_mutex
83 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
85 * ->mmap_lock
86 * ->i_mmap_rwsem
87 * ->page_table_lock or pte_lock (various, mainly in memory.c)
88 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
90 * ->mmap_lock
91 * ->lock_page (access_process_vm)
93 * ->i_mutex (generic_perform_write)
94 * ->mmap_lock (fault_in_pages_readable->do_page_fault)
96 * bdi->wb.list_lock
97 * sb_lock (fs/fs-writeback.c)
98 * ->i_pages lock (__sync_single_inode)
100 * ->i_mmap_rwsem
101 * ->anon_vma.lock (vma_adjust)
103 * ->anon_vma.lock
104 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
106 * ->page_table_lock or pte_lock
107 * ->swap_lock (try_to_unmap_one)
108 * ->private_lock (try_to_unmap_one)
109 * ->i_pages lock (try_to_unmap_one)
110 * ->pgdat->lru_lock (follow_page->mark_page_accessed)
111 * ->pgdat->lru_lock (check_pte_range->isolate_lru_page)
112 * ->private_lock (page_remove_rmap->set_page_dirty)
113 * ->i_pages lock (page_remove_rmap->set_page_dirty)
114 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
115 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
116 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
117 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
118 * ->inode->i_lock (zap_pte_range->set_page_dirty)
119 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
121 * ->i_mmap_rwsem
122 * ->tasklist_lock (memory_failure, collect_procs_ao)
128 XA_STATE(xas, &mapping->i_pages, page->index); in page_cache_delete()
135 xas_set_order(&xas, page->index, compound_order(page)); in page_cache_delete()
146 page->mapping = NULL; in page_cache_delete()
147 /* Leave page->index set: truncation lookup relies upon it */ in page_cache_delete()
150 mapping->nrexceptional += nr; in page_cache_delete()
159 mapping->nrpages -= nr; in page_cache_delete()
183 current->comm, page_to_pfn(page)); in unaccount_page_cache_page()
208 __mod_lruvec_page_state(page, NR_FILE_PAGES, -nr); in unaccount_page_cache_page()
210 __mod_lruvec_page_state(page, NR_SHMEM, -nr); in unaccount_page_cache_page()
229 account_page_cleaned(page, mapping, inode_to_wb(mapping->host)); in unaccount_page_cache_page()
234 * sure the page is locked and that nobody else uses it - or that usage
239 struct address_space *mapping = page->mapping; in __delete_from_page_cache()
252 freepage = mapping->a_ops->freepage; in page_cache_free_page()
265 * delete_from_page_cache - delete page from page cache
278 xa_lock_irqsave(&mapping->i_pages, flags); in delete_from_page_cache()
280 xa_unlock_irqrestore(&mapping->i_pages, flags); in delete_from_page_cache()
287 * page_cache_delete_batch - delete several pages from page cache
291 * The function walks over mapping->i_pages and removes pages passed in @pvec
303 XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index); in page_cache_delete_batch()
323 if (page != pvec->pages[i]) { in page_cache_delete_batch()
324 VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index, in page_cache_delete_batch()
331 if (page->index == xas.xa_index) in page_cache_delete_batch()
332 page->mapping = NULL; in page_cache_delete_batch()
333 /* Leave page->index set: truncation lookup relies on it */ in page_cache_delete_batch()
337 * page or the index is of the last sub-page of this compound in page_cache_delete_batch()
340 if (page->index + compound_nr(page) - 1 == xas.xa_index) in page_cache_delete_batch()
345 mapping->nrpages -= total_pages; in page_cache_delete_batch()
357 xa_lock_irqsave(&mapping->i_pages, flags); in delete_from_page_cache_batch()
359 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); in delete_from_page_cache_batch()
361 unaccount_page_cache_page(mapping, pvec->pages[i]); in delete_from_page_cache_batch()
364 xa_unlock_irqrestore(&mapping->i_pages, flags); in delete_from_page_cache_batch()
367 page_cache_free_page(mapping, pvec->pages[i]); in delete_from_page_cache_batch()
374 if (test_bit(AS_ENOSPC, &mapping->flags) && in filemap_check_errors()
375 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_errors()
376 ret = -ENOSPC; in filemap_check_errors()
377 if (test_bit(AS_EIO, &mapping->flags) && in filemap_check_errors()
378 test_and_clear_bit(AS_EIO, &mapping->flags)) in filemap_check_errors()
379 ret = -EIO; in filemap_check_errors()
387 if (test_bit(AS_EIO, &mapping->flags)) in filemap_check_and_keep_errors()
388 return -EIO; in filemap_check_and_keep_errors()
389 if (test_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_and_keep_errors()
390 return -ENOSPC; in filemap_check_and_keep_errors()
395 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
409 * Return: %0 on success, negative error code otherwise.
426 wbc_attach_fdatawrite_inode(&wbc, mapping->host); in __filemap_fdatawrite_range()
452 * filemap_flush - mostly a non-blocking flush
455 * This is a mostly non-blocking flush. Not suitable for data-integrity
456 * purposes - I/O may not be started against all dirty pages.
458 * Return: %0 on success, negative error code otherwise.
467 * filemap_range_has_page - check if a page exists in range.
482 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); in filemap_range_has_page()
541 * filemap_fdatawait_range - wait for writeback to complete
546 * Walk the list of under-writeback pages of the given address space
565 * filemap_fdatawait_range_keep_errors - wait for writeback to complete
570 * Walk the list of under-writeback pages of the given address space in the
575 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
587 * file_fdatawait_range - wait for writeback to complete
592 * Walk the list of under-writeback pages of the address space that file
594 * status of the address space vs. the file->f_wb_err cursor and return it.
600 * Return: error status of the address space vs. the file->f_wb_err cursor.
604 struct address_space *mapping = file->f_mapping; in file_fdatawait_range()
612 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
615 * Walk the list of under-writeback pages of the given address space
620 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
636 return mapping->nrexceptional; in mapping_needs_writeback()
638 return mapping->nrpages; in mapping_needs_writeback()
642 * filemap_write_and_wait_range - write out & wait on a file range
647 * Write out and wait upon file offsets lstart->lend, inclusive.
650 * that this function can be used to write to the very end-of-file (end = -1).
664 * written partially (e.g. -ENOSPC), so we wait for it. in filemap_write_and_wait_range()
665 * But the -EIO is special case, it may indicate the worst in filemap_write_and_wait_range()
668 if (err != -EIO) { in filemap_write_and_wait_range()
686 errseq_t eseq = errseq_set(&mapping->wb_err, err); in __filemap_set_wb_err()
693 * file_check_and_advance_wb_err - report wb error (if any) that was previously
710 * While we handle mapping->wb_err with atomic operations, the f_wb_err
714 * Return: %0 on success, negative error code otherwise.
719 errseq_t old = READ_ONCE(file->f_wb_err); in file_check_and_advance_wb_err()
720 struct address_space *mapping = file->f_mapping; in file_check_and_advance_wb_err()
723 if (errseq_check(&mapping->wb_err, old)) { in file_check_and_advance_wb_err()
725 spin_lock(&file->f_lock); in file_check_and_advance_wb_err()
726 old = file->f_wb_err; in file_check_and_advance_wb_err()
727 err = errseq_check_and_advance(&mapping->wb_err, in file_check_and_advance_wb_err()
728 &file->f_wb_err); in file_check_and_advance_wb_err()
730 spin_unlock(&file->f_lock); in file_check_and_advance_wb_err()
738 clear_bit(AS_EIO, &mapping->flags); in file_check_and_advance_wb_err()
739 clear_bit(AS_ENOSPC, &mapping->flags); in file_check_and_advance_wb_err()
745 * file_write_and_wait_range - write out & wait on a file range
750 * Write out and wait upon file offsets lstart->lend, inclusive.
753 * that this function can be used to write to the very end-of-file (end = -1).
758 * Return: %0 on success, negative error code otherwise.
763 struct address_space *mapping = file->f_mapping; in file_write_and_wait_range()
769 if (err != -EIO) in file_write_and_wait_range()
780 * replace_page_cache_page - replace a pagecache page with a new one
797 struct address_space *mapping = old->mapping; in replace_page_cache_page()
798 void (*freepage)(struct page *) = mapping->a_ops->freepage; in replace_page_cache_page()
799 pgoff_t offset = old->index; in replace_page_cache_page()
800 XA_STATE(xas, &mapping->i_pages, offset); in replace_page_cache_page()
805 VM_BUG_ON_PAGE(new->mapping, new); in replace_page_cache_page()
808 new->mapping = mapping; in replace_page_cache_page()
809 new->index = offset; in replace_page_cache_page()
816 old->mapping = NULL; in replace_page_cache_page()
840 XA_STATE(xas, &mapping->i_pages, offset); in __add_to_page_cache_locked()
850 page->mapping = mapping; in __add_to_page_cache_locked()
851 page->index = offset; in __add_to_page_cache_locked()
854 error = mem_cgroup_charge(page, current->mm, gfp); in __add_to_page_cache_locked()
873 xas_set_err(&xas, -EEXIST); in __add_to_page_cache_locked()
894 mapping->nrexceptional--; in __add_to_page_cache_locked()
895 mapping->nrpages++; in __add_to_page_cache_locked()
914 page->mapping = NULL; in __add_to_page_cache_locked()
915 /* Leave page->index set: truncation relies upon it */ in __add_to_page_cache_locked()
922 * add_to_page_cache_locked - add a locked page to the pagecache
931 * Return: %0 on success, negative error code otherwise.
1021 * The page wait code treats the "wait->flags" somewhat unusually, because
1068 flags = wait->flags; in wake_page_function()
1070 if (test_bit(key->bit_nr, &key->page->flags)) in wake_page_function()
1071 return -1; in wake_page_function()
1073 if (test_and_set_bit(key->bit_nr, &key->page->flags)) in wake_page_function()
1074 return -1; in wake_page_function()
1080 * We are holding the wait-queue lock, but the waiter that in wake_page_function()
1085 * afterwards to avoid any races. This store-release pairs in wake_page_function()
1086 * with the load-acquire in wait_on_page_bit_common(). in wake_page_function()
1088 smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN); in wake_page_function()
1089 wake_up_state(wait->private, mode); in wake_page_function()
1097 * After this list_del_init(&wait->entry) the wait entry in wake_page_function()
1098 * might be de-allocated and the process might even have in wake_page_function()
1101 list_del_init_careful(&wait->entry); in wake_page_function()
1121 spin_lock_irqsave(&q->lock, flags); in wake_up_page_bit()
1131 spin_unlock_irqrestore(&q->lock, flags); in wake_up_page_bit()
1133 spin_lock_irqsave(&q->lock, flags); in wake_up_page_bit()
1139 * hash, so in that case check for a page match. That prevents a long- in wake_up_page_bit()
1156 spin_unlock_irqrestore(&q->lock, flags); in wake_up_page_bit()
1188 if (wait->flags & WQ_FLAG_EXCLUSIVE) { in trylock_page_bit_common()
1189 if (test_and_set_bit(bit_nr, &page->flags)) in trylock_page_bit_common()
1191 } else if (test_bit(bit_nr, &page->flags)) in trylock_page_bit_common()
1194 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE; in trylock_page_bit_common()
1222 wait->func = wake_page_function; in wait_on_page_bit_common()
1227 wait->flags = 0; in wait_on_page_bit_common()
1229 wait->flags = WQ_FLAG_EXCLUSIVE; in wait_on_page_bit_common()
1230 if (--unfairness < 0) in wait_on_page_bit_common()
1231 wait->flags |= WQ_FLAG_CUSTOM; in wait_on_page_bit_common()
1248 spin_lock_irq(&q->lock); in wait_on_page_bit_common()
1252 spin_unlock_irq(&q->lock); in wait_on_page_bit_common()
1268 * be very careful with the 'wait->flags', because in wait_on_page_bit_common()
1277 flags = smp_load_acquire(&wait->flags); in wait_on_page_bit_common()
1286 /* If we were non-exclusive, we're done */ in wait_on_page_bit_common()
1300 if (unlikely(test_and_set_bit(bit_nr, &page->flags))) in wait_on_page_bit_common()
1303 wait->flags |= WQ_FLAG_DONE; in wait_on_page_bit_common()
1309 * waiter from the wait-queues, but the PageWaiters bit will remain in wait_on_page_bit_common()
1322 * NOTE! The wait->flags weren't stable until we've done the in wait_on_page_bit_common()
1331 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive in wait_on_page_bit_common()
1335 return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; in wait_on_page_bit_common()
1337 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; in wait_on_page_bit_common()
1360 wait->page = page; in __wait_on_page_locked_async()
1361 wait->bit_nr = PG_locked; in __wait_on_page_locked_async()
1363 spin_lock_irq(&q->lock); in __wait_on_page_locked_async()
1364 __add_wait_queue_entry_tail(q, &wait->wait); in __wait_on_page_locked_async()
1377 __remove_wait_queue(q, &wait->wait); in __wait_on_page_locked_async()
1379 ret = -EIOCBQUEUED; in __wait_on_page_locked_async()
1380 spin_unlock_irq(&q->lock); in __wait_on_page_locked_async()
1393 * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
1412 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
1423 spin_lock_irqsave(&q->lock, flags); in add_page_wait_queue()
1426 spin_unlock_irqrestore(&q->lock, flags); in add_page_wait_queue()
1454 * unlock_page - unlock a locked page
1460 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
1463 * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to
1473 if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags)) in unlock_page()
1479 * end_page_writeback - end writeback against a page
1541 * __lock_page - get a lock on the page, assuming we need to sleep to get it
1569 * 1 - page is locked; mmap_lock is still held.
1570 * 0 - page is not locked.
1611 * page_cache_next_miss() - Find the next gap in the page cache.
1616 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1626 * range specified (in which case 'return - index >= max_scan' will be true).
1627 * In the rare case of index wrap-around, 0 will be returned.
1632 XA_STATE(xas, &mapping->i_pages, index); in page_cache_next_miss()
1634 while (max_scan--) { in page_cache_next_miss()
1647 * page_cache_prev_miss() - Find the previous gap in the page cache.
1652 * Search the range [max(index - max_scan + 1, 0), index] for the
1662 * range specified (in which case 'index - return >= max_scan' will be true).
1663 * In the rare case of wrap-around, ULONG_MAX will be returned.
1668 XA_STATE(xas, &mapping->i_pages, index); in page_cache_prev_miss()
1670 while (max_scan--) { in page_cache_prev_miss()
1683 * find_get_entry - find and get a page cache entry
1697 XA_STATE(xas, &mapping->i_pages, index); in find_get_entry()
1732 * find_lock_entry - Locate and lock a page cache entry.
1754 if (unlikely(page->mapping != mapping)) { in find_lock_entry()
1765 * pagecache_get_page - Find and get a reference to a page.
1775 * * %FGP_ACCESSED - The page will be marked accessed.
1776 * * %FGP_LOCK - The page is returned locked.
1777 * * %FGP_HEAD - If the page is present and a THP, return the head page
1779 * * %FGP_CREAT - If no page is present then a new page is allocated using
1782 * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
1785 * * %FGP_WRITE - The page will be written
1786 * * %FGP_NOFS - __GFP_FS will get cleared in gfp mask
1787 * * %FGP_NOWAIT - Don't get blocked by page lock
1822 if (unlikely(page->mapping != mapping)) { in pagecache_get_page()
1863 if (err == -EEXIST) in pagecache_get_page()
1880 * find_get_entries - gang pagecache lookup
1892 * The search returns a group of mapping-contiguous page cache entries
1894 * not-present pages.
1910 XA_STATE(xas, &mapping->i_pages, start); in find_get_entries()
1960 * find_get_pages_range - gang pagecache lookup
1972 * The search returns a group of mapping-contiguous pages with ascending
1973 * indexes. There may be holes in the indices due to not-present pages.
1984 XA_STATE(xas, &mapping->i_pages, *start); in find_get_pages_range()
2021 * breaks the iteration when there is a page at index -1 but that is in find_get_pages_range()
2024 if (end == (pgoff_t)-1) in find_get_pages_range()
2025 *start = (pgoff_t)-1; in find_get_pages_range()
2035 * find_get_pages_contig - gang contiguous pagecache lookup
2049 XA_STATE(xas, &mapping->i_pages, index); in find_get_pages_contig()
2089 * find_get_pages_range_tag - find and return pages in given range matching @tag
2106 XA_STATE(xas, &mapping->i_pages, *index); in find_get_pages_range_tag()
2147 * iteration when there is a page at index -1 but that is already in find_get_pages_range_tag()
2150 if (end == (pgoff_t)-1) in find_get_pages_range_tag()
2151 *index = (pgoff_t)-1; in find_get_pages_range_tag()
2165 * ---R__________________________________________B__________
2178 ra->ra_pages /= 4; in shrink_readahead_size_eio()
2182 * generic_file_buffered_read - generic file read routine
2188 * mapping->a_ops->readpage() function for the actual low-level stuff.
2195 * * negative error code if nothing was copied
2200 struct file *filp = iocb->ki_filp; in generic_file_buffered_read()
2201 struct address_space *mapping = filp->f_mapping; in generic_file_buffered_read()
2202 struct inode *inode = mapping->host; in generic_file_buffered_read()
2203 struct file_ra_state *ra = &filp->f_ra; in generic_file_buffered_read()
2204 loff_t *ppos = &iocb->ki_pos; in generic_file_buffered_read()
2212 if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) in generic_file_buffered_read()
2214 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); in generic_file_buffered_read()
2217 prev_index = ra->prev_pos >> PAGE_SHIFT; in generic_file_buffered_read()
2218 prev_offset = ra->prev_pos & (PAGE_SIZE-1); in generic_file_buffered_read()
2219 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; in generic_file_buffered_read()
2224 * can no longer safely return -EIOCBQUEUED. Hence mark in generic_file_buffered_read()
2227 if (written && (iocb->ki_flags & IOCB_WAITQ)) in generic_file_buffered_read()
2228 iocb->ki_flags |= IOCB_NOWAIT; in generic_file_buffered_read()
2239 error = -EINTR; in generic_file_buffered_read()
2245 if (iocb->ki_flags & IOCB_NOIO) in generic_file_buffered_read()
2249 index, last_index - index); in generic_file_buffered_read()
2255 if (iocb->ki_flags & IOCB_NOIO) { in generic_file_buffered_read()
2261 index, last_index - index); in generic_file_buffered_read()
2269 if (iocb->ki_flags & IOCB_WAITQ) { in generic_file_buffered_read()
2275 iocb->ki_waitq); in generic_file_buffered_read()
2277 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_buffered_read()
2288 if (inode->i_blkbits == PAGE_SHIFT || in generic_file_buffered_read()
2289 !mapping->a_ops->is_partially_uptodate) in generic_file_buffered_read()
2297 if (!page->mapping) in generic_file_buffered_read()
2299 if (!mapping->a_ops->is_partially_uptodate(page, in generic_file_buffered_read()
2300 offset, iter->count)) in generic_file_buffered_read()
2309 * the correct value for "nr", which means the zero-filled in generic_file_buffered_read()
2311 * another truncate extends the file - this is desired though). in generic_file_buffered_read()
2315 end_index = (isize - 1) >> PAGE_SHIFT; in generic_file_buffered_read()
2324 nr = ((isize - 1) & ~PAGE_MASK) + 1; in generic_file_buffered_read()
2330 nr = nr - offset; in generic_file_buffered_read()
2348 * Ok, we have the page, and it's up-to-date, so in generic_file_buffered_read()
2363 error = -EFAULT; in generic_file_buffered_read()
2370 if (iocb->ki_flags & IOCB_WAITQ) { in generic_file_buffered_read()
2375 error = lock_page_async(page, iocb->ki_waitq); in generic_file_buffered_read()
2384 if (!page->mapping) { in generic_file_buffered_read()
2397 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT)) { in generic_file_buffered_read()
2409 error = mapping->a_ops->readpage(filp, page); in generic_file_buffered_read()
2421 if (iocb->ki_flags & IOCB_WAITQ) { in generic_file_buffered_read()
2426 error = lock_page_async(page, iocb->ki_waitq); in generic_file_buffered_read()
2434 if (page->mapping == NULL) { in generic_file_buffered_read()
2444 error = -EIO; in generic_file_buffered_read()
2464 error = -ENOMEM; in generic_file_buffered_read()
2471 if (error == -EEXIST) { in generic_file_buffered_read()
2481 error = -EAGAIN; in generic_file_buffered_read()
2483 ra->prev_pos = prev_index; in generic_file_buffered_read()
2484 ra->prev_pos <<= PAGE_SHIFT; in generic_file_buffered_read()
2485 ra->prev_pos |= prev_offset; in generic_file_buffered_read()
2494 * generic_file_read_iter - generic filesystem read routine
2501 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
2505 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
2507 * can be read, -EAGAIN shall be returned. When readahead would be
2512 * * negative error code (or 0 if IOCB_NOIO) if nothing was read
2523 if (iocb->ki_flags & IOCB_DIRECT) { in generic_file_read_iter()
2524 struct file *file = iocb->ki_filp; in generic_file_read_iter()
2525 struct address_space *mapping = file->f_mapping; in generic_file_read_iter()
2526 struct inode *inode = mapping->host; in generic_file_read_iter()
2530 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_read_iter()
2531 if (filemap_range_has_page(mapping, iocb->ki_pos, in generic_file_read_iter()
2532 iocb->ki_pos + count - 1)) in generic_file_read_iter()
2533 return -EAGAIN; in generic_file_read_iter()
2536 iocb->ki_pos, in generic_file_read_iter()
2537 iocb->ki_pos + count - 1); in generic_file_read_iter()
2544 retval = mapping->a_ops->direct_IO(iocb, iter); in generic_file_read_iter()
2546 iocb->ki_pos += retval; in generic_file_read_iter()
2547 count -= retval; in generic_file_read_iter()
2549 iov_iter_revert(iter, count - iov_iter_count(iter)); in generic_file_read_iter()
2556 * and return. Otherwise fallthrough to buffered io for in generic_file_read_iter()
2557 * the rest of the read. Buffered reads will not work for in generic_file_read_iter()
2560 if (retval < 0 || !count || iocb->ki_pos >= size || in generic_file_read_iter()
2574 * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
2575 * @vmf - the vm_fault for this fault.
2576 * @page - the page to lock.
2577 * @fpin - the pointer to the file we may pin (or is already pinned).
2595 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) in lock_page_maybe_drop_mmap()
2599 if (vmf->flags & FAULT_FLAG_KILLABLE) { in lock_page_maybe_drop_mmap()
2608 mmap_read_unlock(vmf->vma->vm_mm); in lock_page_maybe_drop_mmap()
2626 struct file *file = vmf->vma->vm_file; in do_sync_mmap_readahead()
2627 struct file_ra_state *ra = &file->f_ra; in do_sync_mmap_readahead()
2628 struct address_space *mapping = file->f_mapping; in do_sync_mmap_readahead()
2629 DEFINE_READAHEAD(ractl, file, mapping, vmf->pgoff); in do_sync_mmap_readahead()
2633 /* If we don't want any read-ahead, don't bother */ in do_sync_mmap_readahead()
2634 if (vmf->vma->vm_flags & VM_RAND_READ) in do_sync_mmap_readahead()
2636 if (!ra->ra_pages) in do_sync_mmap_readahead()
2639 if (vmf->vma->vm_flags & VM_SEQ_READ) { in do_sync_mmap_readahead()
2641 page_cache_sync_ra(&ractl, ra, ra->ra_pages); in do_sync_mmap_readahead()
2646 mmap_miss = READ_ONCE(ra->mmap_miss); in do_sync_mmap_readahead()
2648 WRITE_ONCE(ra->mmap_miss, ++mmap_miss); in do_sync_mmap_readahead()
2652 * stop bothering with read-ahead. It will only hurt. in do_sync_mmap_readahead()
2658 * mmap read-around in do_sync_mmap_readahead()
2661 ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); in do_sync_mmap_readahead()
2662 ra->size = ra->ra_pages; in do_sync_mmap_readahead()
2663 ra->async_size = ra->ra_pages / 4; in do_sync_mmap_readahead()
2664 ractl._index = ra->start; in do_sync_mmap_readahead()
2665 do_page_cache_ra(&ractl, ra->size, ra->async_size); in do_sync_mmap_readahead()
2677 struct file *file = vmf->vma->vm_file; in do_async_mmap_readahead()
2678 struct file_ra_state *ra = &file->f_ra; in do_async_mmap_readahead()
2679 struct address_space *mapping = file->f_mapping; in do_async_mmap_readahead()
2682 pgoff_t offset = vmf->pgoff; in do_async_mmap_readahead()
2684 /* If we don't want any read-ahead, don't bother */ in do_async_mmap_readahead()
2685 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) in do_async_mmap_readahead()
2687 mmap_miss = READ_ONCE(ra->mmap_miss); in do_async_mmap_readahead()
2689 WRITE_ONCE(ra->mmap_miss, --mmap_miss); in do_async_mmap_readahead()
2693 page, offset, ra->ra_pages); in do_async_mmap_readahead()
2699 * filemap_fault - read in file data for page fault handling
2711 * Otherwise, vma->vm_mm->mmap_lock must be held on entry.
2721 * Return: bitwise-OR of %VM_FAULT_ codes.
2726 struct file *file = vmf->vma->vm_file; in filemap_fault()
2728 struct address_space *mapping = file->f_mapping; in filemap_fault()
2729 struct file_ra_state *ra = &file->f_ra; in filemap_fault()
2730 struct inode *inode = mapping->host; in filemap_fault()
2731 pgoff_t offset = vmf->pgoff; in filemap_fault()
2737 if (vmf->flags & FAULT_FLAG_SPECULATIVE) { in filemap_fault()
2745 if (unlikely(compound_head(page)->mapping != mapping)) in filemap_fault()
2760 * we'll come back to filemap_fault() non-speculative in filemap_fault()
2765 if (!(vmf->vma->vm_flags & VM_RAND_READ) && ra->ra_pages) { in filemap_fault()
2766 unsigned int mmap_miss = READ_ONCE(ra->mmap_miss); in filemap_fault()
2768 WRITE_ONCE(ra->mmap_miss, --mmap_miss); in filemap_fault()
2771 vmf->page = page; in filemap_fault()
2792 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) { in filemap_fault()
2801 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); in filemap_fault()
2807 vmf->gfp_mask); in filemap_fault()
2819 if (unlikely(compound_head(page)->mapping != mapping)) { in filemap_fault()
2828 * that it's up-to-date. If not, it is going to be due to an error. in filemap_fault()
2835 * time to return to the upper layer and have it re-find the vma and in filemap_fault()
2855 vmf->page = page; in filemap_fault()
2860 * Umm, take care of errors if the page isn't up-to-date. in filemap_fault()
2861 * Try to re-read it _once_. We do this synchronously, in filemap_fault()
2867 error = mapping->a_ops->readpage(file, page); in filemap_fault()
2871 error = -EIO; in filemap_fault()
2886 * re-find the vma and come back and find our hopefully still populated in filemap_fault()
2901 struct mm_struct *mm = vmf->vma->vm_mm; in filemap_map_pmd()
2904 if (pmd_trans_huge(*vmf->pmd)) { in filemap_map_pmd()
2910 if (pmd_none(*vmf->pmd) && PageTransHuge(page)) { in filemap_map_pmd()
2919 if (pmd_none(*vmf->pmd)) { in filemap_map_pmd()
2920 vmf->ptl = pmd_lock(mm, vmf->pmd); in filemap_map_pmd()
2921 if (likely(pmd_none(*vmf->pmd))) { in filemap_map_pmd()
2923 pmd_populate(mm, vmf->pmd, vmf->prealloc_pte); in filemap_map_pmd()
2924 vmf->prealloc_pte = NULL; in filemap_map_pmd()
2926 spin_unlock(vmf->ptl); in filemap_map_pmd()
2930 if (pmd_devmap_trans_unstable(vmf->pmd)) { in filemap_map_pmd()
2965 if (page->mapping != mapping) in next_uptodate_page()
2969 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in next_uptodate_page()
2970 if (xas->xa_index >= max_idx) in next_uptodate_page()
3009 struct vm_area_struct *vma = vmf->vma; in filemap_map_pages()
3010 struct file *file = vma->vm_file; in filemap_map_pages()
3011 struct address_space *mapping = file->f_mapping; in filemap_map_pages()
3014 XA_STATE(xas, &mapping->i_pages, start_pgoff); in filemap_map_pages()
3016 unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); in filemap_map_pages()
3017 vm_fault_t ret = (vmf->flags & FAULT_FLAG_SPECULATIVE) ? in filemap_map_pages()
3025 if (!(vmf->flags & FAULT_FLAG_SPECULATIVE) && in filemap_map_pages()
3031 addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); in filemap_map_pages()
3044 mmap_miss--; in filemap_map_pages()
3046 addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; in filemap_map_pages()
3047 vmf->pte += xas.xa_index - last_pgoff; in filemap_map_pages()
3050 if (!pte_none(*vmf->pte)) in filemap_map_pages()
3054 if (vmf->address == addr) in filemap_map_pages()
3058 /* no need to invalidate: a not-present page won't be cached */ in filemap_map_pages()
3059 update_mmu_cache(vma, addr, vmf->pte); in filemap_map_pages()
3066 pte_unmap_unlock(vmf->pte, vmf->ptl); in filemap_map_pages()
3069 WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); in filemap_map_pages()
3076 struct page *page = vmf->page; in filemap_page_mkwrite()
3077 struct inode *inode = file_inode(vmf->vma->vm_file); in filemap_page_mkwrite()
3080 sb_start_pagefault(inode->i_sb); in filemap_page_mkwrite()
3081 file_update_time(vmf->vma->vm_file); in filemap_page_mkwrite()
3083 if (page->mapping != inode->i_mapping) { in filemap_page_mkwrite()
3096 sb_end_pagefault(inode->i_sb); in filemap_page_mkwrite()
3113 struct address_space *mapping = file->f_mapping; in generic_file_mmap()
3115 if (!mapping->a_ops->readpage) in generic_file_mmap()
3116 return -ENOEXEC; in generic_file_mmap()
3118 vma->vm_ops = &generic_file_vm_ops; in generic_file_mmap()
3123 * This is for filesystems which do not implement ->writepage.
3127 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) in generic_file_readonly_mmap()
3128 return -EINVAL; in generic_file_readonly_mmap()
3138 return -ENOSYS; in generic_file_mmap()
3142 return -ENOSYS; in generic_file_readonly_mmap()
3156 page = ERR_PTR(-EIO); in wait_on_page_read()
3175 return ERR_PTR(-ENOMEM); in do_read_cache_page()
3179 if (err == -EEXIST) in do_read_cache_page()
3189 err = mapping->a_ops->readpage(data, page); in do_read_cache_page()
3243 if (!page->mapping) { in do_read_cache_page()
3270 * read_cache_page - read into page cache, fill it if needed
3279 * If the page does not get brought uptodate, return -EIO.
3294 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
3302 * If the page does not get brought uptodate, return -EIO.
3318 const struct address_space_operations *aops = mapping->a_ops; in pagecache_write_begin()
3320 return aops->write_begin(file, mapping, pos, len, flags, in pagecache_write_begin()
3329 const struct address_space_operations *aops = mapping->a_ops; in pagecache_write_end()
3331 return aops->write_end(file, mapping, pos, len, copied, page, fsdata); in pagecache_write_end()
3345 errseq_set(&inode->i_mapping->wb_err, -EIO); in dio_warn_stale_pagecache()
3350 …alidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n"); in dio_warn_stale_pagecache()
3351 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid, in dio_warn_stale_pagecache()
3352 current->comm); in dio_warn_stale_pagecache()
3359 struct file *file = iocb->ki_filp; in generic_file_direct_write()
3360 struct address_space *mapping = file->f_mapping; in generic_file_direct_write()
3361 struct inode *inode = mapping->host; in generic_file_direct_write()
3362 loff_t pos = iocb->ki_pos; in generic_file_direct_write()
3368 end = (pos + write_len - 1) >> PAGE_SHIFT; in generic_file_direct_write()
3370 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_direct_write()
3372 if (filemap_range_has_page(inode->i_mapping, pos, in generic_file_direct_write()
3373 pos + write_len - 1)) in generic_file_direct_write()
3374 return -EAGAIN; in generic_file_direct_write()
3377 pos + write_len - 1); in generic_file_direct_write()
3383 * After a write we want buffered reads to be sure to go to disk to get in generic_file_direct_write()
3386 * without clobbering -EIOCBQUEUED from ->direct_IO(). in generic_file_direct_write()
3392 * to buffered write. in generic_file_direct_write()
3395 if (written == -EBUSY) in generic_file_direct_write()
3400 written = mapping->a_ops->direct_IO(iocb, from); in generic_file_direct_write()
3404 * cached by non-direct readahead, or faulted in by get_user_pages() in generic_file_direct_write()
3419 if (written > 0 && mapping->nrpages && in generic_file_direct_write()
3425 write_len -= written; in generic_file_direct_write()
3426 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { in generic_file_direct_write()
3430 iocb->ki_pos = pos; in generic_file_direct_write()
3432 iov_iter_revert(from, write_len - iov_iter_count(from)); in generic_file_direct_write()
3440 * page. This function is specifically for buffered writes.
3463 struct address_space *mapping = file->f_mapping; in generic_perform_write()
3464 const struct address_space_operations *a_ops = mapping->a_ops; in generic_perform_write()
3476 offset = (pos & (PAGE_SIZE - 1)); in generic_perform_write()
3477 bytes = min_t(unsigned long, PAGE_SIZE - offset, in generic_perform_write()
3485 * up-to-date. in generic_perform_write()
3492 status = -EFAULT; in generic_perform_write()
3497 status = -EINTR; in generic_perform_write()
3501 status = a_ops->write_begin(file, mapping, pos, bytes, flags, in generic_perform_write()
3512 status = a_ops->write_end(file, mapping, pos, bytes, copied, in generic_perform_write()
3530 bytes = min_t(unsigned long, PAGE_SIZE - offset, in generic_perform_write()
3545 * __generic_file_write_iter - write data to a file
3552 * do direct IO or a standard buffered write.
3563 * * negative error code if no data has been written at all
3567 struct file *file = iocb->ki_filp; in __generic_file_write_iter()
3568 struct address_space * mapping = file->f_mapping; in __generic_file_write_iter()
3569 struct inode *inode = mapping->host; in __generic_file_write_iter()
3575 current->backing_dev_info = inode_to_bdi(inode); in __generic_file_write_iter()
3584 if (iocb->ki_flags & IOCB_DIRECT) { in __generic_file_write_iter()
3590 * buffered writes. Some filesystems do this for writes to in __generic_file_write_iter()
3591 * holes, for example. For DAX files, a buffered write will in __generic_file_write_iter()
3593 * page-cache pages correctly). in __generic_file_write_iter()
3598 status = generic_perform_write(file, from, pos = iocb->ki_pos); in __generic_file_write_iter()
3602 * direct-written, or the error code if that was zero. Note in __generic_file_write_iter()
3603 * that this differs from normal direct-io semantics, which in __generic_file_write_iter()
3604 * will return -EFOO even if some bytes were written. in __generic_file_write_iter()
3615 endbyte = pos + status - 1; in __generic_file_write_iter()
3618 iocb->ki_pos = endbyte + 1; in __generic_file_write_iter()
3626 * the number of bytes which were direct-written in __generic_file_write_iter()
3630 written = generic_perform_write(file, from, iocb->ki_pos); in __generic_file_write_iter()
3632 iocb->ki_pos += written; in __generic_file_write_iter()
3635 current->backing_dev_info = NULL; in __generic_file_write_iter()
3641 * generic_file_write_iter - write data to a file
3649 * * negative error code if no data has been written at all of
3655 struct file *file = iocb->ki_filp; in generic_file_write_iter()
3656 struct inode *inode = file->f_mapping->host; in generic_file_write_iter()
3672 * try_to_release_page() - release old fs-specific metadata on a page
3678 * (presumably at page->private).
3690 struct address_space * const mapping = page->mapping; in try_to_release_page()
3696 if (mapping && mapping->a_ops->releasepage) in try_to_release_page()
3697 return mapping->a_ops->releasepage(page, gfp_mask); in try_to_release_page()