1 // SPDX-License-Identifier: GPL-2.0-only
3  * mm/readahead.c - address_space-level file readahead.
16 #include <linux/backing-dev.h>
23 #include <linux/blk-cgroup.h>
41 	ra->ra_pages = inode_to_bdi(mapping->host)->ra_pages;  in file_ra_state_init()
42 	ra->prev_pos = -1;  in file_ra_state_init()
48  * - the caller of read_cache_pages() may have set PG_private or PG_fscache
50  *   on disk, thus we need to give the fs a chance to clean up in the event of
59 		page->mapping = mapping;  in read_cache_pages_invalidate_page()
61 		page->mapping = NULL;  in read_cache_pages_invalidate_page()
77 		list_del(&victim->lru);  in read_cache_pages_invalidate_pages()
83  * read_cache_pages - populate an address space with some pages & start reads against them
86  *   pages have their ->index populated and are otherwise uninitialised.
102 		list_del(&page->lru);  in read_cache_pages()
103 		if (add_to_page_cache_lru(page, mapping, page->index,  in read_cache_pages()
134 	const struct address_space_operations *aops = rac->mapping->a_ops;  in read_pages()
143 	if (aops->readahead) {  in read_pages()
144 		aops->readahead(rac);  in read_pages()
145 		/* Clean up the remaining pages */  in read_pages()
150 	} else if (aops->readpages) {  in read_pages()
151 		aops->readpages(rac->file, rac->mapping, pages,  in read_pages()
153 		/* Clean up the remaining pages */  in read_pages()
155 		rac->_index += rac->_nr_pages;  in read_pages()
156 		rac->_nr_pages = 0;  in read_pages()
159 			aops->readpage(rac->file, page);  in read_pages()
171 		rac->_index++;  in read_pages()
175  * page_cache_ra_unbounded - Start unchecked readahead.
178  * @lookahead_size: Where to start the next readahead.
180  * This function is for filesystems to call when they want to start
191 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_unbounded()
203 	 * touch file-backed pages, preventing a deadlock.  Most (all?)  in page_cache_ra_unbounded()
213 		struct page *page = xa_load(&mapping->i_pages, index + i);  in page_cache_ra_unbounded()
215 		BUG_ON(index + i != ractl->_index + ractl->_nr_pages);  in page_cache_ra_unbounded()
233 		if (mapping->a_ops->readpages) {  in page_cache_ra_unbounded()
234 			page->index = index + i;  in page_cache_ra_unbounded()
235 			list_add(&page->lru, &page_pool);  in page_cache_ra_unbounded()
242 		if (i == nr_to_read - lookahead_size)  in page_cache_ra_unbounded()
244 		ractl->_nr_pages++;  in page_cache_ra_unbounded()
248 	 * Now start the IO.  We ignore I/O errors - if the page is not  in page_cache_ra_unbounded()
266 	struct inode *inode = ractl->mapping->host;  in do_page_cache_ra()
274 	end_index = (isize - 1) >> PAGE_SHIFT;  in do_page_cache_ra()
278 	if (nr_to_read > end_index - index)  in do_page_cache_ra()
279 		nr_to_read = end_index - index + 1;  in do_page_cache_ra()
291 	struct address_space *mapping = ractl->mapping;  in force_page_cache_ra()
292 	struct backing_dev_info *bdi = inode_to_bdi(mapping->host);  in force_page_cache_ra()
298 	if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages &&  in force_page_cache_ra()
299 			!mapping->a_ops->readahead))  in force_page_cache_ra()
304 	 * be up to the optimal hardware IO size  in force_page_cache_ra()
307 	max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages);  in force_page_cache_ra()
310 	 * or decryption process to ping-pong, lookahead is forcibly enabled.  in force_page_cache_ra()
312 	if (nr_to_read > max_pages && fscrypt_inode_uses_fs_layer_crypto(mapping->host))  in force_page_cache_ra()
321 		ractl->_index = index;  in force_page_cache_ra()
332 		nr_to_read -= this_chunk;  in force_page_cache_ra()
340  * 1-8 page = 32k initial, > 8 page = 128k initial
357  *  Get the previous window size, ramp it up, and
363 	unsigned long cur = ra->size;  in get_next_ra_size()
373  * On-demand readahead design.
375  * The fields in struct file_ra_state represent the most-recently-executed
378  *                        |<----- async_size ---------|
379  *     |------------------- size -------------------->|
381  *     ^start             ^page marked with PG_readahead
392  * page at (start+size-async_size) with PG_readahead, and use it as readahead
394  * readahead-for-nothing fuss, saving pointless page cache lookups.
402  * There is a special-case: if the first page which the application tries to
407  * The code ramps up the readahead size aggressively at first, but slow down as
412  * Count contiguously cached pages from @index-1 to @index-@max,
414  * 	- length of the sequential read sequence, or
415  * 	- thrashing threshold in memory tight systems
423 	head = page_cache_prev_miss(mapping, index - 1, max);  in count_history_pages()
426 	return index - 1 - head;  in count_history_pages()
430  * page cache context based read-ahead
451 	 * it is a strong indication of long-run stream (or whole-file-read)  in try_context_readahead()
456 	ra->start = index;  in try_context_readahead()
457 	ra->size = min(size + req_size, max);  in try_context_readahead()
458 	ra->async_size = 1;  in try_context_readahead()
470 	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);  in ondemand_readahead()
471 	unsigned long max_pages = ra->ra_pages;  in ondemand_readahead()
478 	 * be up to the optimal hardware IO size  in ondemand_readahead()
480 	if (req_size > max_pages && bdi->io_pages > max_pages)  in ondemand_readahead()
481 		max_pages = min(req_size, bdi->io_pages);  in ondemand_readahead()
486 	 * start of file  in ondemand_readahead()
493 	 * Ramp up sizes, and push forward the readahead window.  in ondemand_readahead()
495 	if ((index == (ra->start + ra->size - ra->async_size) ||  in ondemand_readahead()
496 	     index == (ra->start + ra->size))) {  in ondemand_readahead()
497 		ra->start += ra->size;  in ondemand_readahead()
498 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
499 		ra->async_size = ra->size;  in ondemand_readahead()
507 	 * readahead size. Ramp it up and use it as the new readahead size.  in ondemand_readahead()
510 		pgoff_t start;  in ondemand_readahead()  local
513 		start = page_cache_next_miss(ractl->mapping, index + 1,  in ondemand_readahead()
517 		if (!start || start - index > max_pages)  in ondemand_readahead()
520 		ra->start = start;  in ondemand_readahead()
521 		ra->size = start - index;	/* old async_size */  in ondemand_readahead()
522 		ra->size += req_size;  in ondemand_readahead()
523 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
524 		ra->async_size = ra->size;  in ondemand_readahead()
536 	 * trivial case: (index - prev_index) == 1  in ondemand_readahead()
537 	 * unaligned reads: (index - prev_index) == 0  in ondemand_readahead()
539 	prev_index = (unsigned long long)ra->prev_pos >> PAGE_SHIFT;  in ondemand_readahead()
540 	if (index - prev_index <= 1UL)  in ondemand_readahead()
547 	if (try_context_readahead(ractl->mapping, ra, index, req_size,  in ondemand_readahead()
559 	ra->start = index;  in ondemand_readahead()
560 	ra->size = get_init_ra_size(req_size, max_pages);  in ondemand_readahead()
561 	ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;  in ondemand_readahead()
570 	if (index == ra->start && ra->size == ra->async_size) {  in ondemand_readahead()
572 		if (ra->size + add_pages <= max_pages) {  in ondemand_readahead()
573 			ra->async_size = add_pages;  in ondemand_readahead()
574 			ra->size += add_pages;  in ondemand_readahead()
576 			ra->size = max_pages;  in ondemand_readahead()
577 			ra->async_size = max_pages >> 1;  in ondemand_readahead()
581 	ractl->_index = ra->start;  in ondemand_readahead()
582 	do_page_cache_ra(ractl, ra->size, ra->async_size);  in ondemand_readahead()
588 	bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM);  in page_cache_sync_ra()
591 	 * Even if read-ahead is disabled, issue this request as read-ahead  in page_cache_sync_ra()
593 	 * read-ahead will do the right thing and limit the read to just the  in page_cache_sync_ra()
596 	if (!ra->ra_pages || blk_cgroup_congested()) {  in page_cache_sync_ra()
597 		if (!ractl->file)  in page_cache_sync_ra()
609 	/* do read-ahead */  in page_cache_sync_ra()
618 	/* no read-ahead */  in page_cache_async_ra()
619 	if (!ra->ra_pages)  in page_cache_async_ra()
631 	 * Defer asynchronous read-ahead on IO congestion.  in page_cache_async_ra()
633 	if (inode_read_congested(ractl->mapping->host))  in page_cache_async_ra()
639 	/* do read-ahead */  in page_cache_async_ra()
649 	ret = -EBADF;  in ksys_readahead()
651 	if (!f.file || !(f.file->f_mode & FMODE_READ))  in ksys_readahead()
657 	 * on this file, then we must return -EINVAL.  in ksys_readahead()
659 	ret = -EINVAL;  in ksys_readahead()
660 	if (!f.file->f_mapping || !f.file->f_mapping->a_ops ||  in ksys_readahead()
661 	    !S_ISREG(file_inode(f.file)->i_mode))  in ksys_readahead()