Lines Matching +full:sub +full:- +full:sampled
1 // SPDX-License-Identifier: GPL-2.0-only
3 * fs/direct-io.c
16 * added support for non-aligned IO.
60 * is determined on a per-invocation basis. When talking to the filesystem
62 * down by dio->blkfactor. Similarly, fs-blocksize quantities are converted
77 finer. blkfactor=2 means 1/4-block
79 unsigned start_zero_done; /* flag: sub-blocksize zeroing has
164 return sdio->tail - sdio->head; in dio_pages_present()
174 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
175 &sdio->from); in dio_refill_pages()
177 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
184 if (dio->page_errors == 0) in dio_refill_pages()
185 dio->page_errors = ret; in dio_refill_pages()
187 dio->pages[0] = page; in dio_refill_pages()
188 sdio->head = 0; in dio_refill_pages()
189 sdio->tail = 1; in dio_refill_pages()
190 sdio->from = 0; in dio_refill_pages()
191 sdio->to = PAGE_SIZE; in dio_refill_pages()
196 iov_iter_advance(sdio->iter, ret); in dio_refill_pages()
197 ret += sdio->from; in dio_refill_pages()
198 sdio->head = 0; in dio_refill_pages()
199 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; in dio_refill_pages()
200 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; in dio_refill_pages()
223 return dio->pages[sdio->head]; in dio_get_page()
227 * dio_complete() - called when all DIO BIO I/O has been completed
239 loff_t offset = dio->iocb->ki_pos; in dio_complete()
246 * In that case -EIOCBQUEUED is in fact not an error we want in dio_complete()
249 if (ret == -EIOCBQUEUED) in dio_complete()
252 if (dio->result) { in dio_complete()
253 transferred = dio->result; in dio_complete()
256 if ((dio->op == REQ_OP_READ) && in dio_complete()
257 ((offset + transferred) > dio->i_size)) in dio_complete()
258 transferred = dio->i_size - offset; in dio_complete()
260 if (unlikely(ret == -EFAULT) && transferred) in dio_complete()
265 ret = dio->page_errors; in dio_complete()
267 ret = dio->io_error; in dio_complete()
271 if (dio->end_io) { in dio_complete()
273 err = dio->end_io(dio->iocb, offset, ret, dio->private); in dio_complete()
280 * non-direct readahead, or faulted in by get_user_pages() if the source in dio_complete()
285 * And this page cache invalidation has to be after dio->end_io(), as in dio_complete()
291 ret > 0 && dio->op == REQ_OP_WRITE && in dio_complete()
292 dio->inode->i_mapping->nrpages) { in dio_complete()
293 err = invalidate_inode_pages2_range(dio->inode->i_mapping, in dio_complete()
295 (offset + ret - 1) >> PAGE_SHIFT); in dio_complete()
297 dio_warn_stale_pagecache(dio->iocb->ki_filp); in dio_complete()
300 inode_dio_end(dio->inode); in dio_complete()
308 dio->iocb->ki_pos += transferred; in dio_complete()
310 if (ret > 0 && dio->op == REQ_OP_WRITE) in dio_complete()
311 ret = generic_write_sync(dio->iocb, ret); in dio_complete()
312 dio->iocb->ki_complete(dio->iocb, ret, 0); in dio_complete()
333 struct dio *dio = bio->bi_private; in dio_bio_end_aio()
341 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_aio()
342 remaining = --dio->refcount; in dio_bio_end_aio()
343 if (remaining == 1 && dio->waiter) in dio_bio_end_aio()
344 wake_up_process(dio->waiter); in dio_bio_end_aio()
345 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_aio()
356 if (dio->result) in dio_bio_end_aio()
357 defer_completion = dio->defer_completion || in dio_bio_end_aio()
358 (dio->op == REQ_OP_WRITE && in dio_bio_end_aio()
359 dio->inode->i_mapping->nrpages); in dio_bio_end_aio()
361 INIT_WORK(&dio->complete_work, dio_aio_complete_work); in dio_bio_end_aio()
362 queue_work(dio->inode->i_sb->s_dio_done_wq, in dio_bio_end_aio()
363 &dio->complete_work); in dio_bio_end_aio()
371 * The BIO completion handler simply queues the BIO up for the process-context
375 * implement a singly-linked list of completed BIOs, at dio->bio_list.
379 struct dio *dio = bio->bi_private; in dio_bio_end_io()
382 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_io()
383 bio->bi_private = dio->bio_list; in dio_bio_end_io()
384 dio->bio_list = bio; in dio_bio_end_io()
385 if (--dio->refcount == 1 && dio->waiter) in dio_bio_end_io()
386 wake_up_process(dio->waiter); in dio_bio_end_io()
387 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_io()
396 struct inode *inode = dio->inode; in dio_bio_alloc()
405 sdio->cur_page_fs_offset >> inode->i_blkbits, in dio_bio_alloc()
408 bio->bi_iter.bi_sector = first_sector; in dio_bio_alloc()
409 bio_set_op_attrs(bio, dio->op, dio->op_flags); in dio_bio_alloc()
410 if (dio->is_async) in dio_bio_alloc()
411 bio->bi_end_io = dio_bio_end_aio; in dio_bio_alloc()
413 bio->bi_end_io = dio_bio_end_io; in dio_bio_alloc()
415 bio->bi_write_hint = dio->iocb->ki_hint; in dio_bio_alloc()
417 sdio->bio = bio; in dio_bio_alloc()
418 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; in dio_bio_alloc()
426 * bios hold a dio reference between submit_bio and ->end_io.
430 struct bio *bio = sdio->bio; in dio_bio_submit()
433 bio->bi_private = dio; in dio_bio_submit()
435 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_submit()
436 dio->refcount++; in dio_bio_submit()
437 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_submit()
439 if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) in dio_bio_submit()
442 dio->bio_disk = bio->bi_disk; in dio_bio_submit()
444 if (sdio->submit_io) { in dio_bio_submit()
445 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
446 dio->bio_cookie = BLK_QC_T_NONE; in dio_bio_submit()
448 dio->bio_cookie = submit_bio(bio); in dio_bio_submit()
450 sdio->bio = NULL; in dio_bio_submit()
451 sdio->boundary = 0; in dio_bio_submit()
452 sdio->logical_offset_in_bio = 0; in dio_bio_submit()
460 while (sdio->head < sdio->tail) in dio_cleanup()
461 put_page(dio->pages[sdio->head++]); in dio_cleanup()
467 * all bios have been issued so that dio->refcount can only decrease. This
475 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
483 while (dio->refcount > 1 && dio->bio_list == NULL) { in dio_await_one()
485 dio->waiter = current; in dio_await_one()
486 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
487 if (!(dio->iocb->ki_flags & IOCB_HIPRI) || in dio_await_one()
488 !blk_poll(dio->bio_disk->queue, dio->bio_cookie, true)) in dio_await_one()
491 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
492 dio->waiter = NULL; in dio_await_one()
494 if (dio->bio_list) { in dio_await_one()
495 bio = dio->bio_list; in dio_await_one()
496 dio->bio_list = bio->bi_private; in dio_await_one()
498 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
507 blk_status_t err = bio->bi_status; in dio_bio_complete()
508 bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty; in dio_bio_complete()
511 if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT)) in dio_bio_complete()
512 dio->io_error = -EAGAIN; in dio_bio_complete()
514 dio->io_error = -EIO; in dio_bio_complete()
517 if (dio->is_async && should_dirty) { in dio_bio_complete()
527 * Wait on and process all in-flight BIOs. This must only be called once
530 * errors are propagated through dio->io_error and should be propagated via
554 if (sdio->reap_counter++ >= 64) { in dio_bio_reap()
555 while (dio->bio_list) { in dio_bio_reap()
560 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_reap()
561 bio = dio->bio_list; in dio_bio_reap()
562 dio->bio_list = bio->bi_private; in dio_bio_reap()
563 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_reap()
568 sdio->reap_counter = 0; in dio_bio_reap()
584 sb->s_id); in sb_init_dio_done_wq()
586 return -ENOMEM; in sb_init_dio_done_wq()
590 old = cmpxchg(&sb->s_dio_done_wq, NULL, wq); in sb_init_dio_done_wq()
599 struct super_block *sb = dio->inode->i_sb; in dio_set_defer_completion()
601 if (dio->defer_completion) in dio_set_defer_completion()
603 dio->defer_completion = true; in dio_set_defer_completion()
604 if (!sb->s_dio_done_wq) in dio_set_defer_completion()
611 * of available blocks at sdio->blocks_available. These are in units of the
615 * it uses the passed inode-relative block number as the file offset, as usual.
617 * get_block() is passed the number of i_blkbits-sized blocks which direct_io
620 * If the fs has mapped a lot of blocks, it should populate bh->b_size to
622 * bh->b_blocknr.
627 * In the case of filesystem holes: the fs may return an arbitrarily-large
629 * buffer_mapped(). However the direct-io code will only process holes one
630 * block at a time - it will repeatedly call get_block() as it walks the hole.
636 sector_t fs_startblk; /* Into file, in filesystem-sized blocks */ in get_more_blocks()
637 sector_t fs_endblk; /* Into file, in filesystem-sized blocks */ in get_more_blocks()
638 unsigned long fs_count; /* Number of filesystem-sized blocks */ in get_more_blocks()
640 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; in get_more_blocks()
647 ret = dio->page_errors; in get_more_blocks()
649 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); in get_more_blocks()
650 fs_startblk = sdio->block_in_file >> sdio->blkfactor; in get_more_blocks()
651 fs_endblk = (sdio->final_block_in_request - 1) >> in get_more_blocks()
652 sdio->blkfactor; in get_more_blocks()
653 fs_count = fs_endblk - fs_startblk + 1; in get_more_blocks()
655 map_bh->b_state = 0; in get_more_blocks()
656 map_bh->b_size = fs_count << i_blkbits; in get_more_blocks()
669 create = dio->op == REQ_OP_WRITE; in get_more_blocks()
670 if (dio->flags & DIO_SKIP_HOLES) { in get_more_blocks()
671 i_size = i_size_read(dio->inode); in get_more_blocks()
672 if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits) in get_more_blocks()
676 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
680 dio->private = map_bh->b_private; in get_more_blocks()
700 sector = start_sector << (sdio->blkbits - 9); in dio_new_bio()
701 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); in dio_new_bio()
703 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
704 sdio->boundary = 0; in dio_new_bio()
712 * the just-added page.
714 * Return zero on success. Non-zero means the caller needs to start a new BIO.
720 ret = bio_add_page(sdio->bio, sdio->cur_page, in dio_bio_add_page()
721 sdio->cur_page_len, sdio->cur_page_offset); in dio_bio_add_page()
722 if (ret == sdio->cur_page_len) { in dio_bio_add_page()
726 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) in dio_bio_add_page()
727 sdio->pages_in_io--; in dio_bio_add_page()
728 get_page(sdio->cur_page); in dio_bio_add_page()
729 sdio->final_block_in_bio = sdio->cur_page_block + in dio_bio_add_page()
730 (sdio->cur_page_len >> sdio->blkbits); in dio_bio_add_page()
741 * starts on-disk at cur_page_block.
753 if (sdio->bio) { in dio_send_cur_page()
754 loff_t cur_offset = sdio->cur_page_fs_offset; in dio_send_cur_page()
755 loff_t bio_next_offset = sdio->logical_offset_in_bio + in dio_send_cur_page()
756 sdio->bio->bi_iter.bi_size; in dio_send_cur_page()
761 * Btrfs cannot handle having logically non-contiguous requests in dio_send_cur_page()
764 * Logical: [0-4095][HOLE][8192-12287] in dio_send_cur_page()
765 * Physical: [0-4095] [4096-8191] in dio_send_cur_page()
778 if (sdio->final_block_in_bio != sdio->cur_page_block || in dio_send_cur_page()
780 !fscrypt_mergeable_bio(sdio->bio, dio->inode, in dio_send_cur_page()
781 cur_offset >> dio->inode->i_blkbits)) in dio_send_cur_page()
785 if (sdio->bio == NULL) { in dio_send_cur_page()
786 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
793 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
811 * The chunk of page starts on-disk at blocknr.
813 * We perform deferred IO, by recording the last-submitted page inside our
826 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ in submit_page_section()
828 if (dio->op == REQ_OP_WRITE) { in submit_page_section()
838 if (sdio->cur_page == page && in submit_page_section()
839 sdio->cur_page_offset + sdio->cur_page_len == offset && in submit_page_section()
840 sdio->cur_page_block + in submit_page_section()
841 (sdio->cur_page_len >> sdio->blkbits) == blocknr) { in submit_page_section()
842 sdio->cur_page_len += len; in submit_page_section()
849 if (sdio->cur_page) { in submit_page_section()
851 put_page(sdio->cur_page); in submit_page_section()
852 sdio->cur_page = NULL; in submit_page_section()
858 sdio->cur_page = page; in submit_page_section()
859 sdio->cur_page_offset = offset; in submit_page_section()
860 sdio->cur_page_len = len; in submit_page_section()
861 sdio->cur_page_block = blocknr; in submit_page_section()
862 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; in submit_page_section()
870 if (sdio->bio) in submit_page_section()
872 put_page(sdio->cur_page); in submit_page_section()
873 sdio->cur_page = NULL; in submit_page_section()
880 * the block for us, we need to fill-in the unused portion of the
881 * block with zeros. This happens only if user-buffer, fileoffset or
882 * io length is not filesystem block-size multiple.
895 sdio->start_zero_done = 1; in dio_zero_block()
896 if (!sdio->blkfactor || !buffer_new(map_bh)) in dio_zero_block()
899 dio_blocks_per_fs_block = 1 << sdio->blkfactor; in dio_zero_block()
900 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); in dio_zero_block()
910 this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks; in dio_zero_block()
912 this_chunk_bytes = this_chunk_blocks << sdio->blkbits; in dio_zero_block()
916 sdio->next_block_for_io, map_bh)) in dio_zero_block()
919 sdio->next_block_for_io += this_chunk_blocks; in dio_zero_block()
928 * happily perform page-sized but 512-byte aligned IOs. It is important that
931 * So what we do is to permit the ->get_block function to populate bh.b_size
934 * For best results, the blockdev should be set up with 512-byte i_blkbits and
941 const unsigned blkbits = sdio->blkbits; in do_direct_IO()
942 const unsigned i_blkbits = blkbits + sdio->blkfactor; in do_direct_IO()
945 while (sdio->block_in_file < sdio->final_block_in_request) { in do_direct_IO()
954 from = sdio->head ? 0 : sdio->from; in do_direct_IO()
955 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; in do_direct_IO()
956 sdio->head++; in do_direct_IO()
963 if (sdio->blocks_available == 0) { in do_direct_IO()
978 sdio->blocks_available = in do_direct_IO()
979 map_bh->b_size >> blkbits; in do_direct_IO()
980 sdio->next_block_for_io = in do_direct_IO()
981 map_bh->b_blocknr << sdio->blkfactor; in do_direct_IO()
984 map_bh->b_bdev, in do_direct_IO()
985 map_bh->b_blocknr, in do_direct_IO()
986 map_bh->b_size >> i_blkbits); in do_direct_IO()
989 if (!sdio->blkfactor) in do_direct_IO()
992 blkmask = (1 << sdio->blkfactor) - 1; in do_direct_IO()
993 dio_remainder = (sdio->block_in_file & blkmask); in do_direct_IO()
997 * starts partway into a fs-block, in do_direct_IO()
998 * dio_remainder will be non-zero. If the IO in do_direct_IO()
1004 * on-disk in do_direct_IO()
1007 sdio->next_block_for_io += dio_remainder; in do_direct_IO()
1008 sdio->blocks_available -= dio_remainder; in do_direct_IO()
1015 /* AKPM: eargh, -ENOTBLK is a hack */ in do_direct_IO()
1016 if (dio->op == REQ_OP_WRITE) { in do_direct_IO()
1018 return -ENOTBLK; in do_direct_IO()
1025 i_size_aligned = ALIGN(i_size_read(dio->inode), in do_direct_IO()
1027 if (sdio->block_in_file >= in do_direct_IO()
1034 sdio->block_in_file++; in do_direct_IO()
1036 dio->result += 1 << blkbits; in do_direct_IO()
1045 if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) in do_direct_IO()
1052 this_chunk_blocks = sdio->blocks_available; in do_direct_IO()
1053 u = (to - from) >> blkbits; in do_direct_IO()
1056 u = sdio->final_block_in_request - sdio->block_in_file; in do_direct_IO()
1062 if (this_chunk_blocks == sdio->blocks_available) in do_direct_IO()
1063 sdio->boundary = buffer_boundary(map_bh); in do_direct_IO()
1067 sdio->next_block_for_io, in do_direct_IO()
1073 sdio->next_block_for_io += this_chunk_blocks; in do_direct_IO()
1075 sdio->block_in_file += this_chunk_blocks; in do_direct_IO()
1077 dio->result += this_chunk_bytes; in do_direct_IO()
1078 sdio->blocks_available -= this_chunk_blocks; in do_direct_IO()
1080 BUG_ON(sdio->block_in_file > sdio->final_block_in_request); in do_direct_IO()
1081 if (sdio->block_in_file == sdio->final_block_in_request) in do_direct_IO()
1102 * return code that the caller will hand to ->complete(). in drop_refcount()
1108 spin_lock_irqsave(&dio->bio_lock, flags); in drop_refcount()
1109 ret2 = --dio->refcount; in drop_refcount()
1110 spin_unlock_irqrestore(&dio->bio_lock, flags); in drop_refcount()
1118 * - if the flags value contains DIO_LOCKING we use a fancy locking
1123 * - if the flags value does NOT contain DIO_LOCKING we don't use any
1145 unsigned i_blkbits = READ_ONCE(inode->i_blkbits); in do_blockdev_direct_IO()
1147 unsigned blocksize_mask = (1 << blkbits) - 1; in do_blockdev_direct_IO()
1148 ssize_t retval = -EINVAL; in do_blockdev_direct_IO()
1150 loff_t offset = iocb->ki_pos; in do_blockdev_direct_IO()
1169 return -ENOMEM; in do_blockdev_direct_IO()
1177 dio->flags = flags; in do_blockdev_direct_IO()
1178 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in do_blockdev_direct_IO()
1183 /* Once we sampled i_size check for reads beyond EOF */ in do_blockdev_direct_IO()
1184 dio->i_size = i_size_read(inode); in do_blockdev_direct_IO()
1185 if (iov_iter_rw(iter) == READ && offset >= dio->i_size) { in do_blockdev_direct_IO()
1193 blocksize_mask = (1 << blkbits) - 1; in do_blockdev_direct_IO()
1198 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in do_blockdev_direct_IO()
1199 struct address_space *mapping = iocb->ki_filp->f_mapping; in do_blockdev_direct_IO()
1201 retval = filemap_write_and_wait_range(mapping, offset, end - 1); in do_blockdev_direct_IO()
1213 dio->is_async = false; in do_blockdev_direct_IO()
1215 dio->is_async = false; in do_blockdev_direct_IO()
1217 dio->is_async = true; in do_blockdev_direct_IO()
1219 dio->inode = inode; in do_blockdev_direct_IO()
1221 dio->op = REQ_OP_WRITE; in do_blockdev_direct_IO()
1222 dio->op_flags = REQ_SYNC | REQ_IDLE; in do_blockdev_direct_IO()
1223 if (iocb->ki_flags & IOCB_NOWAIT) in do_blockdev_direct_IO()
1224 dio->op_flags |= REQ_NOWAIT; in do_blockdev_direct_IO()
1226 dio->op = REQ_OP_READ; in do_blockdev_direct_IO()
1228 if (iocb->ki_flags & IOCB_HIPRI) in do_blockdev_direct_IO()
1229 dio->op_flags |= REQ_HIPRI; in do_blockdev_direct_IO()
1233 * so that we can call ->fsync. in do_blockdev_direct_IO()
1235 if (dio->is_async && iov_iter_rw(iter) == WRITE) { in do_blockdev_direct_IO()
1237 if (iocb->ki_flags & IOCB_DSYNC) in do_blockdev_direct_IO()
1239 else if (!dio->inode->i_sb->s_dio_done_wq) { in do_blockdev_direct_IO()
1245 retval = sb_init_dio_done_wq(dio->inode->i_sb); in do_blockdev_direct_IO()
1258 sdio.blkfactor = i_blkbits - blkbits; in do_blockdev_direct_IO()
1262 dio->end_io = end_io; in do_blockdev_direct_IO()
1264 sdio.final_block_in_bio = -1; in do_blockdev_direct_IO()
1265 sdio.next_block_for_io = -1; in do_blockdev_direct_IO()
1267 dio->iocb = iocb; in do_blockdev_direct_IO()
1269 spin_lock_init(&dio->bio_lock); in do_blockdev_direct_IO()
1270 dio->refcount = 1; in do_blockdev_direct_IO()
1272 dio->should_dirty = iter_is_iovec(iter) && iov_iter_rw(iter) == READ; in do_blockdev_direct_IO()
1277 * In case of non-aligned buffers, we may need 2 more in do_blockdev_direct_IO()
1291 if (retval == -ENOTBLK) { in do_blockdev_direct_IO()
1299 * There may be some unwritten disk at the end of a part-written in do_blockdev_direct_IO()
1300 * fs-block-sized block. Go zero that now. in do_blockdev_direct_IO()
1329 if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING)) in do_blockdev_direct_IO()
1330 inode_unlock(dio->inode); in do_blockdev_direct_IO()
1336 * call aio_complete is when we return -EIOCBQUEUED, so we key on that. in do_blockdev_direct_IO()
1337 * This had *better* be the only place that raises -EIOCBQUEUED. in do_blockdev_direct_IO()
1339 BUG_ON(retval == -EIOCBQUEUED); in do_blockdev_direct_IO()
1340 if (dio->is_async && retval == 0 && dio->result && in do_blockdev_direct_IO()
1341 (iov_iter_rw(iter) == READ || dio->result == count)) in do_blockdev_direct_IO()
1342 retval = -EIOCBQUEUED; in do_blockdev_direct_IO()
1349 BUG_ON(retval != -EIOCBQUEUED); in do_blockdev_direct_IO()
1354 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) in do_blockdev_direct_IO()
1375 prefetch(&bdev->bd_disk->part_tbl); in __blockdev_direct_IO()
1376 prefetch(bdev->bd_disk->queue); in __blockdev_direct_IO()
1377 prefetch((char *)bdev->bd_disk->queue + SMP_CACHE_BYTES); in __blockdev_direct_IO()