Lines Matching refs:dio
119 struct dio { struct
170 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
174 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
177 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
184 if (dio->page_errors == 0) in dio_refill_pages()
185 dio->page_errors = ret; in dio_refill_pages()
187 dio->pages[0] = page; in dio_refill_pages()
212 static inline struct page *dio_get_page(struct dio *dio, in dio_get_page() argument
218 ret = dio_refill_pages(dio, sdio); in dio_get_page()
223 return dio->pages[sdio->head]; in dio_get_page()
237 static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) in dio_complete() argument
239 loff_t offset = dio->iocb->ki_pos; in dio_complete()
252 if (dio->result) { in dio_complete()
253 transferred = dio->result; in dio_complete()
256 if ((dio->op == REQ_OP_READ) && in dio_complete()
257 ((offset + transferred) > dio->i_size)) in dio_complete()
258 transferred = dio->i_size - offset; in dio_complete()
265 ret = dio->page_errors; in dio_complete()
267 ret = dio->io_error; in dio_complete()
271 if (dio->end_io) { in dio_complete()
273 err = dio->end_io(dio->iocb, offset, ret, dio->private); in dio_complete()
291 ret > 0 && dio->op == REQ_OP_WRITE && in dio_complete()
292 dio->inode->i_mapping->nrpages) { in dio_complete()
293 err = invalidate_inode_pages2_range(dio->inode->i_mapping, in dio_complete()
297 dio_warn_stale_pagecache(dio->iocb->ki_filp); in dio_complete()
300 inode_dio_end(dio->inode); in dio_complete()
308 dio->iocb->ki_pos += transferred; in dio_complete()
310 if (ret > 0 && dio->op == REQ_OP_WRITE) in dio_complete()
311 ret = generic_write_sync(dio->iocb, ret); in dio_complete()
312 dio->iocb->ki_complete(dio->iocb, ret, 0); in dio_complete()
315 kmem_cache_free(dio_cache, dio); in dio_complete()
321 struct dio *dio = container_of(work, struct dio, complete_work); in dio_aio_complete_work() local
323 dio_complete(dio, 0, DIO_COMPLETE_ASYNC | DIO_COMPLETE_INVALIDATE); in dio_aio_complete_work()
326 static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio);
333 struct dio *dio = bio->bi_private; in dio_bio_end_aio() local
339 dio_bio_complete(dio, bio); in dio_bio_end_aio()
341 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_aio()
342 remaining = --dio->refcount; in dio_bio_end_aio()
343 if (remaining == 1 && dio->waiter) in dio_bio_end_aio()
344 wake_up_process(dio->waiter); in dio_bio_end_aio()
345 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_aio()
356 if (dio->result) in dio_bio_end_aio()
357 defer_completion = dio->defer_completion || in dio_bio_end_aio()
358 (dio->op == REQ_OP_WRITE && in dio_bio_end_aio()
359 dio->inode->i_mapping->nrpages); in dio_bio_end_aio()
361 INIT_WORK(&dio->complete_work, dio_aio_complete_work); in dio_bio_end_aio()
362 queue_work(dio->inode->i_sb->s_dio_done_wq, in dio_bio_end_aio()
363 &dio->complete_work); in dio_bio_end_aio()
365 dio_complete(dio, 0, DIO_COMPLETE_ASYNC); in dio_bio_end_aio()
379 struct dio *dio = bio->bi_private; in dio_bio_end_io() local
382 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_io()
383 bio->bi_private = dio->bio_list; in dio_bio_end_io()
384 dio->bio_list = bio; in dio_bio_end_io()
385 if (--dio->refcount == 1 && dio->waiter) in dio_bio_end_io()
386 wake_up_process(dio->waiter); in dio_bio_end_io()
387 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_io()
391 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
396 struct inode *inode = dio->inode; in dio_bio_alloc()
409 bio_set_op_attrs(bio, dio->op, dio->op_flags); in dio_bio_alloc()
410 if (dio->is_async) in dio_bio_alloc()
415 bio->bi_write_hint = dio->iocb->ki_hint; in dio_bio_alloc()
428 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
433 bio->bi_private = dio; in dio_bio_submit()
435 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_submit()
436 dio->refcount++; in dio_bio_submit()
437 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_submit()
439 if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) in dio_bio_submit()
442 dio->bio_disk = bio->bi_disk; in dio_bio_submit()
445 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
446 dio->bio_cookie = BLK_QC_T_NONE; in dio_bio_submit()
448 dio->bio_cookie = submit_bio(bio); in dio_bio_submit()
458 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
461 put_page(dio->pages[sdio->head++]); in dio_cleanup()
470 static struct bio *dio_await_one(struct dio *dio) in dio_await_one() argument
475 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
483 while (dio->refcount > 1 && dio->bio_list == NULL) { in dio_await_one()
485 dio->waiter = current; in dio_await_one()
486 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
487 if (!(dio->iocb->ki_flags & IOCB_HIPRI) || in dio_await_one()
488 !blk_poll(dio->bio_disk->queue, dio->bio_cookie, true)) in dio_await_one()
491 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
492 dio->waiter = NULL; in dio_await_one()
494 if (dio->bio_list) { in dio_await_one()
495 bio = dio->bio_list; in dio_await_one()
496 dio->bio_list = bio->bi_private; in dio_await_one()
498 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
505 static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio) in dio_bio_complete() argument
508 bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty; in dio_bio_complete()
512 dio->io_error = -EAGAIN; in dio_bio_complete()
514 dio->io_error = -EIO; in dio_bio_complete()
517 if (dio->is_async && should_dirty) { in dio_bio_complete()
533 static void dio_await_completion(struct dio *dio) in dio_await_completion() argument
537 bio = dio_await_one(dio); in dio_await_completion()
539 dio_bio_complete(dio, bio); in dio_await_completion()
550 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
555 while (dio->bio_list) { in dio_bio_reap()
560 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_reap()
561 bio = dio->bio_list; in dio_bio_reap()
562 dio->bio_list = bio->bi_private; in dio_bio_reap()
563 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_reap()
564 ret2 = blk_status_to_errno(dio_bio_complete(dio, bio)); in dio_bio_reap()
597 static int dio_set_defer_completion(struct dio *dio) in dio_set_defer_completion() argument
599 struct super_block *sb = dio->inode->i_sb; in dio_set_defer_completion()
601 if (dio->defer_completion) in dio_set_defer_completion()
603 dio->defer_completion = true; in dio_set_defer_completion()
632 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
647 ret = dio->page_errors; in get_more_blocks()
669 create = dio->op == REQ_OP_WRITE; in get_more_blocks()
670 if (dio->flags & DIO_SKIP_HOLES) { in get_more_blocks()
671 i_size = i_size_read(dio->inode); in get_more_blocks()
676 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
680 dio->private = map_bh->b_private; in get_more_blocks()
683 ret = dio_set_defer_completion(dio); in get_more_blocks()
691 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
697 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
703 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
748 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
780 !fscrypt_mergeable_bio(sdio->bio, dio->inode, in dio_send_cur_page()
781 cur_offset >> dio->inode->i_blkbits)) in dio_send_cur_page()
782 dio_bio_submit(dio, sdio); in dio_send_cur_page()
786 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
792 dio_bio_submit(dio, sdio); in dio_send_cur_page()
793 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
821 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
828 if (dio->op == REQ_OP_WRITE) { in submit_page_section()
850 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
869 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
871 dio_bio_submit(dio, sdio); in submit_page_section()
887 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
915 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
938 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
949 page = dio_get_page(dio, sdio); in do_direct_IO()
970 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
1016 if (dio->op == REQ_OP_WRITE) { in do_direct_IO()
1025 i_size_aligned = ALIGN(i_size_read(dio->inode), in do_direct_IO()
1036 dio->result += 1 << blkbits; in do_direct_IO()
1046 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1064 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1077 dio->result += this_chunk_bytes; in do_direct_IO()
1092 static inline int drop_refcount(struct dio *dio) in drop_refcount() argument
1108 spin_lock_irqsave(&dio->bio_lock, flags); in drop_refcount()
1109 ret2 = --dio->refcount; in drop_refcount()
1110 spin_unlock_irqrestore(&dio->bio_lock, flags); in drop_refcount()
1152 struct dio *dio; in do_blockdev_direct_IO() local
1167 dio = kmem_cache_alloc(dio_cache, GFP_KERNEL); in do_blockdev_direct_IO()
1168 if (!dio) in do_blockdev_direct_IO()
1175 memset(dio, 0, offsetof(struct dio, pages)); in do_blockdev_direct_IO()
1177 dio->flags = flags; in do_blockdev_direct_IO()
1178 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in do_blockdev_direct_IO()
1184 dio->i_size = i_size_read(inode); in do_blockdev_direct_IO()
1185 if (iov_iter_rw(iter) == READ && offset >= dio->i_size) { in do_blockdev_direct_IO()
1198 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in do_blockdev_direct_IO()
1213 dio->is_async = false; in do_blockdev_direct_IO()
1215 dio->is_async = false; in do_blockdev_direct_IO()
1217 dio->is_async = true; in do_blockdev_direct_IO()
1219 dio->inode = inode; in do_blockdev_direct_IO()
1221 dio->op = REQ_OP_WRITE; in do_blockdev_direct_IO()
1222 dio->op_flags = REQ_SYNC | REQ_IDLE; in do_blockdev_direct_IO()
1224 dio->op_flags |= REQ_NOWAIT; in do_blockdev_direct_IO()
1226 dio->op = REQ_OP_READ; in do_blockdev_direct_IO()
1229 dio->op_flags |= REQ_HIPRI; in do_blockdev_direct_IO()
1235 if (dio->is_async && iov_iter_rw(iter) == WRITE) { in do_blockdev_direct_IO()
1238 retval = dio_set_defer_completion(dio); in do_blockdev_direct_IO()
1239 else if (!dio->inode->i_sb->s_dio_done_wq) { in do_blockdev_direct_IO()
1245 retval = sb_init_dio_done_wq(dio->inode->i_sb); in do_blockdev_direct_IO()
1262 dio->end_io = end_io; in do_blockdev_direct_IO()
1267 dio->iocb = iocb; in do_blockdev_direct_IO()
1269 spin_lock_init(&dio->bio_lock); in do_blockdev_direct_IO()
1270 dio->refcount = 1; in do_blockdev_direct_IO()
1272 dio->should_dirty = iter_is_iovec(iter) && iov_iter_rw(iter) == READ; in do_blockdev_direct_IO()
1287 retval = do_direct_IO(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1289 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1302 dio_zero_block(dio, &sdio, 1, &map_bh); in do_blockdev_direct_IO()
1307 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1314 dio_bio_submit(dio, &sdio); in do_blockdev_direct_IO()
1322 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1329 if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING)) in do_blockdev_direct_IO()
1330 inode_unlock(dio->inode); in do_blockdev_direct_IO()
1340 if (dio->is_async && retval == 0 && dio->result && in do_blockdev_direct_IO()
1341 (iov_iter_rw(iter) == READ || dio->result == count)) in do_blockdev_direct_IO()
1344 dio_await_completion(dio); in do_blockdev_direct_IO()
1346 if (drop_refcount(dio) == 0) { in do_blockdev_direct_IO()
1347 retval = dio_complete(dio, retval, DIO_COMPLETE_INVALIDATE); in do_blockdev_direct_IO()
1354 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) in do_blockdev_direct_IO()
1357 kmem_cache_free(dio_cache, dio); in do_blockdev_direct_IO()
1387 dio_cache = KMEM_CACHE(dio, SLAB_PANIC); in dio_init()