Lines Matching refs:dio
63 static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap, in iomap_dio_submit_bio() argument
66 atomic_inc(&dio->ref); in iomap_dio_submit_bio()
68 if (dio->iocb->ki_flags & IOCB_HIPRI) in iomap_dio_submit_bio()
69 bio_set_polled(bio, dio->iocb); in iomap_dio_submit_bio()
71 dio->submit.last_queue = bdev_get_queue(iomap->bdev); in iomap_dio_submit_bio()
72 if (dio->dops && dio->dops->submit_io) in iomap_dio_submit_bio()
73 dio->submit.cookie = dio->dops->submit_io( in iomap_dio_submit_bio()
74 file_inode(dio->iocb->ki_filp), in iomap_dio_submit_bio()
77 dio->submit.cookie = submit_bio(bio); in iomap_dio_submit_bio()
80 ssize_t iomap_dio_complete(struct iomap_dio *dio) in iomap_dio_complete() argument
82 const struct iomap_dio_ops *dops = dio->dops; in iomap_dio_complete()
83 struct kiocb *iocb = dio->iocb; in iomap_dio_complete()
86 ssize_t ret = dio->error; in iomap_dio_complete()
89 ret = dops->end_io(iocb, dio->size, ret, dio->flags); in iomap_dio_complete()
92 ret = dio->size; in iomap_dio_complete()
94 if (offset + ret > dio->i_size && in iomap_dio_complete()
95 !(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_complete()
96 ret = dio->i_size - offset; in iomap_dio_complete()
112 if (!dio->error && dio->size && in iomap_dio_complete()
113 (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) { in iomap_dio_complete()
117 (offset + dio->size - 1) >> PAGE_SHIFT); in iomap_dio_complete()
127 if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC)) in iomap_dio_complete()
130 kfree(dio); in iomap_dio_complete()
138 struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work); in iomap_dio_complete_work() local
139 struct kiocb *iocb = dio->iocb; in iomap_dio_complete_work()
141 iocb->ki_complete(iocb, iomap_dio_complete(dio), 0); in iomap_dio_complete_work()
149 static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret) in iomap_dio_set_error() argument
151 cmpxchg(&dio->error, 0, ret); in iomap_dio_set_error()
156 struct iomap_dio *dio = bio->bi_private; in iomap_dio_bio_end_io() local
157 bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY); in iomap_dio_bio_end_io()
160 iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status)); in iomap_dio_bio_end_io()
162 if (atomic_dec_and_test(&dio->ref)) { in iomap_dio_bio_end_io()
163 if (dio->wait_for_completion) { in iomap_dio_bio_end_io()
164 struct task_struct *waiter = dio->submit.waiter; in iomap_dio_bio_end_io()
165 WRITE_ONCE(dio->submit.waiter, NULL); in iomap_dio_bio_end_io()
167 } else if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_bio_end_io()
168 struct inode *inode = file_inode(dio->iocb->ki_filp); in iomap_dio_bio_end_io()
170 INIT_WORK(&dio->aio.work, iomap_dio_complete_work); in iomap_dio_bio_end_io()
171 queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work); in iomap_dio_bio_end_io()
173 iomap_dio_complete_work(&dio->aio.work); in iomap_dio_bio_end_io()
186 iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, in iomap_dio_zero() argument
189 struct inode *inode = file_inode(dio->iocb->ki_filp); in iomap_dio_zero()
199 bio->bi_private = dio; in iomap_dio_zero()
205 iomap_dio_submit_bio(dio, iomap, bio, pos); in iomap_dio_zero()
210 struct iomap_dio *dio, struct iomap *iomap) in iomap_dio_bio_actor() argument
214 unsigned int align = iov_iter_alignment(dio->submit.iter); in iomap_dio_bio_actor()
226 dio->flags |= IOMAP_DIO_UNWRITTEN; in iomap_dio_bio_actor()
231 dio->flags |= IOMAP_DIO_COW; in iomap_dio_bio_actor()
244 (dio->flags & IOMAP_DIO_WRITE_FUA) && in iomap_dio_bio_actor()
254 orig_count = iov_iter_count(dio->submit.iter); in iomap_dio_bio_actor()
255 iov_iter_truncate(dio->submit.iter, length); in iomap_dio_bio_actor()
257 nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES); in iomap_dio_bio_actor()
267 iomap_dio_zero(dio, iomap, pos - pad, pad); in iomap_dio_bio_actor()
272 if (dio->error) { in iomap_dio_bio_actor()
273 iov_iter_revert(dio->submit.iter, copied); in iomap_dio_bio_actor()
283 bio->bi_write_hint = dio->iocb->ki_hint; in iomap_dio_bio_actor()
284 bio->bi_ioprio = dio->iocb->ki_ioprio; in iomap_dio_bio_actor()
285 bio->bi_private = dio; in iomap_dio_bio_actor()
288 ret = bio_iov_iter_get_pages(bio, dio->submit.iter); in iomap_dio_bio_actor()
301 if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_bio_actor()
306 dio->flags &= ~IOMAP_DIO_WRITE_FUA; in iomap_dio_bio_actor()
310 if (dio->flags & IOMAP_DIO_DIRTY) in iomap_dio_bio_actor()
314 dio->size += n; in iomap_dio_bio_actor()
317 nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES); in iomap_dio_bio_actor()
318 iomap_dio_submit_bio(dio, iomap, bio, pos); in iomap_dio_bio_actor()
330 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) { in iomap_dio_bio_actor()
334 iomap_dio_zero(dio, iomap, pos, fs_block_size - pad); in iomap_dio_bio_actor()
338 iov_iter_reexpand(dio->submit.iter, orig_count - copied); in iomap_dio_bio_actor()
345 iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio) in iomap_dio_hole_actor() argument
347 length = iov_iter_zero(length, dio->submit.iter); in iomap_dio_hole_actor()
348 dio->size += length; in iomap_dio_hole_actor()
354 struct iomap_dio *dio, struct iomap *iomap) in iomap_dio_inline_actor() argument
356 struct iov_iter *iter = dio->submit.iter; in iomap_dio_inline_actor()
361 if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_inline_actor()
375 dio->size += copied; in iomap_dio_inline_actor()
383 struct iomap_dio *dio = data; in iomap_dio_actor() local
387 if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_actor()
389 return iomap_dio_hole_actor(length, dio); in iomap_dio_actor()
391 if (!(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_actor()
392 return iomap_dio_hole_actor(length, dio); in iomap_dio_actor()
393 return iomap_dio_bio_actor(inode, pos, length, dio, iomap); in iomap_dio_actor()
395 return iomap_dio_bio_actor(inode, pos, length, dio, iomap); in iomap_dio_actor()
397 return iomap_dio_inline_actor(inode, pos, length, dio, iomap); in iomap_dio_actor()
406 dio->iocb->ki_filp, current->comm); in iomap_dio_actor()
438 struct iomap_dio *dio; in __iomap_dio_rw() local
446 dio = kmalloc(sizeof(*dio), GFP_KERNEL); in __iomap_dio_rw()
447 if (!dio) in __iomap_dio_rw()
450 dio->iocb = iocb; in __iomap_dio_rw()
451 atomic_set(&dio->ref, 1); in __iomap_dio_rw()
452 dio->size = 0; in __iomap_dio_rw()
453 dio->i_size = i_size_read(inode); in __iomap_dio_rw()
454 dio->dops = dops; in __iomap_dio_rw()
455 dio->error = 0; in __iomap_dio_rw()
456 dio->flags = 0; in __iomap_dio_rw()
458 dio->submit.iter = iter; in __iomap_dio_rw()
459 dio->submit.waiter = current; in __iomap_dio_rw()
460 dio->submit.cookie = BLK_QC_T_NONE; in __iomap_dio_rw()
461 dio->submit.last_queue = NULL; in __iomap_dio_rw()
464 if (pos >= dio->i_size) in __iomap_dio_rw()
468 dio->flags |= IOMAP_DIO_DIRTY; in __iomap_dio_rw()
471 dio->flags |= IOMAP_DIO_WRITE; in __iomap_dio_rw()
475 dio->flags |= IOMAP_DIO_NEED_SYNC; in __iomap_dio_rw()
484 dio->flags |= IOMAP_DIO_WRITE_FUA; in __iomap_dio_rw()
523 ret = iomap_apply(inode, pos, count, flags, ops, dio, in __iomap_dio_rw()
535 if (iov_iter_rw(iter) == READ && pos >= dio->i_size) { in __iomap_dio_rw()
541 iov_iter_revert(iter, pos - dio->i_size); in __iomap_dio_rw()
548 iomap_dio_set_error(dio, ret); in __iomap_dio_rw()
554 if (dio->flags & IOMAP_DIO_WRITE_FUA) in __iomap_dio_rw()
555 dio->flags &= ~IOMAP_DIO_NEED_SYNC; in __iomap_dio_rw()
557 WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie); in __iomap_dio_rw()
558 WRITE_ONCE(iocb->private, dio->submit.last_queue); in __iomap_dio_rw()
575 dio->wait_for_completion = wait_for_completion; in __iomap_dio_rw()
576 if (!atomic_dec_and_test(&dio->ref)) { in __iomap_dio_rw()
582 if (!READ_ONCE(dio->submit.waiter)) in __iomap_dio_rw()
586 !dio->submit.last_queue || in __iomap_dio_rw()
587 !blk_poll(dio->submit.last_queue, in __iomap_dio_rw()
588 dio->submit.cookie, true)) in __iomap_dio_rw()
594 return dio; in __iomap_dio_rw()
597 kfree(dio); in __iomap_dio_rw()
609 struct iomap_dio *dio; in iomap_dio_rw() local
611 dio = __iomap_dio_rw(iocb, iter, ops, dops, wait_for_completion); in iomap_dio_rw()
612 if (IS_ERR_OR_NULL(dio)) in iomap_dio_rw()
613 return PTR_ERR_OR_ZERO(dio); in iomap_dio_rw()
614 return iomap_dio_complete(dio); in iomap_dio_rw()