337 	struct blkdev_dio *dio = bio->bi_private;  in blkdev_bio_end_io()  local
338 	bool should_dirty = dio->should_dirty;  in blkdev_bio_end_io()
340 	if (bio->bi_status && !dio->bio.bi_status)  in blkdev_bio_end_io()
341 		dio->bio.bi_status = bio->bi_status;  in blkdev_bio_end_io()
343 	if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) {  in blkdev_bio_end_io()
344 		if (!dio->is_sync) {  in blkdev_bio_end_io()
345 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()
348 			if (likely(!dio->bio.bi_status)) {  in blkdev_bio_end_io()
349 				ret = dio->size;  in blkdev_bio_end_io()
352 				ret = blk_status_to_errno(dio->bio.bi_status);  in blkdev_bio_end_io()
355 			dio->iocb->ki_complete(iocb, ret, 0);  in blkdev_bio_end_io()
356 			if (dio->multi_bio)  in blkdev_bio_end_io()
357 				bio_put(&dio->bio);  in blkdev_bio_end_io()
359 			struct task_struct *waiter = dio->waiter;  in blkdev_bio_end_io()
361 			WRITE_ONCE(dio->waiter, NULL);  in blkdev_bio_end_io()
381 	struct blkdev_dio *dio;  in __blkdev_direct_IO()  local
395 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO()
396 	dio->is_sync = is_sync = is_sync_kiocb(iocb);  in __blkdev_direct_IO()
397 	if (dio->is_sync) {  in __blkdev_direct_IO()
398 		dio->waiter = current;  in __blkdev_direct_IO()
401 		dio->iocb = iocb;  in __blkdev_direct_IO()
404 	dio->size = 0;  in __blkdev_direct_IO()
405 	dio->multi_bio = false;  in __blkdev_direct_IO()
406 	dio->should_dirty = is_read && iter_is_iovec(iter);  in __blkdev_direct_IO()
419 		bio->bi_private = dio;  in __blkdev_direct_IO()
432 			if (dio->should_dirty)  in __blkdev_direct_IO()
441 		dio->size += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
460 		if (!dio->multi_bio) {  in __blkdev_direct_IO()
468 			dio->multi_bio = true;  in __blkdev_direct_IO()
469 			atomic_set(&dio->ref, 2);  in __blkdev_direct_IO()
471 			atomic_inc(&dio->ref);  in __blkdev_direct_IO()
486 		if (!READ_ONCE(dio->waiter))  in __blkdev_direct_IO()
496 		ret = blk_status_to_errno(dio->bio.bi_status);  in __blkdev_direct_IO()
498 		ret = dio->size;  in __blkdev_direct_IO()
500 	bio_put(&dio->bio);  in __blkdev_direct_IO()