39 static bool ext4_dio_supported(struct kiocb *iocb, struct iov_iter *iter)  in ext4_dio_supported()  argument
41 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_supported()
43 	if (!fscrypt_dio_supported(iocb, iter))  in ext4_dio_supported()
54 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dio_read_iter()  argument
57 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_read_iter()
59 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_read_iter()
66 	if (!ext4_dio_supported(iocb, to)) {  in ext4_dio_read_iter()
75 		iocb->ki_flags &= ~IOCB_DIRECT;  in ext4_dio_read_iter()
76 		return generic_file_read_iter(iocb, to);  in ext4_dio_read_iter()
79 	ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL,  in ext4_dio_read_iter()
80 			   is_sync_kiocb(iocb));  in ext4_dio_read_iter()
83 	file_accessed(iocb->ki_filp);  in ext4_dio_read_iter()
88 static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dax_read_iter()  argument
90 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_read_iter()
93 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_read_iter()
106 		return generic_file_read_iter(iocb, to);  in ext4_dax_read_iter()
108 	ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);  in ext4_dax_read_iter()
111 	file_accessed(iocb->ki_filp);  in ext4_dax_read_iter()
116 static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_file_read_iter()  argument
118 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_read_iter()
128 		return ext4_dax_read_iter(iocb, to);  in ext4_file_read_iter()
130 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_read_iter()
131 		return ext4_dio_read_iter(iocb, to);  in ext4_file_read_iter()
133 	return generic_file_read_iter(iocb, to);  in ext4_file_read_iter()
214 static ssize_t ext4_generic_write_checks(struct kiocb *iocb,  in ext4_generic_write_checks()  argument
217 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_generic_write_checks()
223 	ret = generic_write_checks(iocb, from);  in ext4_generic_write_checks()
234 		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)  in ext4_generic_write_checks()
236 		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);  in ext4_generic_write_checks()
242 static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)  in ext4_write_checks()  argument
246 	count = ext4_generic_write_checks(iocb, from);  in ext4_write_checks()
250 	ret = file_modified(iocb->ki_filp);  in ext4_write_checks()
256 static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,  in ext4_buffered_write_iter()  argument
260 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_buffered_write_iter()
262 	if (iocb->ki_flags & IOCB_NOWAIT)  in ext4_buffered_write_iter()
267 	ret = ext4_write_checks(iocb, from);  in ext4_buffered_write_iter()
272 	ret = generic_perform_write(iocb->ki_filp, from, iocb->ki_pos);  in ext4_buffered_write_iter()
279 		iocb->ki_pos += ret;  in ext4_buffered_write_iter()
280 		ret = generic_write_sync(iocb, ret);  in ext4_buffered_write_iter()
374 static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,  in ext4_dio_write_end_io()  argument
377 	loff_t pos = iocb->ki_pos;  in ext4_dio_write_end_io()
378 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_end_io()
425 static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,  in ext4_dio_write_checks()  argument
428 	struct file *file = iocb->ki_filp;  in ext4_dio_write_checks()
435 	ret = ext4_generic_write_checks(iocb, from);  in ext4_dio_write_checks()
439 	offset = iocb->ki_pos;  in ext4_dio_write_checks()
451 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
474 static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dio_write_iter()  argument
478 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_iter()
479 	loff_t offset = iocb->ki_pos;  in ext4_dio_write_iter()
501 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_iter()
517 	if (!ext4_dio_supported(iocb, from)) {  in ext4_dio_write_iter()
522 		return ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
525 	ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend);  in ext4_dio_write_iter()
530 	if ((iocb->ki_flags & IOCB_NOWAIT) && (unaligned_io || extend)) {  in ext4_dio_write_iter()
541 	offset = iocb->ki_pos;  in ext4_dio_write_iter()
577 	ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,  in ext4_dio_write_iter()
578 			   is_sync_kiocb(iocb) || unaligned_io || extend);  in ext4_dio_write_iter()
595 		offset = iocb->ki_pos;  in ext4_dio_write_iter()
596 		err = ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
609 		err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
612 			invalidate_mapping_pages(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
622 ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dax_write_iter()  argument
629 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_write_iter()
631 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_write_iter()
638 	ret = ext4_write_checks(iocb, from);  in ext4_dax_write_iter()
642 	offset = iocb->ki_pos;  in ext4_dax_write_iter()
662 	ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);  in ext4_dax_write_iter()
669 		ret = generic_write_sync(iocb, ret);  in ext4_dax_write_iter()
675 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_file_write_iter()  argument
677 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_write_iter()
684 		return ext4_dax_write_iter(iocb, from);  in ext4_file_write_iter()
686 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_write_iter()
687 		return ext4_dio_write_iter(iocb, from);  in ext4_file_write_iter()
689 		return ext4_buffered_write_iter(iocb, from);  in ext4_file_write_iter()