570 	struct kiocb			kiocb;  member
2520 		struct kiocb *kiocb = &req->rw.kiocb;  in io_do_iopoll()  local
2535 		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);  in io_do_iopoll()
2697 	if (req->rw.kiocb.ki_flags & IOCB_WRITE) {  in __io_complete_rw_common()
2755 static void io_complete_rw(struct kiocb *kiocb, long res, long res2)  in io_complete_rw()  argument
2757 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw()
2766 static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)  in io_complete_rw_iopoll()  argument
2768 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw_iopoll()
2770 	if (kiocb->ki_flags & IOCB_WRITE)  in io_complete_rw_iopoll()
2817 			queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);  in io_iopoll_req_issued()
2818 			queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);  in io_iopoll_req_issued()
2905 	struct kiocb *kiocb = &req->rw.kiocb;  in io_prep_rw()  local
2913 	kiocb->ki_pos = READ_ONCE(sqe->off);  in io_prep_rw()
2914 	if (kiocb->ki_pos == -1) {  in io_prep_rw()
2917 			kiocb->ki_pos = file->f_pos;  in io_prep_rw()
2919 			kiocb->ki_pos = 0;  in io_prep_rw()
2922 	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));  in io_prep_rw()
2923 	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);  in io_prep_rw()
2924 	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));  in io_prep_rw()
2933 	if ((kiocb->ki_flags & IOCB_NOWAIT) ||  in io_prep_rw()
2943 		kiocb->ki_ioprio = ioprio;  in io_prep_rw()
2945 		kiocb->ki_ioprio = get_current_ioprio();  in io_prep_rw()
2948 		if (!(kiocb->ki_flags & IOCB_DIRECT) ||  in io_prep_rw()
2949 		    !kiocb->ki_filp->f_op->iopoll)  in io_prep_rw()
2952 		kiocb->ki_flags |= IOCB_HIPRI;  in io_prep_rw()
2953 		kiocb->ki_complete = io_complete_rw_iopoll;  in io_prep_rw()
2956 		if (kiocb->ki_flags & IOCB_HIPRI)  in io_prep_rw()
2958 		kiocb->ki_complete = io_complete_rw;  in io_prep_rw()
2982 static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)  in io_rw_done()  argument
2999 		kiocb->ki_complete(kiocb, ret, 0);  in io_rw_done()
3003 static void kiocb_done(struct kiocb *kiocb, ssize_t ret,  in kiocb_done()  argument
3006 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in kiocb_done()
3009 		req->file->f_pos = kiocb->ki_pos;  in kiocb_done()
3010 	if (ret >= 0 && (kiocb->ki_complete == io_complete_rw))  in kiocb_done()
3013 		io_rw_done(kiocb, ret);  in kiocb_done()
3281 static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)  in io_kiocb_ppos()  argument
3283 	return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;  in io_kiocb_ppos()
3292 	struct kiocb *kiocb = &req->rw.kiocb;  in loop_rw_iter()  local
3301 	if (kiocb->ki_flags & IOCB_HIPRI)  in loop_rw_iter()
3303 	if (kiocb->ki_flags & IOCB_NOWAIT)  in loop_rw_iter()
3319 					      iovec.iov_len, io_kiocb_ppos(kiocb));  in loop_rw_iter()
3322 					       iovec.iov_len, io_kiocb_ppos(kiocb));  in loop_rw_iter()
3449 	req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;  in io_async_buf_func()
3471 	struct kiocb *kiocb = &req->rw.kiocb;  in io_rw_should_retry()  local
3478 	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))  in io_rw_should_retry()
3492 	kiocb->ki_flags |= IOCB_WAITQ;  in io_rw_should_retry()
3493 	kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_rw_should_retry()
3494 	kiocb->ki_waitq = wait;  in io_rw_should_retry()
3501 		return call_read_iter(req->file, &req->rw.kiocb, iter);  in io_iter_do_read()
3517 	struct kiocb *kiocb = &req->rw.kiocb;  in io_read()  local
3545 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_read()
3547 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_read()
3555 	ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result);  in io_read()
3616 			kiocb->ki_flags &= ~IOCB_WAITQ;  in io_read()
3631 		kiocb->ki_flags &= ~IOCB_WAITQ;  in io_read()
3635 	kiocb_done(kiocb, ret, issue_flags);  in io_read()
3653 	struct kiocb *kiocb = &req->rw.kiocb;  in io_write()  local
3676 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_write()
3678 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_write()
3685 	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&  in io_write()
3689 	ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result);  in io_write()
3705 	kiocb->ki_flags |= IOCB_WRITE;  in io_write()
3708 		ret2 = call_write_iter(req->file, kiocb, iter);  in io_write()
3723 	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))  in io_write()
3733 		kiocb_done(kiocb, ret2, issue_flags);  in io_write()
3739 			if (kiocb->ki_flags & IOCB_WRITE)  in io_write()