518 	struct io_kiocb			*req;  member
909 	struct io_kiocb		*req;  member
1147 #define req_ref_zero_or_close_to_overflow(req)	\  argument
1150 static inline bool req_ref_inc_not_zero(struct io_kiocb *req)  in req_ref_inc_not_zero()
1156 static inline bool req_ref_put_and_test(struct io_kiocb *req)  in req_ref_put_and_test()
1165 static inline void req_ref_get(struct io_kiocb *req)  in req_ref_get()
1172 static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)  in __io_req_set_refcount()
1180 static inline void io_req_set_refcount(struct io_kiocb *req)  in io_req_set_refcount()
1185 static inline void io_req_set_rsrc_node(struct io_kiocb *req)  in io_req_set_rsrc_node()
1211 	struct io_kiocb *req;  in io_match_task()  local
1227 	struct io_kiocb *req;  in io_match_linked()  local
1263 static inline void req_set_fail(struct io_kiocb *req)  in req_set_fail()
1268 static inline void req_fail_link_node(struct io_kiocb *req, int res)  in req_fail_link_node()
1281 static inline bool io_is_timeout_noseq(struct io_kiocb *req)  in io_is_timeout_noseq()
1291 	struct io_kiocb *req, *tmp;  in io_fallback_req_func()  local
1381 static bool req_need_defer(struct io_kiocb *req, u32 seq)  in req_need_defer()
1401 static inline bool io_req_ffs_set(struct io_kiocb *req)  in io_req_ffs_set()
1406 static void io_req_track_inflight(struct io_kiocb *req)  in io_req_track_inflight()
1414 static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)  in __io_prep_linked_timeout()
1428 static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)  in io_prep_linked_timeout()
1435 static void io_prep_async_work(struct io_kiocb *req)  in io_prep_async_work()
1459 static void io_prep_async_link(struct io_kiocb *req)  in io_prep_async_link()
1476 static void io_queue_async_work(struct io_kiocb *req, bool *locked)  in io_queue_async_work()
1508 static void io_kill_timeout(struct io_kiocb *req, int status)  in io_kill_timeout()
1543 	struct io_kiocb *req, *tmp;  in io_flush_timeouts()  local
1821 static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)  in io_fill_cqe_req()
1833 static void io_req_complete_post(struct io_kiocb *req, s32 res,  in io_req_complete_post()
1870 static inline bool io_req_needs_clean(struct io_kiocb *req)  in io_req_needs_clean()
1875 static inline void io_req_complete_state(struct io_kiocb *req, s32 res,  in io_req_complete_state()
1885 static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,  in __io_req_complete()
1894 static inline void io_req_complete(struct io_kiocb *req, s32 res)  in io_req_complete()
1899 static void io_req_complete_failed(struct io_kiocb *req, s32 res)  in io_req_complete_failed()
1905 static void io_req_complete_fail_submit(struct io_kiocb *req)  in io_req_complete_fail_submit()
1920 static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)  in io_preinit_req()
1954 		struct io_kiocb *req = list_first_entry(&state->free_list,  in io_flush_cached_reqs()  local
2013 static void io_dismantle_req(struct io_kiocb *req)  in io_dismantle_req()
2029 static void __io_free_req(struct io_kiocb *req)  in __io_free_req()
2044 static inline void io_remove_next_linked(struct io_kiocb *req)  in io_remove_next_linked()
2052 static bool io_kill_linked_timeout(struct io_kiocb *req)  in io_kill_linked_timeout()
2073 static void io_fail_links(struct io_kiocb *req)  in io_fail_links()
2095 static bool io_disarm_next(struct io_kiocb *req)  in io_disarm_next()
2125 static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)  in __io_req_find_next()
2152 static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)  in io_req_find_next()
2196 			struct io_kiocb *req = container_of(node, struct io_kiocb,  in tctx_task_work()  local
2220 static void io_req_task_work_add(struct io_kiocb *req)  in io_req_task_work_add()
2269 static void io_req_task_cancel(struct io_kiocb *req, bool *locked)  in io_req_task_cancel()
2278 static void io_req_task_submit(struct io_kiocb *req, bool *locked)  in io_req_task_submit()
2290 static void io_req_task_queue_fail(struct io_kiocb *req, int ret)  in io_req_task_queue_fail()
2297 static void io_req_task_queue(struct io_kiocb *req)  in io_req_task_queue()
2303 static void io_req_task_queue_reissue(struct io_kiocb *req)  in io_req_task_queue_reissue()
2309 static inline void io_queue_next(struct io_kiocb *req)  in io_queue_next()
2317 static void io_free_req(struct io_kiocb *req)  in io_free_req()
2323 static void io_free_req_work(struct io_kiocb *req, bool *locked)  in io_free_req_work()
2350 static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,  in io_req_free_batch()
2380 		struct io_kiocb *req = state->compl_reqs[i];  in io_submit_flush_completions()  local
2391 		struct io_kiocb *req = state->compl_reqs[i];  in io_submit_flush_completions()  local
2405 static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)  in io_put_req_find_next()
2416 static inline void io_put_req(struct io_kiocb *req)  in io_put_req()
2422 static inline void io_put_req_deferred(struct io_kiocb *req)  in io_put_req_deferred()
2445 static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)  in io_put_kbuf()
2456 static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)  in io_put_rw_kbuf()
2484 	struct io_kiocb *req;  in io_iopoll_complete()  local
2509 	struct io_kiocb *req, *tmp;  in io_do_iopoll()  local
2634 static void kiocb_end_write(struct io_kiocb *req)  in kiocb_end_write()
2649 static bool io_resubmit_prep(struct io_kiocb *req)  in io_resubmit_prep()
2659 static bool io_rw_should_reissue(struct io_kiocb *req)  in io_rw_should_reissue()
2685 static bool io_resubmit_prep(struct io_kiocb *req)  in io_resubmit_prep()
2689 static bool io_rw_should_reissue(struct io_kiocb *req)  in io_rw_should_reissue()
2695 static bool __io_complete_rw_common(struct io_kiocb *req, long res)  in __io_complete_rw_common()
2715 static inline int io_fixup_rw_res(struct io_kiocb *req, unsigned res)  in io_fixup_rw_res()
2729 static void io_req_task_complete(struct io_kiocb *req, bool *locked)  in io_req_task_complete()
2747 static void __io_complete_rw(struct io_kiocb *req, long res, long res2,  in __io_complete_rw()
2757 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw()  local
2768 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw_iopoll()  local
2791 static void io_iopoll_req_issued(struct io_kiocb *req)  in io_iopoll_req_issued()
2891 static bool io_file_supports_nowait(struct io_kiocb *req, int rw)  in io_file_supports_nowait()
2901 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,  in io_prep_rw()
3006 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in kiocb_done()  local
3036 static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,  in __io_import_fixed()
3094 static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)  in io_import_fixed()
3119 static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,  in io_buffer_select()
3153 static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,  in io_rw_buffer_select()
3170 static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,  in io_compat_import()
3196 static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,  in __io_iov_buffer_select()
3217 static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,  in io_iov_buffer_select()
3239 static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,  in io_import_iovec()
3290 static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)  in loop_rw_iter()
3346 static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,  in io_req_map_rw()
3373 static inline int io_alloc_async_data(struct io_kiocb *req)  in io_alloc_async_data()
3380 static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,  in io_setup_async_rw()
3402 static inline int io_rw_prep_async(struct io_kiocb *req, int rw)  in io_rw_prep_async()
3420 static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_read_prep()
3441 	struct io_kiocb *req = wait->private;  in io_async_buf_func()  local
3467 static bool io_rw_should_retry(struct io_kiocb *req)  in io_rw_should_retry()
3498 static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)  in io_iter_do_read()
3508 static bool need_read_all(struct io_kiocb *req)  in need_read_all()
3514 static int io_read(struct io_kiocb *req, unsigned int issue_flags)  in io_read()
3643 static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_write_prep()
3650 static int io_write(struct io_kiocb *req, unsigned int issue_flags)  in io_write()
3752 static int io_renameat_prep(struct io_kiocb *req,  in io_renameat_prep()
3785 static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)  in io_renameat()
3803 static int io_unlinkat_prep(struct io_kiocb *req,  in io_unlinkat_prep()
3832 static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)  in io_unlinkat()
3852 static int io_shutdown_prep(struct io_kiocb *req,  in io_shutdown_prep()
3869 static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)  in io_shutdown()
3892 static int __io_splice_prep(struct io_kiocb *req,  in __io_splice_prep()
3909 static int io_tee_prep(struct io_kiocb *req,  in io_tee_prep()
3917 static int io_tee(struct io_kiocb *req, unsigned int issue_flags)  in io_tee()
3947 static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_splice_prep()
3956 static int io_splice(struct io_kiocb *req, unsigned int issue_flags)  in io_splice()
3993 static int io_nop(struct io_kiocb *req, unsigned int issue_flags)  in io_nop()
4004 static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_fsync_prep()
4023 static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)  in io_fsync()
4041 static int io_fallocate_prep(struct io_kiocb *req,  in io_fallocate_prep()
4056 static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)  in io_fallocate()
4073 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in __io_openat_prep()
4107 static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_openat_prep()
4116 static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_openat2_prep()
4135 static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)  in io_openat2()
4201 static int io_openat(struct io_kiocb *req, unsigned int issue_flags)  in io_openat()
4206 static int io_remove_buffers_prep(struct io_kiocb *req,  in io_remove_buffers_prep()
4253 static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)  in io_remove_buffers()
4278 static int io_provide_buffers_prep(struct io_kiocb *req,  in io_provide_buffers_prep()
4341 static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)  in io_provide_buffers()
4370 static int io_epoll_ctl_prep(struct io_kiocb *req,  in io_epoll_ctl_prep()
4397 static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)  in io_epoll_ctl()
4417 static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_madvise_prep()
4434 static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)  in io_madvise()
4453 static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_fadvise_prep()
4466 static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)  in io_fadvise()
4489 static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_statx_prep()
4507 static int io_statx(struct io_kiocb *req, unsigned int issue_flags)  in io_statx()
4524 static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_close_prep()
4542 static int io_close(struct io_kiocb *req, unsigned int issue_flags)  in io_close()
4593 static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_sfr_prep()
4609 static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)  in io_sync_file_range()
4626 static int io_setup_async_msg(struct io_kiocb *req,  in io_setup_async_msg()
4649 static int io_sendmsg_copy_hdr(struct io_kiocb *req,  in io_sendmsg_copy_hdr()
4658 static int io_sendmsg_prep_async(struct io_kiocb *req)  in io_sendmsg_prep_async()
4668 static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_sendmsg_prep()
4692 static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)  in io_sendmsg()
4734 static int io_send(struct io_kiocb *req, unsigned int issue_flags)  in io_send()
4776 static int __io_recvmsg_copy_hdr(struct io_kiocb *req,  in __io_recvmsg_copy_hdr()
4809 static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,  in __io_compat_recvmsg_copy_hdr()
4850 static int io_recvmsg_copy_hdr(struct io_kiocb *req,  in io_recvmsg_copy_hdr()
4863 static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,  in io_recv_buffer_select()
4878 static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)  in io_put_recv_kbuf()
4883 static int io_recvmsg_prep_async(struct io_kiocb *req)  in io_recvmsg_prep_async()
4893 static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_recvmsg_prep()
4918 static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)  in io_recvmsg()
4975 static int io_recv(struct io_kiocb *req, unsigned int issue_flags)  in io_recv()
5030 static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_accept_prep()
5054 static int io_accept(struct io_kiocb *req, unsigned int issue_flags)  in io_accept()
5094 static int io_connect_prep_async(struct io_kiocb *req)  in io_connect_prep_async()
5102 static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_connect_prep()
5117 static int io_connect(struct io_kiocb *req, unsigned int issue_flags)  in io_connect()
5188 	struct io_kiocb *req;  member
5203 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)  in io_poll_get_ownership_slowpath()
5224 static inline bool io_poll_get_ownership(struct io_kiocb *req)  in io_poll_get_ownership()
5231 static void io_poll_mark_cancelled(struct io_kiocb *req)  in io_poll_mark_cancelled()
5236 static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)  in io_poll_get_double()
5244 static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)  in io_poll_get_single()
5251 static void io_poll_req_insert(struct io_kiocb *req)  in io_poll_req_insert()
5283 static void io_poll_remove_entries(struct io_kiocb *req)  in io_poll_remove_entries()
5318 static int io_poll_check_events(struct io_kiocb *req)  in io_poll_check_events()
5390 static void io_poll_task_func(struct io_kiocb *req, bool *locked)  in io_poll_task_func()
5413 static void io_apoll_task_func(struct io_kiocb *req, bool *locked)  in io_apoll_task_func()
5433 static void __io_poll_execute(struct io_kiocb *req, int mask)  in __io_poll_execute()
5445 static inline void io_poll_execute(struct io_kiocb *req, int res)  in io_poll_execute()
5451 static void io_poll_cancel_req(struct io_kiocb *req)  in io_poll_cancel_req()
5461 	struct io_kiocb *req = wait->private;  in io_poll_wake()  local
5512 	struct io_kiocb *req = pt->req;  in __io_queue_proc()  local
5561 static int __io_arm_poll_handler(struct io_kiocb *req,  in __io_arm_poll_handler()
5634 static int io_arm_poll_handler(struct io_kiocb *req)  in io_arm_poll_handler()
5685 	struct io_kiocb *req;  in io_poll_remove_all()  local
5711 	struct io_kiocb *req;  in io_poll_find()  local
5724 static bool io_poll_disarm(struct io_kiocb *req)  in io_poll_disarm()
5738 	struct io_kiocb *req = io_poll_find(ctx, sqe_addr, poll_only);  in io_poll_cancel()  local
5760 static int io_poll_update_prep(struct io_kiocb *req,  in io_poll_update_prep()
5793 static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_poll_add_prep()
5811 static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_add()
5828 static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_update()
5868 static void io_req_task_timeout(struct io_kiocb *req, bool *locked)  in io_req_task_timeout()
5878 	struct io_kiocb *req = data->req;  in io_timeout_fn()  local
5898 	struct io_kiocb *req;  in io_timeout_extract()  local
5920 	struct io_kiocb *req = io_timeout_extract(ctx, user_data);  in io_timeout_cancel()  local
5952 	struct io_kiocb *req;  in io_linked_timeout_update()  local
5976 	struct io_kiocb *req = io_timeout_extract(ctx, user_data);  in io_timeout_update()  local
5991 static int io_timeout_remove_prep(struct io_kiocb *req,  in io_timeout_remove_prep()
6032 static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout_remove()
6061 static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,  in io_timeout_prep()
6114 static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout()
6171 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_cancel_cb()  local
6203 static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)  in io_try_cancel_userdata()
6226 static int io_async_cancel_prep(struct io_kiocb *req,  in io_async_cancel_prep()
6241 static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)  in io_async_cancel()
6270 static int io_rsrc_update_prep(struct io_kiocb *req,  in io_rsrc_update_prep()
6286 static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)  in io_files_update()
6310 static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_req_prep()
6388 static int io_req_prep_async(struct io_kiocb *req)  in io_req_prep_async()
6414 static u32 io_get_sequence(struct io_kiocb *req)  in io_get_sequence()
6424 static bool io_drain_req(struct io_kiocb *req)  in io_drain_req()
6499 static void io_clean_op(struct io_kiocb *req)  in io_clean_op()
6565 static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)  in io_issue_sqe()
6693 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_wq_free_work()  local
6701 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_wq_submit_work()  local
6766 					     struct io_kiocb *req, int fd)  in io_file_get_fixed()
6784 				       struct io_kiocb *req, int fd)  in io_file_get_normal()
6797 				       struct io_kiocb *req, int fd, bool fixed)  in io_file_get()
6805 static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)  in io_req_task_link_timeout()
6824 	struct io_kiocb *prev, *req = data->req;  in io_link_timeout_fn()  local
6850 static void io_queue_linked_timeout(struct io_kiocb *req)  in io_queue_linked_timeout()
6872 static void __io_queue_sqe(struct io_kiocb *req)  in __io_queue_sqe()
6923 static inline void io_queue_sqe(struct io_kiocb *req)  in io_queue_sqe()
6949 					struct io_kiocb *req,  in io_check_restriction()
6969 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_init_req()
7031 static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_submit_sqe()
7197 		struct io_kiocb *req;  in io_submit_sqes()  local
8303 static int io_install_fixed_file(struct io_kiocb *req, struct file *file,  in io_install_fixed_file()
8359 static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)  in io_close_fixed()
9174 	struct io_kiocb *req, *nxt;  in io_req_cache_free()  local
9332 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_cancel_ctx_cb()  local
9409 	struct io_kiocb *req, *tmp;  in io_kill_timeouts()  local
9474 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_cancel_task_cb()  local
10053 		struct io_kiocb *req;  in __io_uring_show_fdinfo()  local