210 struct io_ring_ctx;
245 typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
248 	struct io_ring_ctx		*ctx;
330 struct io_ring_ctx {  struct
478 	const struct io_ring_ctx *last;
876 	struct io_ring_ctx		*ctx;
904 	struct io_ring_ctx	*ctx;
1085 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1096 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
1100 static struct file *io_file_get(struct io_ring_ctx *ctx,
1106 static void io_submit_flush_completions(struct io_ring_ctx *ctx);
1123 		struct io_ring_ctx *ctx = file->private_data;  in io_uring_get_socket()
1132 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)  in io_tw_lock()
1187 	struct io_ring_ctx *ctx = req->ctx;  in io_req_set_rsrc_node()
1251 		struct io_ring_ctx *ctx = head->ctx;  in io_match_task_safe()
1276 	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);  in io_ring_ctx_ref_free()
1288 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,  in io_fallback_req_func()
1307 static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)  in io_ring_ctx_alloc()
1309 	struct io_ring_ctx *ctx;  in io_ring_ctx_alloc()
1373 static void io_account_cq_overflow(struct io_ring_ctx *ctx)  in io_account_cq_overflow()
1384 		struct io_ring_ctx *ctx = req->ctx;  in req_need_defer()
1438 	struct io_ring_ctx *ctx = req->ctx;  in io_prep_async_work()
1464 		struct io_ring_ctx *ctx = req->ctx;  in io_prep_async_link()
1478 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_async_work()
1525 static void io_queue_deferred(struct io_ring_ctx *ctx)  in io_queue_deferred()
1539 static void io_flush_timeouts(struct io_ring_ctx *ctx)  in io_flush_timeouts()
1570 static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)  in __io_commit_cqring_flush()
1578 static inline void io_commit_cqring(struct io_ring_ctx *ctx)  in io_commit_cqring()
1586 static inline bool io_sqring_full(struct io_ring_ctx *ctx)  in io_sqring_full()
1593 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)  in __io_cqring_events()
1598 static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)  in io_get_cqe()
1615 static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)  in io_should_trigger_evfd()
1631 static void io_cqring_ev_posted(struct io_ring_ctx *ctx)  in io_cqring_ev_posted()
1650 static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)  in io_cqring_ev_posted_iopoll()
1668 static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)  in __io_cqring_overflow_flush()
1710 static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)  in io_cqring_overflow_flush()
1771 static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,  in io_cqring_event_overflow()
1799 static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data,  in __io_fill_cqe()
1826 static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,  in io_fill_cqe_aux()
1836 	struct io_ring_ctx *ctx = req->ctx;  in io_req_complete_post()
1920 static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)  in io_preinit_req()
1929 static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,  in io_flush_cached_locked_reqs()
1939 static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)  in io_flush_cached_reqs()
1973 static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)  in io_alloc_req()
2031 	struct io_ring_ctx *ctx = req->ctx;  in __io_free_req()
2111 		struct io_ring_ctx *ctx = req->ctx;  in io_disarm_next()
2136 		struct io_ring_ctx *ctx = req->ctx;  in __io_req_find_next()
2159 static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)  in ctx_flush_and_put()
2175 	struct io_ring_ctx *ctx = NULL;  in tctx_task_work()
2271 	struct io_ring_ctx *ctx = req->ctx;  in io_req_task_cancel()
2280 	struct io_ring_ctx *ctx = req->ctx;  in io_req_task_submit()
2341 static void io_req_free_batch_finish(struct io_ring_ctx *ctx,  in io_req_free_batch_finish()
2371 static void io_submit_flush_completions(struct io_ring_ctx *ctx)  in io_submit_flush_completions()
2430 static unsigned io_cqring_events(struct io_ring_ctx *ctx)  in io_cqring_events()
2437 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)  in io_sqring_entries()
2480 static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,  in io_iopoll_complete()
2506 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,  in io_do_iopoll()
2556 static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)  in io_iopoll_try_reap_events()
2584 static int io_iopoll_check(struct io_ring_ctx *ctx, long min)  in io_iopoll_check()
2662 	struct io_ring_ctx *ctx = req->ctx;  in io_rw_should_reissue()
2735 		struct io_ring_ctx *ctx = req->ctx;  in io_req_task_complete()
2793 	struct io_ring_ctx *ctx = req->ctx;  in io_iopoll_req_issued()
2904 	struct io_ring_ctx *ctx = req->ctx;  in io_prep_rw()
2967 		struct io_ring_ctx *ctx = req->ctx;  in io_prep_rw()
3021 			struct io_ring_ctx *ctx = req->ctx;  in kiocb_done()
3101 static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)  in io_ring_submit_unlock()
3107 static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)  in io_ring_submit_lock()
3995 	struct io_ring_ctx *ctx = req->ctx;  in io_nop()
4006 	struct io_ring_ctx *ctx = req->ctx;  in io_fsync_prep()
4226 static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,  in __io_remove_buffers()
4256 	struct io_ring_ctx *ctx = req->ctx;  in io_remove_buffers()
4344 	struct io_ring_ctx *ctx = req->ctx;  in io_provide_buffers()
4595 	struct io_ring_ctx *ctx = req->ctx;  in io_sfr_prep()
5253 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_req_insert()
5320 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_check_events()
5392 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_task_func()
5415 	struct io_ring_ctx *ctx = req->ctx;  in io_apoll_task_func()
5565 	struct io_ring_ctx *ctx = req->ctx;  in __io_arm_poll_handler()
5637 	struct io_ring_ctx *ctx = req->ctx;  in io_arm_poll_handler()
5681 static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,  in io_poll_remove_all()
5706 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,  in io_poll_find()
5734 static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,  in io_poll_cancel()
5830 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_update()
5879 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_fn()
5893 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,  in io_timeout_extract()
5916 static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)  in io_timeout_cancel()
5947 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,  in io_linked_timeout_update()
5972 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,  in io_timeout_update()
6035 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_remove()
6116 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout()
6165 	struct io_ring_ctx *ctx;
6178 			       struct io_ring_ctx *ctx)  in io_async_cancel_one()
6205 	struct io_ring_ctx *ctx = req->ctx;  in io_try_cancel_userdata()
6243 	struct io_ring_ctx *ctx = req->ctx;  in io_async_cancel()
6288 	struct io_ring_ctx *ctx = req->ctx;  in io_files_update()
6427 	struct io_ring_ctx *ctx = req->ctx;  in io_drain_req()
6567 	struct io_ring_ctx *ctx = req->ctx;  in io_issue_sqe()
6744 static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,  in io_file_from_index()
6765 static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,  in io_file_get_fixed()
6783 static struct file *io_file_get_normal(struct io_ring_ctx *ctx,  in io_file_get_normal()
6796 static inline struct file *io_file_get(struct io_ring_ctx *ctx,  in io_file_get()
6825 	struct io_ring_ctx *ctx = req->ctx;  in io_link_timeout_fn()
6852 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_linked_timeout()
6887 			struct io_ring_ctx *ctx = req->ctx;  in __io_queue_sqe()
6948 static inline bool io_check_restriction(struct io_ring_ctx *ctx,  in io_check_restriction()
6969 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_init_req()
7031 static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_submit_sqe()
7117 				struct io_ring_ctx *ctx)  in io_submit_state_end()
7139 static void io_commit_sqring(struct io_ring_ctx *ctx)  in io_commit_sqring()
7159 static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)  in io_get_sqe()
7183 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)  in io_submit_sqes()
7236 static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)  in io_ring_set_wakeup_flag()
7245 static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)  in io_ring_clear_wakeup_flag()
7253 static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)  in __io_sq_thread()
7294 	struct io_ring_ctx *ctx;  in io_sqd_update_thread_idle()
7321 	struct io_ring_ctx *ctx;  in io_sq_thread()
7406 	struct io_ring_ctx *ctx;
7413 	struct io_ring_ctx *ctx = iowq->ctx;  in io_should_wake()
7451 static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,  in io_cqring_wait_schedule()
7474 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,  in io_cqring_wait()
7579 	struct io_ring_ctx *ctx = node->rsrc_data->ctx;  in io_rsrc_node_ref_zero()
7606 static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)  in io_rsrc_node_alloc()
7625 static void io_rsrc_node_switch(struct io_ring_ctx *ctx,  in io_rsrc_node_switch()
7650 static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)  in io_rsrc_node_switch_start()
7658 static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)  in io_rsrc_ref_quiesce()
7722 static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,  in io_rsrc_data_alloc()
7775 static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)  in __io_sqe_files_unregister()
7802 static int io_sqe_files_unregister(struct io_ring_ctx *ctx)  in io_sqe_files_unregister()
7872 static void io_sq_thread_finish(struct io_ring_ctx *ctx)  in io_sq_thread_finish()
7889 	struct io_ring_ctx *ctx_attach;  in io_attach_sq_data()
7953 static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)  in __io_sqe_files_scm()
8013 static int io_sqe_files_scm(struct io_ring_ctx *ctx)  in io_sqe_files_scm()
8044 static int io_sqe_files_scm(struct io_ring_ctx *ctx)  in io_sqe_files_scm()
8050 static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)  in io_rsrc_file_put()
8116 	struct io_ring_ctx *ctx = rsrc_data->ctx;  in __io_rsrc_put_work()
8145 	struct io_ring_ctx *ctx;  in io_rsrc_put_work()
8148 	ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);  in io_rsrc_put_work()
8161 static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,  in io_sqe_files_register()
8243 static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,  in io_sqe_file_register()
8306 	struct io_ring_ctx *ctx = req->ctx;  in io_install_fixed_file()
8362 	struct io_ring_ctx *ctx = req->ctx;  in io_close_fixed()
8397 static int __io_sqe_files_update(struct io_ring_ctx *ctx,  in __io_sqe_files_update()
8476 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,  in io_init_wq_offload()
8509 				       struct io_ring_ctx *ctx)  in io_uring_alloc_task_context()
8556 static int io_sq_offload_create(struct io_ring_ctx *ctx,  in io_sq_offload_create()
8667 static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)  in io_unaccount_mem()
8676 static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)  in io_account_mem()
8740 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)  in io_buffer_unmap()
8755 static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)  in io_rsrc_buf_put()
8761 static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)  in __io_sqe_buffers_unregister()
8774 static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)  in io_sqe_buffers_unregister()
8794 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,  in io_copy_iov()
8828 static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,  in headpage_already_acct()
8856 static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,  in io_buffer_account_pin()
8888 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,  in io_sqe_buffer_register()
8988 static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)  in io_buffers_map_alloc()
9018 static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,  in io_sqe_buffers_register()
9070 static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,  in __io_sqe_buffers_update()
9130 static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)  in io_eventfd_register()
9152 static int io_eventfd_unregister(struct io_ring_ctx *ctx)  in io_eventfd_unregister()
9163 static void io_destroy_buffers(struct io_ring_ctx *ctx)  in io_destroy_buffers()
9182 static void io_req_caches_free(struct io_ring_ctx *ctx)  in io_req_caches_free()
9204 static void io_ring_ctx_free(struct io_ring_ctx *ctx)  in io_ring_ctx_free()
9263 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_poll()
9294 static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)  in io_unregister_personality()
9310 	struct io_ring_ctx		*ctx;
9339 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);  in io_ring_exit_work()
9406 static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,  in io_kill_timeouts()
9429 static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)  in io_ring_ctx_wait_and_kill()
9460 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_release()
9480 static bool io_cancel_defer_files(struct io_ring_ctx *ctx,  in io_cancel_defer_files()
9506 static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)  in io_uring_try_cancel_iowq()
9530 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,  in io_uring_try_cancel_requests()
9573 static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)  in __io_uring_add_tctx_node()
9619 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)  in io_uring_add_tctx_node()
9688 	struct io_ring_ctx *ctx;  in io_uring_cancel_generic()
9758 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_validate_mmap_request()
9825 static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)  in io_sqpoll_wait_sq()
9879 	struct io_ring_ctx *ctx;  in SYSCALL_DEFINE6()
10003 static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)  in __io_uring_show_fdinfo()
10066 	struct io_ring_ctx *ctx = f->private_data;  in io_uring_show_fdinfo()
10088 static int io_allocate_scq_urings(struct io_ring_ctx *ctx,  in io_allocate_scq_urings()
10130 static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)  in io_uring_install_fd()
10153 static struct file *io_uring_get_file(struct io_ring_ctx *ctx)  in io_uring_get_file()
10181 	struct io_ring_ctx *ctx;  in io_uring_create()
10337 static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)  in io_probe()
10376 static int io_register_personality(struct io_ring_ctx *ctx)  in io_register_personality()
10393 static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,  in io_register_restrictions()
10463 static int io_register_enable_rings(struct io_ring_ctx *ctx)  in io_register_enable_rings()
10477 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,  in __io_register_rsrc_update()
10499 static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,  in io_register_files_update()
10514 static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,  in io_register_rsrc_update()
10528 static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,  in io_register_rsrc()
10554 static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,  in io_register_iowq_aff()
10593 static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)  in io_unregister_iowq_aff()
10603 static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,  in io_register_iowq_max_workers()
10711 static int io_ctx_quiesce(struct io_ring_ctx *ctx)  in io_ctx_quiesce()
10738 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,  in __io_uring_register()
10883 	struct io_ring_ctx *ctx;  in SYSCALL_DEFINE4()