72 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_has_pending()
82 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,  in blk_mq_hctx_mark_pending()
91 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,  in blk_mq_hctx_clear_pending()
104 static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,  in blk_mq_check_inflight()
226 	struct blk_mq_hw_ctx *hctx;  in blk_mq_quiesce_queue()
261 	struct blk_mq_hw_ctx *hctx;  in blk_mq_wake_waiters()
496 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in __blk_mq_free_request()
515 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_free_request()
708 static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)  in hctx_unlock()
717 static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)  in hctx_lock()
874 static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq,  in blk_mq_rq_inflight()
943 static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,  in blk_mq_check_expired()
965 	struct blk_mq_hw_ctx *hctx;  in blk_mq_timeout_work()
1005 	struct blk_mq_hw_ctx *hctx;
1012 	struct blk_mq_hw_ctx *hctx = flush_data->hctx;  in flush_busy_ctx()
1027 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)  in blk_mq_flush_busy_ctxs()
1039 	struct blk_mq_hw_ctx *hctx;
1047 	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;  in dispatch_rq_from_ctx()
1063 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,  in blk_mq_dequeue_from_ctx()
1112 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_get_driver_tag()
1129 	struct blk_mq_hw_ctx *hctx;  in blk_mq_dispatch_wake()
1131 	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);  in blk_mq_dispatch_wake()
1153 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,  in blk_mq_mark_tag_wait()
1226 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)  in blk_mq_update_dispatch_busy()
1284 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_prep_dispatch_rq()
1326 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,  in blk_mq_dispatch_rq_list()
1486 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)  in __blk_mq_run_hw_queue()
1528 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)  in blk_mq_first_mapped_cpu()
1543 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_next_cpu()
1592 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,  in __blk_mq_delay_run_hw_queue()
1620 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)  in blk_mq_delay_run_hw_queue()
1635 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)  in blk_mq_run_hw_queue()
1676 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)  in blk_mq_get_sq_hctx()
1686 	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, 0, ctx);  in blk_mq_get_sq_hctx()
1700 	struct blk_mq_hw_ctx *hctx, *sq_hctx;  in blk_mq_run_hw_queues()
1728 	struct blk_mq_hw_ctx *hctx, *sq_hctx;  in blk_mq_delay_run_hw_queues()
1758 	struct blk_mq_hw_ctx *hctx;  in blk_mq_queue_stopped()
1778 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)  in blk_mq_stop_hw_queue()
1797 	struct blk_mq_hw_ctx *hctx;  in blk_mq_stop_hw_queues()
1805 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)  in blk_mq_start_hw_queue()
1815 	struct blk_mq_hw_ctx *hctx;  in blk_mq_start_hw_queues()
1823 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)  in blk_mq_start_stopped_hw_queue()
1835 	struct blk_mq_hw_ctx *hctx;  in blk_mq_start_stopped_hw_queues()
1845 	struct blk_mq_hw_ctx *hctx;  in blk_mq_run_work_fn()
1847 	hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);  in blk_mq_run_work_fn()
1858 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,  in __blk_mq_insert_req_list()
1875 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,  in __blk_mq_insert_request()
1898 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_request_bypass_insert()
1911 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,  in blk_mq_insert_requests()
1963 		struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx;  in blk_mq_flush_plug_list()
2001 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,  in __blk_mq_issue_directly()
2040 static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,  in __blk_mq_try_issue_directly()
2093 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,  in blk_mq_try_issue_directly()
2117 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_request_issue_directly()
2126 void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,  in blk_mq_try_issue_list_directly()
2544 	struct blk_mq_hw_ctx *hctx;
2558 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_has_requests()
2571 		struct blk_mq_hw_ctx *hctx)  in blk_mq_last_cpu_in_hctx()
2582 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,  in blk_mq_hctx_notify_offline()
2583 			struct blk_mq_hw_ctx, cpuhp_online);  in blk_mq_hctx_notify_offline()
2615 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,  in blk_mq_hctx_notify_online()
2616 			struct blk_mq_hw_ctx, cpuhp_online);  in blk_mq_hctx_notify_online()
2630 	struct blk_mq_hw_ctx *hctx;  in blk_mq_hctx_notify_dead()
2635 	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);  in blk_mq_hctx_notify_dead()
2660 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)  in blk_mq_remove_cpuhp()
2701 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in blk_mq_exit_hctx()
2726 	struct blk_mq_hw_ctx *hctx;  in blk_mq_exit_hw_queues()
2739 	int hw_ctx_size = sizeof(struct blk_mq_hw_ctx);  in blk_mq_hw_ctx_size()
2741 	BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu),  in blk_mq_hw_ctx_size()
2742 			   __alignof__(struct blk_mq_hw_ctx)) !=  in blk_mq_hw_ctx_size()
2743 		     sizeof(struct blk_mq_hw_ctx));  in blk_mq_hw_ctx_size()
2753 		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)  in blk_mq_init_hctx()
2781 static struct blk_mq_hw_ctx *
2785 	struct blk_mq_hw_ctx *hctx;  in blk_mq_alloc_hctx()
2856 		struct blk_mq_hw_ctx *hctx;  in blk_mq_init_cpu_queues()
2914 	struct blk_mq_hw_ctx *hctx;  in blk_mq_map_swqueue()
3019 	struct blk_mq_hw_ctx *hctx;  in queue_set_hctx_shared()
3117 	struct blk_mq_hw_ctx *hctx, *next;  in blk_mq_release()
3200 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(  in blk_mq_alloc_and_init_hctx()
3204 	struct blk_mq_hw_ctx *hctx = NULL, *tmp;  in blk_mq_alloc_and_init_hctx()
3238 	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;  in blk_mq_realloc_hw_ctxs()
3241 		struct blk_mq_hw_ctx **new_hctxs;  in blk_mq_realloc_hw_ctxs()
3260 		struct blk_mq_hw_ctx *hctx;  in blk_mq_realloc_hw_ctxs()
3299 		struct blk_mq_hw_ctx *hctx = hctxs[j];  in blk_mq_realloc_hw_ctxs()
3632 	struct blk_mq_hw_ctx *hctx;  in blk_mq_update_nr_requests()
3940 			       struct blk_mq_hw_ctx *hctx, blk_qc_t cookie)  in blk_mq_poll_hybrid()
3978 	struct blk_mq_hw_ctx *hctx;  in blk_poll()