62 	struct request_queue *q = rq->q;  in elv_iosched_allow_bio_merge()  local
63 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
140 static struct elevator_type *elevator_get(struct request_queue *q,  in elevator_get()  argument
147 	e = elevator_find(name, q->required_elevator_features);  in elevator_get()
152 		e = elevator_find(name, q->required_elevator_features);  in elevator_get()
164 struct elevator_queue *elevator_alloc(struct request_queue *q,  in elevator_alloc()  argument
169 	eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);  in elevator_alloc()
191 void __elevator_exit(struct request_queue *q, struct elevator_queue *e)  in __elevator_exit()  argument
194 	blk_mq_exit_sched(q, e);  in __elevator_exit()
206 void elv_rqhash_del(struct request_queue *q, struct request *rq)  in elv_rqhash_del()  argument
213 void elv_rqhash_add(struct request_queue *q, struct request *rq)  in elv_rqhash_add()  argument
215 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()
223 void elv_rqhash_reposition(struct request_queue *q, struct request *rq)  in elv_rqhash_reposition()  argument
226 	elv_rqhash_add(q, rq);  in elv_rqhash_reposition()
229 struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)  in elv_rqhash_find()  argument
231 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()
303 enum elv_merge elv_merge(struct request_queue *q, struct request **req,  in elv_merge()  argument
306 	struct elevator_queue *e = q->elevator;  in elv_merge()
315 	if (blk_queue_nomerges(q) || !bio_mergeable(bio))  in elv_merge()
321 	if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {  in elv_merge()
322 		enum elv_merge ret = blk_try_merge(q->last_merge, bio);  in elv_merge()
325 			*req = q->last_merge;  in elv_merge()
330 	if (blk_queue_noxmerges(q))  in elv_merge()
336 	__rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);  in elv_merge()
346 		return e->type->ops.request_merge(q, req, bio);  in elv_merge()
358 bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq)  in elv_attempt_insert_merge()  argument
363 	if (blk_queue_nomerges(q))  in elv_attempt_insert_merge()
369 	if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))  in elv_attempt_insert_merge()
372 	if (blk_queue_noxmerges(q))  in elv_attempt_insert_merge()
380 		__rq = elv_rqhash_find(q, blk_rq_pos(rq));  in elv_attempt_insert_merge()
381 		if (!__rq || !blk_attempt_req_merge(q, __rq, rq))  in elv_attempt_insert_merge()
392 void elv_merged_request(struct request_queue *q, struct request *rq,  in elv_merged_request()  argument
395 	struct elevator_queue *e = q->elevator;  in elv_merged_request()
398 		e->type->ops.request_merged(q, rq, type);  in elv_merged_request()
401 		elv_rqhash_reposition(q, rq);  in elv_merged_request()
403 	q->last_merge = rq;  in elv_merged_request()
406 void elv_merge_requests(struct request_queue *q, struct request *rq,  in elv_merge_requests()  argument
409 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()
412 		e->type->ops.requests_merged(q, rq, next);  in elv_merge_requests()
414 	elv_rqhash_reposition(q, rq);  in elv_merge_requests()
415 	q->last_merge = rq;  in elv_merge_requests()
418 struct request *elv_latter_request(struct request_queue *q, struct request *rq)  in elv_latter_request()  argument
420 	struct elevator_queue *e = q->elevator;  in elv_latter_request()
423 		return e->type->ops.next_request(q, rq);  in elv_latter_request()
428 struct request *elv_former_request(struct request_queue *q, struct request *rq)  in elv_former_request()  argument
430 	struct elevator_queue *e = q->elevator;  in elv_former_request()
433 		return e->type->ops.former_request(q, rq);  in elv_former_request()
485 int elv_register_queue(struct request_queue *q, bool uevent)  in elv_register_queue()  argument
487 	struct elevator_queue *e = q->elevator;  in elv_register_queue()
490 	lockdep_assert_held(&q->sysfs_lock);  in elv_register_queue()
492 	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");  in elv_register_queue()
510 void elv_unregister_queue(struct request_queue *q)  in elv_unregister_queue()  argument
512 	lockdep_assert_held(&q->sysfs_lock);  in elv_unregister_queue()
514 	if (q) {  in elv_unregister_queue()
515 		struct elevator_queue *e = q->elevator;  in elv_unregister_queue()
575 int elevator_switch_mq(struct request_queue *q,  in elevator_switch_mq()  argument
580 	lockdep_assert_held(&q->sysfs_lock);  in elevator_switch_mq()
582 	if (q->elevator) {  in elevator_switch_mq()
583 		if (q->elevator->registered)  in elevator_switch_mq()
584 			elv_unregister_queue(q);  in elevator_switch_mq()
586 		ioc_clear_queue(q);  in elevator_switch_mq()
587 		elevator_exit(q, q->elevator);  in elevator_switch_mq()
590 	ret = blk_mq_init_sched(q, new_e);  in elevator_switch_mq()
595 		ret = elv_register_queue(q, true);  in elevator_switch_mq()
597 			elevator_exit(q, q->elevator);  in elevator_switch_mq()
603 		blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);  in elevator_switch_mq()
605 		blk_add_trace_msg(q, "elv switch: none");  in elevator_switch_mq()
611 static inline bool elv_support_iosched(struct request_queue *q)  in elv_support_iosched()  argument
613 	if (!queue_is_mq(q) ||  in elv_support_iosched()
614 	    (q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))  in elv_support_iosched()
623 static struct elevator_type *elevator_get_default(struct request_queue *q)  in elevator_get_default()  argument
625 	if (q->tag_set && q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT)  in elevator_get_default()
628 	if (q->nr_hw_queues != 1)  in elevator_get_default()
631 	return elevator_get(q, "mq-deadline", false);  in elevator_get_default()
638 static struct elevator_type *elevator_get_by_features(struct request_queue *q)  in elevator_get_by_features()  argument
646 					 q->required_elevator_features)) {  in elevator_get_by_features()
665 void elevator_init_mq(struct request_queue *q)  in elevator_init_mq()  argument
670 	if (!elv_support_iosched(q))  in elevator_init_mq()
673 	WARN_ON_ONCE(blk_queue_registered(q));  in elevator_init_mq()
675 	if (unlikely(q->elevator))  in elevator_init_mq()
678 	if (!q->required_elevator_features)  in elevator_init_mq()
679 		e = elevator_get_default(q);  in elevator_init_mq()
681 		e = elevator_get_by_features(q);  in elevator_init_mq()
685 	blk_mq_freeze_queue(q);  in elevator_init_mq()
686 	blk_mq_quiesce_queue(q);  in elevator_init_mq()
688 	err = blk_mq_init_sched(q, e);  in elevator_init_mq()
690 	blk_mq_unquiesce_queue(q);  in elevator_init_mq()
691 	blk_mq_unfreeze_queue(q);  in elevator_init_mq()
707 static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)  in elevator_switch()  argument
711 	lockdep_assert_held(&q->sysfs_lock);  in elevator_switch()
713 	blk_mq_freeze_queue(q);  in elevator_switch()
714 	blk_mq_quiesce_queue(q);  in elevator_switch()
716 	err = elevator_switch_mq(q, new_e);  in elevator_switch()
718 	blk_mq_unquiesce_queue(q);  in elevator_switch()
719 	blk_mq_unfreeze_queue(q);  in elevator_switch()
727 static int __elevator_change(struct request_queue *q, const char *name)  in __elevator_change()  argument
733 	if (!blk_queue_registered(q))  in __elevator_change()
740 		if (!q->elevator)  in __elevator_change()
742 		return elevator_switch(q, NULL);  in __elevator_change()
746 	e = elevator_get(q, strstrip(elevator_name), true);  in __elevator_change()
750 	if (q->elevator &&  in __elevator_change()
751 	    elevator_match(q->elevator->type, elevator_name, 0)) {  in __elevator_change()
756 	return elevator_switch(q, e);  in __elevator_change()
759 ssize_t elv_iosched_store(struct request_queue *q, const char *name,  in elv_iosched_store()  argument
764 	if (!elv_support_iosched(q))  in elv_iosched_store()
767 	ret = __elevator_change(q, name);  in elv_iosched_store()
774 ssize_t elv_iosched_show(struct request_queue *q, char *name)  in elv_iosched_show()  argument
776 	struct elevator_queue *e = q->elevator;  in elv_iosched_show()
781 	if (!queue_is_mq(q))  in elv_iosched_show()
784 	if (!q->elevator)  in elv_iosched_show()
795 		if (elv_support_iosched(q) &&  in elv_iosched_show()
797 				   q->required_elevator_features))  in elv_iosched_show()
802 	if (q->elevator)  in elv_iosched_show()
809 struct request *elv_rb_former_request(struct request_queue *q,  in elv_rb_former_request()  argument
821 struct request *elv_rb_latter_request(struct request_queue *q,  in elv_rb_latter_request()  argument