83 	struct dd_per_prio per_prio[DD_PRIO_COUNT];  member
142 deadline_rb_root(struct dd_per_prio *per_prio, struct request *rq)  in deadline_rb_root()  argument
144 	return &per_prio->sort_list[rq_data_dir(rq)];  in deadline_rb_root()
171 deadline_add_rq_rb(struct dd_per_prio *per_prio, struct request *rq)  in deadline_add_rq_rb()  argument
173 	struct rb_root *root = deadline_rb_root(per_prio, rq);  in deadline_add_rq_rb()
179 deadline_del_rq_rb(struct dd_per_prio *per_prio, struct request *rq)  in deadline_del_rq_rb()  argument
183 	if (per_prio->next_rq[data_dir] == rq)  in deadline_del_rq_rb()
184 		per_prio->next_rq[data_dir] = deadline_latter_request(rq);  in deadline_del_rq_rb()
186 	elv_rb_del(deadline_rb_root(per_prio, rq), rq);  in deadline_del_rq_rb()
193 				    struct dd_per_prio *per_prio,  in deadline_remove_request()  argument
202 		deadline_del_rq_rb(per_prio, rq);  in deadline_remove_request()
215 	struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_request_merged()  local
221 		elv_rb_del(deadline_rb_root(per_prio, req), req);  in dd_request_merged()
222 		deadline_add_rq_rb(per_prio, req);  in dd_request_merged()
255 	deadline_remove_request(q, &dd->per_prio[prio], next);  in dd_merged_requests()
262 deadline_move_request(struct deadline_data *dd, struct dd_per_prio *per_prio,  in deadline_move_request()  argument
267 	per_prio->next_rq[data_dir] = deadline_latter_request(rq);  in deadline_move_request()
272 	deadline_remove_request(rq->q, per_prio, rq);  in deadline_move_request()
285 static inline int deadline_check_fifo(struct dd_per_prio *per_prio,  in deadline_check_fifo()  argument
288 	struct request *rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);  in deadline_check_fifo()
304 deadline_fifo_request(struct deadline_data *dd, struct dd_per_prio *per_prio,  in deadline_fifo_request()  argument
310 	if (list_empty(&per_prio->fifo_list[data_dir]))  in deadline_fifo_request()
313 	rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);  in deadline_fifo_request()
322 	list_for_each_entry(rq, &per_prio->fifo_list[DD_WRITE], queuelist) {  in deadline_fifo_request()
338 deadline_next_request(struct deadline_data *dd, struct dd_per_prio *per_prio,  in deadline_next_request()  argument
344 	rq = per_prio->next_rq[data_dir];  in deadline_next_request()
371 					     struct dd_per_prio *per_prio,  in __dd_dispatch_request()  argument
382 	if (!list_empty(&per_prio->dispatch)) {  in __dd_dispatch_request()
383 		rq = list_first_entry(&per_prio->dispatch, struct request,  in __dd_dispatch_request()
394 	rq = deadline_next_request(dd, per_prio, dd->last_dir);  in __dd_dispatch_request()
404 	if (!list_empty(&per_prio->fifo_list[DD_READ])) {  in __dd_dispatch_request()
405 		BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_READ]));  in __dd_dispatch_request()
407 		if (deadline_fifo_request(dd, per_prio, DD_WRITE) &&  in __dd_dispatch_request()
420 	if (!list_empty(&per_prio->fifo_list[DD_WRITE])) {  in __dd_dispatch_request()
422 		BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_WRITE]));  in __dd_dispatch_request()
437 	next_rq = deadline_next_request(dd, per_prio, data_dir);  in __dd_dispatch_request()
438 	if (deadline_check_fifo(per_prio, data_dir) || !next_rq) {  in __dd_dispatch_request()
444 		rq = deadline_fifo_request(dd, per_prio, data_dir);  in __dd_dispatch_request()
470 	deadline_move_request(dd, per_prio, rq);  in __dd_dispatch_request()
506 		rq = __dd_dispatch_request(dd, &dd->per_prio[prio], now_ns -  in dd_dispatch_request()
516 		rq = __dd_dispatch_request(dd, &dd->per_prio[prio], now_ns);  in dd_dispatch_request()
573 		struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_exit_sched()  local
575 		WARN_ON_ONCE(!list_empty(&per_prio->fifo_list[DD_READ]));  in dd_exit_sched()
576 		WARN_ON_ONCE(!list_empty(&per_prio->fifo_list[DD_WRITE]));  in dd_exit_sched()
618 		struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_init_sched()  local
620 		INIT_LIST_HEAD(&per_prio->dispatch);  in dd_init_sched()
621 		INIT_LIST_HEAD(&per_prio->fifo_list[DD_READ]);  in dd_init_sched()
622 		INIT_LIST_HEAD(&per_prio->fifo_list[DD_WRITE]);  in dd_init_sched()
623 		per_prio->sort_list[DD_READ] = RB_ROOT;  in dd_init_sched()
624 		per_prio->sort_list[DD_WRITE] = RB_ROOT;  in dd_init_sched()
665 	struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_request_merge()  local
672 	__rq = elv_rb_find(&per_prio->sort_list[bio_data_dir(bio)], sector);  in dd_request_merge()
719 	struct dd_per_prio *per_prio;  in dd_insert_request()  local
749 	per_prio = &dd->per_prio[prio];  in dd_insert_request()
751 		list_add(&rq->queuelist, &per_prio->dispatch);  in dd_insert_request()
754 		deadline_add_rq_rb(per_prio, rq);  in dd_insert_request()
766 		list_add_tail(&rq->queuelist, &per_prio->fifo_list[data_dir]);  in dd_insert_request()
819 	struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_finish_request()  local
829 		if (!list_empty(&per_prio->fifo_list[DD_WRITE]))  in dd_finish_request()
835 static bool dd_has_work_for_prio(struct dd_per_prio *per_prio)  in dd_has_work_for_prio()  argument
837 	return !list_empty_careful(&per_prio->dispatch) ||  in dd_has_work_for_prio()
838 		!list_empty_careful(&per_prio->fifo_list[DD_READ]) ||  in dd_has_work_for_prio()
839 		!list_empty_careful(&per_prio->fifo_list[DD_WRITE]);  in dd_has_work_for_prio()
848 		if (dd_has_work_for_prio(&dd->per_prio[prio]))  in dd_has_work()
928 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
931 	return seq_list_start(&per_prio->fifo_list[data_dir], *pos);	\
939 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
941 	return seq_list_next(v, &per_prio->fifo_list[data_dir], pos);	\
965 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
966 	struct request *rq = per_prio->next_rq[data_dir];		\
1044 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
1047 	return seq_list_start(&per_prio->dispatch, *pos);		\
1055 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
1057 	return seq_list_next(v, &per_prio->dispatch, pos);		\