kernel/block/bfq-iosched.h

1 /* SPDX-License-Identifier: GPL-2.0-or-later */
11 #include <linux/blk-cgroup.h>
13 #include "blk-cgroup-rwstat.h"
31  * Soft real-time applications are extremely more latency sensitive
32  * than interactive ones. Over-raise the weight of the former to
40  * struct bfq_service_tree - per ioprio_class service tree.
42  * Each service tree represents a B-WF2Q+ scheduler on its own.  Each
65  * struct bfq_sched_data - multi-class scheduler.
75  * queue requests are served according to B-WF2Q+.
80  * before the current in-service entity is expired, 2) the in-service
82  * in_service_entity is not a queue, then the in-service child entity
89  * in-service entity with those of such best candidate.
96 	/* head-of-line entity (see comments above) */
106  * struct bfq_weight_counter - counter of the number of all active queues
119  * struct bfq_entity - schedulable entity.
124  * hierarchy.  Non-leaf entities have also their own sched_data, stored
138  * "well-behaved" queues (i.e., queues that do not spend too much
155 	/* B-WF2Q+ start and finish timestamps [sectors/weight] */
173 	/* device weight, if non-zero, it overrides the default weight of
188 	 * For non-leaf nodes in the hierarchy, the associated
205  * struct bfq_ttime - per process thinktime stats.
220  * struct bfq_queue - leaf schedulable entity.
241 	/* last total-service-time sample, see bfq_update_inject_limit() */
253 	/* request-position tree member (see bfq_group's @rq_pos_tree) */
255 	/* request-position tree root (see bfq_group's @rq_pos_tree) */
297 	/* node for the device's burst list */
320 	/* current maximum weight-raising time for this queue */
326 	 * queue it is deemed as soft real-time (see the comments on
331 	 * Start time of the current weight-raising period if
332 	 * the @bfq-queue is being weight-raised, otherwise
333 	 * finish time of the last weight-raising period.
350 	 * last transition to weight-raised state.
363 	/* max service rate measured so far */
386  * struct bfq_io_cq - per (request_queue, io_context) structure.
412 	 * field keeping the queue's belonging to a large burst
416 	 * True if the queue belonged to a burst list before its merge
441  * struct bfq_data - per-device data structure.
459 	 * weight-raised @bfq_queue (see the comments to the functions
484 	 * should be decremented when the in-flight request of the
489 	 * pending-request counter must be added to entities, and must
513 	 * Per-class (RT, BE, IDLE) number of bfq_queues containing
518 	/* number of weight-raised busy @bfq_queues */
550 	/* on-disk position of the last served request */
553 	/* position of the last served request for the in-service queue */
562 	/* time of last transition from empty to non-empty (ns) */
567 	 * of a just-arrived first I/O request (see
599 	/* max rq size seen during current observation interval (sectors) */
605 	 * [(sectors/usec) / 2^BFQ_RATE_SHIFT]. The left-shift by
607 	 * fixed-point calculations.
631 	/* user-configured max budget value (0 for auto-tuning) */
636 	 * sequential or quasi-sequential ones (this also implies that
640 	 * without service-domain guarantees).
647 	 * was marked as non-I/O-bound (see the definition of the
661 	 * Last time at which a queue entered the current burst of
664 	 * a burst of activations, see the comments on the function
673 	/* number of queues in the current burst of queue activations */
676 	/* common parent entity for the queues in the burst */
678 	/* Maximum burst size above which the current queue-activation
679 	 * burst is deemed as 'large'.
682 	/* true if a large queue-activation burst is in progress */
685 	 * Head of the burst list (as for the above fields, more
690 	/* if set to true, low-latency heuristics are enabled */
693 	 * Maximum factor by which the weight of a weight-raised queue
697 	/* maximum duration of a weight-raising period (jiffies) */
700 	/* Maximum weight-raising duration for soft real-time processes */
703 	 * Minimum idle period after which weight-raising may be
709 	 * weight-raising may be reactivated for an already busy async
714 	/* Max service-rate for a soft real-time queue, in sectors/sec */
731 	 * be able to perform the bic lookup, needed by bio-merge
733 	 * avoid taking the request-queue lock while the scheduler
759 				 * bfqq has timed-out at least once
764 				 * bfqq activated in a large burst,
768 				 * may need softrt-next-start
851  * struct bfq_group_data - per-blkcg storage for the blkio subsystem.
864  * struct bfq_group - per (device, cgroup) data structure.
887  * there is a set of bfq_groups, each one collecting the lower-level
938 /* --------------- main algorithm interface ----------------- */
965 /* ------------ end of main algorithm interface -------------- */
967 /* ---------------- cgroups-support interface ---------------- */
999 /* ------------- end of cgroups-support interface ------------- */
1001 /* - interface of the internal hierarchical B-WF2Q+ scheduler - */
1006 	for (; entity ; entity = entity->parent)
1015 	for (; entity && ({ parent = entity->parent; 1; }); entity = parent)
1059 /* --------------- end of interface of B-WF2Q+ ---------------- */
1062 static inline void bfq_pid_to_str(int pid, char *str, int len)  in bfq_pid_to_str()  argument
1064 	if (pid != -1)  in bfq_pid_to_str()
1065 		snprintf(str, len, "%d", pid);  in bfq_pid_to_str()
1067 		snprintf(str, len, "SHARED-");  in bfq_pid_to_str()
1075 	if (likely(!blk_trace_note_message_enabled((bfqd)->queue)))	\
1077 	bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH);	\
1078 	blk_add_cgroup_trace_msg((bfqd)->queue,				\
1079 			bfqg_to_blkg(bfqq_group(bfqq))->blkcg,		\
1085 	blk_add_cgroup_trace_msg((bfqd)->queue,				\
1086 		bfqg_to_blkg(bfqg)->blkcg, fmt, ##args);		\
1093 	if (likely(!blk_trace_note_message_enabled((bfqd)->queue)))	\
1095 	bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH);	\
1096 	blk_add_trace_msg((bfqd)->queue, "bfq%s%c " fmt, pid_str,	\
1105 	blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args)