211 				  unsigned int sched_domain, unsigned int type)  in flush_latency_buckets()  argument
213 	unsigned int *buckets = kqd->latency_buckets[sched_domain][type];  in flush_latency_buckets()
214 	atomic_t *cpu_buckets = cpu_latency->buckets[sched_domain][type];  in flush_latency_buckets()
226 				unsigned int sched_domain, unsigned int type,  in calculate_percentile()  argument
229 	unsigned int *buckets = kqd->latency_buckets[sched_domain][type];  in calculate_percentile()
242 	if (!kqd->latency_timeout[sched_domain])  in calculate_percentile()
243 		kqd->latency_timeout[sched_domain] = max(jiffies + HZ, 1UL);  in calculate_percentile()
245 	    time_is_after_jiffies(kqd->latency_timeout[sched_domain])) {  in calculate_percentile()
248 	kqd->latency_timeout[sched_domain] = 0;  in calculate_percentile()
256 	memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type]));  in calculate_percentile()
258 	trace_kyber_latency(kqd->q, kyber_domain_names[sched_domain],  in calculate_percentile()
266 				unsigned int sched_domain, unsigned int depth)  in kyber_resize_domain()  argument
268 	depth = clamp(depth, 1U, kyber_depth[sched_domain]);  in kyber_resize_domain()
269 	if (depth != kqd->domain_tokens[sched_domain].sb.depth) {  in kyber_resize_domain()
270 		sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth);  in kyber_resize_domain()
271 		trace_kyber_adjust(kqd->q, kyber_domain_names[sched_domain],  in kyber_resize_domain()
279 	unsigned int sched_domain;  in kyber_timer_fn()  local
288 		for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {  in kyber_timer_fn()
289 			flush_latency_buckets(kqd, cpu_latency, sched_domain,  in kyber_timer_fn()
291 			flush_latency_buckets(kqd, cpu_latency, sched_domain,  in kyber_timer_fn()
301 	for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {  in kyber_timer_fn()
304 		p90 = calculate_percentile(kqd, sched_domain, KYBER_IO_LATENCY,  in kyber_timer_fn()
315 	for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {  in kyber_timer_fn()
319 		p99 = calculate_percentile(kqd, sched_domain,  in kyber_timer_fn()
331 				p99 = kqd->domain_p99[sched_domain];  in kyber_timer_fn()
332 			kqd->domain_p99[sched_domain] = -1;  in kyber_timer_fn()
334 			kqd->domain_p99[sched_domain] = p99;  in kyber_timer_fn()
349 			orig_depth = kqd->domain_tokens[sched_domain].sb.depth;  in kyber_timer_fn()
351 			kyber_resize_domain(kqd, sched_domain, depth);  in kyber_timer_fn()
541 	unsigned int sched_domain;  in rq_clear_domain_token()  local
546 		sched_domain = kyber_sched_domain(rq->cmd_flags);  in rq_clear_domain_token()
547 		sbitmap_queue_clear(&kqd->domain_tokens[sched_domain], nr,  in rq_clear_domain_token()
572 	unsigned int sched_domain = kyber_sched_domain(bio->bi_opf);  in kyber_bio_merge()  local
573 	struct list_head *rq_list = &kcq->rq_list[sched_domain];  in kyber_bio_merge()
595 		unsigned int sched_domain = kyber_sched_domain(rq->cmd_flags);  in kyber_insert_requests()  local
597 		struct list_head *head = &kcq->rq_list[sched_domain];  in kyber_insert_requests()
604 		sbitmap_set_bit(&khd->kcq_map[sched_domain],  in kyber_insert_requests()
619 			       unsigned int sched_domain, unsigned int type,  in add_latency_sample()  argument
633 	atomic_inc(&cpu_latency->buckets[sched_domain][type][bucket]);  in add_latency_sample()
640 	unsigned int sched_domain;  in kyber_completed_request()  local
643 	sched_domain = kyber_sched_domain(rq->cmd_flags);  in kyber_completed_request()
644 	if (sched_domain == KYBER_OTHER)  in kyber_completed_request()
648 	target = kqd->latency_targets[sched_domain];  in kyber_completed_request()
649 	add_latency_sample(cpu_latency, sched_domain, KYBER_TOTAL_LATENCY,  in kyber_completed_request()
651 	add_latency_sample(cpu_latency, sched_domain, KYBER_IO_LATENCY, target,  in kyber_completed_request()
660 	unsigned int sched_domain;  member
670 	list_splice_tail_init(&kcq->rq_list[flush_data->sched_domain],  in flush_busy_kcq()
679 				  unsigned int sched_domain,  in kyber_flush_busy_kcqs()  argument
684 		.sched_domain = sched_domain,  in kyber_flush_busy_kcqs()
688 	sbitmap_for_each_set(&khd->kcq_map[sched_domain],  in kyber_flush_busy_kcqs()
707 	unsigned int sched_domain = khd->cur_domain;  in kyber_get_domain_token()  local
708 	struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];  in kyber_get_domain_token()
709 	struct sbq_wait *wait = &khd->domain_wait[sched_domain];  in kyber_get_domain_token()
722 				  &khd->wait_index[sched_domain]);  in kyber_get_domain_token()
723 		khd->domain_ws[sched_domain] = ws;  in kyber_get_domain_token()
741 		ws = khd->domain_ws[sched_domain];  in kyber_get_domain_token()