209 static void flush_latency_buckets(struct kyber_queue_data *kqd,  in flush_latency_buckets()  argument
213 	unsigned int *buckets = kqd->latency_buckets[sched_domain][type];  in flush_latency_buckets()
225 static int calculate_percentile(struct kyber_queue_data *kqd,  in calculate_percentile()  argument
229 	unsigned int *buckets = kqd->latency_buckets[sched_domain][type];  in calculate_percentile()
242 	if (!kqd->latency_timeout[sched_domain])  in calculate_percentile()
243 		kqd->latency_timeout[sched_domain] = max(jiffies + HZ, 1UL);  in calculate_percentile()
245 	    time_is_after_jiffies(kqd->latency_timeout[sched_domain])) {  in calculate_percentile()
248 	kqd->latency_timeout[sched_domain] = 0;  in calculate_percentile()
256 	memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type]));  in calculate_percentile()
258 	trace_kyber_latency(kqd->q, kyber_domain_names[sched_domain],  in calculate_percentile()
265 static void kyber_resize_domain(struct kyber_queue_data *kqd,  in kyber_resize_domain()  argument
269 	if (depth != kqd->domain_tokens[sched_domain].sb.depth) {  in kyber_resize_domain()
270 		sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth);  in kyber_resize_domain()
271 		trace_kyber_adjust(kqd->q, kyber_domain_names[sched_domain],  in kyber_resize_domain()
278 	struct kyber_queue_data *kqd = from_timer(kqd, t, timer);  in kyber_timer_fn()  local
287 		cpu_latency = per_cpu_ptr(kqd->cpu_latency, cpu);  in kyber_timer_fn()
289 			flush_latency_buckets(kqd, cpu_latency, sched_domain,  in kyber_timer_fn()
291 			flush_latency_buckets(kqd, cpu_latency, sched_domain,  in kyber_timer_fn()
304 		p90 = calculate_percentile(kqd, sched_domain, KYBER_IO_LATENCY,  in kyber_timer_fn()
319 		p99 = calculate_percentile(kqd, sched_domain,  in kyber_timer_fn()
331 				p99 = kqd->domain_p99[sched_domain];  in kyber_timer_fn()
332 			kqd->domain_p99[sched_domain] = -1;  in kyber_timer_fn()
334 			kqd->domain_p99[sched_domain] = p99;  in kyber_timer_fn()
349 			orig_depth = kqd->domain_tokens[sched_domain].sb.depth;  in kyber_timer_fn()
351 			kyber_resize_domain(kqd, sched_domain, depth);  in kyber_timer_fn()
367 	struct kyber_queue_data *kqd;  in kyber_queue_data_alloc()  local
372 	kqd = kzalloc_node(sizeof(*kqd), GFP_KERNEL, q->node);  in kyber_queue_data_alloc()
373 	if (!kqd)  in kyber_queue_data_alloc()
376 	kqd->q = q;  in kyber_queue_data_alloc()
378 	kqd->cpu_latency = alloc_percpu_gfp(struct kyber_cpu_latency,  in kyber_queue_data_alloc()
380 	if (!kqd->cpu_latency)  in kyber_queue_data_alloc()
383 	timer_setup(&kqd->timer, kyber_timer_fn, 0);  in kyber_queue_data_alloc()
388 		ret = sbitmap_queue_init_node(&kqd->domain_tokens[i],  in kyber_queue_data_alloc()
393 				sbitmap_queue_free(&kqd->domain_tokens[i]);  in kyber_queue_data_alloc()
399 		kqd->domain_p99[i] = -1;  in kyber_queue_data_alloc()
400 		kqd->latency_targets[i] = kyber_latency_targets[i];  in kyber_queue_data_alloc()
404 	kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U;  in kyber_queue_data_alloc()
406 	return kqd;  in kyber_queue_data_alloc()
409 	free_percpu(kqd->cpu_latency);  in kyber_queue_data_alloc()
411 	kfree(kqd);  in kyber_queue_data_alloc()
418 	struct kyber_queue_data *kqd;  in kyber_init_sched()  local
425 	kqd = kyber_queue_data_alloc(q);  in kyber_init_sched()
426 	if (IS_ERR(kqd)) {  in kyber_init_sched()
428 		return PTR_ERR(kqd);  in kyber_init_sched()
433 	eq->elevator_data = kqd;  in kyber_init_sched()
441 	struct kyber_queue_data *kqd = e->elevator_data;  in kyber_exit_sched()  local
444 	del_timer_sync(&kqd->timer);  in kyber_exit_sched()
447 		sbitmap_queue_free(&kqd->domain_tokens[i]);  in kyber_exit_sched()
448 	free_percpu(kqd->cpu_latency);  in kyber_exit_sched()
449 	kfree(kqd);  in kyber_exit_sched()
463 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_init_hctx()  local
506 					kqd->async_depth);  in kyber_init_hctx()
538 static void rq_clear_domain_token(struct kyber_queue_data *kqd,  in rq_clear_domain_token()  argument
547 		sbitmap_queue_clear(&kqd->domain_tokens[sched_domain], nr,  in rq_clear_domain_token()
559 		struct kyber_queue_data *kqd = data->q->elevator->elevator_data;  in kyber_limit_depth()  local
561 		data->shallow_depth = kqd->async_depth;  in kyber_limit_depth()
613 	struct kyber_queue_data *kqd = rq->q->elevator->elevator_data;  in kyber_finish_request()  local
615 	rq_clear_domain_token(kqd, rq);  in kyber_finish_request()
638 	struct kyber_queue_data *kqd = rq->q->elevator->elevator_data;  in kyber_completed_request()  local
647 	cpu_latency = get_cpu_ptr(kqd->cpu_latency);  in kyber_completed_request()
648 	target = kqd->latency_targets[sched_domain];  in kyber_completed_request()
653 	put_cpu_ptr(kqd->cpu_latency);  in kyber_completed_request()
655 	timer_reduce(&kqd->timer, jiffies + HZ / 10);  in kyber_completed_request()
703 static int kyber_get_domain_token(struct kyber_queue_data *kqd,  in kyber_get_domain_token()  argument
708 	struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];  in kyber_get_domain_token()
751 kyber_dispatch_cur_domain(struct kyber_queue_data *kqd,  in kyber_dispatch_cur_domain()  argument
771 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
778 			trace_kyber_throttled(kqd->q,  in kyber_dispatch_cur_domain()
782 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
791 			trace_kyber_throttled(kqd->q,  in kyber_dispatch_cur_domain()
802 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_dispatch_request()  local
814 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
835 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
864 	struct kyber_queue_data *kqd = e->elevator_data;		\
866 	return sprintf(page, "%llu\n", kqd->latency_targets[domain]);	\
872 	struct kyber_queue_data *kqd = e->elevator_data;		\
880 	kqd->latency_targets[domain] = nsec;				\
901 	struct kyber_queue_data *kqd = q->elevator->elevator_data;	\
903 	sbitmap_queue_show(&kqd->domain_tokens[domain], m);		\
960 	struct kyber_queue_data *kqd = q->elevator->elevator_data;  in KYBER_DEBUGFS_DOMAIN_ATTRS()  local
962 	seq_printf(m, "%u\n", kqd->async_depth);  in KYBER_DEBUGFS_DOMAIN_ATTRS()