329 				struct child_latency_info *lat_info,  in scale_cookie_change()  argument
334 	unsigned long old = atomic_read(&lat_info->scale_cookie);  in scale_cookie_change()
343 			atomic_set(&lat_info->scale_cookie,  in scale_cookie_change()
346 			atomic_inc(&lat_info->scale_cookie);  in scale_cookie_change()
348 			atomic_add(scale, &lat_info->scale_cookie);  in scale_cookie_change()
358 				atomic_dec(&lat_info->scale_cookie);  in scale_cookie_change()
360 			atomic_sub(scale, &lat_info->scale_cookie);  in scale_cookie_change()
399 	struct child_latency_info *lat_info;  in check_scale_change()  local
413 	lat_info = &parent->child_lat;  in check_scale_change()
414 	cur_cookie = atomic_read(&lat_info->scale_cookie);  in check_scale_change()
415 	scale_lat = READ_ONCE(lat_info->scale_lat);  in check_scale_change()
442 		samples_thresh = lat_info->nr_samples * 5;  in check_scale_change()
529 	struct child_latency_info *lat_info;  in iolatency_check_latencies()  local
548 	lat_info = &parent->child_lat;  in iolatency_check_latencies()
554 	    atomic_read(&lat_info->scale_cookie) == DEFAULT_SCALE_COOKIE)  in iolatency_check_latencies()
558 	spin_lock_irqsave(&lat_info->lock, flags);  in iolatency_check_latencies()
561 	lat_info->nr_samples -= iolat->nr_samples;  in iolatency_check_latencies()
562 	lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
565 	if ((lat_info->last_scale_event >= now ||  in iolatency_check_latencies()
566 	    now - lat_info->last_scale_event < BLKIOLATENCY_MIN_ADJUST_TIME))  in iolatency_check_latencies()
574 		if (lat_info->scale_grp == iolat) {  in iolatency_check_latencies()
575 			lat_info->last_scale_event = now;  in iolatency_check_latencies()
576 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in iolatency_check_latencies()
578 	} else if (lat_info->scale_lat == 0 ||  in iolatency_check_latencies()
579 		   lat_info->scale_lat >= iolat->min_lat_nsec) {  in iolatency_check_latencies()
580 		lat_info->last_scale_event = now;  in iolatency_check_latencies()
581 		if (!lat_info->scale_grp ||  in iolatency_check_latencies()
582 		    lat_info->scale_lat > iolat->min_lat_nsec) {  in iolatency_check_latencies()
583 			WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec);  in iolatency_check_latencies()
584 			lat_info->scale_grp = iolat;  in iolatency_check_latencies()
586 		scale_cookie_change(iolat->blkiolat, lat_info, false);  in iolatency_check_latencies()
590 	spin_unlock_irqrestore(&lat_info->lock, flags);  in iolatency_check_latencies()
672 		struct child_latency_info *lat_info;  in blkiolatency_timer_fn()  local
687 		lat_info = &iolat->child_lat;  in blkiolatency_timer_fn()
688 		cookie = atomic_read(&lat_info->scale_cookie);  in blkiolatency_timer_fn()
693 		spin_lock_irqsave(&lat_info->lock, flags);  in blkiolatency_timer_fn()
694 		if (lat_info->last_scale_event >= now)  in blkiolatency_timer_fn()
701 		if (lat_info->scale_grp == NULL) {  in blkiolatency_timer_fn()
702 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in blkiolatency_timer_fn()
711 		if (now - lat_info->last_scale_event >=  in blkiolatency_timer_fn()
713 			lat_info->scale_grp = NULL;  in blkiolatency_timer_fn()
715 		spin_unlock_irqrestore(&lat_info->lock, flags);  in blkiolatency_timer_fn()
816 		struct child_latency_info *lat_info;  in iolatency_clear_scaling()  local
820 		lat_info = &iolat->child_lat;  in iolatency_clear_scaling()
821 		spin_lock(&lat_info->lock);  in iolatency_clear_scaling()
822 		atomic_set(&lat_info->scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_clear_scaling()
823 		lat_info->last_scale_event = 0;  in iolatency_clear_scaling()
824 		lat_info->scale_grp = NULL;  in iolatency_clear_scaling()
825 		lat_info->scale_lat = 0;  in iolatency_clear_scaling()
826 		spin_unlock(&lat_info->lock);  in iolatency_clear_scaling()