138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,  in bfqg_stats_set_start_group_wait_time()
139 						 struct bfq_group *curr_bfqg)  in bfqg_stats_set_start_group_wait_time()
166 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)  in bfqg_stats_update_dequeue()
171 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_empty_time()
190 void bfqg_stats_update_idle_time(struct bfq_group *bfqg)  in bfqg_stats_update_idle_time()
204 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_idle_time()
212 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)  in bfqg_stats_update_avg_queue_size()
222 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()
231 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op)  in bfqg_stats_update_io_remove()
236 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)  in bfqg_stats_update_io_merged()
241 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()
257 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()
259 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op) { }  in bfqg_stats_update_io_remove()
260 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op) { }  in bfqg_stats_update_io_merged()
261 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()
263 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }  in bfqg_stats_update_dequeue()
264 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_empty_time()
265 void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_update_idle_time()
266 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_idle_time()
267 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { }  in bfqg_stats_update_avg_queue_size()
279 static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd)  in pd_to_bfqg()
281 	return pd ? container_of(pd, struct bfq_group, pd) : NULL;  in pd_to_bfqg()
284 struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)  in bfqg_to_blkg()
289 static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)  in blkg_to_bfqg()
301 static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)  in bfqg_parent()
308 struct bfq_group *bfqq_group(struct bfq_queue *bfqq)  in bfqq_group()
312 	return group_entity ? container_of(group_entity, struct bfq_group,  in bfqq_group()
322 static void bfqg_get(struct bfq_group *bfqg)  in bfqg_get()
327 static void bfqg_put(struct bfq_group *bfqg)  in bfqg_put()
335 static void bfqg_and_blkg_get(struct bfq_group *bfqg)  in bfqg_and_blkg_get()
343 void bfqg_and_blkg_put(struct bfq_group *bfqg)  in bfqg_and_blkg_put()
352 	struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);  in bfqg_stats_update_legacy_io()
406 static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)  in bfqg_stats_xfer_dead()
408 	struct bfq_group *parent;  in bfqg_stats_xfer_dead()
424 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()
524 	struct bfq_group *bfqg;  in bfq_pd_alloc()
543 	struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_pd_init()
562 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_free()
570 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_reset_stats()
575 static void bfq_group_set_parent(struct bfq_group *bfqg,  in bfq_group_set_parent()
576 					struct bfq_group *parent)  in bfq_group_set_parent()
585 static void bfq_link_bfqg(struct bfq_data *bfqd, struct bfq_group *bfqg)  in bfq_link_bfqg()
587 	struct bfq_group *parent;  in bfq_link_bfqg()
597 		struct bfq_group *curr_bfqg = container_of(entity,  in bfq_link_bfqg()
598 						struct bfq_group, entity);  in bfq_link_bfqg()
608 struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)  in bfq_bio_bfqg()
611 	struct bfq_group *bfqg;  in bfq_bio_bfqg()
645 		   struct bfq_group *bfqg)  in bfq_bfqq_move()
706 				     struct bfq_group *bfqg)  in __bfq_bic_change_cgroup()
763 	struct bfq_group *bfqg = bfq_bio_bfqg(bfqd, bio);  in bfq_bic_update_cgroup()
883 				       struct bfq_group *bfqg,  in bfq_reparent_active_queues()
910 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_offline()
977 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_end_wr_async()
1001 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_weight_device()
1019 static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)  in bfq_group_set_weight()
1068 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_io_set_weight_legacy()
1085 	struct bfq_group *bfqg;  in bfq_io_set_device_weight()
1204 	struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);  in bfqg_prfill_sectors()
1223 			offsetof(struct bfq_group, stats.bytes), &tmp);  in bfqg_prfill_sectors_recursive()
1240 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_avg_queue_size()
1262 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()
1306 		.private = offsetof(struct bfq_group, stats.bytes),
1311 		.private = offsetof(struct bfq_group, stats.ios),
1317 		.private = offsetof(struct bfq_group, stats.time),
1326 		.private = offsetof(struct bfq_group, stats.service_time),
1331 		.private = offsetof(struct bfq_group, stats.wait_time),
1336 		.private = offsetof(struct bfq_group, stats.merged),
1341 		.private = offsetof(struct bfq_group, stats.queued),
1349 		.private = offsetof(struct bfq_group, stats.bytes),
1354 		.private = offsetof(struct bfq_group, stats.ios),
1360 		.private = offsetof(struct bfq_group, stats.time),
1369 		.private = offsetof(struct bfq_group, stats.service_time),
1374 		.private = offsetof(struct bfq_group, stats.wait_time),
1379 		.private = offsetof(struct bfq_group, stats.merged),
1384 		.private = offsetof(struct bfq_group, stats.queued),
1393 		.private = offsetof(struct bfq_group, stats.group_wait_time),
1398 		.private = offsetof(struct bfq_group, stats.idle_time),
1403 		.private = offsetof(struct bfq_group, stats.empty_time),
1408 		.private = offsetof(struct bfq_group, stats.dequeue),
1428 		   struct bfq_group *bfqg) {}  in bfq_bfqq_move()
1430 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()
1450 struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)  in bfq_bio_bfqg()
1455 struct bfq_group *bfqq_group(struct bfq_queue *bfqq)  in bfqq_group()
1460 void bfqg_and_blkg_get(struct bfq_group *bfqg) {}  in bfqg_and_blkg_get()
1462 void bfqg_and_blkg_put(struct bfq_group *bfqg) {}  in bfqg_and_blkg_put()
1464 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()
1466 	struct bfq_group *bfqg;  in bfq_create_group_hierarchy()