71 static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl,  in get_discard_list()  argument
74 	return &discard_ctl->discard_list[block_group->discard_index];  in get_discard_list()
77 static void __add_to_discard_list(struct btrfs_discard_ctl *discard_ctl,  in __add_to_discard_list()  argument
80 	if (!btrfs_run_discard_work(discard_ctl))  in __add_to_discard_list()
93 		       get_discard_list(discard_ctl, block_group));  in __add_to_discard_list()
96 static void add_to_discard_list(struct btrfs_discard_ctl *discard_ctl,  in add_to_discard_list()  argument
102 	spin_lock(&discard_ctl->lock);  in add_to_discard_list()
103 	__add_to_discard_list(discard_ctl, block_group);  in add_to_discard_list()
104 	spin_unlock(&discard_ctl->lock);  in add_to_discard_list()
107 static void add_to_discard_unused_list(struct btrfs_discard_ctl *discard_ctl,  in add_to_discard_unused_list()  argument
110 	spin_lock(&discard_ctl->lock);  in add_to_discard_unused_list()
112 	if (!btrfs_run_discard_work(discard_ctl)) {  in add_to_discard_unused_list()
113 		spin_unlock(&discard_ctl->lock);  in add_to_discard_unused_list()
124 		      &discard_ctl->discard_list[BTRFS_DISCARD_INDEX_UNUSED]);  in add_to_discard_unused_list()
126 	spin_unlock(&discard_ctl->lock);  in add_to_discard_unused_list()
129 static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl,  in remove_from_discard_list()  argument
134 	spin_lock(&discard_ctl->lock);  in remove_from_discard_list()
136 	if (block_group == discard_ctl->block_group) {  in remove_from_discard_list()
138 		discard_ctl->block_group = NULL;  in remove_from_discard_list()
144 	spin_unlock(&discard_ctl->lock);  in remove_from_discard_list()
158 					struct btrfs_discard_ctl *discard_ctl,  in find_next_block_group()  argument
165 		struct list_head *discard_list = &discard_ctl->discard_list[i];  in find_next_block_group()
200 					struct btrfs_discard_ctl *discard_ctl,  in peek_discard_list()  argument
206 	spin_lock(&discard_ctl->lock);  in peek_discard_list()
208 	block_group = find_next_block_group(discard_ctl, now);  in peek_discard_list()
214 				__add_to_discard_list(discard_ctl, block_group);  in peek_discard_list()
223 		discard_ctl->block_group = block_group;  in peek_discard_list()
229 	spin_unlock(&discard_ctl->lock);  in peek_discard_list()
247 	struct btrfs_discard_ctl *discard_ctl;  in btrfs_discard_check_filter()  local
253 	discard_ctl = &block_group->fs_info->discard_ctl;  in btrfs_discard_check_filter()
259 		remove_from_discard_list(discard_ctl, block_group);  in btrfs_discard_check_filter()
265 				add_to_discard_list(discard_ctl, block_group);  in btrfs_discard_check_filter()
280 static void btrfs_update_discard_index(struct btrfs_discard_ctl *discard_ctl,  in btrfs_update_discard_index()  argument
289 	add_to_discard_list(discard_ctl, block_group);  in btrfs_update_discard_index()
300 void btrfs_discard_cancel_work(struct btrfs_discard_ctl *discard_ctl,  in btrfs_discard_cancel_work()  argument
303 	if (remove_from_discard_list(discard_ctl, block_group)) {  in btrfs_discard_cancel_work()
304 		cancel_delayed_work_sync(&discard_ctl->work);  in btrfs_discard_cancel_work()
305 		btrfs_discard_schedule_work(discard_ctl, true);  in btrfs_discard_cancel_work()
316 void btrfs_discard_queue_work(struct btrfs_discard_ctl *discard_ctl,  in btrfs_discard_queue_work()  argument
323 		add_to_discard_unused_list(discard_ctl, block_group);  in btrfs_discard_queue_work()
325 		add_to_discard_list(discard_ctl, block_group);  in btrfs_discard_queue_work()
327 	if (!delayed_work_pending(&discard_ctl->work))  in btrfs_discard_queue_work()
328 		btrfs_discard_schedule_work(discard_ctl, false);  in btrfs_discard_queue_work()
331 static void __btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,  in __btrfs_discard_schedule_work()  argument
336 	if (!btrfs_run_discard_work(discard_ctl))  in __btrfs_discard_schedule_work()
338 	if (!override && delayed_work_pending(&discard_ctl->work))  in __btrfs_discard_schedule_work()
341 	block_group = find_next_block_group(discard_ctl, now);  in __btrfs_discard_schedule_work()
343 		unsigned long delay = discard_ctl->delay;  in __btrfs_discard_schedule_work()
344 		u32 kbps_limit = READ_ONCE(discard_ctl->kbps_limit);  in __btrfs_discard_schedule_work()
351 		if (kbps_limit && discard_ctl->prev_discard) {  in __btrfs_discard_schedule_work()
353 			u64 bps_delay = div64_u64(discard_ctl->prev_discard *  in __btrfs_discard_schedule_work()
369 		mod_delayed_work(discard_ctl->discard_workers,  in __btrfs_discard_schedule_work()
370 				 &discard_ctl->work, delay);  in __btrfs_discard_schedule_work()
383 void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,  in btrfs_discard_schedule_work()  argument
388 	spin_lock(&discard_ctl->lock);  in btrfs_discard_schedule_work()
389 	__btrfs_discard_schedule_work(discard_ctl, now, override);  in btrfs_discard_schedule_work()
390 	spin_unlock(&discard_ctl->lock);  in btrfs_discard_schedule_work()
403 static void btrfs_finish_discard_pass(struct btrfs_discard_ctl *discard_ctl,  in btrfs_finish_discard_pass()  argument
406 	remove_from_discard_list(discard_ctl, block_group);  in btrfs_finish_discard_pass()
412 			add_to_discard_unused_list(discard_ctl, block_group);  in btrfs_finish_discard_pass()
414 		btrfs_update_discard_index(discard_ctl, block_group);  in btrfs_finish_discard_pass()
428 	struct btrfs_discard_ctl *discard_ctl;  in btrfs_discard_workfn()  local
436 	discard_ctl = container_of(work, struct btrfs_discard_ctl, work.work);  in btrfs_discard_workfn()
438 	block_group = peek_discard_list(discard_ctl, &discard_state,  in btrfs_discard_workfn()
440 	if (!block_group || !btrfs_run_discard_work(discard_ctl))  in btrfs_discard_workfn()
443 		btrfs_discard_schedule_work(discard_ctl, false);  in btrfs_discard_workfn()
466 		discard_ctl->discard_bitmap_bytes += trimmed;  in btrfs_discard_workfn()
472 		discard_ctl->discard_extent_bytes += trimmed;  in btrfs_discard_workfn()
475 	discard_ctl->prev_discard = trimmed;  in btrfs_discard_workfn()
480 			btrfs_finish_discard_pass(discard_ctl, block_group);  in btrfs_discard_workfn()
483 			spin_lock(&discard_ctl->lock);  in btrfs_discard_workfn()
488 			spin_unlock(&discard_ctl->lock);  in btrfs_discard_workfn()
492 	spin_lock(&discard_ctl->lock);  in btrfs_discard_workfn()
493 	discard_ctl->block_group = NULL;  in btrfs_discard_workfn()
494 	__btrfs_discard_schedule_work(discard_ctl, now, false);  in btrfs_discard_workfn()
495 	spin_unlock(&discard_ctl->lock);  in btrfs_discard_workfn()
504 bool btrfs_run_discard_work(struct btrfs_discard_ctl *discard_ctl)  in btrfs_run_discard_work()  argument
506 	struct btrfs_fs_info *fs_info = container_of(discard_ctl,  in btrfs_run_discard_work()
508 						     discard_ctl);  in btrfs_run_discard_work()
522 void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)  in btrfs_discard_calc_delay()  argument
530 	discardable_extents = atomic_read(&discard_ctl->discardable_extents);  in btrfs_discard_calc_delay()
534 	spin_lock(&discard_ctl->lock);  in btrfs_discard_calc_delay()
545 			   &discard_ctl->discardable_extents);  in btrfs_discard_calc_delay()
547 	discardable_bytes = atomic64_read(&discard_ctl->discardable_bytes);  in btrfs_discard_calc_delay()
550 			     &discard_ctl->discardable_bytes);  in btrfs_discard_calc_delay()
553 		spin_unlock(&discard_ctl->lock);  in btrfs_discard_calc_delay()
557 	iops_limit = READ_ONCE(discard_ctl->iops_limit);  in btrfs_discard_calc_delay()
564 	discard_ctl->delay = msecs_to_jiffies(delay);  in btrfs_discard_calc_delay()
566 	spin_unlock(&discard_ctl->lock);  in btrfs_discard_calc_delay()
581 	struct btrfs_discard_ctl *discard_ctl;  in btrfs_discard_update_discardable()  local
590 	discard_ctl = &block_group->fs_info->discard_ctl;  in btrfs_discard_update_discardable()
595 		atomic_add(extents_delta, &discard_ctl->discardable_extents);  in btrfs_discard_update_discardable()
603 		atomic64_add(bytes_delta, &discard_ctl->discardable_bytes);  in btrfs_discard_update_discardable()
629 		btrfs_discard_queue_work(&fs_info->discard_ctl, block_group);  in btrfs_discard_punt_unused_bgs_list()
644 static void btrfs_discard_purge_list(struct btrfs_discard_ctl *discard_ctl)  in btrfs_discard_purge_list()  argument
649 	spin_lock(&discard_ctl->lock);  in btrfs_discard_purge_list()
652 					 &discard_ctl->discard_list[i],  in btrfs_discard_purge_list()
655 			spin_unlock(&discard_ctl->lock);  in btrfs_discard_purge_list()
658 			spin_lock(&discard_ctl->lock);  in btrfs_discard_purge_list()
661 	spin_unlock(&discard_ctl->lock);  in btrfs_discard_purge_list()
683 	struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl;  in btrfs_discard_init()  local
686 	spin_lock_init(&discard_ctl->lock);  in btrfs_discard_init()
687 	INIT_DELAYED_WORK(&discard_ctl->work, btrfs_discard_workfn);  in btrfs_discard_init()
690 		INIT_LIST_HEAD(&discard_ctl->discard_list[i]);  in btrfs_discard_init()
692 	discard_ctl->prev_discard = 0;  in btrfs_discard_init()
693 	atomic_set(&discard_ctl->discardable_extents, 0);  in btrfs_discard_init()
694 	atomic64_set(&discard_ctl->discardable_bytes, 0);  in btrfs_discard_init()
695 	discard_ctl->max_discard_size = BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE;  in btrfs_discard_init()
696 	discard_ctl->delay = BTRFS_DISCARD_MAX_DELAY_MSEC;  in btrfs_discard_init()
697 	discard_ctl->iops_limit = BTRFS_DISCARD_MAX_IOPS;  in btrfs_discard_init()
698 	discard_ctl->kbps_limit = 0;  in btrfs_discard_init()
699 	discard_ctl->discard_extent_bytes = 0;  in btrfs_discard_init()
700 	discard_ctl->discard_bitmap_bytes = 0;  in btrfs_discard_init()
701 	atomic64_set(&discard_ctl->discard_bytes_saved, 0);  in btrfs_discard_init()
707 	cancel_delayed_work_sync(&fs_info->discard_ctl.work);  in btrfs_discard_cleanup()
708 	btrfs_discard_purge_list(&fs_info->discard_ctl);  in btrfs_discard_cleanup()