168 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)  in f2fs_need_SSR()  argument
170 	int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);  in f2fs_need_SSR()
171 	int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);  in f2fs_need_SSR()
172 	int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);  in f2fs_need_SSR()
174 	if (f2fs_lfs_mode(sbi))  in f2fs_need_SSR()
176 	if (sbi->gc_mode == GC_URGENT_HIGH)  in f2fs_need_SSR()
178 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in f2fs_need_SSR()
181 	return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +  in f2fs_need_SSR()
182 			SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));  in f2fs_need_SSR()
211 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __revoke_inmem_pages()  local
254 			err = f2fs_get_node_info(sbi, dn.nid, &ni, false);  in __revoke_inmem_pages()
261 				f2fs_invalidate_blocks(sbi, dn.data_blkaddr);  in __revoke_inmem_pages()
264 				f2fs_replace_block(sbi, &dn, dn.data_blkaddr,  in __revoke_inmem_pages()
285 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)  in f2fs_drop_inmem_pages_all()  argument
287 	struct list_head *head = &sbi->inode_list[ATOMIC_FILE];  in f2fs_drop_inmem_pages_all()
290 	unsigned int count = sbi->atomic_files;  in f2fs_drop_inmem_pages_all()
293 	spin_lock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages_all()
295 		spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages_all()
302 	spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages_all()
325 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_drop_inmem_pages()  local
333 			spin_lock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages()
338 				sbi->atomic_files--;  in f2fs_drop_inmem_pages()
340 			spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages()
354 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_drop_inmem_page()  local
359 	f2fs_bug_on(sbi, !page_private_atomic(page));  in f2fs_drop_inmem_page()
369 	f2fs_bug_on(sbi, !cur);  in f2fs_drop_inmem_page()
373 	dec_page_count(sbi, F2FS_INMEM_PAGES);  in f2fs_drop_inmem_page()
388 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __f2fs_commit_inmem_pages()  local
392 		.sbi = sbi,  in __f2fs_commit_inmem_pages()
444 		f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);  in __f2fs_commit_inmem_pages()
471 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_commit_inmem_pages()  local
475 	f2fs_balance_fs(sbi, true);  in f2fs_commit_inmem_pages()
479 	f2fs_lock_op(sbi);  in f2fs_commit_inmem_pages()
488 	f2fs_unlock_op(sbi);  in f2fs_commit_inmem_pages()
498 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)  in f2fs_balance_fs()  argument
500 	if (time_to_inject(sbi, FAULT_CHECKPOINT)) {  in f2fs_balance_fs()
501 		f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);  in f2fs_balance_fs()
502 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);  in f2fs_balance_fs()
506 	if (need && excess_cached_nats(sbi))  in f2fs_balance_fs()
507 		f2fs_balance_fs_bg(sbi, false);  in f2fs_balance_fs()
509 	if (!f2fs_is_checkpoint_ready(sbi))  in f2fs_balance_fs()
516 	if (has_not_enough_free_secs(sbi, 0, 0)) {  in f2fs_balance_fs()
517 		if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&  in f2fs_balance_fs()
518 					sbi->gc_thread->f2fs_gc_task) {  in f2fs_balance_fs()
521 			prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,  in f2fs_balance_fs()
523 			wake_up(&sbi->gc_thread->gc_wait_queue_head);  in f2fs_balance_fs()
525 			finish_wait(&sbi->gc_thread->fggc_wq, &wait);  in f2fs_balance_fs()
527 			f2fs_down_write(&sbi->gc_lock);  in f2fs_balance_fs()
528 			f2fs_gc(sbi, false, false, false, NULL_SEGNO);  in f2fs_balance_fs()
533 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)  in f2fs_balance_fs_bg()  argument
535 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))  in f2fs_balance_fs_bg()
539 	if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE))  in f2fs_balance_fs_bg()
540 		f2fs_shrink_read_extent_tree(sbi,  in f2fs_balance_fs_bg()
544 	if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE))  in f2fs_balance_fs_bg()
545 		f2fs_shrink_age_extent_tree(sbi,  in f2fs_balance_fs_bg()
549 	if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))  in f2fs_balance_fs_bg()
550 		f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);  in f2fs_balance_fs_bg()
552 	if (!f2fs_available_free_memory(sbi, FREE_NIDS))  in f2fs_balance_fs_bg()
553 		f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);  in f2fs_balance_fs_bg()
555 		f2fs_build_free_nids(sbi, false, false);  in f2fs_balance_fs_bg()
557 	if (excess_dirty_nats(sbi) || excess_dirty_nodes(sbi) ||  in f2fs_balance_fs_bg()
558 		excess_prefree_segs(sbi))  in f2fs_balance_fs_bg()
562 	if (is_inflight_io(sbi, REQ_TIME) ||  in f2fs_balance_fs_bg()
563 		(!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))  in f2fs_balance_fs_bg()
567 	if (f2fs_time_over(sbi, CP_TIME))  in f2fs_balance_fs_bg()
571 	if (f2fs_available_free_memory(sbi, NAT_ENTRIES) ||  in f2fs_balance_fs_bg()
572 		f2fs_available_free_memory(sbi, INO_ENTRIES))  in f2fs_balance_fs_bg()
576 	if (test_opt(sbi, DATA_FLUSH) && from_bg) {  in f2fs_balance_fs_bg()
579 		mutex_lock(&sbi->flush_lock);  in f2fs_balance_fs_bg()
582 		f2fs_sync_dirty_inodes(sbi, FILE_INODE, NULL);  in f2fs_balance_fs_bg()
585 		mutex_unlock(&sbi->flush_lock);  in f2fs_balance_fs_bg()
587 	f2fs_sync_fs(sbi->sb, true);  in f2fs_balance_fs_bg()
588 	stat_inc_bg_cp_count(sbi->stat_info);  in f2fs_balance_fs_bg()
591 static int __submit_flush_wait(struct f2fs_sb_info *sbi,  in __submit_flush_wait()  argument
596 	trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),  in __submit_flush_wait()
597 				test_opt(sbi, FLUSH_MERGE), ret);  in __submit_flush_wait()
601 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)  in submit_flush_wait()  argument
606 	if (!f2fs_is_multi_device(sbi))  in submit_flush_wait()
607 		return __submit_flush_wait(sbi, sbi->sb->s_bdev);  in submit_flush_wait()
609 	for (i = 0; i < sbi->s_ndevs; i++) {  in submit_flush_wait()
610 		if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))  in submit_flush_wait()
612 		ret = __submit_flush_wait(sbi, FDEV(i).bdev);  in submit_flush_wait()
621 	struct f2fs_sb_info *sbi = data;  in issue_flush_thread()  local
622 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in issue_flush_thread()
637 		ret = submit_flush_wait(sbi, cmd->ino);  in issue_flush_thread()
653 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)  in f2fs_issue_flush()  argument
655 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in f2fs_issue_flush()
659 	if (test_opt(sbi, NOBARRIER))  in f2fs_issue_flush()
662 	if (!test_opt(sbi, FLUSH_MERGE)) {  in f2fs_issue_flush()
664 		ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
671 	    f2fs_is_multi_device(sbi)) {  in f2fs_issue_flush()
672 		ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
707 			ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
724 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)  in f2fs_create_flush_cmd_control()  argument
726 	dev_t dev = sbi->sb->s_bdev->bd_dev;  in f2fs_create_flush_cmd_control()
730 	if (SM_I(sbi)->fcc_info) {  in f2fs_create_flush_cmd_control()
731 		fcc = SM_I(sbi)->fcc_info;  in f2fs_create_flush_cmd_control()
737 	fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);  in f2fs_create_flush_cmd_control()
744 	SM_I(sbi)->fcc_info = fcc;  in f2fs_create_flush_cmd_control()
745 	if (!test_opt(sbi, FLUSH_MERGE))  in f2fs_create_flush_cmd_control()
749 	fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,  in f2fs_create_flush_cmd_control()
754 		SM_I(sbi)->fcc_info = NULL;  in f2fs_create_flush_cmd_control()
761 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)  in f2fs_destroy_flush_cmd_control()  argument
763 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in f2fs_destroy_flush_cmd_control()
773 		SM_I(sbi)->fcc_info = NULL;  in f2fs_destroy_flush_cmd_control()
777 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)  in f2fs_flush_device_cache()  argument
781 	if (!f2fs_is_multi_device(sbi))  in f2fs_flush_device_cache()
784 	if (test_opt(sbi, NOBARRIER))  in f2fs_flush_device_cache()
787 	for (i = 1; i < sbi->s_ndevs; i++) {  in f2fs_flush_device_cache()
788 		if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))  in f2fs_flush_device_cache()
790 		ret = __submit_flush_wait(sbi, FDEV(i).bdev);  in f2fs_flush_device_cache()
792 			f2fs_stop_checkpoint(sbi, false,  in f2fs_flush_device_cache()
797 		spin_lock(&sbi->dev_lock);  in f2fs_flush_device_cache()
798 		f2fs_clear_bit(i, (char *)&sbi->dirty_device);  in f2fs_flush_device_cache()
799 		spin_unlock(&sbi->dev_lock);  in f2fs_flush_device_cache()
805 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,  in __locate_dirty_segment()  argument
808 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in __locate_dirty_segment()
811 	if (IS_CURSEG(sbi, segno))  in __locate_dirty_segment()
818 		struct seg_entry *sentry = get_seg_entry(sbi, segno);  in __locate_dirty_segment()
822 			f2fs_bug_on(sbi, 1);  in __locate_dirty_segment()
828 		if (__is_large_section(sbi)) {  in __locate_dirty_segment()
829 			unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);  in __locate_dirty_segment()
831 				get_valid_blocks(sbi, segno, true);  in __locate_dirty_segment()
833 			f2fs_bug_on(sbi, unlikely(!valid_blocks ||  in __locate_dirty_segment()
834 					valid_blocks == BLKS_PER_SEC(sbi)));  in __locate_dirty_segment()
836 			if (!IS_CURSEC(sbi, secno))  in __locate_dirty_segment()
842 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,  in __remove_dirty_segment()  argument
845 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in __remove_dirty_segment()
852 		struct seg_entry *sentry = get_seg_entry(sbi, segno);  in __remove_dirty_segment()
858 		valid_blocks = get_valid_blocks(sbi, segno, true);  in __remove_dirty_segment()
860 			clear_bit(GET_SEC_FROM_SEG(sbi, segno),  in __remove_dirty_segment()
863 			clear_bit(segno, SIT_I(sbi)->invalid_segmap);  in __remove_dirty_segment()
866 		if (__is_large_section(sbi)) {  in __remove_dirty_segment()
867 			unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);  in __remove_dirty_segment()
870 					valid_blocks == BLKS_PER_SEC(sbi)) {  in __remove_dirty_segment()
875 			if (!IS_CURSEC(sbi, secno))  in __remove_dirty_segment()
886 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)  in locate_dirty_segment()  argument
888 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in locate_dirty_segment()
892 	if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))  in locate_dirty_segment()
895 	usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);  in locate_dirty_segment()
898 	valid_blocks = get_valid_blocks(sbi, segno, false);  in locate_dirty_segment()
899 	ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);  in locate_dirty_segment()
901 	if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||  in locate_dirty_segment()
903 		__locate_dirty_segment(sbi, segno, PRE);  in locate_dirty_segment()
904 		__remove_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
906 		__locate_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
909 		__remove_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
916 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)  in f2fs_dirty_to_prefree()  argument
918 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_dirty_to_prefree()
922 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in f2fs_dirty_to_prefree()
923 		if (get_valid_blocks(sbi, segno, false))  in f2fs_dirty_to_prefree()
925 		if (IS_CURSEG(sbi, segno))  in f2fs_dirty_to_prefree()
927 		__locate_dirty_segment(sbi, segno, PRE);  in f2fs_dirty_to_prefree()
928 		__remove_dirty_segment(sbi, segno, DIRTY);  in f2fs_dirty_to_prefree()
933 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)  in f2fs_get_unusable_blocks()  argument
936 		(overprovision_segments(sbi) - reserved_segments(sbi));  in f2fs_get_unusable_blocks()
937 	block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;  in f2fs_get_unusable_blocks()
938 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_get_unusable_blocks()
945 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in f2fs_get_unusable_blocks()
946 		se = get_seg_entry(sbi, segno);  in f2fs_get_unusable_blocks()
948 			holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -  in f2fs_get_unusable_blocks()
951 			holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -  in f2fs_get_unusable_blocks()
962 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)  in f2fs_disable_cp_again()  argument
965 		(overprovision_segments(sbi) - reserved_segments(sbi));  in f2fs_disable_cp_again()
966 	if (unusable > F2FS_OPTION(sbi).unusable_cap)  in f2fs_disable_cp_again()
968 	if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&  in f2fs_disable_cp_again()
969 		dirty_segments(sbi) > ovp_hole_segs)  in f2fs_disable_cp_again()
975 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)  in get_free_segment()  argument
977 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in get_free_segment()
981 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in get_free_segment()
982 		if (get_valid_blocks(sbi, segno, false))  in get_free_segment()
984 		if (get_ckpt_valid_blocks(sbi, segno, false))  in get_free_segment()
993 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,  in __create_discard_cmd()  argument
997 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __create_discard_cmd()
1001 	f2fs_bug_on(sbi, !len);  in __create_discard_cmd()
1025 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,  in __attach_discard_cmd()  argument
1031 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __attach_discard_cmd()
1034 	dc = __create_discard_cmd(sbi, bdev, lstart, start, len);  in __attach_discard_cmd()
1057 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,  in __remove_discard_cmd()  argument
1060 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __remove_discard_cmd()
1072 	f2fs_bug_on(sbi, dc->ref);  in __remove_discard_cmd()
1080 			KERN_INFO, sbi->sb->s_id,  in __remove_discard_cmd()
1102 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,  in __check_sit_bitmap()  argument
1109 	unsigned long offset, size, max_blocks = sbi->blocks_per_seg;  in __check_sit_bitmap()
1113 		segno = GET_SEGNO(sbi, blk);  in __check_sit_bitmap()
1114 		sentry = get_seg_entry(sbi, segno);  in __check_sit_bitmap()
1115 		offset = GET_BLKOFF_FROM_SEG0(sbi, blk);  in __check_sit_bitmap()
1117 		if (end < START_BLOCK(sbi, segno + 1))  in __check_sit_bitmap()
1118 			size = GET_BLKOFF_FROM_SEG0(sbi, end);  in __check_sit_bitmap()
1123 		f2fs_bug_on(sbi, offset != size);  in __check_sit_bitmap()
1124 		blk = START_BLOCK(sbi, segno + 1);  in __check_sit_bitmap()
1129 static void __init_discard_policy(struct f2fs_sb_info *sbi,  in __init_discard_policy()  argument
1133 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __init_discard_policy()
1152 		if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {  in __init_discard_policy()
1173 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1177 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,  in __submit_discard_cmd()  argument
1186 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __submit_discard_cmd()
1196 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))  in __submit_discard_cmd()
1224 		if (time_to_inject(sbi, FAULT_DISCARD)) {  in __submit_discard_cmd()
1225 			f2fs_show_injection_info(sbi, FAULT_DISCARD);  in __submit_discard_cmd()
1243 		f2fs_bug_on(sbi, !bio);  in __submit_discard_cmd()
1262 		__check_sit_bitmap(sbi, lstart, lstart + len);  in __submit_discard_cmd()
1271 		f2fs_update_iostat(sbi, FS_DISCARD, 1);  in __submit_discard_cmd()
1281 		__update_discard_tree_range(sbi, bdev, lstart, start, len);  in __submit_discard_cmd()
1286 static void __insert_discard_tree(struct f2fs_sb_info *sbi,  in __insert_discard_tree()  argument
1292 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __insert_discard_tree()
1303 	p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,  in __insert_discard_tree()
1306 	__attach_discard_cmd(sbi, bdev, lstart, start, len, parent,  in __insert_discard_tree()
1316 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,  in __punch_discard_cmd()  argument
1319 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __punch_discard_cmd()
1324 		__remove_discard_cmd(sbi, dc);  in __punch_discard_cmd()
1339 			__insert_discard_tree(sbi, dc->bdev, blkaddr + 1,  in __punch_discard_cmd()
1353 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,  in __update_discard_tree_range()  argument
1357 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __update_discard_tree_range()
1426 				__remove_discard_cmd(sbi, tdc);  in __update_discard_tree_range()
1431 			__insert_discard_tree(sbi, bdev, di.lstart, di.start,  in __update_discard_tree_range()
1444 static int __queue_discard_cmd(struct f2fs_sb_info *sbi,  in __queue_discard_cmd()  argument
1454 	if (f2fs_is_multi_device(sbi)) {  in __queue_discard_cmd()
1455 		int devi = f2fs_target_device_index(sbi, blkstart);  in __queue_discard_cmd()
1459 	mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);  in __queue_discard_cmd()
1460 	__update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);  in __queue_discard_cmd()
1461 	mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);  in __queue_discard_cmd()
1465 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,  in __issue_discard_cmd_orderly()  argument
1468 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_orderly()
1495 		if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {  in __issue_discard_cmd_orderly()
1501 		err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);  in __issue_discard_cmd_orderly()
1508 			__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_orderly()
1524 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1527 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,  in __issue_discard_cmd()  argument
1530 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd()
1538 		f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);  in __issue_discard_cmd()
1544 				f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))  in __issue_discard_cmd()
1551 			return __issue_discard_cmd_orderly(sbi, dpolicy);  in __issue_discard_cmd()
1559 			f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,  in __issue_discard_cmd()
1563 			f2fs_bug_on(sbi, dc->state != D_PREP);  in __issue_discard_cmd()
1566 				f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))  in __issue_discard_cmd()
1570 						!is_idle(sbi, DISCARD_TIME)) {  in __issue_discard_cmd()
1575 			__submit_discard_cmd(sbi, dpolicy, dc, &issued);  in __issue_discard_cmd()
1589 		__wait_all_discard_cmd(sbi, dpolicy);  in __issue_discard_cmd()
1599 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)  in __drop_discard_cmd()  argument
1601 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __drop_discard_cmd()
1611 			f2fs_bug_on(sbi, dc->state != D_PREP);  in __drop_discard_cmd()
1612 			__remove_discard_cmd(sbi, dc);  in __drop_discard_cmd()
1621 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)  in f2fs_drop_discard_cmd()  argument
1623 	__drop_discard_cmd(sbi);  in f2fs_drop_discard_cmd()
1626 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,  in __wait_one_discard_bio()  argument
1629 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_one_discard_bio()
1634 	f2fs_bug_on(sbi, dc->state != D_DONE);  in __wait_one_discard_bio()
1639 		__remove_discard_cmd(sbi, dc);  in __wait_one_discard_bio()
1646 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,  in __wait_discard_cmd_range()  argument
1650 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_discard_cmd_range()
1670 			__remove_discard_cmd(sbi, dc);  in __wait_discard_cmd_range()
1680 		trimmed += __wait_one_discard_bio(sbi, dc);  in __wait_discard_cmd_range()
1687 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,  in __wait_all_discard_cmd()  argument
1694 		return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);  in __wait_all_discard_cmd()
1697 	__init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);  in __wait_all_discard_cmd()
1698 	discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);  in __wait_all_discard_cmd()
1699 	__init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);  in __wait_all_discard_cmd()
1700 	discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);  in __wait_all_discard_cmd()
1706 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)  in f2fs_wait_discard_bio()  argument
1708 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_wait_discard_bio()
1717 			__punch_discard_cmd(sbi, dc, blkaddr);  in f2fs_wait_discard_bio()
1726 		__wait_one_discard_bio(sbi, dc);  in f2fs_wait_discard_bio()
1729 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)  in f2fs_stop_discard_thread()  argument
1731 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_stop_discard_thread()
1742 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)  in f2fs_issue_discard_timeout()  argument
1744 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_issue_discard_timeout()
1748 	__init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,  in f2fs_issue_discard_timeout()
1750 	__issue_discard_cmd(sbi, &dpolicy);  in f2fs_issue_discard_timeout()
1751 	dropped = __drop_discard_cmd(sbi);  in f2fs_issue_discard_timeout()
1754 	__wait_all_discard_cmd(sbi, NULL);  in f2fs_issue_discard_timeout()
1756 	f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));  in f2fs_issue_discard_timeout()
1762 	struct f2fs_sb_info *sbi = data;  in issue_discard_thread()  local
1763 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in issue_discard_thread()
1772 		if (sbi->gc_mode == GC_URGENT_HIGH ||  in issue_discard_thread()
1773 			!f2fs_available_free_memory(sbi, DISCARD_CACHE))  in issue_discard_thread()
1774 			__init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);  in issue_discard_thread()
1776 			__init_discard_policy(sbi, &dpolicy, DPOLICY_BG,  in issue_discard_thread()
1792 			__wait_all_discard_cmd(sbi, NULL);  in issue_discard_thread()
1796 		if (f2fs_readonly(sbi->sb))  in issue_discard_thread()
1800 		if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {  in issue_discard_thread()
1807 		sb_start_intwrite(sbi->sb);  in issue_discard_thread()
1809 		issued = __issue_discard_cmd(sbi, &dpolicy);  in issue_discard_thread()
1811 			__wait_all_discard_cmd(sbi, &dpolicy);  in issue_discard_thread()
1814 			wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);  in issue_discard_thread()
1821 		sb_end_intwrite(sbi->sb);  in issue_discard_thread()
1828 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,  in __f2fs_issue_discard_zone()  argument
1835 	if (f2fs_is_multi_device(sbi)) {  in __f2fs_issue_discard_zone()
1836 		devi = f2fs_target_device_index(sbi, blkstart);  in __f2fs_issue_discard_zone()
1839 			f2fs_err(sbi, "Invalid block %x", blkstart);  in __f2fs_issue_discard_zone()
1846 	if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {  in __f2fs_issue_discard_zone()
1852 			f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",  in __f2fs_issue_discard_zone()
1853 				 devi, sbi->s_ndevs ? FDEV(devi).path : "",  in __f2fs_issue_discard_zone()
1863 	return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);  in __f2fs_issue_discard_zone()
1867 static int __issue_discard_async(struct f2fs_sb_info *sbi,  in __issue_discard_async()  argument
1871 	if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))  in __issue_discard_async()
1872 		return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);  in __issue_discard_async()
1874 	return __queue_discard_cmd(sbi, bdev, blkstart, blklen);  in __issue_discard_async()
1877 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,  in f2fs_issue_discard()  argument
1887 	bdev = f2fs_target_device(sbi, blkstart, NULL);  in f2fs_issue_discard()
1892 				f2fs_target_device(sbi, i, NULL);  in f2fs_issue_discard()
1895 				err = __issue_discard_async(sbi, bdev,  in f2fs_issue_discard()
1905 		se = get_seg_entry(sbi, GET_SEGNO(sbi, i));  in f2fs_issue_discard()
1906 		offset = GET_BLKOFF_FROM_SEG0(sbi, i);  in f2fs_issue_discard()
1909 			sbi->discard_blks--;  in f2fs_issue_discard()
1913 		err = __issue_discard_async(sbi, bdev, start, len);  in f2fs_issue_discard()
1917 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,  in add_discard_addrs()  argument
1921 	int max_blocks = sbi->blocks_per_seg;  in add_discard_addrs()
1922 	struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);  in add_discard_addrs()
1926 	unsigned long *dmap = SIT_I(sbi)->tmp_map;  in add_discard_addrs()
1930 	struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;  in add_discard_addrs()
1933 	if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi))  in add_discard_addrs()
1937 		if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||  in add_discard_addrs()
1938 			SM_I(sbi)->dcc_info->nr_discards >=  in add_discard_addrs()
1939 				SM_I(sbi)->dcc_info->max_discards)  in add_discard_addrs()
1948 	while (force || SM_I(sbi)->dcc_info->nr_discards <=  in add_discard_addrs()
1949 				SM_I(sbi)->dcc_info->max_discards) {  in add_discard_addrs()
1965 			de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);  in add_discard_addrs()
1972 		SM_I(sbi)->dcc_info->nr_discards += end - start;  in add_discard_addrs()
1983 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)  in f2fs_release_discard_addrs()  argument
1985 	struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);  in f2fs_release_discard_addrs()
1996 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)  in set_prefree_as_free_segments()  argument
1998 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in set_prefree_as_free_segments()
2002 	for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))  in set_prefree_as_free_segments()
2003 		__set_test_and_free(sbi, segno, false);  in set_prefree_as_free_segments()
2007 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,  in f2fs_clear_prefree_segments()  argument
2010 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_clear_prefree_segments()
2013 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_clear_prefree_segments()
2018 	bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);  in f2fs_clear_prefree_segments()
2027 		start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);  in f2fs_clear_prefree_segments()
2028 		if (start >= MAIN_SEGS(sbi))  in f2fs_clear_prefree_segments()
2030 		end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),  in f2fs_clear_prefree_segments()
2034 			start = rounddown(start, sbi->segs_per_sec);  in f2fs_clear_prefree_segments()
2035 			end = roundup(end, sbi->segs_per_sec);  in f2fs_clear_prefree_segments()
2043 		if (!f2fs_realtime_discard_enable(sbi))  in f2fs_clear_prefree_segments()
2050 		if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {  in f2fs_clear_prefree_segments()
2051 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start),  in f2fs_clear_prefree_segments()
2052 				(end - start) << sbi->log_blocks_per_seg);  in f2fs_clear_prefree_segments()
2056 		secno = GET_SEC_FROM_SEG(sbi, start);  in f2fs_clear_prefree_segments()
2057 		start_segno = GET_SEG_FROM_SEC(sbi, secno);  in f2fs_clear_prefree_segments()
2058 		if (!IS_CURSEC(sbi, secno) &&  in f2fs_clear_prefree_segments()
2059 			!get_valid_blocks(sbi, start, true))  in f2fs_clear_prefree_segments()
2060 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),  in f2fs_clear_prefree_segments()
2061 				sbi->segs_per_sec << sbi->log_blocks_per_seg);  in f2fs_clear_prefree_segments()
2063 		start = start_segno + sbi->segs_per_sec;  in f2fs_clear_prefree_segments()
2079 					sbi->blocks_per_seg, cur_pos);  in f2fs_clear_prefree_segments()
2082 			if (f2fs_sb_has_blkzoned(sbi) ||  in f2fs_clear_prefree_segments()
2086 			f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,  in f2fs_clear_prefree_segments()
2091 					sbi->blocks_per_seg, cur_pos);  in f2fs_clear_prefree_segments()
2097 		if (cur_pos < sbi->blocks_per_seg)  in f2fs_clear_prefree_segments()
2104 	wake_up_discard_thread(sbi, false);  in f2fs_clear_prefree_segments()
2107 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)  in create_discard_cmd_control()  argument
2109 	dev_t dev = sbi->sb->s_bdev->bd_dev;  in create_discard_cmd_control()
2113 	if (SM_I(sbi)->dcc_info) {  in create_discard_cmd_control()
2114 		dcc = SM_I(sbi)->dcc_info;  in create_discard_cmd_control()
2118 	dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);  in create_discard_cmd_control()
2133 	dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;  in create_discard_cmd_control()
2140 	SM_I(sbi)->dcc_info = dcc;  in create_discard_cmd_control()
2142 	dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,  in create_discard_cmd_control()
2147 		SM_I(sbi)->dcc_info = NULL;  in create_discard_cmd_control()
2154 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)  in destroy_discard_cmd_control()  argument
2156 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in destroy_discard_cmd_control()
2161 	f2fs_stop_discard_thread(sbi);  in destroy_discard_cmd_control()
2168 		f2fs_issue_discard_timeout(sbi);  in destroy_discard_cmd_control()
2171 	SM_I(sbi)->dcc_info = NULL;  in destroy_discard_cmd_control()
2174 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)  in __mark_sit_entry_dirty()  argument
2176 	struct sit_info *sit_i = SIT_I(sbi);  in __mark_sit_entry_dirty()
2186 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,  in __set_sit_entry_type()  argument
2189 	struct seg_entry *se = get_seg_entry(sbi, segno);  in __set_sit_entry_type()
2193 		__mark_sit_entry_dirty(sbi, segno);  in __set_sit_entry_type()
2196 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,  in get_segment_mtime()  argument
2199 	unsigned int segno = GET_SEGNO(sbi, blkaddr);  in get_segment_mtime()
2203 	return get_seg_entry(sbi, segno)->mtime;  in get_segment_mtime()
2206 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,  in update_segment_mtime()  argument
2210 	unsigned int segno = GET_SEGNO(sbi, blkaddr);  in update_segment_mtime()
2211 	unsigned long long ctime = get_mtime(sbi, false);  in update_segment_mtime()
2217 	se = get_seg_entry(sbi, segno);  in update_segment_mtime()
2225 	if (ctime > SIT_I(sbi)->max_mtime)  in update_segment_mtime()
2226 		SIT_I(sbi)->max_mtime = ctime;  in update_segment_mtime()
2229 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)  in update_sit_entry()  argument
2239 	segno = GET_SEGNO(sbi, blkaddr);  in update_sit_entry()
2241 	se = get_seg_entry(sbi, segno);  in update_sit_entry()
2243 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);  in update_sit_entry()
2245 	f2fs_bug_on(sbi, (new_vblocks < 0 ||  in update_sit_entry()
2246 			(new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));  in update_sit_entry()
2257 			f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",  in update_sit_entry()
2259 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2263 			f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",  in update_sit_entry()
2265 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2271 			sbi->discard_blks--;  in update_sit_entry()
2277 		if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {  in update_sit_entry()
2287 			f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",  in update_sit_entry()
2289 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2293 			f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",  in update_sit_entry()
2295 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2298 		} else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {  in update_sit_entry()
2306 				spin_lock(&sbi->stat_lock);  in update_sit_entry()
2307 				sbi->unusable_block_count++;  in update_sit_entry()
2308 				spin_unlock(&sbi->stat_lock);  in update_sit_entry()
2313 			sbi->discard_blks++;  in update_sit_entry()
2318 	__mark_sit_entry_dirty(sbi, segno);  in update_sit_entry()
2321 	SIT_I(sbi)->written_valid_blocks += del;  in update_sit_entry()
2323 	if (__is_large_section(sbi))  in update_sit_entry()
2324 		get_sec_entry(sbi, segno)->valid_blocks += del;  in update_sit_entry()
2327 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)  in f2fs_invalidate_blocks()  argument
2329 	unsigned int segno = GET_SEGNO(sbi, addr);  in f2fs_invalidate_blocks()
2330 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_invalidate_blocks()
2332 	f2fs_bug_on(sbi, addr == NULL_ADDR);  in f2fs_invalidate_blocks()
2336 	invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);  in f2fs_invalidate_blocks()
2337 	f2fs_invalidate_compress_page(sbi, addr);  in f2fs_invalidate_blocks()
2342 	update_segment_mtime(sbi, addr, 0);  in f2fs_invalidate_blocks()
2343 	update_sit_entry(sbi, addr, -1);  in f2fs_invalidate_blocks()
2346 	locate_dirty_segment(sbi, segno);  in f2fs_invalidate_blocks()
2351 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)  in f2fs_is_checkpointed_data()  argument
2353 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_is_checkpointed_data()
2363 	segno = GET_SEGNO(sbi, blkaddr);  in f2fs_is_checkpointed_data()
2364 	se = get_seg_entry(sbi, segno);  in f2fs_is_checkpointed_data()
2365 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);  in f2fs_is_checkpointed_data()
2378 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,  in __add_sum_entry()  argument
2381 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __add_sum_entry()
2391 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)  in f2fs_npages_for_summary_flush()  argument
2397 		if (sbi->ckpt->alloc_type[i] == SSR)  in f2fs_npages_for_summary_flush()
2398 			valid_sum_count += sbi->blocks_per_seg;  in f2fs_npages_for_summary_flush()
2402 					F2FS_CKPT(sbi)->cur_data_blkoff[i]);  in f2fs_npages_for_summary_flush()
2404 				valid_sum_count += curseg_blkoff(sbi, i);  in f2fs_npages_for_summary_flush()
2421 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)  in f2fs_get_sum_page()  argument
2423 	if (unlikely(f2fs_cp_error(sbi)))  in f2fs_get_sum_page()
2425 	return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));  in f2fs_get_sum_page()
2428 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,  in f2fs_update_meta_page()  argument
2431 	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);  in f2fs_update_meta_page()
2438 static void write_sum_page(struct f2fs_sb_info *sbi,  in write_sum_page()  argument
2441 	f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);  in write_sum_page()
2444 static void write_current_sum_page(struct f2fs_sb_info *sbi,  in write_current_sum_page()  argument
2447 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in write_current_sum_page()
2448 	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);  in write_current_sum_page()
2470 static int is_next_segment_free(struct f2fs_sb_info *sbi,  in is_next_segment_free()  argument
2474 	struct free_segmap_info *free_i = FREE_I(sbi);  in is_next_segment_free()
2476 	if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)  in is_next_segment_free()
2485 static void get_new_segment(struct f2fs_sb_info *sbi,  in get_new_segment()  argument
2488 	struct free_segmap_info *free_i = FREE_I(sbi);  in get_new_segment()
2490 	unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;  in get_new_segment()
2491 	unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);  in get_new_segment()
2492 	unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);  in get_new_segment()
2500 	if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {  in get_new_segment()
2502 			GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);  in get_new_segment()
2503 		if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))  in get_new_segment()
2507 	secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);  in get_new_segment()
2508 	if (secno >= MAIN_SECS(sbi)) {  in get_new_segment()
2511 							MAIN_SECS(sbi), 0);  in get_new_segment()
2512 			f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));  in get_new_segment()
2527 							MAIN_SECS(sbi), 0);  in get_new_segment()
2528 		f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));  in get_new_segment()
2533 	segno = GET_SEG_FROM_SEC(sbi, secno);  in get_new_segment()
2534 	zoneno = GET_ZONE_FROM_SEC(sbi, secno);  in get_new_segment()
2539 	if (sbi->secs_per_zone == 1)  in get_new_segment()
2550 		if (CURSEG_I(sbi, i)->zone == zoneno)  in get_new_segment()
2556 			hint = zoneno * sbi->secs_per_zone - 1;  in get_new_segment()
2560 			hint = (zoneno + 1) * sbi->secs_per_zone;  in get_new_segment()
2566 	f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));  in get_new_segment()
2567 	__set_inuse(sbi, segno);  in get_new_segment()
2572 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)  in reset_curseg()  argument
2574 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in reset_curseg()
2580 	curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);  in reset_curseg()
2587 	sanity_check_seg_type(sbi, seg_type);  in reset_curseg()
2593 	__set_sit_entry_type(sbi, seg_type, curseg->segno, modified);  in reset_curseg()
2596 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)  in __get_next_segno()  argument
2598 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __get_next_segno()
2601 	sanity_check_seg_type(sbi, seg_type);  in __get_next_segno()
2604 	if (__is_large_section(sbi))  in __get_next_segno()
2611 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in __get_next_segno()
2614 	if (test_opt(sbi, NOHEAP) &&  in __get_next_segno()
2618 	if (SIT_I(sbi)->last_victim[ALLOC_NEXT])  in __get_next_segno()
2619 		return SIT_I(sbi)->last_victim[ALLOC_NEXT];  in __get_next_segno()
2622 	if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)  in __get_next_segno()
2632 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)  in new_curseg()  argument
2634 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in new_curseg()
2640 		write_sum_page(sbi, curseg->sum_blk,  in new_curseg()
2641 				GET_SUM_BLOCK(sbi, segno));  in new_curseg()
2645 	if (test_opt(sbi, NOHEAP))  in new_curseg()
2648 	segno = __get_next_segno(sbi, type);  in new_curseg()
2649 	get_new_segment(sbi, &segno, new_sec, dir);  in new_curseg()
2651 	reset_curseg(sbi, type, 1);  in new_curseg()
2655 static int __next_free_blkoff(struct f2fs_sb_info *sbi,  in __next_free_blkoff()  argument
2658 	struct seg_entry *se = get_seg_entry(sbi, segno);  in __next_free_blkoff()
2660 	unsigned long *target_map = SIT_I(sbi)->tmp_map;  in __next_free_blkoff()
2668 	return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);  in __next_free_blkoff()
2676 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,  in __refresh_next_blkoff()  argument
2681 			__next_free_blkoff(sbi, seg->segno,  in __refresh_next_blkoff()
2687 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)  in f2fs_segment_has_free_slot()  argument
2689 	return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;  in f2fs_segment_has_free_slot()
2696 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)  in change_curseg()  argument
2698 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in change_curseg()
2699 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in change_curseg()
2705 		write_sum_page(sbi, curseg->sum_blk,  in change_curseg()
2706 					GET_SUM_BLOCK(sbi, curseg->segno));  in change_curseg()
2708 	__set_test_and_inuse(sbi, new_segno);  in change_curseg()
2711 	__remove_dirty_segment(sbi, new_segno, PRE);  in change_curseg()
2712 	__remove_dirty_segment(sbi, new_segno, DIRTY);  in change_curseg()
2715 	reset_curseg(sbi, type, 1);  in change_curseg()
2717 	curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);  in change_curseg()
2719 	sum_page = f2fs_get_sum_page(sbi, new_segno);  in change_curseg()
2730 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2733 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,  in get_atssr_segment()  argument
2737 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in get_atssr_segment()
2741 	if (get_ssr_segment(sbi, type, alloc_mode, age)) {  in get_atssr_segment()
2742 		struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);  in get_atssr_segment()
2745 		change_curseg(sbi, type, true);  in get_atssr_segment()
2749 		new_curseg(sbi, type, true);  in get_atssr_segment()
2751 	stat_inc_seg_type(sbi, curseg);  in get_atssr_segment()
2754 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)  in __f2fs_init_atgc_curseg()  argument
2756 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);  in __f2fs_init_atgc_curseg()
2758 	if (!sbi->am.atgc_enabled)  in __f2fs_init_atgc_curseg()
2761 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in __f2fs_init_atgc_curseg()
2764 	down_write(&SIT_I(sbi)->sentry_lock);  in __f2fs_init_atgc_curseg()
2766 	get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);  in __f2fs_init_atgc_curseg()
2768 	up_write(&SIT_I(sbi)->sentry_lock);  in __f2fs_init_atgc_curseg()
2771 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in __f2fs_init_atgc_curseg()
2774 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_init_inmem_curseg()  argument
2776 	__f2fs_init_atgc_curseg(sbi);  in f2fs_init_inmem_curseg()
2779 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)  in __f2fs_save_inmem_curseg()  argument
2781 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __f2fs_save_inmem_curseg()
2787 	if (get_valid_blocks(sbi, curseg->segno, false)) {  in __f2fs_save_inmem_curseg()
2788 		write_sum_page(sbi, curseg->sum_blk,  in __f2fs_save_inmem_curseg()
2789 				GET_SUM_BLOCK(sbi, curseg->segno));  in __f2fs_save_inmem_curseg()
2791 		mutex_lock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_save_inmem_curseg()
2792 		__set_test_and_free(sbi, curseg->segno, true);  in __f2fs_save_inmem_curseg()
2793 		mutex_unlock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_save_inmem_curseg()
2799 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_save_inmem_curseg()  argument
2801 	__f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);  in f2fs_save_inmem_curseg()
2803 	if (sbi->am.atgc_enabled)  in f2fs_save_inmem_curseg()
2804 		__f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);  in f2fs_save_inmem_curseg()
2807 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)  in __f2fs_restore_inmem_curseg()  argument
2809 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __f2fs_restore_inmem_curseg()
2814 	if (get_valid_blocks(sbi, curseg->segno, false))  in __f2fs_restore_inmem_curseg()
2817 	mutex_lock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_restore_inmem_curseg()
2818 	__set_test_and_inuse(sbi, curseg->segno);  in __f2fs_restore_inmem_curseg()
2819 	mutex_unlock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_restore_inmem_curseg()
2824 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_restore_inmem_curseg()  argument
2826 	__f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);  in f2fs_restore_inmem_curseg()
2828 	if (sbi->am.atgc_enabled)  in f2fs_restore_inmem_curseg()
2829 		__f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);  in f2fs_restore_inmem_curseg()
2832 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,  in get_ssr_segment()  argument
2835 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in get_ssr_segment()
2836 	const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;  in get_ssr_segment()
2842 	sanity_check_seg_type(sbi, seg_type);  in get_ssr_segment()
2845 	if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {  in get_ssr_segment()
2872 		if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {  in get_ssr_segment()
2879 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {  in get_ssr_segment()
2880 		segno = get_free_segment(sbi);  in get_ssr_segment()
2893 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,  in allocate_segment_by_default()  argument
2896 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in allocate_segment_by_default()
2899 		new_curseg(sbi, type, true);  in allocate_segment_by_default()
2900 	else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&  in allocate_segment_by_default()
2902 		new_curseg(sbi, type, false);  in allocate_segment_by_default()
2904 			is_next_segment_free(sbi, curseg, type) &&  in allocate_segment_by_default()
2905 			likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in allocate_segment_by_default()
2906 		new_curseg(sbi, type, false);  in allocate_segment_by_default()
2907 	else if (f2fs_need_SSR(sbi) &&  in allocate_segment_by_default()
2908 			get_ssr_segment(sbi, type, SSR, 0))  in allocate_segment_by_default()
2909 		change_curseg(sbi, type, true);  in allocate_segment_by_default()
2911 		new_curseg(sbi, type, false);  in allocate_segment_by_default()
2913 	stat_inc_seg_type(sbi, curseg);  in allocate_segment_by_default()
2916 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,  in f2fs_allocate_segment_for_resize()  argument
2919 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in f2fs_allocate_segment_for_resize()
2922 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_segment_for_resize()
2924 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_segment_for_resize()
2926 	segno = CURSEG_I(sbi, type)->segno;  in f2fs_allocate_segment_for_resize()
2930 	if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))  in f2fs_allocate_segment_for_resize()
2931 		change_curseg(sbi, type, true);  in f2fs_allocate_segment_for_resize()
2933 		new_curseg(sbi, type, true);  in f2fs_allocate_segment_for_resize()
2935 	stat_inc_seg_type(sbi, curseg);  in f2fs_allocate_segment_for_resize()
2937 	locate_dirty_segment(sbi, segno);  in f2fs_allocate_segment_for_resize()
2939 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_segment_for_resize()
2942 		f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",  in f2fs_allocate_segment_for_resize()
2946 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_segment_for_resize()
2949 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,  in __allocate_new_segment()  argument
2952 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __allocate_new_segment()
2959 		get_valid_blocks(sbi, curseg->segno, new_sec))  in __allocate_new_segment()
2962 	if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))  in __allocate_new_segment()
2966 	SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);  in __allocate_new_segment()
2967 	locate_dirty_segment(sbi, old_segno);  in __allocate_new_segment()
2970 static void __allocate_new_section(struct f2fs_sb_info *sbi,  in __allocate_new_section()  argument
2973 	__allocate_new_segment(sbi, type, true, force);  in __allocate_new_section()
2976 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)  in f2fs_allocate_new_section()  argument
2978 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_new_section()
2979 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_section()
2980 	__allocate_new_section(sbi, type, force);  in f2fs_allocate_new_section()
2981 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_section()
2982 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_new_section()
2985 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)  in f2fs_allocate_new_segments()  argument
2989 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_new_segments()
2990 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_segments()
2992 		__allocate_new_segment(sbi, i, false, false);  in f2fs_allocate_new_segments()
2993 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_segments()
2994 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_new_segments()
3001 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,  in f2fs_exist_trim_candidates()  argument
3007 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_exist_trim_candidates()
3009 		if (add_discard_addrs(sbi, cpc, true)) {  in f2fs_exist_trim_candidates()
3014 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_exist_trim_candidates()
3020 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,  in __issue_discard_cmd_range()  argument
3024 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_range()
3037 		f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,  in __issue_discard_cmd_range()
3062 		err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);  in __issue_discard_cmd_range()
3068 				__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_range()
3072 			trimmed += __wait_all_discard_cmd(sbi, NULL);  in __issue_discard_cmd_range()
3079 			__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_range()
3092 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)  in f2fs_trim_fs()  argument
3102 	bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);  in f2fs_trim_fs()
3104 	if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)  in f2fs_trim_fs()
3107 	if (end < MAIN_BLKADDR(sbi))  in f2fs_trim_fs()
3110 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {  in f2fs_trim_fs()
3111 		f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");  in f2fs_trim_fs()
3116 	start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);  in f2fs_trim_fs()
3117 	end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :  in f2fs_trim_fs()
3118 						GET_SEGNO(sbi, end);  in f2fs_trim_fs()
3120 		start_segno = rounddown(start_segno, sbi->segs_per_sec);  in f2fs_trim_fs()
3121 		end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;  in f2fs_trim_fs()
3129 	if (sbi->discard_blks == 0)  in f2fs_trim_fs()
3132 	f2fs_down_write(&sbi->gc_lock);  in f2fs_trim_fs()
3133 	err = f2fs_write_checkpoint(sbi, &cpc);  in f2fs_trim_fs()
3134 	f2fs_up_write(&sbi->gc_lock);  in f2fs_trim_fs()
3144 	if (f2fs_realtime_discard_enable(sbi))  in f2fs_trim_fs()
3147 	start_block = START_BLOCK(sbi, start_segno);  in f2fs_trim_fs()
3148 	end_block = START_BLOCK(sbi, end_segno + 1);  in f2fs_trim_fs()
3150 	__init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);  in f2fs_trim_fs()
3151 	trimmed = __issue_discard_cmd_range(sbi, &dpolicy,  in f2fs_trim_fs()
3154 	trimmed += __wait_discard_cmd_range(sbi, &dpolicy,  in f2fs_trim_fs()
3162 static bool __has_curseg_space(struct f2fs_sb_info *sbi,  in __has_curseg_space()  argument
3165 	return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,  in __has_curseg_space()
3242 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,  in f2fs_io_type_to_rw_hint()  argument
3245 	if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {  in f2fs_io_type_to_rw_hint()
3256 	} else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {  in f2fs_io_type_to_rw_hint()
3303 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __get_age_segment_type()  local
3309 		if (ei.age <= sbi->hot_data_age_threshold)  in __get_age_segment_type()
3311 		if (ei.age <= sbi->warm_data_age_threshold)  in __get_age_segment_type()
3328 			if (fio->sbi->am.atgc_enabled &&  in __get_segment_type_6()
3330 				(fio->sbi->gc_mode != GC_URGENT_HIGH))  in __get_segment_type_6()
3360 	switch (F2FS_OPTION(fio->sbi).active_logs) {  in __get_segment_type()
3371 		f2fs_bug_on(fio->sbi, true);  in __get_segment_type()
3383 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,  in f2fs_allocate_data_block()  argument
3388 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_allocate_data_block()
3389 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in f2fs_allocate_data_block()
3394 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_data_block()
3400 		f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);  in f2fs_allocate_data_block()
3401 		se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_allocate_data_block()
3402 		sanity_check_seg_type(sbi, se->type);  in f2fs_allocate_data_block()
3403 		f2fs_bug_on(sbi, IS_NODESEG(se->type));  in f2fs_allocate_data_block()
3405 	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);  in f2fs_allocate_data_block()
3407 	f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);  in f2fs_allocate_data_block()
3409 	f2fs_wait_discard_bio(sbi, *new_blkaddr);  in f2fs_allocate_data_block()
3416 	__add_sum_entry(sbi, type, sum);  in f2fs_allocate_data_block()
3418 	__refresh_next_blkoff(sbi, curseg);  in f2fs_allocate_data_block()
3420 	stat_inc_block_count(sbi, curseg);  in f2fs_allocate_data_block()
3423 		old_mtime = get_segment_mtime(sbi, old_blkaddr);  in f2fs_allocate_data_block()
3425 		update_segment_mtime(sbi, old_blkaddr, 0);  in f2fs_allocate_data_block()
3428 	update_segment_mtime(sbi, *new_blkaddr, old_mtime);  in f2fs_allocate_data_block()
3434 	update_sit_entry(sbi, *new_blkaddr, 1);  in f2fs_allocate_data_block()
3435 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)  in f2fs_allocate_data_block()
3436 		update_sit_entry(sbi, old_blkaddr, -1);  in f2fs_allocate_data_block()
3438 	if (!__has_curseg_space(sbi, curseg)) {  in f2fs_allocate_data_block()
3440 			get_atssr_segment(sbi, type, se->type,  in f2fs_allocate_data_block()
3443 			sit_i->s_ops->allocate_segment(sbi, type, false);  in f2fs_allocate_data_block()
3450 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_allocate_data_block()
3451 	locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));  in f2fs_allocate_data_block()
3454 		atomic64_inc(&sbi->allocated_data_blocks);  in f2fs_allocate_data_block()
3459 		fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));  in f2fs_allocate_data_block()
3461 		f2fs_inode_chksum_set(sbi, page);  in f2fs_allocate_data_block()
3467 		if (F2FS_IO_ALIGNED(sbi))  in f2fs_allocate_data_block()
3472 		io = sbi->write_io[fio->type] + fio->temp;  in f2fs_allocate_data_block()
3480 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_data_block()
3485 	struct f2fs_sb_info *sbi = fio->sbi;  in update_device_state()  local
3488 	if (!f2fs_is_multi_device(sbi))  in update_device_state()
3491 	devidx = f2fs_target_device_index(sbi, fio->new_blkaddr);  in update_device_state()
3494 	f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO);  in update_device_state()
3497 	if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {  in update_device_state()
3498 		spin_lock(&sbi->dev_lock);  in update_device_state()
3499 		f2fs_set_bit(devidx, (char *)&sbi->dirty_device);  in update_device_state()
3500 		spin_unlock(&sbi->dev_lock);  in update_device_state()
3507 	bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);  in do_write_page()
3510 		f2fs_down_read(&fio->sbi->io_order_lock);  in do_write_page()
3512 	f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,  in do_write_page()
3514 	if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {  in do_write_page()
3515 		invalidate_mapping_pages(META_MAPPING(fio->sbi),  in do_write_page()
3517 		f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);  in do_write_page()
3530 		f2fs_up_read(&fio->sbi->io_order_lock);  in do_write_page()
3533 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,  in f2fs_do_write_meta_page()  argument
3537 		.sbi = sbi,  in f2fs_do_write_meta_page()
3549 	if (unlikely(page->index >= MAIN_BLKADDR(sbi)))  in f2fs_do_write_meta_page()
3556 	stat_inc_meta_count(sbi, page->index);  in f2fs_do_write_meta_page()
3557 	f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);  in f2fs_do_write_meta_page()
3567 	f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);  in f2fs_do_write_node_page()
3573 	struct f2fs_sb_info *sbi = fio->sbi;  in f2fs_outplace_write_data()  local
3576 	f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);  in f2fs_outplace_write_data()
3583 	f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);  in f2fs_outplace_write_data()
3589 	struct f2fs_sb_info *sbi = fio->sbi;  in f2fs_inplace_write_data()  local
3596 	segno = GET_SEGNO(sbi, fio->new_blkaddr);  in f2fs_inplace_write_data()
3598 	if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {  in f2fs_inplace_write_data()
3599 		set_sbi_flag(sbi, SBI_NEED_FSCK);  in f2fs_inplace_write_data()
3600 		f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",  in f2fs_inplace_write_data()
3606 	if (f2fs_cp_error(sbi)) {  in f2fs_inplace_write_data()
3612 		invalidate_mapping_pages(META_MAPPING(sbi),  in f2fs_inplace_write_data()
3615 	stat_inc_inplace_blocks(fio->sbi);  in f2fs_inplace_write_data()
3617 	if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))  in f2fs_inplace_write_data()
3623 		f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);  in f2fs_inplace_write_data()
3638 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,  in __f2fs_get_curseg()  argument
3644 		if (CURSEG_I(sbi, i)->segno == segno)  in __f2fs_get_curseg()
3650 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,  in f2fs_do_replace_block()  argument
3655 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_do_replace_block()
3663 	segno = GET_SEGNO(sbi, new_blkaddr);  in f2fs_do_replace_block()
3664 	se = get_seg_entry(sbi, segno);  in f2fs_do_replace_block()
3667 	f2fs_down_write(&SM_I(sbi)->curseg_lock);  in f2fs_do_replace_block()
3671 		if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {  in f2fs_do_replace_block()
3678 		if (IS_CURSEG(sbi, segno)) {  in f2fs_do_replace_block()
3680 			type = __f2fs_get_curseg(sbi, segno);  in f2fs_do_replace_block()
3681 			f2fs_bug_on(sbi, type == NO_CHECK_TYPE);  in f2fs_do_replace_block()
3687 	f2fs_bug_on(sbi, !IS_DATASEG(type));  in f2fs_do_replace_block()
3688 	curseg = CURSEG_I(sbi, type);  in f2fs_do_replace_block()
3700 		change_curseg(sbi, type, true);  in f2fs_do_replace_block()
3703 	curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);  in f2fs_do_replace_block()
3704 	__add_sum_entry(sbi, type, sum);  in f2fs_do_replace_block()
3708 			update_segment_mtime(sbi, new_blkaddr, 0);  in f2fs_do_replace_block()
3709 		update_sit_entry(sbi, new_blkaddr, 1);  in f2fs_do_replace_block()
3711 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {  in f2fs_do_replace_block()
3712 		invalidate_mapping_pages(META_MAPPING(sbi),  in f2fs_do_replace_block()
3714 		f2fs_invalidate_compress_page(sbi, old_blkaddr);  in f2fs_do_replace_block()
3716 			update_segment_mtime(sbi, old_blkaddr, 0);  in f2fs_do_replace_block()
3717 		update_sit_entry(sbi, old_blkaddr, -1);  in f2fs_do_replace_block()
3720 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_do_replace_block()
3721 	locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));  in f2fs_do_replace_block()
3723 	locate_dirty_segment(sbi, old_cursegno);  in f2fs_do_replace_block()
3728 			change_curseg(sbi, type, true);  in f2fs_do_replace_block()
3736 	f2fs_up_write(&SM_I(sbi)->curseg_lock);  in f2fs_do_replace_block()
3739 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,  in f2fs_replace_block()  argument
3748 	f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,  in f2fs_replace_block()
3758 		struct f2fs_sb_info *sbi = F2FS_P_SB(page);  in f2fs_wait_on_page_writeback()  local
3761 		f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);  in f2fs_wait_on_page_writeback()
3763 		f2fs_submit_merged_ipu_write(sbi, NULL, page);  in f2fs_wait_on_page_writeback()
3766 			f2fs_bug_on(sbi, locked && PageWriteback(page));  in f2fs_wait_on_page_writeback()
3775 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_wait_on_block_writeback()  local
3784 	cpage = find_lock_page(META_MAPPING(sbi), blkaddr);  in f2fs_wait_on_block_writeback()
3794 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_wait_on_block_writeback_range()  local
3803 	invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr + len - 1);  in f2fs_wait_on_block_writeback_range()
3806 static int read_compacted_summaries(struct f2fs_sb_info *sbi)  in read_compacted_summaries()  argument
3808 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in read_compacted_summaries()
3815 	start = start_sum_block(sbi);  in read_compacted_summaries()
3817 	page = f2fs_get_meta_page(sbi, start++);  in read_compacted_summaries()
3823 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);  in read_compacted_summaries()
3827 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);  in read_compacted_summaries()
3836 		seg_i = CURSEG_I(sbi, i);  in read_compacted_summaries()
3840 		reset_curseg(sbi, i, 0);  in read_compacted_summaries()
3845 			blk_off = sbi->blocks_per_seg;  in read_compacted_summaries()
3860 			page = f2fs_get_meta_page(sbi, start++);  in read_compacted_summaries()
3871 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)  in read_normal_summaries()  argument
3873 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in read_normal_summaries()
3887 		if (__exist_node_summaries(sbi))  in read_normal_summaries()
3888 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);  in read_normal_summaries()
3890 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);  in read_normal_summaries()
3896 		if (__exist_node_summaries(sbi))  in read_normal_summaries()
3897 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,  in read_normal_summaries()
3900 			blk_addr = GET_SUM_BLOCK(sbi, segno);  in read_normal_summaries()
3903 	new = f2fs_get_meta_page(sbi, blk_addr);  in read_normal_summaries()
3909 		if (__exist_node_summaries(sbi)) {  in read_normal_summaries()
3913 			for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {  in read_normal_summaries()
3918 			err = f2fs_restore_node_summary(sbi, segno, sum);  in read_normal_summaries()
3925 	curseg = CURSEG_I(sbi, type);  in read_normal_summaries()
3936 	reset_curseg(sbi, type, 0);  in read_normal_summaries()
3945 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)  in restore_curseg_summaries()  argument
3947 	struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;  in restore_curseg_summaries()
3948 	struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;  in restore_curseg_summaries()
3952 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {  in restore_curseg_summaries()
3953 		int npages = f2fs_npages_for_summary_flush(sbi, true);  in restore_curseg_summaries()
3956 			f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,  in restore_curseg_summaries()
3960 		err = read_compacted_summaries(sbi);  in restore_curseg_summaries()
3966 	if (__exist_node_summaries(sbi))  in restore_curseg_summaries()
3967 		f2fs_ra_meta_pages(sbi,  in restore_curseg_summaries()
3968 				sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),  in restore_curseg_summaries()
3972 		err = read_normal_summaries(sbi, type);  in restore_curseg_summaries()
3980 		f2fs_err(sbi, "invalid journal entries nats %u sits %u",  in restore_curseg_summaries()
3988 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)  in write_compacted_summaries()  argument
3997 	page = f2fs_grab_meta_page(sbi, blkaddr++);  in write_compacted_summaries()
4002 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);  in write_compacted_summaries()
4007 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);  in write_compacted_summaries()
4015 		seg_i = CURSEG_I(sbi, i);  in write_compacted_summaries()
4016 		if (sbi->ckpt->alloc_type[i] == SSR)  in write_compacted_summaries()
4017 			blkoff = sbi->blocks_per_seg;  in write_compacted_summaries()
4019 			blkoff = curseg_blkoff(sbi, i);  in write_compacted_summaries()
4023 				page = f2fs_grab_meta_page(sbi, blkaddr++);  in write_compacted_summaries()
4047 static void write_normal_summaries(struct f2fs_sb_info *sbi,  in write_normal_summaries()  argument
4058 		write_current_sum_page(sbi, i, blkaddr + (i - type));  in write_normal_summaries()
4061 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)  in f2fs_write_data_summaries()  argument
4063 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))  in f2fs_write_data_summaries()
4064 		write_compacted_summaries(sbi, start_blk);  in f2fs_write_data_summaries()
4066 		write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);  in f2fs_write_data_summaries()
4069 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)  in f2fs_write_node_summaries()  argument
4071 	write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);  in f2fs_write_node_summaries()
4096 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,  in get_current_sit_page()  argument
4099 	return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));  in get_current_sit_page()
4102 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,  in get_next_sit_page()  argument
4105 	struct sit_info *sit_i = SIT_I(sbi);  in get_next_sit_page()
4109 	src_off = current_sit_addr(sbi, start);  in get_next_sit_page()
4110 	dst_off = next_sit_addr(sbi, src_off);  in get_next_sit_page()
4112 	page = f2fs_grab_meta_page(sbi, dst_off);  in get_next_sit_page()
4113 	seg_info_to_sit_page(sbi, page, start);  in get_next_sit_page()
4172 static void add_sits_in_set(struct f2fs_sb_info *sbi)  in add_sits_in_set()  argument
4174 	struct f2fs_sm_info *sm_info = SM_I(sbi);  in add_sits_in_set()
4176 	unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;  in add_sits_in_set()
4179 	for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))  in add_sits_in_set()
4183 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)  in remove_sits_in_journal()  argument
4185 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in remove_sits_in_journal()
4195 		dirtied = __mark_sit_entry_dirty(sbi, segno);  in remove_sits_in_journal()
4198 			add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);  in remove_sits_in_journal()
4208 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)  in f2fs_flush_sit_entries()  argument
4210 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_flush_sit_entries()
4212 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in f2fs_flush_sit_entries()
4215 	struct list_head *head = &SM_I(sbi)->sit_entry_set;  in f2fs_flush_sit_entries()
4216 	bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);  in f2fs_flush_sit_entries()
4228 	add_sits_in_set(sbi);  in f2fs_flush_sit_entries()
4237 		remove_sits_in_journal(sbi);  in f2fs_flush_sit_entries()
4249 						(unsigned long)MAIN_SEGS(sbi));  in f2fs_flush_sit_entries()
4259 			page = get_next_sit_page(sbi, start_segno);  in f2fs_flush_sit_entries()
4267 			se = get_seg_entry(sbi, segno);  in f2fs_flush_sit_entries()
4271 				f2fs_bug_on(sbi, 1);  in f2fs_flush_sit_entries()
4277 				add_discard_addrs(sbi, cpc, false);  in f2fs_flush_sit_entries()
4283 				f2fs_bug_on(sbi, offset < 0);  in f2fs_flush_sit_entries()
4288 				check_block_count(sbi, segno,  in f2fs_flush_sit_entries()
4294 				check_block_count(sbi, segno,  in f2fs_flush_sit_entries()
4308 		f2fs_bug_on(sbi, ses->entry_cnt);  in f2fs_flush_sit_entries()
4312 	f2fs_bug_on(sbi, !list_empty(head));  in f2fs_flush_sit_entries()
4313 	f2fs_bug_on(sbi, sit_i->dirty_sentries);  in f2fs_flush_sit_entries()
4319 			add_discard_addrs(sbi, cpc, false);  in f2fs_flush_sit_entries()
4325 	set_prefree_as_free_segments(sbi);  in f2fs_flush_sit_entries()
4328 static int build_sit_info(struct f2fs_sb_info *sbi)  in build_sit_info()  argument
4330 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);  in build_sit_info()
4337 	sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);  in build_sit_info()
4341 	SM_I(sbi)->sit_info = sit_i;  in build_sit_info()
4344 		f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),  in build_sit_info()
4345 					      MAIN_SEGS(sbi)),  in build_sit_info()
4350 	main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_sit_info()
4351 	sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,  in build_sit_info()
4357 	bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4;  in build_sit_info()
4359 	bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3;  in build_sit_info()
4361 	sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);  in build_sit_info()
4367 	for (start = 0; start < MAIN_SEGS(sbi); start++) {  in build_sit_info()
4383 	sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);  in build_sit_info()
4387 	if (__is_large_section(sbi)) {  in build_sit_info()
4389 			f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),  in build_sit_info()
4390 						      MAIN_SECS(sbi)),  in build_sit_info()
4400 	sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);  in build_sit_info()
4401 	src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);  in build_sit_info()
4413 	sit_i->invalid_segmap = f2fs_kvzalloc(sbi,  in build_sit_info()
4423 	sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;  in build_sit_info()
4428 	sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);  in build_sit_info()
4434 static int build_free_segmap(struct f2fs_sb_info *sbi)  in build_free_segmap()  argument
4440 	free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);  in build_free_segmap()
4444 	SM_I(sbi)->free_info = free_i;  in build_free_segmap()
4446 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_free_segmap()
4447 	free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);  in build_free_segmap()
4451 	sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in build_free_segmap()
4452 	free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);  in build_free_segmap()
4461 	free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));  in build_free_segmap()
4468 static int build_curseg(struct f2fs_sb_info *sbi)  in build_curseg()  argument
4473 	array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,  in build_curseg()
4478 	SM_I(sbi)->curseg_array = array;  in build_curseg()
4482 		array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);  in build_curseg()
4486 		array[i].journal = f2fs_kzalloc(sbi,  in build_curseg()
4500 	return restore_curseg_summaries(sbi);  in build_curseg()
4503 static int build_sit_entries(struct f2fs_sb_info *sbi)  in build_sit_entries()  argument
4505 	struct sit_info *sit_i = SIT_I(sbi);  in build_sit_entries()
4506 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in build_sit_entries()
4510 	int sit_blk_cnt = SIT_BLK_CNT(sbi);  in build_sit_entries()
4517 		readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,  in build_sit_entries()
4523 		for (; start < end && start < MAIN_SEGS(sbi); start++) {  in build_sit_entries()
4528 			page = get_current_sit_page(sbi, start);  in build_sit_entries()
4535 			err = check_block_count(sbi, start, &sit);  in build_sit_entries()
4541 				f2fs_err(sbi, "Invalid segment type: %u, segno: %u",  in build_sit_entries()
4549 			if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {  in build_sit_entries()
4556 				sbi->discard_blks +=  in build_sit_entries()
4557 					sbi->blocks_per_seg -  in build_sit_entries()
4561 			if (__is_large_section(sbi))  in build_sit_entries()
4562 				get_sec_entry(sbi, start)->valid_blocks +=  in build_sit_entries()
4573 		if (start >= MAIN_SEGS(sbi)) {  in build_sit_entries()
4574 			f2fs_err(sbi, "Wrong journal entry on segno %u",  in build_sit_entries()
4587 		err = check_block_count(sbi, start, &sit);  in build_sit_entries()
4593 			f2fs_err(sbi, "Invalid segment type: %u, segno: %u",  in build_sit_entries()
4601 		if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {  in build_sit_entries()
4606 			sbi->discard_blks += old_valid_blocks;  in build_sit_entries()
4607 			sbi->discard_blks -= se->valid_blocks;  in build_sit_entries()
4610 		if (__is_large_section(sbi)) {  in build_sit_entries()
4611 			get_sec_entry(sbi, start)->valid_blocks +=  in build_sit_entries()
4613 			get_sec_entry(sbi, start)->valid_blocks -=  in build_sit_entries()
4622 	if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {  in build_sit_entries()
4623 		f2fs_err(sbi, "SIT is corrupted node# %u vs %u",  in build_sit_entries()
4624 			 sit_valid_blocks[NODE], valid_node_count(sbi));  in build_sit_entries()
4629 				valid_user_blocks(sbi)) {  in build_sit_entries()
4630 		f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",  in build_sit_entries()
4632 			 valid_user_blocks(sbi));  in build_sit_entries()
4639 static void init_free_segmap(struct f2fs_sb_info *sbi)  in init_free_segmap()  argument
4645 	for (start = 0; start < MAIN_SEGS(sbi); start++) {  in init_free_segmap()
4646 		if (f2fs_usable_blks_in_seg(sbi, start) == 0)  in init_free_segmap()
4648 		sentry = get_seg_entry(sbi, start);  in init_free_segmap()
4650 			__set_free(sbi, start);  in init_free_segmap()
4652 			SIT_I(sbi)->written_valid_blocks +=  in init_free_segmap()
4658 		struct curseg_info *curseg_t = CURSEG_I(sbi, type);  in init_free_segmap()
4660 		__set_test_and_inuse(sbi, curseg_t->segno);  in init_free_segmap()
4664 static void init_dirty_segmap(struct f2fs_sb_info *sbi)  in init_dirty_segmap()  argument
4666 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in init_dirty_segmap()
4667 	struct free_segmap_info *free_i = FREE_I(sbi);  in init_dirty_segmap()
4670 	block_t blks_per_sec = BLKS_PER_SEC(sbi);  in init_dirty_segmap()
4674 		segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);  in init_dirty_segmap()
4675 		if (segno >= MAIN_SEGS(sbi))  in init_dirty_segmap()
4678 		valid_blocks = get_valid_blocks(sbi, segno, false);  in init_dirty_segmap()
4679 		usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);  in init_dirty_segmap()
4683 			f2fs_bug_on(sbi, 1);  in init_dirty_segmap()
4687 		__locate_dirty_segment(sbi, segno, DIRTY);  in init_dirty_segmap()
4691 	if (!__is_large_section(sbi))  in init_dirty_segmap()
4695 	for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {  in init_dirty_segmap()
4696 		valid_blocks = get_valid_blocks(sbi, segno, true);  in init_dirty_segmap()
4697 		secno = GET_SEC_FROM_SEG(sbi, segno);  in init_dirty_segmap()
4701 		if (IS_CURSEC(sbi, secno))  in init_dirty_segmap()
4708 static int init_victim_secmap(struct f2fs_sb_info *sbi)  in init_victim_secmap()  argument
4710 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in init_victim_secmap()
4711 	unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in init_victim_secmap()
4713 	dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);  in init_victim_secmap()
4719 static int build_dirty_segmap(struct f2fs_sb_info *sbi)  in build_dirty_segmap()  argument
4725 	dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),  in build_dirty_segmap()
4730 	SM_I(sbi)->dirty_info = dirty_i;  in build_dirty_segmap()
4733 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_dirty_segmap()
4736 		dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,  in build_dirty_segmap()
4742 	if (__is_large_section(sbi)) {  in build_dirty_segmap()
4743 		bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in build_dirty_segmap()
4744 		dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,  in build_dirty_segmap()
4750 	init_dirty_segmap(sbi);  in build_dirty_segmap()
4751 	return init_victim_secmap(sbi);  in build_dirty_segmap()
4754 static int sanity_check_curseg(struct f2fs_sb_info *sbi)  in sanity_check_curseg()  argument
4763 		struct curseg_info *curseg = CURSEG_I(sbi, i);  in sanity_check_curseg()
4764 		struct seg_entry *se = get_seg_entry(sbi, curseg->segno);  in sanity_check_curseg()
4767 		if (f2fs_sb_has_readonly(sbi) &&  in sanity_check_curseg()
4771 		sanity_check_seg_type(sbi, curseg->seg_type);  in sanity_check_curseg()
4774 			f2fs_err(sbi,  in sanity_check_curseg()
4786 		for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {  in sanity_check_curseg()
4790 			f2fs_err(sbi,  in sanity_check_curseg()
4802 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,  in check_zone_write_pointer()  argument
4808 	unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;  in check_zone_write_pointer()
4816 	wp_segno = GET_SEGNO(sbi, wp_block);  in check_zone_write_pointer()
4817 	wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);  in check_zone_write_pointer()
4819 	zone_segno = GET_SEGNO(sbi, zone_block);  in check_zone_write_pointer()
4820 	zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);  in check_zone_write_pointer()
4822 	if (zone_segno >= MAIN_SEGS(sbi))  in check_zone_write_pointer()
4830 		if (zone_secno == GET_SEC_FROM_SEG(sbi,  in check_zone_write_pointer()
4831 						   CURSEG_I(sbi, i)->segno))  in check_zone_write_pointer()
4838 	for (s = sbi->segs_per_sec - 1; s >= 0; s--) {  in check_zone_write_pointer()
4840 		se = get_seg_entry(sbi, segno);  in check_zone_write_pointer()
4841 		for (b = sbi->blocks_per_seg - 1; b >= 0; b--)  in check_zone_write_pointer()
4843 				last_valid_block = START_BLOCK(sbi, segno) + b;  in check_zone_write_pointer()
4857 		f2fs_notice(sbi, "Valid block beyond write pointer: "  in check_zone_write_pointer()
4859 			    GET_SEGNO(sbi, last_valid_block),  in check_zone_write_pointer()
4860 			    GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),  in check_zone_write_pointer()
4870 		f2fs_notice(sbi,  in check_zone_write_pointer()
4874 		ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,  in check_zone_write_pointer()
4877 			f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",  in check_zone_write_pointer()
4886 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,  in get_target_zoned_dev()  argument
4891 	for (i = 0; i < sbi->s_ndevs; i++) {  in get_target_zoned_dev()
4894 		if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&  in get_target_zoned_dev()
4909 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)  in fix_curseg_write_pointer()  argument
4911 	struct curseg_info *cs = CURSEG_I(sbi, type);  in fix_curseg_write_pointer()
4916 	unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;  in fix_curseg_write_pointer()
4920 	cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);  in fix_curseg_write_pointer()
4921 	cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));  in fix_curseg_write_pointer()
4923 	zbd = get_target_zoned_dev(sbi, cs_zone_block);  in fix_curseg_write_pointer()
4933 		f2fs_err(sbi, "Report zone failed: %s errno=(%d)",  in fix_curseg_write_pointer()
4942 	wp_segno = GET_SEGNO(sbi, wp_block);  in fix_curseg_write_pointer()
4943 	wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);  in fix_curseg_write_pointer()
4950 	f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "  in fix_curseg_write_pointer()
4954 	f2fs_notice(sbi, "Assign new section to curseg[%d]: "  in fix_curseg_write_pointer()
4957 	f2fs_allocate_new_section(sbi, type, true);  in fix_curseg_write_pointer()
4960 	if (check_zone_write_pointer(sbi, zbd, &zone))  in fix_curseg_write_pointer()
4964 	cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);  in fix_curseg_write_pointer()
4965 	cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));  in fix_curseg_write_pointer()
4967 	zbd = get_target_zoned_dev(sbi, cs_zone_block);  in fix_curseg_write_pointer()
4976 		f2fs_err(sbi, "Report zone failed: %s errno=(%d)",  in fix_curseg_write_pointer()
4985 		f2fs_notice(sbi,  in fix_curseg_write_pointer()
4989 		err = __f2fs_issue_discard_zone(sbi, zbd->bdev,  in fix_curseg_write_pointer()
4993 			f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",  in fix_curseg_write_pointer()
5002 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_fix_curseg_write_pointer()  argument
5007 		ret = fix_curseg_write_pointer(sbi, i);  in f2fs_fix_curseg_write_pointer()
5016 	struct f2fs_sb_info *sbi;  member
5027 	return check_zone_write_pointer(args->sbi, args->fdev, zone);  in check_zone_write_pointer_cb()
5030 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_check_write_pointer()  argument
5035 	for (i = 0; i < sbi->s_ndevs; i++) {  in f2fs_check_write_pointer()
5039 		args.sbi = sbi;  in f2fs_check_write_pointer()
5050 static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx,  in is_conv_zone()  argument
5059 static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno,  in get_zone_idx()  argument
5062 	block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));  in get_zone_idx()
5065 						sbi->log_blocks_per_blkz;  in get_zone_idx()
5073 		struct f2fs_sb_info *sbi, unsigned int segno)  in f2fs_usable_zone_segs_in_sec()  argument
5077 	dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));  in f2fs_usable_zone_segs_in_sec()
5078 	zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx);  in f2fs_usable_zone_segs_in_sec()
5081 	if (is_conv_zone(sbi, zone_idx, dev_idx))  in f2fs_usable_zone_segs_in_sec()
5082 		return sbi->segs_per_sec;  in f2fs_usable_zone_segs_in_sec()
5089 		return sbi->segs_per_sec;  in f2fs_usable_zone_segs_in_sec()
5092 	unusable_segs_in_sec = (sbi->blocks_per_blkz -  in f2fs_usable_zone_segs_in_sec()
5094 				sbi->log_blocks_per_seg;  in f2fs_usable_zone_segs_in_sec()
5095 	return sbi->segs_per_sec - unusable_segs_in_sec;  in f2fs_usable_zone_segs_in_sec()
5107 			struct f2fs_sb_info *sbi, unsigned int segno)  in f2fs_usable_zone_blks_in_seg()  argument
5112 	secno = GET_SEC_FROM_SEG(sbi, segno);  in f2fs_usable_zone_blks_in_seg()
5113 	seg_start = START_BLOCK(sbi, segno);  in f2fs_usable_zone_blks_in_seg()
5114 	dev_idx = f2fs_target_device_index(sbi, seg_start);  in f2fs_usable_zone_blks_in_seg()
5115 	zone_idx = get_zone_idx(sbi, secno, dev_idx);  in f2fs_usable_zone_blks_in_seg()
5121 	if (is_conv_zone(sbi, zone_idx, dev_idx))  in f2fs_usable_zone_blks_in_seg()
5122 		return sbi->blocks_per_seg;  in f2fs_usable_zone_blks_in_seg()
5125 		return sbi->blocks_per_seg;  in f2fs_usable_zone_blks_in_seg()
5127 	sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));  in f2fs_usable_zone_blks_in_seg()
5139 	if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)  in f2fs_usable_zone_blks_in_seg()
5142 	return sbi->blocks_per_seg;  in f2fs_usable_zone_blks_in_seg()
5145 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_fix_curseg_write_pointer()  argument
5150 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_check_write_pointer()  argument
5155 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,  in f2fs_usable_zone_blks_in_seg()  argument
5161 static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi,  in f2fs_usable_zone_segs_in_sec()  argument
5167 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,  in f2fs_usable_blks_in_seg()  argument
5170 	if (f2fs_sb_has_blkzoned(sbi))  in f2fs_usable_blks_in_seg()
5171 		return f2fs_usable_zone_blks_in_seg(sbi, segno);  in f2fs_usable_blks_in_seg()
5173 	return sbi->blocks_per_seg;  in f2fs_usable_blks_in_seg()
5176 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,  in f2fs_usable_segs_in_sec()  argument
5179 	if (f2fs_sb_has_blkzoned(sbi))  in f2fs_usable_segs_in_sec()
5180 		return f2fs_usable_zone_segs_in_sec(sbi, segno);  in f2fs_usable_segs_in_sec()
5182 	return sbi->segs_per_sec;  in f2fs_usable_segs_in_sec()
5188 static void init_min_max_mtime(struct f2fs_sb_info *sbi)  in init_min_max_mtime()  argument
5190 	struct sit_info *sit_i = SIT_I(sbi);  in init_min_max_mtime()
5197 	for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {  in init_min_max_mtime()
5201 		for (i = 0; i < sbi->segs_per_sec; i++)  in init_min_max_mtime()
5202 			mtime += get_seg_entry(sbi, segno + i)->mtime;  in init_min_max_mtime()
5204 		mtime = div_u64(mtime, sbi->segs_per_sec);  in init_min_max_mtime()
5209 	sit_i->max_mtime = get_mtime(sbi, false);  in init_min_max_mtime()
5214 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)  in f2fs_build_segment_manager()  argument
5216 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);  in f2fs_build_segment_manager()
5217 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in f2fs_build_segment_manager()
5221 	sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);  in f2fs_build_segment_manager()
5226 	sbi->sm_info = sm_info;  in f2fs_build_segment_manager()
5239 	if (!f2fs_lfs_mode(sbi))  in f2fs_build_segment_manager()
5243 	sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec;  in f2fs_build_segment_manager()
5245 	sm_info->min_ssr_sections = reserved_sections(sbi);  in f2fs_build_segment_manager()
5251 	if (!f2fs_readonly(sbi->sb)) {  in f2fs_build_segment_manager()
5252 		err = f2fs_create_flush_cmd_control(sbi);  in f2fs_build_segment_manager()
5257 	err = create_discard_cmd_control(sbi);  in f2fs_build_segment_manager()
5261 	err = build_sit_info(sbi);  in f2fs_build_segment_manager()
5264 	err = build_free_segmap(sbi);  in f2fs_build_segment_manager()
5267 	err = build_curseg(sbi);  in f2fs_build_segment_manager()
5272 	err = build_sit_entries(sbi);  in f2fs_build_segment_manager()
5276 	init_free_segmap(sbi);  in f2fs_build_segment_manager()
5277 	err = build_dirty_segmap(sbi);  in f2fs_build_segment_manager()
5281 	err = sanity_check_curseg(sbi);  in f2fs_build_segment_manager()
5285 	init_min_max_mtime(sbi);  in f2fs_build_segment_manager()
5289 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,  in discard_dirty_segmap()  argument
5292 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in discard_dirty_segmap()
5300 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)  in destroy_victim_secmap()  argument
5302 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in destroy_victim_secmap()
5307 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)  in destroy_dirty_segmap()  argument
5309 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in destroy_dirty_segmap()
5317 		discard_dirty_segmap(sbi, i);  in destroy_dirty_segmap()
5319 	if (__is_large_section(sbi)) {  in destroy_dirty_segmap()
5325 	destroy_victim_secmap(sbi);  in destroy_dirty_segmap()
5326 	SM_I(sbi)->dirty_info = NULL;  in destroy_dirty_segmap()
5330 static void destroy_curseg(struct f2fs_sb_info *sbi)  in destroy_curseg()  argument
5332 	struct curseg_info *array = SM_I(sbi)->curseg_array;  in destroy_curseg()
5337 	SM_I(sbi)->curseg_array = NULL;  in destroy_curseg()
5345 static void destroy_free_segmap(struct f2fs_sb_info *sbi)  in destroy_free_segmap()  argument
5347 	struct free_segmap_info *free_i = SM_I(sbi)->free_info;  in destroy_free_segmap()
5351 	SM_I(sbi)->free_info = NULL;  in destroy_free_segmap()
5357 static void destroy_sit_info(struct f2fs_sb_info *sbi)  in destroy_sit_info()  argument
5359 	struct sit_info *sit_i = SIT_I(sbi);  in destroy_sit_info()
5372 	SM_I(sbi)->sit_info = NULL;  in destroy_sit_info()
5381 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)  in f2fs_destroy_segment_manager()  argument
5383 	struct f2fs_sm_info *sm_info = SM_I(sbi);  in f2fs_destroy_segment_manager()
5387 	f2fs_destroy_flush_cmd_control(sbi, true);  in f2fs_destroy_segment_manager()
5388 	destroy_discard_cmd_control(sbi);  in f2fs_destroy_segment_manager()
5389 	destroy_dirty_segmap(sbi);  in f2fs_destroy_segment_manager()
5390 	destroy_curseg(sbi);  in f2fs_destroy_segment_manager()
5391 	destroy_free_segmap(sbi);  in f2fs_destroy_segment_manager()
5392 	destroy_sit_info(sbi);  in f2fs_destroy_segment_manager()
5393 	sbi->sm_info = NULL;  in f2fs_destroy_segment_manager()