38 	struct mount_info *mi =  in zstd_free_workspace()  local
41 	mutex_lock(&mi->mi_zstd_workspace_mutex);  in zstd_free_workspace()
42 	kvfree(mi->mi_zstd_workspace);  in zstd_free_workspace()
43 	mi->mi_zstd_workspace = NULL;  in zstd_free_workspace()
44 	mi->mi_zstd_stream = NULL;  in zstd_free_workspace()
45 	mutex_unlock(&mi->mi_zstd_workspace_mutex);  in zstd_free_workspace()
52 	struct mount_info *mi = NULL;  in incfs_alloc_mount_info()  local
56 	mi = kzalloc(sizeof(*mi), GFP_NOFS);  in incfs_alloc_mount_info()
57 	if (!mi)  in incfs_alloc_mount_info()
60 	mi->mi_sb = sb;  in incfs_alloc_mount_info()
61 	mi->mi_backing_dir_path = *backing_dir_path;  in incfs_alloc_mount_info()
62 	mi->mi_owner = get_current_cred();  in incfs_alloc_mount_info()
63 	path_get(&mi->mi_backing_dir_path);  in incfs_alloc_mount_info()
64 	mutex_init(&mi->mi_dir_struct_mutex);  in incfs_alloc_mount_info()
65 	init_waitqueue_head(&mi->mi_pending_reads_notif_wq);  in incfs_alloc_mount_info()
66 	init_waitqueue_head(&mi->mi_log.ml_notif_wq);  in incfs_alloc_mount_info()
67 	init_waitqueue_head(&mi->mi_blocks_written_notif_wq);  in incfs_alloc_mount_info()
68 	atomic_set(&mi->mi_blocks_written, 0);  in incfs_alloc_mount_info()
69 	INIT_DELAYED_WORK(&mi->mi_log.ml_wakeup_work, log_wake_up_all);  in incfs_alloc_mount_info()
70 	spin_lock_init(&mi->mi_log.rl_lock);  in incfs_alloc_mount_info()
71 	spin_lock_init(&mi->pending_read_lock);  in incfs_alloc_mount_info()
72 	INIT_LIST_HEAD(&mi->mi_reads_list_head);  in incfs_alloc_mount_info()
73 	spin_lock_init(&mi->mi_per_uid_read_timeouts_lock);  in incfs_alloc_mount_info()
74 	mutex_init(&mi->mi_zstd_workspace_mutex);  in incfs_alloc_mount_info()
75 	INIT_DELAYED_WORK(&mi->mi_zstd_cleanup_work, zstd_free_workspace);  in incfs_alloc_mount_info()
76 	mutex_init(&mi->mi_le_mutex);  in incfs_alloc_mount_info()
78 	node = incfs_add_sysfs_node(options->sysfs_name, mi);  in incfs_alloc_mount_info()
83 	mi->mi_sysfs_node = node;  in incfs_alloc_mount_info()
85 	error = incfs_realloc_mount_info(mi, options);  in incfs_alloc_mount_info()
89 	return mi;  in incfs_alloc_mount_info()
92 	incfs_free_mount_info(mi);  in incfs_alloc_mount_info()
96 int incfs_realloc_mount_info(struct mount_info *mi,  in incfs_realloc_mount_info()  argument
103 	if (options->read_log_pages != mi->mi_options.read_log_pages) {  in incfs_realloc_mount_info()
119 		spin_lock(&mi->mi_log.rl_lock);  in incfs_realloc_mount_info()
120 		old_buffer = mi->mi_log.rl_ring_buf;  in incfs_realloc_mount_info()
121 		mi->mi_log.rl_ring_buf = new_buffer;  in incfs_realloc_mount_info()
122 		mi->mi_log.rl_size = new_buffer_size;  in incfs_realloc_mount_info()
124 			.generation_id = mi->mi_log.rl_head.generation_id + 1,  in incfs_realloc_mount_info()
126 		mi->mi_log.rl_head = log_state;  in incfs_realloc_mount_info()
127 		mi->mi_log.rl_tail = log_state;  in incfs_realloc_mount_info()
128 		spin_unlock(&mi->mi_log.rl_lock);  in incfs_realloc_mount_info()
133 	if (options->sysfs_name && !mi->mi_sysfs_node)  in incfs_realloc_mount_info()
134 		mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,  in incfs_realloc_mount_info()
135 							 mi);  in incfs_realloc_mount_info()
136 	else if (!options->sysfs_name && mi->mi_sysfs_node) {  in incfs_realloc_mount_info()
137 		incfs_free_sysfs_node(mi->mi_sysfs_node);  in incfs_realloc_mount_info()
138 		mi->mi_sysfs_node = NULL;  in incfs_realloc_mount_info()
141 		       kobject_name(&mi->mi_sysfs_node->isn_sysfs_node))) {  in incfs_realloc_mount_info()
142 		incfs_free_sysfs_node(mi->mi_sysfs_node);  in incfs_realloc_mount_info()
143 		mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,  in incfs_realloc_mount_info()
144 							 mi);  in incfs_realloc_mount_info()
147 	if (IS_ERR(mi->mi_sysfs_node)) {  in incfs_realloc_mount_info()
148 		int err = PTR_ERR(mi->mi_sysfs_node);  in incfs_realloc_mount_info()
150 		mi->mi_sysfs_node = NULL;  in incfs_realloc_mount_info()
154 	mi->mi_options = *options;  in incfs_realloc_mount_info()
158 void incfs_free_mount_info(struct mount_info *mi)  in incfs_free_mount_info()  argument
161 	if (!mi)  in incfs_free_mount_info()
164 	flush_delayed_work(&mi->mi_log.ml_wakeup_work);  in incfs_free_mount_info()
165 	flush_delayed_work(&mi->mi_zstd_cleanup_work);  in incfs_free_mount_info()
167 	dput(mi->mi_index_dir);  in incfs_free_mount_info()
168 	dput(mi->mi_incomplete_dir);  in incfs_free_mount_info()
169 	path_put(&mi->mi_backing_dir_path);  in incfs_free_mount_info()
170 	mutex_destroy(&mi->mi_dir_struct_mutex);  in incfs_free_mount_info()
171 	mutex_destroy(&mi->mi_zstd_workspace_mutex);  in incfs_free_mount_info()
172 	put_cred(mi->mi_owner);  in incfs_free_mount_info()
173 	kfree(mi->mi_log.rl_ring_buf);  in incfs_free_mount_info()
174 	for (i = 0; i < ARRAY_SIZE(mi->pseudo_file_xattr); ++i)  in incfs_free_mount_info()
175 		kfree(mi->pseudo_file_xattr[i].data);  in incfs_free_mount_info()
176 	kfree(mi->mi_per_uid_read_timeouts);  in incfs_free_mount_info()
177 	incfs_free_sysfs_node(mi->mi_sysfs_node);  in incfs_free_mount_info()
178 	kfree(mi);  in incfs_free_mount_info()
220 static struct data_file *handle_mapped_file(struct mount_info *mi,  in handle_mapped_file()  argument
234 	index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir,  in handle_mapped_file()
247 		.mnt = mi->mi_backing_dir_path.mnt,  in handle_mapped_file()
251 	old_cred = override_creds(mi->mi_owner);  in handle_mapped_file()
261 	result = incfs_open_data_file(mi, bf);  in handle_mapped_file()
273 struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)  in incfs_open_data_file()  argument
282 	if (!bf || !mi)  in incfs_open_data_file()
288 	bfc = incfs_alloc_bfc(mi, bf);  in incfs_open_data_file()
301 	df->df_mount_info = mi;  in incfs_open_data_file()
316 		struct data_file *mapped_df = handle_mapped_file(mi, df);  in incfs_open_data_file()
375 int make_inode_ready_for_data_ops(struct mount_info *mi,  in make_inode_ready_for_data_ops()  argument
386 			df = incfs_open_data_file(mi, backing_file);  in make_inode_ready_for_data_ops()
399 struct dir_file *incfs_open_dir_file(struct mount_info *mi, struct file *bf)  in incfs_open_dir_file()  argument
411 	dir->mount_info = mi;  in incfs_open_dir_file()
424 static ssize_t zstd_decompress_safe(struct mount_info *mi,  in zstd_decompress_safe()  argument
431 	result = mutex_lock_interruptible(&mi->mi_zstd_workspace_mutex);  in zstd_decompress_safe()
435 	if (!mi->mi_zstd_stream) {  in zstd_decompress_safe()
454 		mi->mi_zstd_workspace = workspace;  in zstd_decompress_safe()
455 		mi->mi_zstd_stream = stream;  in zstd_decompress_safe()
458 	result = ZSTD_decompressStream(mi->mi_zstd_stream, &outbuf, &inbuf) ?  in zstd_decompress_safe()
461 	mod_delayed_work(system_wq, &mi->mi_zstd_cleanup_work,  in zstd_decompress_safe()
465 	mutex_unlock(&mi->mi_zstd_workspace_mutex);  in zstd_decompress_safe()
469 static ssize_t decompress(struct mount_info *mi,  in decompress()  argument
483 		return zstd_decompress_safe(mi, src, dst);  in decompress()
551 static void log_block_read(struct mount_info *mi, incfs_uuid_t *id,  in log_block_read()  argument
554 	struct read_log *log = &mi->mi_log;  in log_block_read()
1024 	struct mount_info *mi = NULL;  in add_pending_read()  local
1027 	mi = df->df_mount_info;  in add_pending_read()
1038 	spin_lock(&mi->pending_read_lock);  in add_pending_read()
1040 	result->serial_number = ++mi->mi_last_pending_read_number;  in add_pending_read()
1041 	mi->mi_pending_reads_count++;  in add_pending_read()
1043 	list_add_rcu(&result->mi_reads_list, &mi->mi_reads_list_head);  in add_pending_read()
1046 	spin_unlock(&mi->pending_read_lock);  in add_pending_read()
1048 	wake_up_all(&mi->mi_pending_reads_notif_wq);  in add_pending_read()
1064 	struct mount_info *mi = NULL;  in remove_pending_read()  local
1072 	mi = df->df_mount_info;  in remove_pending_read()
1074 	spin_lock(&mi->pending_read_lock);  in remove_pending_read()
1079 	mi->mi_pending_reads_count--;  in remove_pending_read()
1081 	spin_unlock(&mi->pending_read_lock);  in remove_pending_read()
1087 static void notify_pending_reads(struct mount_info *mi,  in notify_pending_reads()  argument
1103 	atomic_inc(&mi->mi_blocks_written);  in notify_pending_reads()
1104 	wake_up_all(&mi->mi_blocks_written_notif_wq);  in notify_pending_reads()
1130 	struct mount_info *mi = NULL;  in wait_for_data_block()  local
1145 	mi = df->df_mount_info;  in wait_for_data_block()
1176 			log_block_read(mi, &df->df_id, block_index);  in wait_for_data_block()
1198 		log_block_read(mi, &df->df_id, block_index);  in wait_for_data_block()
1248 		mi->mi_reads_delayed_pending++;  in wait_for_data_block()
1249 		mi->mi_reads_delayed_pending_us +=  in wait_for_data_block()
1254 		mi->mi_reads_delayed_min++;  in wait_for_data_block()
1255 		mi->mi_reads_delayed_min_us += delayed_min_us;  in wait_for_data_block()
1262 				struct mount_info *mi, struct data_file *df)  in incfs_update_sysfs_error()  argument
1269 	error = mutex_lock_interruptible(&mi->mi_le_mutex);  in incfs_update_sysfs_error()
1273 	mi->mi_le_file_id = df->df_id;  in incfs_update_sysfs_error()
1274 	mi->mi_le_time_us = ktime_to_us(ktime_get());  in incfs_update_sysfs_error()
1275 	mi->mi_le_page = index;  in incfs_update_sysfs_error()
1276 	mi->mi_le_errno = result;  in incfs_update_sysfs_error()
1277 	mi->mi_le_uid = current_uid().val;  in incfs_update_sysfs_error()
1278 	mutex_unlock(&mi->mi_le_mutex);  in incfs_update_sysfs_error()
1290 	struct mount_info *mi = NULL;  in incfs_read_data_file_block()  local
1301 	mi = df->df_mount_info;  in incfs_read_data_file_block()
1321 				decompress(mi, range(tmp.data, bytes_to_read),  in incfs_read_data_file_block()
1344 		log_block_read(mi, &df->df_id, index);  in incfs_read_data_file_block()
1348 		mi->mi_reads_failed_timed_out++;  in incfs_read_data_file_block()
1350 		mi->mi_reads_failed_hash_verification++;  in incfs_read_data_file_block()
1352 		mi->mi_reads_failed_other++;  in incfs_read_data_file_block()
1354 	incfs_update_sysfs_error(f, index, result, mi, df);  in incfs_read_data_file_block()
1384 	struct mount_info *mi = NULL;  in incfs_process_new_data_block()  local
1395 	mi = df->df_mount_info;  in incfs_process_new_data_block()
1438 		notify_pending_reads(mi, segment, block->block_index);  in incfs_process_new_data_block()
1480 	struct mount_info *mi = NULL;  in incfs_process_new_hash_block()  local
1494 	mi = df->df_mount_info;  in incfs_process_new_hash_block()
1733 bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number)  in incfs_fresh_pending_reads_exist()  argument
1737 	spin_lock(&mi->pending_read_lock);  in incfs_fresh_pending_reads_exist()
1738 	result = (mi->mi_last_pending_read_number > last_number) &&  in incfs_fresh_pending_reads_exist()
1739 		(mi->mi_pending_reads_count > 0);  in incfs_fresh_pending_reads_exist()
1740 	spin_unlock(&mi->pending_read_lock);  in incfs_fresh_pending_reads_exist()
1744 int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,  in incfs_collect_pending_reads()  argument
1752 	if (!mi)  in incfs_collect_pending_reads()
1758 	if (!incfs_fresh_pending_reads_exist(mi, sn_lowerbound))  in incfs_collect_pending_reads()
1763 	list_for_each_entry_rcu(entry, &mi->mi_reads_list_head, mi_reads_list) {  in incfs_collect_pending_reads()
1799 struct read_log_state incfs_get_log_state(struct mount_info *mi)  in incfs_get_log_state()  argument
1801 	struct read_log *log = &mi->mi_log;  in incfs_get_log_state()
1810 int incfs_get_uncollected_logs_count(struct mount_info *mi,  in incfs_get_uncollected_logs_count()  argument
1813 	struct read_log *log = &mi->mi_log;  in incfs_get_uncollected_logs_count()
1829 int incfs_collect_logged_reads(struct mount_info *mi,  in incfs_collect_logged_reads()  argument
1836 	struct read_log *log = &mi->mi_log;  in incfs_collect_logged_reads()