1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2019 Google LLC
4 */
5 #include <linux/crc32.h>
6 #include <linux/delay.h>
7 #include <linux/file.h>
8 #include <linux/fsverity.h>
9 #include <linux/gfp.h>
10 #include <linux/kobject.h>
11 #include <linux/ktime.h>
12 #include <linux/lz4.h>
13 #include <linux/mm.h>
14 #include <linux/namei.h>
15 #include <linux/pagemap.h>
16 #include <linux/slab.h>
17 #include <linux/types.h>
18 #include <linux/workqueue.h>
19
20 #include "data_mgmt.h"
21 #include "format.h"
22 #include "integrity.h"
23 #include "sysfs.h"
24 #include "verity.h"
25
26 static int incfs_scan_metadata_chain(struct data_file *df);
27
log_wake_up_all(struct work_struct * work)28 static void log_wake_up_all(struct work_struct *work)
29 {
30 struct delayed_work *dw = container_of(work, struct delayed_work, work);
31 struct read_log *rl = container_of(dw, struct read_log, ml_wakeup_work);
32 wake_up_all(&rl->ml_notif_wq);
33 }
34
zstd_free_workspace(struct work_struct * work)35 static void zstd_free_workspace(struct work_struct *work)
36 {
37 struct delayed_work *dw = container_of(work, struct delayed_work, work);
38 struct mount_info *mi =
39 container_of(dw, struct mount_info, mi_zstd_cleanup_work);
40
41 mutex_lock(&mi->mi_zstd_workspace_mutex);
42 kvfree(mi->mi_zstd_workspace);
43 mi->mi_zstd_workspace = NULL;
44 mi->mi_zstd_stream = NULL;
45 mutex_unlock(&mi->mi_zstd_workspace_mutex);
46 }
47
incfs_alloc_mount_info(struct super_block * sb,struct mount_options * options,struct path * backing_dir_path)48 struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
49 struct mount_options *options,
50 struct path *backing_dir_path)
51 {
52 struct mount_info *mi = NULL;
53 int error = 0;
54 struct incfs_sysfs_node *node;
55
56 mi = kzalloc(sizeof(*mi), GFP_NOFS);
57 if (!mi)
58 return ERR_PTR(-ENOMEM);
59
60 mi->mi_sb = sb;
61 mi->mi_backing_dir_path = *backing_dir_path;
62 mi->mi_owner = get_current_cred();
63 path_get(&mi->mi_backing_dir_path);
64 mutex_init(&mi->mi_dir_struct_mutex);
65 init_waitqueue_head(&mi->mi_pending_reads_notif_wq);
66 init_waitqueue_head(&mi->mi_log.ml_notif_wq);
67 init_waitqueue_head(&mi->mi_blocks_written_notif_wq);
68 atomic_set(&mi->mi_blocks_written, 0);
69 INIT_DELAYED_WORK(&mi->mi_log.ml_wakeup_work, log_wake_up_all);
70 spin_lock_init(&mi->mi_log.rl_lock);
71 spin_lock_init(&mi->pending_read_lock);
72 INIT_LIST_HEAD(&mi->mi_reads_list_head);
73 spin_lock_init(&mi->mi_per_uid_read_timeouts_lock);
74 mutex_init(&mi->mi_zstd_workspace_mutex);
75 INIT_DELAYED_WORK(&mi->mi_zstd_cleanup_work, zstd_free_workspace);
76 mutex_init(&mi->mi_le_mutex);
77
78 node = incfs_add_sysfs_node(options->sysfs_name, mi);
79 if (IS_ERR(node)) {
80 error = PTR_ERR(node);
81 goto err;
82 }
83 mi->mi_sysfs_node = node;
84
85 error = incfs_realloc_mount_info(mi, options);
86 if (error)
87 goto err;
88
89 return mi;
90
91 err:
92 incfs_free_mount_info(mi);
93 return ERR_PTR(error);
94 }
95
incfs_realloc_mount_info(struct mount_info * mi,struct mount_options * options)96 int incfs_realloc_mount_info(struct mount_info *mi,
97 struct mount_options *options)
98 {
99 void *new_buffer = NULL;
100 void *old_buffer;
101 size_t new_buffer_size = 0;
102
103 if (options->read_log_pages != mi->mi_options.read_log_pages) {
104 struct read_log_state log_state;
105 /*
106 * Even though having two buffers allocated at once isn't
107 * usually good, allocating a multipage buffer under a spinlock
108 * is even worse, so let's optimize for the shorter lock
109 * duration. It's not end of the world if we fail to increase
110 * the buffer size anyway.
111 */
112 if (options->read_log_pages > 0) {
113 new_buffer_size = PAGE_SIZE * options->read_log_pages;
114 new_buffer = kzalloc(new_buffer_size, GFP_NOFS);
115 if (!new_buffer)
116 return -ENOMEM;
117 }
118
119 spin_lock(&mi->mi_log.rl_lock);
120 old_buffer = mi->mi_log.rl_ring_buf;
121 mi->mi_log.rl_ring_buf = new_buffer;
122 mi->mi_log.rl_size = new_buffer_size;
123 log_state = (struct read_log_state){
124 .generation_id = mi->mi_log.rl_head.generation_id + 1,
125 };
126 mi->mi_log.rl_head = log_state;
127 mi->mi_log.rl_tail = log_state;
128 spin_unlock(&mi->mi_log.rl_lock);
129
130 kfree(old_buffer);
131 }
132
133 if (options->sysfs_name && !mi->mi_sysfs_node)
134 mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,
135 mi);
136 else if (!options->sysfs_name && mi->mi_sysfs_node) {
137 incfs_free_sysfs_node(mi->mi_sysfs_node);
138 mi->mi_sysfs_node = NULL;
139 } else if (options->sysfs_name &&
140 strcmp(options->sysfs_name,
141 kobject_name(&mi->mi_sysfs_node->isn_sysfs_node))) {
142 incfs_free_sysfs_node(mi->mi_sysfs_node);
143 mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,
144 mi);
145 }
146
147 if (IS_ERR(mi->mi_sysfs_node)) {
148 int err = PTR_ERR(mi->mi_sysfs_node);
149
150 mi->mi_sysfs_node = NULL;
151 return err;
152 }
153
154 mi->mi_options = *options;
155 return 0;
156 }
157
incfs_free_mount_info(struct mount_info * mi)158 void incfs_free_mount_info(struct mount_info *mi)
159 {
160 int i;
161 if (!mi)
162 return;
163
164 flush_delayed_work(&mi->mi_log.ml_wakeup_work);
165 flush_delayed_work(&mi->mi_zstd_cleanup_work);
166
167 dput(mi->mi_index_dir);
168 dput(mi->mi_incomplete_dir);
169 path_put(&mi->mi_backing_dir_path);
170 mutex_destroy(&mi->mi_dir_struct_mutex);
171 mutex_destroy(&mi->mi_zstd_workspace_mutex);
172 put_cred(mi->mi_owner);
173 kfree(mi->mi_log.rl_ring_buf);
174 for (i = 0; i < ARRAY_SIZE(mi->pseudo_file_xattr); ++i)
175 kfree(mi->pseudo_file_xattr[i].data);
176 kfree(mi->mi_per_uid_read_timeouts);
177 incfs_free_sysfs_node(mi->mi_sysfs_node);
178 kfree(mi);
179 }
180
data_file_segment_init(struct data_file_segment * segment)181 static void data_file_segment_init(struct data_file_segment *segment)
182 {
183 init_waitqueue_head(&segment->new_data_arrival_wq);
184 init_rwsem(&segment->rwsem);
185 INIT_LIST_HEAD(&segment->reads_list_head);
186 }
187
file_id_to_str(incfs_uuid_t id)188 char *file_id_to_str(incfs_uuid_t id)
189 {
190 char *result = kmalloc(1 + sizeof(id.bytes) * 2, GFP_NOFS);
191 char *end;
192
193 if (!result)
194 return NULL;
195
196 end = bin2hex(result, id.bytes, sizeof(id.bytes));
197 *end = 0;
198 return result;
199 }
200
incfs_lookup_dentry(struct dentry * parent,const char * name)201 struct dentry *incfs_lookup_dentry(struct dentry *parent, const char *name)
202 {
203 struct inode *inode;
204 struct dentry *result = NULL;
205
206 if (!parent)
207 return ERR_PTR(-EFAULT);
208
209 inode = d_inode(parent);
210 inode_lock_nested(inode, I_MUTEX_PARENT);
211 result = lookup_one_len(name, parent, strlen(name));
212 inode_unlock(inode);
213
214 if (IS_ERR(result))
215 pr_warn("%s err:%ld\n", __func__, PTR_ERR(result));
216
217 return result;
218 }
219
handle_mapped_file(struct mount_info * mi,struct data_file * df)220 static struct data_file *handle_mapped_file(struct mount_info *mi,
221 struct data_file *df)
222 {
223 char *file_id_str;
224 struct dentry *index_file_dentry;
225 struct path path;
226 struct file *bf;
227 struct data_file *result = NULL;
228 const struct cred *old_cred;
229
230 file_id_str = file_id_to_str(df->df_id);
231 if (!file_id_str)
232 return ERR_PTR(-ENOENT);
233
234 index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir,
235 file_id_str);
236 kfree(file_id_str);
237 if (!index_file_dentry)
238 return ERR_PTR(-ENOENT);
239 if (IS_ERR(index_file_dentry))
240 return (struct data_file *)index_file_dentry;
241 if (!d_really_is_positive(index_file_dentry)) {
242 result = ERR_PTR(-ENOENT);
243 goto out;
244 }
245
246 path = (struct path) {
247 .mnt = mi->mi_backing_dir_path.mnt,
248 .dentry = index_file_dentry
249 };
250
251 old_cred = override_creds(mi->mi_owner);
252 bf = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE,
253 current_cred());
254 revert_creds(old_cred);
255
256 if (IS_ERR(bf)) {
257 result = (struct data_file *)bf;
258 goto out;
259 }
260
261 result = incfs_open_data_file(mi, bf);
262 fput(bf);
263 if (IS_ERR(result))
264 goto out;
265
266 result->df_mapped_offset = df->df_metadata_off;
267
268 out:
269 dput(index_file_dentry);
270 return result;
271 }
272
incfs_open_data_file(struct mount_info * mi,struct file * bf)273 struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)
274 {
275 struct data_file *df = NULL;
276 struct backing_file_context *bfc = NULL;
277 int md_records;
278 u64 size;
279 int error = 0;
280 int i;
281
282 if (!bf || !mi)
283 return ERR_PTR(-EFAULT);
284
285 if (!S_ISREG(bf->f_inode->i_mode))
286 return ERR_PTR(-EBADF);
287
288 bfc = incfs_alloc_bfc(mi, bf);
289 if (IS_ERR(bfc))
290 return ERR_CAST(bfc);
291
292 df = kzalloc(sizeof(*df), GFP_NOFS);
293 if (!df) {
294 error = -ENOMEM;
295 goto out;
296 }
297
298 mutex_init(&df->df_enable_verity);
299
300 df->df_backing_file_context = bfc;
301 df->df_mount_info = mi;
302 for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
303 data_file_segment_init(&df->df_segments[i]);
304
305 error = incfs_read_file_header(bfc, &df->df_metadata_off, &df->df_id,
306 &size, &df->df_header_flags);
307
308 if (error)
309 goto out;
310
311 df->df_size = size;
312 if (size > 0)
313 df->df_data_block_count = get_blocks_count_for_size(size);
314
315 if (df->df_header_flags & INCFS_FILE_MAPPED) {
316 struct data_file *mapped_df = handle_mapped_file(mi, df);
317
318 incfs_free_data_file(df);
319 return mapped_df;
320 }
321
322 md_records = incfs_scan_metadata_chain(df);
323 if (md_records < 0)
324 error = md_records;
325
326 out:
327 if (error) {
328 incfs_free_bfc(bfc);
329 if (df)
330 df->df_backing_file_context = NULL;
331 incfs_free_data_file(df);
332 return ERR_PTR(error);
333 }
334 return df;
335 }
336
incfs_free_data_file(struct data_file * df)337 void incfs_free_data_file(struct data_file *df)
338 {
339 u32 data_blocks_written, hash_blocks_written;
340
341 if (!df)
342 return;
343
344 data_blocks_written = atomic_read(&df->df_data_blocks_written);
345 hash_blocks_written = atomic_read(&df->df_hash_blocks_written);
346
347 if (data_blocks_written != df->df_initial_data_blocks_written ||
348 hash_blocks_written != df->df_initial_hash_blocks_written) {
349 struct backing_file_context *bfc = df->df_backing_file_context;
350 int error = -1;
351
352 if (bfc && !mutex_lock_interruptible(&bfc->bc_mutex)) {
353 error = incfs_write_status_to_backing_file(
354 df->df_backing_file_context,
355 df->df_status_offset,
356 data_blocks_written,
357 hash_blocks_written);
358 mutex_unlock(&bfc->bc_mutex);
359 }
360
361 if (error)
362 /* Nothing can be done, just warn */
363 pr_warn("incfs: failed to write status to backing file\n");
364 }
365
366 incfs_free_mtree(df->df_hash_tree);
367 incfs_free_bfc(df->df_backing_file_context);
368 kfree(df->df_signature);
369 kfree(df->df_verity_file_digest.data);
370 kfree(df->df_verity_signature);
371 mutex_destroy(&df->df_enable_verity);
372 kfree(df);
373 }
374
make_inode_ready_for_data_ops(struct mount_info * mi,struct inode * inode,struct file * backing_file)375 int make_inode_ready_for_data_ops(struct mount_info *mi,
376 struct inode *inode,
377 struct file *backing_file)
378 {
379 struct inode_info *node = get_incfs_node(inode);
380 struct data_file *df = NULL;
381 int err = 0;
382
383 inode_lock(inode);
384 if (S_ISREG(inode->i_mode)) {
385 if (!node->n_file) {
386 df = incfs_open_data_file(mi, backing_file);
387
388 if (IS_ERR(df))
389 err = PTR_ERR(df);
390 else
391 node->n_file = df;
392 }
393 } else
394 err = -EBADF;
395 inode_unlock(inode);
396 return err;
397 }
398
incfs_open_dir_file(struct mount_info * mi,struct file * bf)399 struct dir_file *incfs_open_dir_file(struct mount_info *mi, struct file *bf)
400 {
401 struct dir_file *dir = NULL;
402
403 if (!S_ISDIR(bf->f_inode->i_mode))
404 return ERR_PTR(-EBADF);
405
406 dir = kzalloc(sizeof(*dir), GFP_NOFS);
407 if (!dir)
408 return ERR_PTR(-ENOMEM);
409
410 dir->backing_dir = get_file(bf);
411 dir->mount_info = mi;
412 return dir;
413 }
414
incfs_free_dir_file(struct dir_file * dir)415 void incfs_free_dir_file(struct dir_file *dir)
416 {
417 if (!dir)
418 return;
419 if (dir->backing_dir)
420 fput(dir->backing_dir);
421 kfree(dir);
422 }
423
zstd_decompress_safe(struct mount_info * mi,struct mem_range src,struct mem_range dst)424 static ssize_t zstd_decompress_safe(struct mount_info *mi,
425 struct mem_range src, struct mem_range dst)
426 {
427 ssize_t result;
428 ZSTD_inBuffer inbuf = {.src = src.data, .size = src.len};
429 ZSTD_outBuffer outbuf = {.dst = dst.data, .size = dst.len};
430
431 result = mutex_lock_interruptible(&mi->mi_zstd_workspace_mutex);
432 if (result)
433 return result;
434
435 if (!mi->mi_zstd_stream) {
436 unsigned int workspace_size = ZSTD_DStreamWorkspaceBound(
437 INCFS_DATA_FILE_BLOCK_SIZE);
438 void *workspace = kvmalloc(workspace_size, GFP_NOFS);
439 ZSTD_DStream *stream;
440
441 if (!workspace) {
442 result = -ENOMEM;
443 goto out;
444 }
445
446 stream = ZSTD_initDStream(INCFS_DATA_FILE_BLOCK_SIZE, workspace,
447 workspace_size);
448 if (!stream) {
449 kvfree(workspace);
450 result = -EIO;
451 goto out;
452 }
453
454 mi->mi_zstd_workspace = workspace;
455 mi->mi_zstd_stream = stream;
456 }
457
458 result = ZSTD_decompressStream(mi->mi_zstd_stream, &outbuf, &inbuf) ?
459 -EBADMSG : outbuf.pos;
460
461 mod_delayed_work(system_wq, &mi->mi_zstd_cleanup_work,
462 msecs_to_jiffies(5000));
463
464 out:
465 mutex_unlock(&mi->mi_zstd_workspace_mutex);
466 return result;
467 }
468
decompress(struct mount_info * mi,struct mem_range src,struct mem_range dst,int alg)469 static ssize_t decompress(struct mount_info *mi,
470 struct mem_range src, struct mem_range dst, int alg)
471 {
472 int result;
473
474 switch (alg) {
475 case INCFS_BLOCK_COMPRESSED_LZ4:
476 result = LZ4_decompress_safe(src.data, dst.data, src.len,
477 dst.len);
478 if (result < 0)
479 return -EBADMSG;
480 return result;
481
482 case INCFS_BLOCK_COMPRESSED_ZSTD:
483 return zstd_decompress_safe(mi, src, dst);
484
485 default:
486 WARN_ON(true);
487 return -EOPNOTSUPP;
488 }
489 }
490
log_read_one_record(struct read_log * rl,struct read_log_state * rs)491 static void log_read_one_record(struct read_log *rl, struct read_log_state *rs)
492 {
493 union log_record *record =
494 (union log_record *)((u8 *)rl->rl_ring_buf + rs->next_offset);
495 size_t record_size;
496
497 switch (record->full_record.type) {
498 case FULL:
499 rs->base_record = record->full_record;
500 record_size = sizeof(record->full_record);
501 break;
502
503 case SAME_FILE:
504 rs->base_record.block_index =
505 record->same_file.block_index;
506 rs->base_record.absolute_ts_us +=
507 record->same_file.relative_ts_us;
508 rs->base_record.uid = record->same_file.uid;
509 record_size = sizeof(record->same_file);
510 break;
511
512 case SAME_FILE_CLOSE_BLOCK:
513 rs->base_record.block_index +=
514 record->same_file_close_block.block_index_delta;
515 rs->base_record.absolute_ts_us +=
516 record->same_file_close_block.relative_ts_us;
517 record_size = sizeof(record->same_file_close_block);
518 break;
519
520 case SAME_FILE_CLOSE_BLOCK_SHORT:
521 rs->base_record.block_index +=
522 record->same_file_close_block_short.block_index_delta;
523 rs->base_record.absolute_ts_us +=
524 record->same_file_close_block_short.relative_ts_tens_us * 10;
525 record_size = sizeof(record->same_file_close_block_short);
526 break;
527
528 case SAME_FILE_NEXT_BLOCK:
529 ++rs->base_record.block_index;
530 rs->base_record.absolute_ts_us +=
531 record->same_file_next_block.relative_ts_us;
532 record_size = sizeof(record->same_file_next_block);
533 break;
534
535 case SAME_FILE_NEXT_BLOCK_SHORT:
536 ++rs->base_record.block_index;
537 rs->base_record.absolute_ts_us +=
538 record->same_file_next_block_short.relative_ts_tens_us * 10;
539 record_size = sizeof(record->same_file_next_block_short);
540 break;
541 }
542
543 rs->next_offset += record_size;
544 if (rs->next_offset > rl->rl_size - sizeof(*record)) {
545 rs->next_offset = 0;
546 ++rs->current_pass_no;
547 }
548 ++rs->current_record_no;
549 }
550
log_block_read(struct mount_info * mi,incfs_uuid_t * id,int block_index)551 static void log_block_read(struct mount_info *mi, incfs_uuid_t *id,
552 int block_index)
553 {
554 struct read_log *log = &mi->mi_log;
555 struct read_log_state *head, *tail;
556 s64 now_us;
557 s64 relative_us;
558 union log_record record;
559 size_t record_size;
560 uid_t uid = current_uid().val;
561 int block_delta;
562 bool same_file, same_uid;
563 bool next_block, close_block, very_close_block;
564 bool close_time, very_close_time, very_very_close_time;
565
566 /*
567 * This may read the old value, but it's OK to delay the logging start
568 * right after the configuration update.
569 */
570 if (READ_ONCE(log->rl_size) == 0)
571 return;
572
573 now_us = ktime_to_us(ktime_get());
574
575 spin_lock(&log->rl_lock);
576 if (log->rl_size == 0) {
577 spin_unlock(&log->rl_lock);
578 return;
579 }
580
581 head = &log->rl_head;
582 tail = &log->rl_tail;
583 relative_us = now_us - head->base_record.absolute_ts_us;
584
585 same_file = !memcmp(id, &head->base_record.file_id,
586 sizeof(incfs_uuid_t));
587 same_uid = uid == head->base_record.uid;
588
589 block_delta = block_index - head->base_record.block_index;
590 next_block = block_delta == 1;
591 very_close_block = block_delta >= S8_MIN && block_delta <= S8_MAX;
592 close_block = block_delta >= S16_MIN && block_delta <= S16_MAX;
593
594 very_very_close_time = relative_us < (1 << 5) * 10;
595 very_close_time = relative_us < (1 << 13);
596 close_time = relative_us < (1 << 16);
597
598 if (same_file && same_uid && next_block && very_very_close_time) {
599 record.same_file_next_block_short =
600 (struct same_file_next_block_short){
601 .type = SAME_FILE_NEXT_BLOCK_SHORT,
602 .relative_ts_tens_us = div_s64(relative_us, 10),
603 };
604 record_size = sizeof(struct same_file_next_block_short);
605 } else if (same_file && same_uid && next_block && very_close_time) {
606 record.same_file_next_block = (struct same_file_next_block){
607 .type = SAME_FILE_NEXT_BLOCK,
608 .relative_ts_us = relative_us,
609 };
610 record_size = sizeof(struct same_file_next_block);
611 } else if (same_file && same_uid && very_close_block &&
612 very_very_close_time) {
613 record.same_file_close_block_short =
614 (struct same_file_close_block_short){
615 .type = SAME_FILE_CLOSE_BLOCK_SHORT,
616 .relative_ts_tens_us = div_s64(relative_us, 10),
617 .block_index_delta = block_delta,
618 };
619 record_size = sizeof(struct same_file_close_block_short);
620 } else if (same_file && same_uid && close_block && very_close_time) {
621 record.same_file_close_block = (struct same_file_close_block){
622 .type = SAME_FILE_CLOSE_BLOCK,
623 .relative_ts_us = relative_us,
624 .block_index_delta = block_delta,
625 };
626 record_size = sizeof(struct same_file_close_block);
627 } else if (same_file && close_time) {
628 record.same_file = (struct same_file){
629 .type = SAME_FILE,
630 .block_index = block_index,
631 .relative_ts_us = relative_us,
632 .uid = uid,
633 };
634 record_size = sizeof(struct same_file);
635 } else {
636 record.full_record = (struct full_record){
637 .type = FULL,
638 .block_index = block_index,
639 .file_id = *id,
640 .absolute_ts_us = now_us,
641 .uid = uid,
642 };
643 head->base_record.file_id = *id;
644 record_size = sizeof(struct full_record);
645 }
646
647 head->base_record.block_index = block_index;
648 head->base_record.absolute_ts_us = now_us;
649
650 /* Advance tail beyond area we are going to overwrite */
651 while (tail->current_pass_no < head->current_pass_no &&
652 tail->next_offset < head->next_offset + record_size)
653 log_read_one_record(log, tail);
654
655 memcpy(((u8 *)log->rl_ring_buf) + head->next_offset, &record,
656 record_size);
657 head->next_offset += record_size;
658 if (head->next_offset > log->rl_size - sizeof(record)) {
659 head->next_offset = 0;
660 ++head->current_pass_no;
661 }
662 ++head->current_record_no;
663
664 spin_unlock(&log->rl_lock);
665 schedule_delayed_work(&log->ml_wakeup_work, msecs_to_jiffies(16));
666 }
667
validate_hash_tree(struct backing_file_context * bfc,struct file * f,int block_index,struct mem_range data,u8 * buf)668 static int validate_hash_tree(struct backing_file_context *bfc, struct file *f,
669 int block_index, struct mem_range data, u8 *buf)
670 {
671 struct data_file *df = get_incfs_data_file(f);
672 u8 stored_digest[INCFS_MAX_HASH_SIZE] = {};
673 u8 calculated_digest[INCFS_MAX_HASH_SIZE] = {};
674 struct mtree *tree = NULL;
675 struct incfs_df_signature *sig = NULL;
676 int digest_size;
677 int hash_block_index = block_index;
678 int lvl;
679 int res;
680 loff_t hash_block_offset[INCFS_MAX_MTREE_LEVELS];
681 size_t hash_offset_in_block[INCFS_MAX_MTREE_LEVELS];
682 int hash_per_block;
683 pgoff_t file_pages;
684
685 /*
686 * Memory barrier to make sure tree is fully present if added via enable
687 * verity
688 */
689 tree = smp_load_acquire(&df->df_hash_tree);
690 sig = df->df_signature;
691 if (!tree || !sig)
692 return 0;
693
694 digest_size = tree->alg->digest_size;
695 hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
696 for (lvl = 0; lvl < tree->depth; lvl++) {
697 loff_t lvl_off = tree->hash_level_suboffset[lvl];
698
699 hash_block_offset[lvl] =
700 lvl_off + round_down(hash_block_index * digest_size,
701 INCFS_DATA_FILE_BLOCK_SIZE);
702 hash_offset_in_block[lvl] = hash_block_index * digest_size %
703 INCFS_DATA_FILE_BLOCK_SIZE;
704 hash_block_index /= hash_per_block;
705 }
706
707 memcpy(stored_digest, tree->root_hash, digest_size);
708
709 file_pages = DIV_ROUND_UP(df->df_size, INCFS_DATA_FILE_BLOCK_SIZE);
710 for (lvl = tree->depth - 1; lvl >= 0; lvl--) {
711 pgoff_t hash_page =
712 file_pages +
713 hash_block_offset[lvl] / INCFS_DATA_FILE_BLOCK_SIZE;
714 struct page *page = find_get_page_flags(
715 f->f_inode->i_mapping, hash_page, FGP_ACCESSED);
716
717 if (page && PageChecked(page)) {
718 u8 *addr = kmap_atomic(page);
719
720 memcpy(stored_digest, addr + hash_offset_in_block[lvl],
721 digest_size);
722 kunmap_atomic(addr);
723 put_page(page);
724 continue;
725 }
726
727 if (page)
728 put_page(page);
729
730 res = incfs_kread(bfc, buf, INCFS_DATA_FILE_BLOCK_SIZE,
731 hash_block_offset[lvl] + sig->hash_offset);
732 if (res < 0)
733 return res;
734 if (res != INCFS_DATA_FILE_BLOCK_SIZE)
735 return -EIO;
736 res = incfs_calc_digest(tree->alg,
737 range(buf, INCFS_DATA_FILE_BLOCK_SIZE),
738 range(calculated_digest, digest_size));
739 if (res)
740 return res;
741
742 if (memcmp(stored_digest, calculated_digest, digest_size)) {
743 int i;
744 bool zero = true;
745
746 pr_warn("incfs: Hash mismatch lvl:%d blk:%d\n",
747 lvl, block_index);
748 for (i = 0; i < digest_size; i++)
749 if (stored_digest[i]) {
750 zero = false;
751 break;
752 }
753
754 if (zero)
755 pr_debug("Note saved_digest all zero - did you forget to load the hashes?\n");
756 return -EBADMSG;
757 }
758
759 memcpy(stored_digest, buf + hash_offset_in_block[lvl],
760 digest_size);
761
762 page = grab_cache_page(f->f_inode->i_mapping, hash_page);
763 if (page) {
764 u8 *addr = kmap_atomic(page);
765
766 memcpy(addr, buf, INCFS_DATA_FILE_BLOCK_SIZE);
767 kunmap_atomic(addr);
768 SetPageChecked(page);
769 unlock_page(page);
770 put_page(page);
771 }
772 }
773
774 res = incfs_calc_digest(tree->alg, data,
775 range(calculated_digest, digest_size));
776 if (res)
777 return res;
778
779 if (memcmp(stored_digest, calculated_digest, digest_size)) {
780 pr_debug("Leaf hash mismatch blk:%d\n", block_index);
781 return -EBADMSG;
782 }
783
784 return 0;
785 }
786
get_file_segment(struct data_file * df,int block_index)787 static struct data_file_segment *get_file_segment(struct data_file *df,
788 int block_index)
789 {
790 int seg_idx = block_index % ARRAY_SIZE(df->df_segments);
791
792 return &df->df_segments[seg_idx];
793 }
794
is_data_block_present(struct data_file_block * block)795 static bool is_data_block_present(struct data_file_block *block)
796 {
797 return (block->db_backing_file_data_offset != 0) &&
798 (block->db_stored_size != 0);
799 }
800
convert_data_file_block(struct incfs_blockmap_entry * bme,struct data_file_block * res_block)801 static void convert_data_file_block(struct incfs_blockmap_entry *bme,
802 struct data_file_block *res_block)
803 {
804 u16 flags = le16_to_cpu(bme->me_flags);
805
806 res_block->db_backing_file_data_offset =
807 le16_to_cpu(bme->me_data_offset_hi);
808 res_block->db_backing_file_data_offset <<= 32;
809 res_block->db_backing_file_data_offset |=
810 le32_to_cpu(bme->me_data_offset_lo);
811 res_block->db_stored_size = le16_to_cpu(bme->me_data_size);
812 res_block->db_comp_alg = flags & INCFS_BLOCK_COMPRESSED_MASK;
813 }
814
get_data_file_block(struct data_file * df,int index,struct data_file_block * res_block)815 static int get_data_file_block(struct data_file *df, int index,
816 struct data_file_block *res_block)
817 {
818 struct incfs_blockmap_entry bme = {};
819 struct backing_file_context *bfc = NULL;
820 loff_t blockmap_off = 0;
821 int error = 0;
822
823 if (!df || !res_block)
824 return -EFAULT;
825
826 blockmap_off = df->df_blockmap_off;
827 bfc = df->df_backing_file_context;
828
829 if (index < 0 || blockmap_off == 0)
830 return -EINVAL;
831
832 error = incfs_read_blockmap_entry(bfc, index, blockmap_off, &bme);
833 if (error)
834 return error;
835
836 convert_data_file_block(&bme, res_block);
837 return 0;
838 }
839
check_room_for_one_range(u32 size,u32 size_out)840 static int check_room_for_one_range(u32 size, u32 size_out)
841 {
842 if (size_out + sizeof(struct incfs_filled_range) > size)
843 return -ERANGE;
844 return 0;
845 }
846
copy_one_range(struct incfs_filled_range * range,void __user * buffer,u32 size,u32 * size_out)847 static int copy_one_range(struct incfs_filled_range *range, void __user *buffer,
848 u32 size, u32 *size_out)
849 {
850 int error = check_room_for_one_range(size, *size_out);
851 if (error)
852 return error;
853
854 if (copy_to_user(((char __user *)buffer) + *size_out, range,
855 sizeof(*range)))
856 return -EFAULT;
857
858 *size_out += sizeof(*range);
859 return 0;
860 }
861
862 #define READ_BLOCKMAP_ENTRIES 512
incfs_get_filled_blocks(struct data_file * df,struct incfs_file_data * fd,struct incfs_get_filled_blocks_args * arg)863 int incfs_get_filled_blocks(struct data_file *df,
864 struct incfs_file_data *fd,
865 struct incfs_get_filled_blocks_args *arg)
866 {
867 int error = 0;
868 bool in_range = false;
869 struct incfs_filled_range range;
870 void __user *buffer = u64_to_user_ptr(arg->range_buffer);
871 u32 size = arg->range_buffer_size;
872 u32 end_index =
873 arg->end_index ? arg->end_index : df->df_total_block_count;
874 u32 *size_out = &arg->range_buffer_size_out;
875 int i = READ_BLOCKMAP_ENTRIES - 1;
876 int entries_read = 0;
877 struct incfs_blockmap_entry *bme;
878 int data_blocks_filled = 0;
879 int hash_blocks_filled = 0;
880
881 *size_out = 0;
882 if (end_index > df->df_total_block_count)
883 end_index = df->df_total_block_count;
884 arg->total_blocks_out = df->df_total_block_count;
885 arg->data_blocks_out = df->df_data_block_count;
886
887 if (atomic_read(&df->df_data_blocks_written) ==
888 df->df_data_block_count) {
889 pr_debug("File marked full, fast get_filled_blocks");
890 if (arg->start_index > end_index) {
891 arg->index_out = arg->start_index;
892 return 0;
893 }
894 arg->index_out = arg->start_index;
895
896 error = check_room_for_one_range(size, *size_out);
897 if (error)
898 return error;
899
900 range = (struct incfs_filled_range){
901 .begin = arg->start_index,
902 .end = end_index,
903 };
904
905 error = copy_one_range(&range, buffer, size, size_out);
906 if (error)
907 return error;
908 arg->index_out = end_index;
909 return 0;
910 }
911
912 bme = kzalloc(sizeof(*bme) * READ_BLOCKMAP_ENTRIES,
913 GFP_NOFS | __GFP_COMP);
914 if (!bme)
915 return -ENOMEM;
916
917 for (arg->index_out = arg->start_index; arg->index_out < end_index;
918 ++arg->index_out) {
919 struct data_file_block dfb;
920
921 if (++i == READ_BLOCKMAP_ENTRIES) {
922 entries_read = incfs_read_blockmap_entries(
923 df->df_backing_file_context, bme,
924 arg->index_out, READ_BLOCKMAP_ENTRIES,
925 df->df_blockmap_off);
926 if (entries_read < 0) {
927 error = entries_read;
928 break;
929 }
930
931 i = 0;
932 }
933
934 if (i >= entries_read) {
935 error = -EIO;
936 break;
937 }
938
939 convert_data_file_block(bme + i, &dfb);
940
941 if (is_data_block_present(&dfb)) {
942 if (arg->index_out >= df->df_data_block_count)
943 ++hash_blocks_filled;
944 else
945 ++data_blocks_filled;
946 }
947
948 if (is_data_block_present(&dfb) == in_range)
949 continue;
950
951 if (!in_range) {
952 error = check_room_for_one_range(size, *size_out);
953 if (error)
954 break;
955 in_range = true;
956 range.begin = arg->index_out;
957 } else {
958 range.end = arg->index_out;
959 error = copy_one_range(&range, buffer, size, size_out);
960 if (error) {
961 /* there will be another try out of the loop,
962 * it will reset the index_out if it fails too
963 */
964 break;
965 }
966 in_range = false;
967 }
968 }
969
970 if (in_range) {
971 range.end = arg->index_out;
972 error = copy_one_range(&range, buffer, size, size_out);
973 if (error)
974 arg->index_out = range.begin;
975 }
976
977 if (arg->start_index == 0) {
978 fd->fd_get_block_pos = 0;
979 fd->fd_filled_data_blocks = 0;
980 fd->fd_filled_hash_blocks = 0;
981 }
982
983 if (arg->start_index == fd->fd_get_block_pos) {
984 fd->fd_get_block_pos = arg->index_out + 1;
985 fd->fd_filled_data_blocks += data_blocks_filled;
986 fd->fd_filled_hash_blocks += hash_blocks_filled;
987 }
988
989 if (fd->fd_get_block_pos == df->df_total_block_count + 1) {
990 if (fd->fd_filled_data_blocks >
991 atomic_read(&df->df_data_blocks_written))
992 atomic_set(&df->df_data_blocks_written,
993 fd->fd_filled_data_blocks);
994
995 if (fd->fd_filled_hash_blocks >
996 atomic_read(&df->df_hash_blocks_written))
997 atomic_set(&df->df_hash_blocks_written,
998 fd->fd_filled_hash_blocks);
999 }
1000
1001 kfree(bme);
1002 return error;
1003 }
1004
is_read_done(struct pending_read * read)1005 static bool is_read_done(struct pending_read *read)
1006 {
1007 return atomic_read_acquire(&read->done) != 0;
1008 }
1009
set_read_done(struct pending_read * read)1010 static void set_read_done(struct pending_read *read)
1011 {
1012 atomic_set_release(&read->done, 1);
1013 }
1014
1015 /*
1016 * Notifies a given data file about pending read from a given block.
1017 * Returns a new pending read entry.
1018 */
add_pending_read(struct data_file * df,int block_index)1019 static struct pending_read *add_pending_read(struct data_file *df,
1020 int block_index)
1021 {
1022 struct pending_read *result = NULL;
1023 struct data_file_segment *segment = NULL;
1024 struct mount_info *mi = NULL;
1025
1026 segment = get_file_segment(df, block_index);
1027 mi = df->df_mount_info;
1028
1029 result = kzalloc(sizeof(*result), GFP_NOFS);
1030 if (!result)
1031 return NULL;
1032
1033 result->file_id = df->df_id;
1034 result->block_index = block_index;
1035 result->timestamp_us = ktime_to_us(ktime_get());
1036 result->uid = current_uid().val;
1037
1038 spin_lock(&mi->pending_read_lock);
1039
1040 result->serial_number = ++mi->mi_last_pending_read_number;
1041 mi->mi_pending_reads_count++;
1042
1043 list_add_rcu(&result->mi_reads_list, &mi->mi_reads_list_head);
1044 list_add_rcu(&result->segment_reads_list, &segment->reads_list_head);
1045
1046 spin_unlock(&mi->pending_read_lock);
1047
1048 wake_up_all(&mi->mi_pending_reads_notif_wq);
1049 return result;
1050 }
1051
free_pending_read_entry(struct rcu_head * entry)1052 static void free_pending_read_entry(struct rcu_head *entry)
1053 {
1054 struct pending_read *read;
1055
1056 read = container_of(entry, struct pending_read, rcu);
1057
1058 kfree(read);
1059 }
1060
1061 /* Notifies a given data file that pending read is completed. */
remove_pending_read(struct data_file * df,struct pending_read * read)1062 static void remove_pending_read(struct data_file *df, struct pending_read *read)
1063 {
1064 struct mount_info *mi = NULL;
1065
1066 if (!df || !read) {
1067 WARN_ON(!df);
1068 WARN_ON(!read);
1069 return;
1070 }
1071
1072 mi = df->df_mount_info;
1073
1074 spin_lock(&mi->pending_read_lock);
1075
1076 list_del_rcu(&read->mi_reads_list);
1077 list_del_rcu(&read->segment_reads_list);
1078
1079 mi->mi_pending_reads_count--;
1080
1081 spin_unlock(&mi->pending_read_lock);
1082
1083 /* Don't free. Wait for readers */
1084 call_rcu(&read->rcu, free_pending_read_entry);
1085 }
1086
notify_pending_reads(struct mount_info * mi,struct data_file_segment * segment,int index)1087 static void notify_pending_reads(struct mount_info *mi,
1088 struct data_file_segment *segment,
1089 int index)
1090 {
1091 struct pending_read *entry = NULL;
1092
1093 /* Notify pending reads waiting for this block. */
1094 rcu_read_lock();
1095 list_for_each_entry_rcu(entry, &segment->reads_list_head,
1096 segment_reads_list) {
1097 if (entry->block_index == index)
1098 set_read_done(entry);
1099 }
1100 rcu_read_unlock();
1101 wake_up_all(&segment->new_data_arrival_wq);
1102
1103 atomic_inc(&mi->mi_blocks_written);
1104 wake_up_all(&mi->mi_blocks_written_notif_wq);
1105 }
1106
usleep_interruptible(u32 us)1107 static int usleep_interruptible(u32 us)
1108 {
1109 /* See:
1110 * https://www.kernel.org/doc/Documentation/timers/timers-howto.txt
1111 * for explanation
1112 */
1113 if (us < 10) {
1114 udelay(us);
1115 return 0;
1116 } else if (us < 20000) {
1117 usleep_range(us, us + us / 10);
1118 return 0;
1119 } else
1120 return msleep_interruptible(us / 1000);
1121 }
1122
wait_for_data_block(struct data_file * df,int block_index,struct data_file_block * res_block,struct incfs_read_data_file_timeouts * timeouts)1123 static int wait_for_data_block(struct data_file *df, int block_index,
1124 struct data_file_block *res_block,
1125 struct incfs_read_data_file_timeouts *timeouts)
1126 {
1127 struct data_file_block block = {};
1128 struct data_file_segment *segment = NULL;
1129 struct pending_read *read = NULL;
1130 struct mount_info *mi = NULL;
1131 int error;
1132 int wait_res = 0;
1133 unsigned int delayed_pending_us = 0, delayed_min_us = 0;
1134 bool delayed_pending = false;
1135
1136 if (!df || !res_block)
1137 return -EFAULT;
1138
1139 if (block_index < 0 || block_index >= df->df_data_block_count)
1140 return -EINVAL;
1141
1142 if (df->df_blockmap_off <= 0 || !df->df_mount_info)
1143 return -ENODATA;
1144
1145 mi = df->df_mount_info;
1146 segment = get_file_segment(df, block_index);
1147
1148 error = down_read_killable(&segment->rwsem);
1149 if (error)
1150 return error;
1151
1152 /* Look up the given block */
1153 error = get_data_file_block(df, block_index, &block);
1154
1155 up_read(&segment->rwsem);
1156
1157 if (error)
1158 return error;
1159
1160 /* If the block was found, just return it. No need to wait. */
1161 if (is_data_block_present(&block)) {
1162 *res_block = block;
1163 if (timeouts && timeouts->min_time_us) {
1164 delayed_min_us = timeouts->min_time_us;
1165 error = usleep_interruptible(delayed_min_us);
1166 goto out;
1167 }
1168 return 0;
1169 } else {
1170 /* If it's not found, create a pending read */
1171 if (timeouts && timeouts->max_pending_time_us) {
1172 read = add_pending_read(df, block_index);
1173 if (!read)
1174 return -ENOMEM;
1175 } else {
1176 log_block_read(mi, &df->df_id, block_index);
1177 return -ETIME;
1178 }
1179 }
1180
1181 /* Rest of function only applies if timeouts != NULL */
1182 if (!timeouts) {
1183 pr_warn("incfs: timeouts unexpectedly NULL\n");
1184 return -EFSCORRUPTED;
1185 }
1186
1187 /* Wait for notifications about block's arrival */
1188 wait_res =
1189 wait_event_interruptible_timeout(segment->new_data_arrival_wq,
1190 (is_read_done(read)),
1191 usecs_to_jiffies(timeouts->max_pending_time_us));
1192
1193 /* Woke up, the pending read is no longer needed. */
1194 remove_pending_read(df, read);
1195
1196 if (wait_res == 0) {
1197 /* Wait has timed out */
1198 log_block_read(mi, &df->df_id, block_index);
1199 return -ETIME;
1200 }
1201 if (wait_res < 0) {
1202 /*
1203 * Only ERESTARTSYS is really expected here when a signal
1204 * comes while we wait.
1205 */
1206 return wait_res;
1207 }
1208
1209 delayed_pending = true;
1210 delayed_pending_us = timeouts->max_pending_time_us -
1211 jiffies_to_usecs(wait_res);
1212 if (timeouts->min_pending_time_us > delayed_pending_us) {
1213 delayed_min_us = timeouts->min_pending_time_us -
1214 delayed_pending_us;
1215 error = usleep_interruptible(delayed_min_us);
1216 if (error)
1217 return error;
1218 }
1219
1220 error = down_read_killable(&segment->rwsem);
1221 if (error)
1222 return error;
1223
1224 /*
1225 * Re-read blocks info now, it has just arrived and
1226 * should be available.
1227 */
1228 error = get_data_file_block(df, block_index, &block);
1229 if (!error) {
1230 if (is_data_block_present(&block))
1231 *res_block = block;
1232 else {
1233 /*
1234 * Somehow wait finished successfully but block still
1235 * can't be found. It's not normal.
1236 */
1237 pr_warn("incfs: Wait succeeded but block not found.\n");
1238 error = -ENODATA;
1239 }
1240 }
1241 up_read(&segment->rwsem);
1242
1243 out:
1244 if (error)
1245 return error;
1246
1247 if (delayed_pending) {
1248 mi->mi_reads_delayed_pending++;
1249 mi->mi_reads_delayed_pending_us +=
1250 delayed_pending_us;
1251 }
1252
1253 if (delayed_min_us) {
1254 mi->mi_reads_delayed_min++;
1255 mi->mi_reads_delayed_min_us += delayed_min_us;
1256 }
1257
1258 return 0;
1259 }
1260
incfs_update_sysfs_error(struct file * file,int index,int result,struct mount_info * mi,struct data_file * df)1261 static int incfs_update_sysfs_error(struct file *file, int index, int result,
1262 struct mount_info *mi, struct data_file *df)
1263 {
1264 int error;
1265
1266 if (result >= 0)
1267 return 0;
1268
1269 error = mutex_lock_interruptible(&mi->mi_le_mutex);
1270 if (error)
1271 return error;
1272
1273 mi->mi_le_file_id = df->df_id;
1274 mi->mi_le_time_us = ktime_to_us(ktime_get());
1275 mi->mi_le_page = index;
1276 mi->mi_le_errno = result;
1277 mi->mi_le_uid = current_uid().val;
1278 mutex_unlock(&mi->mi_le_mutex);
1279
1280 return 0;
1281 }
1282
incfs_read_data_file_block(struct mem_range dst,struct file * f,int index,struct mem_range tmp,struct incfs_read_data_file_timeouts * timeouts)1283 ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
1284 int index, struct mem_range tmp,
1285 struct incfs_read_data_file_timeouts *timeouts)
1286 {
1287 loff_t pos;
1288 ssize_t result;
1289 size_t bytes_to_read;
1290 struct mount_info *mi = NULL;
1291 struct backing_file_context *bfc = NULL;
1292 struct data_file_block block = {};
1293 struct data_file *df = get_incfs_data_file(f);
1294
1295 if (!dst.data || !df || !tmp.data)
1296 return -EFAULT;
1297
1298 if (tmp.len < 2 * INCFS_DATA_FILE_BLOCK_SIZE)
1299 return -ERANGE;
1300
1301 mi = df->df_mount_info;
1302 bfc = df->df_backing_file_context;
1303
1304 result = wait_for_data_block(df, index, &block, timeouts);
1305 if (result < 0)
1306 goto out;
1307
1308 pos = block.db_backing_file_data_offset;
1309 if (block.db_comp_alg == COMPRESSION_NONE) {
1310 bytes_to_read = min(dst.len, block.db_stored_size);
1311 result = incfs_kread(bfc, dst.data, bytes_to_read, pos);
1312
1313 /* Some data was read, but not enough */
1314 if (result >= 0 && result != bytes_to_read)
1315 result = -EIO;
1316 } else {
1317 bytes_to_read = min(tmp.len, block.db_stored_size);
1318 result = incfs_kread(bfc, tmp.data, bytes_to_read, pos);
1319 if (result == bytes_to_read) {
1320 result =
1321 decompress(mi, range(tmp.data, bytes_to_read),
1322 dst, block.db_comp_alg);
1323 if (result < 0) {
1324 const char *name =
1325 bfc->bc_file->f_path.dentry->d_name.name;
1326
1327 pr_warn_once("incfs: Decompression error. %s",
1328 name);
1329 }
1330 } else if (result >= 0) {
1331 /* Some data was read, but not enough */
1332 result = -EIO;
1333 }
1334 }
1335
1336 if (result > 0) {
1337 int err = validate_hash_tree(bfc, f, index, dst, tmp.data);
1338
1339 if (err < 0)
1340 result = err;
1341 }
1342
1343 if (result >= 0)
1344 log_block_read(mi, &df->df_id, index);
1345
1346 out:
1347 if (result == -ETIME)
1348 mi->mi_reads_failed_timed_out++;
1349 else if (result == -EBADMSG)
1350 mi->mi_reads_failed_hash_verification++;
1351 else if (result < 0)
1352 mi->mi_reads_failed_other++;
1353
1354 incfs_update_sysfs_error(f, index, result, mi, df);
1355
1356 return result;
1357 }
1358
incfs_read_merkle_tree_blocks(struct mem_range dst,struct data_file * df,size_t offset)1359 ssize_t incfs_read_merkle_tree_blocks(struct mem_range dst,
1360 struct data_file *df, size_t offset)
1361 {
1362 struct backing_file_context *bfc = NULL;
1363 struct incfs_df_signature *sig = NULL;
1364 size_t to_read = dst.len;
1365
1366 if (!dst.data || !df)
1367 return -EFAULT;
1368
1369 sig = df->df_signature;
1370 bfc = df->df_backing_file_context;
1371
1372 if (offset > sig->hash_size)
1373 return -ERANGE;
1374
1375 if (offset + to_read > sig->hash_size)
1376 to_read = sig->hash_size - offset;
1377
1378 return incfs_kread(bfc, dst.data, to_read, sig->hash_offset + offset);
1379 }
1380
incfs_process_new_data_block(struct data_file * df,struct incfs_fill_block * block,u8 * data)1381 int incfs_process_new_data_block(struct data_file *df,
1382 struct incfs_fill_block *block, u8 *data)
1383 {
1384 struct mount_info *mi = NULL;
1385 struct backing_file_context *bfc = NULL;
1386 struct data_file_segment *segment = NULL;
1387 struct data_file_block existing_block = {};
1388 u16 flags = 0;
1389 int error = 0;
1390
1391 if (!df || !block)
1392 return -EFAULT;
1393
1394 bfc = df->df_backing_file_context;
1395 mi = df->df_mount_info;
1396
1397 if (block->block_index >= df->df_data_block_count)
1398 return -ERANGE;
1399
1400 segment = get_file_segment(df, block->block_index);
1401 if (!segment)
1402 return -EFAULT;
1403
1404 if (block->compression == COMPRESSION_LZ4)
1405 flags |= INCFS_BLOCK_COMPRESSED_LZ4;
1406 else if (block->compression == COMPRESSION_ZSTD)
1407 flags |= INCFS_BLOCK_COMPRESSED_ZSTD;
1408 else if (block->compression)
1409 return -EINVAL;
1410
1411 error = down_read_killable(&segment->rwsem);
1412 if (error)
1413 return error;
1414
1415 error = get_data_file_block(df, block->block_index, &existing_block);
1416
1417 up_read(&segment->rwsem);
1418
1419 if (error)
1420 return error;
1421 if (is_data_block_present(&existing_block)) {
1422 /* Block is already present, nothing to do here */
1423 return 0;
1424 }
1425
1426 error = down_write_killable(&segment->rwsem);
1427 if (error)
1428 return error;
1429
1430 error = mutex_lock_interruptible(&bfc->bc_mutex);
1431 if (!error) {
1432 error = incfs_write_data_block_to_backing_file(
1433 bfc, range(data, block->data_len), block->block_index,
1434 df->df_blockmap_off, flags);
1435 mutex_unlock(&bfc->bc_mutex);
1436 }
1437 if (!error) {
1438 notify_pending_reads(mi, segment, block->block_index);
1439 atomic_inc(&df->df_data_blocks_written);
1440 }
1441
1442 up_write(&segment->rwsem);
1443
1444 if (error)
1445 pr_debug("%d error: %d\n", block->block_index, error);
1446 return error;
1447 }
1448
incfs_read_file_signature(struct data_file * df,struct mem_range dst)1449 int incfs_read_file_signature(struct data_file *df, struct mem_range dst)
1450 {
1451 struct backing_file_context *bfc = df->df_backing_file_context;
1452 struct incfs_df_signature *sig;
1453 int read_res = 0;
1454
1455 if (!dst.data)
1456 return -EFAULT;
1457
1458 sig = df->df_signature;
1459 if (!sig)
1460 return 0;
1461
1462 if (dst.len < sig->sig_size)
1463 return -E2BIG;
1464
1465 read_res = incfs_kread(bfc, dst.data, sig->sig_size, sig->sig_offset);
1466
1467 if (read_res < 0)
1468 return read_res;
1469
1470 if (read_res != sig->sig_size)
1471 return -EIO;
1472
1473 return read_res;
1474 }
1475
incfs_process_new_hash_block(struct data_file * df,struct incfs_fill_block * block,u8 * data)1476 int incfs_process_new_hash_block(struct data_file *df,
1477 struct incfs_fill_block *block, u8 *data)
1478 {
1479 struct backing_file_context *bfc = NULL;
1480 struct mount_info *mi = NULL;
1481 struct mtree *hash_tree = NULL;
1482 struct incfs_df_signature *sig = NULL;
1483 loff_t hash_area_base = 0;
1484 loff_t hash_area_size = 0;
1485 int error = 0;
1486
1487 if (!df || !block)
1488 return -EFAULT;
1489
1490 if (!(block->flags & INCFS_BLOCK_FLAGS_HASH))
1491 return -EINVAL;
1492
1493 bfc = df->df_backing_file_context;
1494 mi = df->df_mount_info;
1495
1496 if (!df)
1497 return -ENOENT;
1498
1499 hash_tree = df->df_hash_tree;
1500 sig = df->df_signature;
1501 if (!hash_tree || !sig || sig->hash_offset == 0)
1502 return -ENOTSUPP;
1503
1504 hash_area_base = sig->hash_offset;
1505 hash_area_size = sig->hash_size;
1506 if (hash_area_size < block->block_index * INCFS_DATA_FILE_BLOCK_SIZE
1507 + block->data_len) {
1508 /* Hash block goes beyond dedicated hash area of this file. */
1509 return -ERANGE;
1510 }
1511
1512 error = mutex_lock_interruptible(&bfc->bc_mutex);
1513 if (!error) {
1514 error = incfs_write_hash_block_to_backing_file(
1515 bfc, range(data, block->data_len), block->block_index,
1516 hash_area_base, df->df_blockmap_off, df->df_size);
1517 mutex_unlock(&bfc->bc_mutex);
1518 }
1519 if (!error)
1520 atomic_inc(&df->df_hash_blocks_written);
1521
1522 return error;
1523 }
1524
process_blockmap_md(struct incfs_blockmap * bm,struct metadata_handler * handler)1525 static int process_blockmap_md(struct incfs_blockmap *bm,
1526 struct metadata_handler *handler)
1527 {
1528 struct data_file *df = handler->context;
1529 int error = 0;
1530 loff_t base_off = le64_to_cpu(bm->m_base_offset);
1531 u32 block_count = le32_to_cpu(bm->m_block_count);
1532
1533 if (!df)
1534 return -EFAULT;
1535
1536 if (df->df_data_block_count > block_count)
1537 return -EBADMSG;
1538
1539 df->df_total_block_count = block_count;
1540 df->df_blockmap_off = base_off;
1541 return error;
1542 }
1543
process_file_signature_md(struct incfs_file_signature * sg,struct metadata_handler * handler)1544 static int process_file_signature_md(struct incfs_file_signature *sg,
1545 struct metadata_handler *handler)
1546 {
1547 struct data_file *df = handler->context;
1548 struct mtree *hash_tree = NULL;
1549 int error = 0;
1550 struct incfs_df_signature *signature =
1551 kzalloc(sizeof(*signature), GFP_NOFS);
1552 void *buf = NULL;
1553 ssize_t read;
1554
1555 if (!signature)
1556 return -ENOMEM;
1557
1558 if (!df || !df->df_backing_file_context ||
1559 !df->df_backing_file_context->bc_file) {
1560 error = -ENOENT;
1561 goto out;
1562 }
1563
1564 signature->hash_offset = le64_to_cpu(sg->sg_hash_tree_offset);
1565 signature->hash_size = le32_to_cpu(sg->sg_hash_tree_size);
1566 signature->sig_offset = le64_to_cpu(sg->sg_sig_offset);
1567 signature->sig_size = le32_to_cpu(sg->sg_sig_size);
1568
1569 buf = kzalloc(signature->sig_size, GFP_NOFS);
1570 if (!buf) {
1571 error = -ENOMEM;
1572 goto out;
1573 }
1574
1575 read = incfs_kread(df->df_backing_file_context, buf,
1576 signature->sig_size, signature->sig_offset);
1577 if (read < 0) {
1578 error = read;
1579 goto out;
1580 }
1581
1582 if (read != signature->sig_size) {
1583 error = -EINVAL;
1584 goto out;
1585 }
1586
1587 hash_tree = incfs_alloc_mtree(range(buf, signature->sig_size),
1588 df->df_data_block_count);
1589 if (IS_ERR(hash_tree)) {
1590 error = PTR_ERR(hash_tree);
1591 hash_tree = NULL;
1592 goto out;
1593 }
1594 if (hash_tree->hash_tree_area_size != signature->hash_size) {
1595 error = -EINVAL;
1596 goto out;
1597 }
1598 if (signature->hash_size > 0 &&
1599 handler->md_record_offset <= signature->hash_offset) {
1600 error = -EINVAL;
1601 goto out;
1602 }
1603 if (handler->md_record_offset <= signature->sig_offset) {
1604 error = -EINVAL;
1605 goto out;
1606 }
1607 df->df_hash_tree = hash_tree;
1608 hash_tree = NULL;
1609 df->df_signature = signature;
1610 signature = NULL;
1611 out:
1612 incfs_free_mtree(hash_tree);
1613 kfree(signature);
1614 kfree(buf);
1615
1616 return error;
1617 }
1618
process_status_md(struct incfs_status * is,struct metadata_handler * handler)1619 static int process_status_md(struct incfs_status *is,
1620 struct metadata_handler *handler)
1621 {
1622 struct data_file *df = handler->context;
1623
1624 df->df_initial_data_blocks_written =
1625 le32_to_cpu(is->is_data_blocks_written);
1626 atomic_set(&df->df_data_blocks_written,
1627 df->df_initial_data_blocks_written);
1628
1629 df->df_initial_hash_blocks_written =
1630 le32_to_cpu(is->is_hash_blocks_written);
1631 atomic_set(&df->df_hash_blocks_written,
1632 df->df_initial_hash_blocks_written);
1633
1634 df->df_status_offset = handler->md_record_offset;
1635 return 0;
1636 }
1637
process_file_verity_signature_md(struct incfs_file_verity_signature * vs,struct metadata_handler * handler)1638 static int process_file_verity_signature_md(
1639 struct incfs_file_verity_signature *vs,
1640 struct metadata_handler *handler)
1641 {
1642 struct data_file *df = handler->context;
1643 struct incfs_df_verity_signature *verity_signature;
1644
1645 if (!df)
1646 return -EFAULT;
1647
1648 verity_signature = kzalloc(sizeof(*verity_signature), GFP_NOFS);
1649 if (!verity_signature)
1650 return -ENOMEM;
1651
1652 verity_signature->offset = le64_to_cpu(vs->vs_offset);
1653 verity_signature->size = le32_to_cpu(vs->vs_size);
1654 if (verity_signature->size > FS_VERITY_MAX_SIGNATURE_SIZE) {
1655 kfree(verity_signature);
1656 return -EFAULT;
1657 }
1658
1659 df->df_verity_signature = verity_signature;
1660 return 0;
1661 }
1662
incfs_scan_metadata_chain(struct data_file * df)1663 static int incfs_scan_metadata_chain(struct data_file *df)
1664 {
1665 struct metadata_handler *handler = NULL;
1666 int result = 0;
1667 int records_count = 0;
1668 int error = 0;
1669 struct backing_file_context *bfc = NULL;
1670 int nondata_block_count;
1671
1672 if (!df || !df->df_backing_file_context)
1673 return -EFAULT;
1674
1675 bfc = df->df_backing_file_context;
1676
1677 handler = kzalloc(sizeof(*handler), GFP_NOFS);
1678 if (!handler)
1679 return -ENOMEM;
1680
1681 handler->md_record_offset = df->df_metadata_off;
1682 handler->context = df;
1683 handler->handle_blockmap = process_blockmap_md;
1684 handler->handle_signature = process_file_signature_md;
1685 handler->handle_status = process_status_md;
1686 handler->handle_verity_signature = process_file_verity_signature_md;
1687
1688 while (handler->md_record_offset > 0) {
1689 error = incfs_read_next_metadata_record(bfc, handler);
1690 if (error) {
1691 pr_warn("incfs: Error during reading incfs-metadata record. Offset: %lld Record #%d Error code: %d\n",
1692 handler->md_record_offset, records_count + 1,
1693 -error);
1694 break;
1695 }
1696 records_count++;
1697 }
1698 if (error) {
1699 pr_warn("incfs: Error %d after reading %d incfs-metadata records.\n",
1700 -error, records_count);
1701 result = error;
1702 } else
1703 result = records_count;
1704
1705 nondata_block_count = df->df_total_block_count -
1706 df->df_data_block_count;
1707 if (df->df_hash_tree) {
1708 int hash_block_count = get_blocks_count_for_size(
1709 df->df_hash_tree->hash_tree_area_size);
1710
1711 /*
1712 * Files that were created with a hash tree have the hash tree
1713 * included in the block map, i.e. nondata_block_count ==
1714 * hash_block_count. Files whose hash tree was added by
1715 * FS_IOC_ENABLE_VERITY will still have the original block
1716 * count, i.e. nondata_block_count == 0.
1717 */
1718 if (nondata_block_count != hash_block_count &&
1719 nondata_block_count != 0)
1720 result = -EINVAL;
1721 } else if (nondata_block_count != 0) {
1722 result = -EINVAL;
1723 }
1724
1725 kfree(handler);
1726 return result;
1727 }
1728
1729 /*
1730 * Quickly checks if there are pending reads with a serial number larger
1731 * than a given one.
1732 */
incfs_fresh_pending_reads_exist(struct mount_info * mi,int last_number)1733 bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number)
1734 {
1735 bool result = false;
1736
1737 spin_lock(&mi->pending_read_lock);
1738 result = (mi->mi_last_pending_read_number > last_number) &&
1739 (mi->mi_pending_reads_count > 0);
1740 spin_unlock(&mi->pending_read_lock);
1741 return result;
1742 }
1743
incfs_collect_pending_reads(struct mount_info * mi,int sn_lowerbound,struct incfs_pending_read_info * reads,struct incfs_pending_read_info2 * reads2,int reads_size,int * new_max_sn)1744 int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
1745 struct incfs_pending_read_info *reads,
1746 struct incfs_pending_read_info2 *reads2,
1747 int reads_size, int *new_max_sn)
1748 {
1749 int reported_reads = 0;
1750 struct pending_read *entry = NULL;
1751
1752 if (!mi)
1753 return -EFAULT;
1754
1755 if (reads_size <= 0)
1756 return 0;
1757
1758 if (!incfs_fresh_pending_reads_exist(mi, sn_lowerbound))
1759 return 0;
1760
1761 rcu_read_lock();
1762
1763 list_for_each_entry_rcu(entry, &mi->mi_reads_list_head, mi_reads_list) {
1764 if (entry->serial_number <= sn_lowerbound)
1765 continue;
1766
1767 if (reads) {
1768 reads[reported_reads].file_id = entry->file_id;
1769 reads[reported_reads].block_index = entry->block_index;
1770 reads[reported_reads].serial_number =
1771 entry->serial_number;
1772 reads[reported_reads].timestamp_us =
1773 entry->timestamp_us;
1774 }
1775
1776 if (reads2) {
1777 reads2[reported_reads].file_id = entry->file_id;
1778 reads2[reported_reads].block_index = entry->block_index;
1779 reads2[reported_reads].serial_number =
1780 entry->serial_number;
1781 reads2[reported_reads].timestamp_us =
1782 entry->timestamp_us;
1783 reads2[reported_reads].uid = entry->uid;
1784 }
1785
1786 if (entry->serial_number > *new_max_sn)
1787 *new_max_sn = entry->serial_number;
1788
1789 reported_reads++;
1790 if (reported_reads >= reads_size)
1791 break;
1792 }
1793
1794 rcu_read_unlock();
1795
1796 return reported_reads;
1797 }
1798
incfs_get_log_state(struct mount_info * mi)1799 struct read_log_state incfs_get_log_state(struct mount_info *mi)
1800 {
1801 struct read_log *log = &mi->mi_log;
1802 struct read_log_state result;
1803
1804 spin_lock(&log->rl_lock);
1805 result = log->rl_head;
1806 spin_unlock(&log->rl_lock);
1807 return result;
1808 }
1809
incfs_get_uncollected_logs_count(struct mount_info * mi,const struct read_log_state * state)1810 int incfs_get_uncollected_logs_count(struct mount_info *mi,
1811 const struct read_log_state *state)
1812 {
1813 struct read_log *log = &mi->mi_log;
1814 u32 generation;
1815 u64 head_no, tail_no;
1816
1817 spin_lock(&log->rl_lock);
1818 tail_no = log->rl_tail.current_record_no;
1819 head_no = log->rl_head.current_record_no;
1820 generation = log->rl_head.generation_id;
1821 spin_unlock(&log->rl_lock);
1822
1823 if (generation != state->generation_id)
1824 return head_no - tail_no;
1825 else
1826 return head_no - max_t(u64, tail_no, state->current_record_no);
1827 }
1828
incfs_collect_logged_reads(struct mount_info * mi,struct read_log_state * state,struct incfs_pending_read_info * reads,struct incfs_pending_read_info2 * reads2,int reads_size)1829 int incfs_collect_logged_reads(struct mount_info *mi,
1830 struct read_log_state *state,
1831 struct incfs_pending_read_info *reads,
1832 struct incfs_pending_read_info2 *reads2,
1833 int reads_size)
1834 {
1835 int dst_idx;
1836 struct read_log *log = &mi->mi_log;
1837 struct read_log_state *head, *tail;
1838
1839 spin_lock(&log->rl_lock);
1840 head = &log->rl_head;
1841 tail = &log->rl_tail;
1842
1843 if (state->generation_id != head->generation_id) {
1844 pr_debug("read ptr is wrong generation: %u/%u",
1845 state->generation_id, head->generation_id);
1846
1847 *state = (struct read_log_state){
1848 .generation_id = head->generation_id,
1849 };
1850 }
1851
1852 if (state->current_record_no < tail->current_record_no) {
1853 pr_debug("read ptr is behind, moving: %u/%u -> %u/%u\n",
1854 (u32)state->next_offset,
1855 (u32)state->current_pass_no,
1856 (u32)tail->next_offset, (u32)tail->current_pass_no);
1857
1858 *state = *tail;
1859 }
1860
1861 for (dst_idx = 0; dst_idx < reads_size; dst_idx++) {
1862 if (state->current_record_no == head->current_record_no)
1863 break;
1864
1865 log_read_one_record(log, state);
1866
1867 if (reads)
1868 reads[dst_idx] = (struct incfs_pending_read_info) {
1869 .file_id = state->base_record.file_id,
1870 .block_index = state->base_record.block_index,
1871 .serial_number = state->current_record_no,
1872 .timestamp_us =
1873 state->base_record.absolute_ts_us,
1874 };
1875
1876 if (reads2)
1877 reads2[dst_idx] = (struct incfs_pending_read_info2) {
1878 .file_id = state->base_record.file_id,
1879 .block_index = state->base_record.block_index,
1880 .serial_number = state->current_record_no,
1881 .timestamp_us =
1882 state->base_record.absolute_ts_us,
1883 .uid = state->base_record.uid,
1884 };
1885 }
1886
1887 spin_unlock(&log->rl_lock);
1888 return dst_idx;
1889 }
1890
1891