Lines Matching full:ic
99 #define journal_entry_tag(ic, je) ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block]) argument
290 struct dm_integrity_c *ic; member
308 struct dm_integrity_c *ic; member
320 struct dm_integrity_c *ic; member
378 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err) in dm_integrity_io_error() argument
381 atomic64_inc(&ic->number_of_mismatches); in dm_integrity_io_error()
382 if (!cmpxchg(&ic->failed, 0, err)) in dm_integrity_io_error()
386 static int dm_integrity_failed(struct dm_integrity_c *ic) in dm_integrity_failed() argument
388 return READ_ONCE(ic->failed); in dm_integrity_failed()
391 static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic) in dm_integrity_disable_recalculate() argument
393 if ((ic->internal_hash_alg.key || ic->journal_mac_alg.key) && in dm_integrity_disable_recalculate()
394 !ic->legacy_recalculate) in dm_integrity_disable_recalculate()
399 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i, in dm_integrity_commit_id() argument
406 return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j); in dm_integrity_commit_id()
409 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector, in get_area_and_offset() argument
412 if (!ic->meta_dev) { in get_area_and_offset()
413 __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors; in get_area_and_offset()
422 #define sector_to_block(ic, n) \ argument
424 BUG_ON((n) & (unsigned)((ic)->sectors_per_block - 1)); \
425 (n) >>= (ic)->sb->log2_sectors_per_block; \
428 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area, in get_metadata_sector_and_offset() argument
434 ms = area << ic->sb->log2_interleave_sectors; in get_metadata_sector_and_offset()
435 if (likely(ic->log2_metadata_run >= 0)) in get_metadata_sector_and_offset()
436 ms += area << ic->log2_metadata_run; in get_metadata_sector_and_offset()
438 ms += area * ic->metadata_run; in get_metadata_sector_and_offset()
439 ms >>= ic->log2_buffer_sectors; in get_metadata_sector_and_offset()
441 sector_to_block(ic, offset); in get_metadata_sector_and_offset()
443 if (likely(ic->log2_tag_size >= 0)) { in get_metadata_sector_and_offset()
444 ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size); in get_metadata_sector_and_offset()
445 mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1); in get_metadata_sector_and_offset()
447 ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors); in get_metadata_sector_and_offset()
448 mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1); in get_metadata_sector_and_offset()
454 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset) in get_data_sector() argument
458 if (ic->meta_dev) in get_data_sector()
461 result = area << ic->sb->log2_interleave_sectors; in get_data_sector()
462 if (likely(ic->log2_metadata_run >= 0)) in get_data_sector()
463 result += (area + 1) << ic->log2_metadata_run; in get_data_sector()
465 result += (area + 1) * ic->metadata_run; in get_data_sector()
467 result += (sector_t)ic->initial_sectors + offset; in get_data_sector()
468 result += ic->start; in get_data_sector()
473 static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr) in wraparound_section() argument
475 if (unlikely(*sec_ptr >= ic->journal_sections)) in wraparound_section()
476 *sec_ptr -= ic->journal_sections; in wraparound_section()
479 static void sb_set_version(struct dm_integrity_c *ic) in sb_set_version() argument
481 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) in sb_set_version()
482 ic->sb->version = SB_VERSION_4; in sb_set_version()
483 else if (ic->mode == 'B' || ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) in sb_set_version()
484 ic->sb->version = SB_VERSION_3; in sb_set_version()
485 else if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) in sb_set_version()
486 ic->sb->version = SB_VERSION_2; in sb_set_version()
488 ic->sb->version = SB_VERSION_1; in sb_set_version()
491 static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags) in sync_rw_sb() argument
499 io_req.mem.ptr.addr = ic->sb; in sync_rw_sb()
501 io_req.client = ic->io; in sync_rw_sb()
502 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; in sync_rw_sb()
503 io_loc.sector = ic->start; in sync_rw_sb()
507 sb_set_version(ic); in sync_rw_sb()
517 static bool block_bitmap_op(struct dm_integrity_c *ic, struct page_list *bitmap, in block_bitmap_op() argument
523 if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) { in block_bitmap_op()
527 ic->sb->log2_sectors_per_block, in block_bitmap_op()
528 ic->log2_blocks_per_bitmap_bit, in block_bitmap_op()
536 bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in block_bitmap_op()
538 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in block_bitmap_op()
622 static void block_bitmap_copy(struct dm_integrity_c *ic, struct page_list *dst, struct page_list *s… in block_bitmap_copy() argument
624 unsigned n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE); in block_bitmap_copy()
634 static struct bitmap_block_status *sector_to_bitmap_block(struct dm_integrity_c *ic, sector_t secto… in sector_to_bitmap_block() argument
636 unsigned bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in sector_to_bitmap_block()
639 BUG_ON(bitmap_block >= ic->n_bitmap_blocks); in sector_to_bitmap_block()
640 return &ic->bbs[bitmap_block]; in sector_to_bitmap_block()
643 static void access_journal_check(struct dm_integrity_c *ic, unsigned section, unsigned offset, in access_journal_check() argument
647 unsigned limit = e ? ic->journal_section_entries : ic->journal_section_sectors; in access_journal_check()
649 if (unlikely(section >= ic->journal_sections) || in access_journal_check()
652 function, section, offset, ic->journal_sections, limit); in access_journal_check()
658 static void page_list_location(struct dm_integrity_c *ic, unsigned section, unsigned offset, in page_list_location() argument
663 access_journal_check(ic, section, offset, false, "page_list_location"); in page_list_location()
665 sector = section * ic->journal_section_sectors + offset; in page_list_location()
671 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl, in access_page_list() argument
677 page_list_location(ic, section, offset, &pl_index, &pl_offset); in access_page_list()
687 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned section, unsigned … in access_journal() argument
689 return access_page_list(ic, ic->journal, section, offset, NULL); in access_journal()
692 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned section, unsi… in access_journal_entry() argument
697 access_journal_check(ic, section, n, true, "access_journal_entry"); in access_journal_entry()
702 js = access_journal(ic, section, rel_sector); in access_journal_entry()
703 return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size); in access_journal_entry()
706 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned section, unsi… in access_journal_data() argument
708 n <<= ic->sb->log2_sectors_per_block; in access_journal_data()
712 access_journal_check(ic, section, n, false, "access_journal_data"); in access_journal_data()
714 return access_journal(ic, section, n); in access_journal_data()
717 static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result[JOURNAL_MAC_SIZE]) in section_mac() argument
719 SHASH_DESC_ON_STACK(desc, ic->journal_mac); in section_mac()
723 desc->tfm = ic->journal_mac; in section_mac()
727 dm_integrity_io_error(ic, "crypto_shash_init", r); in section_mac()
731 for (j = 0; j < ic->journal_section_entries; j++) { in section_mac()
732 struct journal_entry *je = access_journal_entry(ic, section, j); in section_mac()
735 dm_integrity_io_error(ic, "crypto_shash_update", r); in section_mac()
740 size = crypto_shash_digestsize(ic->journal_mac); in section_mac()
745 dm_integrity_io_error(ic, "crypto_shash_final", r); in section_mac()
753 dm_integrity_io_error(ic, "digest_size", -EINVAL); in section_mac()
758 dm_integrity_io_error(ic, "crypto_shash_final", r); in section_mac()
769 static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr) in rw_section_mac() argument
774 if (!ic->journal_mac) in rw_section_mac()
777 section_mac(ic, section, result); in rw_section_mac()
780 struct journal_sector *js = access_journal(ic, section, j); in rw_section_mac()
786 dm_integrity_io_error(ic, "journal mac", -EILSEQ); in rw_section_mac()
799 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, in xor_journal() argument
803 size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT; in xor_journal()
808 source_pl = ic->journal; in xor_journal()
809 target_pl = ic->journal_io; in xor_journal()
811 source_pl = ic->journal_io; in xor_journal()
812 target_pl = ic->journal; in xor_journal()
815 page_list_location(ic, section, 0, &pl_index, &pl_offset); in xor_journal()
831 rw_section_mac(ic, section, true); in xor_journal()
836 page_list_location(ic, section, 0, §ion_index, &dummy); in xor_journal()
842 src_pages[1] = ic->journal_xor[pl_index].page; in xor_journal()
861 complete(&comp->ic->crypto_backoff); in complete_journal_encrypt()
864 dm_integrity_io_error(comp->ic, "asynchronous encrypt", err); in complete_journal_encrypt()
883 wait_for_completion(&comp->ic->crypto_backoff); in do_crypt()
884 reinit_completion(&comp->ic->crypto_backoff); in do_crypt()
887 dm_integrity_io_error(comp->ic, "encrypt", r); in do_crypt()
891 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, in crypt_journal() argument
900 source_sg = ic->journal_scatterlist; in crypt_journal()
901 target_sg = ic->journal_io_scatterlist; in crypt_journal()
903 source_sg = ic->journal_io_scatterlist; in crypt_journal()
904 target_sg = ic->journal_scatterlist; in crypt_journal()
913 rw_section_mac(ic, section, true); in crypt_journal()
915 req = ic->sk_requests[section]; in crypt_journal()
916 ivsize = crypto_skcipher_ivsize(ic->journal_crypt); in crypt_journal()
935 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, in encrypt_journal() argument
938 if (ic->journal_xor) in encrypt_journal()
939 return xor_journal(ic, encrypt, section, n_sections, comp); in encrypt_journal()
941 return crypt_journal(ic, encrypt, section, n_sections, comp); in encrypt_journal()
948 dm_integrity_io_error(comp->ic, "writing journal", -EIO); in complete_journal_io()
952 static void rw_journal_sectors(struct dm_integrity_c *ic, int op, int op_flags, in rw_journal_sectors() argument
960 if (unlikely(dm_integrity_failed(ic))) { in rw_journal_sectors()
972 if (ic->journal_io) in rw_journal_sectors()
973 io_req.mem.ptr.pl = &ic->journal_io[pl_index]; in rw_journal_sectors()
975 io_req.mem.ptr.pl = &ic->journal[pl_index]; in rw_journal_sectors()
983 io_req.client = ic->io; in rw_journal_sectors()
984 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; in rw_journal_sectors()
985 io_loc.sector = ic->start + SB_SECTORS + sector; in rw_journal_sectors()
990 dm_integrity_io_error(ic, op == REQ_OP_READ ? "reading journal" : "writing journal", r); in rw_journal_sectors()
998 static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned section, in rw_journal() argument
1003 sector = section * ic->journal_section_sectors; in rw_journal()
1004 n_sectors = n_sections * ic->journal_section_sectors; in rw_journal()
1006 rw_journal_sectors(ic, op, op_flags, sector, n_sectors, comp); in rw_journal()
1009 static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsigned commit_section… in write_journal() argument
1016 io_comp.ic = ic; in write_journal()
1019 if (commit_start + commit_sections <= ic->journal_sections) { in write_journal()
1021 if (ic->journal_io) { in write_journal()
1022 crypt_comp_1.ic = ic; in write_journal()
1025 encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1); in write_journal()
1029 rw_section_mac(ic, commit_start + i, true); in write_journal()
1031 rw_journal(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, commit_start, in write_journal()
1036 to_end = ic->journal_sections - commit_start; in write_journal()
1037 if (ic->journal_io) { in write_journal()
1038 crypt_comp_1.ic = ic; in write_journal()
1041 encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1); in write_journal()
1043 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); in write_journal()
1046 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1); in write_journal()
1049 crypt_comp_2.ic = ic; in write_journal()
1052 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2); in write_journal()
1054 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); in write_journal()
1059 rw_section_mac(ic, commit_start + i, true); in write_journal()
1060 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); in write_journal()
1062 rw_section_mac(ic, i, true); in write_journal()
1064 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, 0, commit_sections - to_end, &io_comp); in write_journal()
1070 static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset, in copy_from_journal() argument
1078 BUG_ON((target | n_sectors | offset) & (unsigned)(ic->sectors_per_block - 1)); in copy_from_journal()
1080 if (unlikely(dm_integrity_failed(ic))) { in copy_from_journal()
1085 sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset; in copy_from_journal()
1093 io_req.mem.ptr.pl = &ic->journal[pl_index]; in copy_from_journal()
1097 io_req.client = ic->io; in copy_from_journal()
1098 io_loc.bdev = ic->dev->bdev; in copy_from_journal()
1115 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool che… in add_new_range() argument
1117 struct rb_node **n = &ic->in_progress.rb_node; in add_new_range()
1120 BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned)(ic->sectors_per_block - 1)); in add_new_range()
1124 list_for_each_entry(range, &ic->wait_list, wait_entry) { in add_new_range()
1146 rb_insert_color(&new_range->node, &ic->in_progress); in add_new_range()
1151 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range) in remove_range_unlocked() argument
1153 rb_erase(&range->node, &ic->in_progress); in remove_range_unlocked()
1154 while (unlikely(!list_empty(&ic->wait_list))) { in remove_range_unlocked()
1156 list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry); in remove_range_unlocked()
1160 if (!add_new_range(ic, last_range, false)) { in remove_range_unlocked()
1162 list_add(&last_range->wait_entry, &ic->wait_list); in remove_range_unlocked()
1170 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range) in remove_range() argument
1174 spin_lock_irqsave(&ic->endio_wait.lock, flags); in remove_range()
1175 remove_range_unlocked(ic, range); in remove_range()
1176 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); in remove_range()
1179 static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) in wait_and_add_new_range() argument
1182 list_add_tail(&new_range->wait_entry, &ic->wait_list); in wait_and_add_new_range()
1186 spin_unlock_irq(&ic->endio_wait.lock); in wait_and_add_new_range()
1188 spin_lock_irq(&ic->endio_wait.lock); in wait_and_add_new_range()
1192 static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) in add_new_range_and_wait() argument
1194 if (unlikely(!add_new_range(ic, new_range, true))) in add_new_range_and_wait()
1195 wait_and_add_new_range(ic, new_range); in add_new_range_and_wait()
1204 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector) in add_journal_node() argument
1212 link = &ic->journal_tree_root.rb_node; in add_journal_node()
1226 rb_insert_color(&node->node, &ic->journal_tree_root); in add_journal_node()
1229 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node) in remove_journal_node() argument
1232 rb_erase(&node->node, &ic->journal_tree_root); in remove_journal_node()
1238 static unsigned find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector) in find_journal_node() argument
1240 struct rb_node *n = ic->journal_tree_root.rb_node; in find_journal_node()
1246 found = j - ic->journal_tree; in find_journal_node()
1259 static bool test_journal_node(struct dm_integrity_c *ic, unsigned pos, sector_t sector) in test_journal_node() argument
1264 if (unlikely(pos >= ic->journal_entries)) in test_journal_node()
1266 node = &ic->journal_tree[pos]; in test_journal_node()
1280 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node) in find_newer_committed_node() argument
1297 next_section = (unsigned)(next_node - ic->journal_tree) / ic->journal_section_entries; in find_newer_committed_node()
1298 if (next_section >= ic->committed_section && in find_newer_committed_node()
1299 next_section < ic->committed_section + ic->n_committed_sections) in find_newer_committed_node()
1301 if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections) in find_newer_committed_node()
1311 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_bl… in dm_integrity_rw_tag() argument
1317 unsigned may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0); in dm_integrity_rw_tag()
1325 r = dm_integrity_failed(ic); in dm_integrity_rw_tag()
1329 data = dm_bufio_read(ic->bufio, *metadata_block, &b); in dm_integrity_rw_tag()
1333 to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size); in dm_integrity_rw_tag()
1343 if (likely(is_power_of_2(ic->tag_size))) { in dm_integrity_rw_tag()
1345 if (unlikely(!ic->discard) || in dm_integrity_rw_tag()
1360 if (unlikely(hash_offset == ic->tag_size)) { in dm_integrity_rw_tag()
1366 may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0); in dm_integrity_rw_tag()
1375 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) { in dm_integrity_rw_tag()
1380 if (unlikely(!is_power_of_2(ic->tag_size))) { in dm_integrity_rw_tag()
1381 hash_offset = (hash_offset + to_copy) % ic->tag_size; in dm_integrity_rw_tag()
1395 struct dm_integrity_c *ic; member
1403 dm_integrity_io_error(fr->ic, "flusing disk cache", -EIO); in flush_notify()
1407 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data) in dm_integrity_flush_buffers() argument
1413 if (!ic->meta_dev) in dm_integrity_flush_buffers()
1422 fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio), in dm_integrity_flush_buffers()
1423 fr.io_reg.bdev = ic->dev->bdev, in dm_integrity_flush_buffers()
1426 fr.ic = ic; in dm_integrity_flush_buffers()
1432 r = dm_bufio_write_dirty_buffers(ic->bufio); in dm_integrity_flush_buffers()
1434 dm_integrity_io_error(ic, "writing tags", r); in dm_integrity_flush_buffers()
1440 static void sleep_on_endio_wait(struct dm_integrity_c *ic) in sleep_on_endio_wait() argument
1443 __add_wait_queue(&ic->endio_wait, &wait); in sleep_on_endio_wait()
1445 spin_unlock_irq(&ic->endio_wait.lock); in sleep_on_endio_wait()
1447 spin_lock_irq(&ic->endio_wait.lock); in sleep_on_endio_wait()
1448 __remove_wait_queue(&ic->endio_wait, &wait); in sleep_on_endio_wait()
1453 struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer); in autocommit_fn() local
1455 if (likely(!dm_integrity_failed(ic))) in autocommit_fn()
1456 queue_work(ic->commit_wq, &ic->commit_work); in autocommit_fn()
1459 static void schedule_autocommit(struct dm_integrity_c *ic) in schedule_autocommit() argument
1461 if (!timer_pending(&ic->autocommit_timer)) in schedule_autocommit()
1462 mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies); in schedule_autocommit()
1465 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in submit_flush_bio() argument
1470 spin_lock_irqsave(&ic->endio_wait.lock, flags); in submit_flush_bio()
1472 bio_list_add(&ic->flush_bio_list, bio); in submit_flush_bio()
1473 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); in submit_flush_bio()
1475 queue_work(ic->commit_wq, &ic->commit_work); in submit_flush_bio()
1478 static void do_endio(struct dm_integrity_c *ic, struct bio *bio) in do_endio() argument
1480 int r = dm_integrity_failed(ic); in do_endio()
1483 if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) { in do_endio()
1485 spin_lock_irqsave(&ic->endio_wait.lock, flags); in do_endio()
1486 bio_list_add(&ic->synchronous_bios, bio); in do_endio()
1487 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); in do_endio()
1488 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); in do_endio()
1494 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in do_endio_flush() argument
1498 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) in do_endio_flush()
1499 submit_flush_bio(ic, dio); in do_endio_flush()
1501 do_endio(ic, bio); in do_endio_flush()
1507 struct dm_integrity_c *ic = dio->ic; in dec_in_flight() local
1510 remove_range(ic, &dio->range); in dec_in_flight()
1513 schedule_autocommit(ic); in dec_in_flight()
1523 queue_work(ic->offload_wq, &dio->work); in dec_in_flight()
1526 do_endio_flush(ic, dio); in dec_in_flight()
1544 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector, in integrity_sector_checksum() argument
1548 SHASH_DESC_ON_STACK(req, ic->internal_hash); in integrity_sector_checksum()
1552 req->tfm = ic->internal_hash; in integrity_sector_checksum()
1556 dm_integrity_io_error(ic, "crypto_shash_init", r); in integrity_sector_checksum()
1562 dm_integrity_io_error(ic, "crypto_shash_update", r); in integrity_sector_checksum()
1566 r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT); in integrity_sector_checksum()
1568 dm_integrity_io_error(ic, "crypto_shash_update", r); in integrity_sector_checksum()
1574 dm_integrity_io_error(ic, "crypto_shash_final", r); in integrity_sector_checksum()
1578 digest_size = crypto_shash_digestsize(ic->internal_hash); in integrity_sector_checksum()
1579 if (unlikely(digest_size < ic->tag_size)) in integrity_sector_checksum()
1580 memset(result + digest_size, 0, ic->tag_size - digest_size); in integrity_sector_checksum()
1586 get_random_bytes(result, ic->tag_size); in integrity_sector_checksum()
1592 struct dm_integrity_c *ic = dio->ic; in integrity_metadata() local
1596 if (ic->internal_hash) { in integrity_metadata()
1599 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash); in integrity_metadata()
1602 unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; in integrity_metadata()
1607 if (unlikely(ic->mode == 'R')) in integrity_metadata()
1611 …checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size +… in integrity_metadata()
1628 unsigned max_blocks = max_size / ic->tag_size; in integrity_metadata()
1632 unsigned this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); in integrity_metadata()
1634 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1635 this_step_blocks * ic->tag_size, TAG_WRITE); in integrity_metadata()
1642 /*if (bi_size < this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block)) { in integrity_metadata()
1647 bi_size -= this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); in integrity_metadata()
1648 bi_sector += this_step_blocks << ic->sb->log2_sectors_per_block; in integrity_metadata()
1668 integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr); in integrity_metadata()
1669 checksums_ptr += ic->tag_size; in integrity_metadata()
1670 sectors_to_process -= ic->sectors_per_block; in integrity_metadata()
1671 pos += ic->sectors_per_block << SECTOR_SHIFT; in integrity_metadata()
1672 sector += ic->sectors_per_block; in integrity_metadata()
1676 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1682 (sector - ((r + ic->tag_size - 1) / ic->tag_size))); in integrity_metadata()
1684 atomic64_inc(&ic->number_of_mismatches); in integrity_metadata()
1710 sector_to_block(ic, data_to_process); in integrity_metadata()
1711 data_to_process *= ic->tag_size; in integrity_metadata()
1720 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1740 struct dm_integrity_c *ic = ti->private; in dm_integrity_map() local
1746 dio->ic = ic; in dm_integrity_map()
1764 submit_flush_bio(ic, dio); in dm_integrity_map()
1777 if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { in dm_integrity_map()
1780 ic->provided_data_sectors); in dm_integrity_map()
1783 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1… in dm_integrity_map()
1785 ic->sectors_per_block, in dm_integrity_map()
1790 if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) { in dm_integrity_map()
1794 if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) { in dm_integrity_map()
1796 bv.bv_offset, bv.bv_len, ic->sectors_per_block); in dm_integrity_map()
1803 if (!ic->internal_hash) { in dm_integrity_map()
1805 unsigned wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block; in dm_integrity_map()
1806 if (ic->log2_tag_size >= 0) in dm_integrity_map()
1807 wanted_tag_size <<= ic->log2_tag_size; in dm_integrity_map()
1809 wanted_tag_size *= ic->tag_size; in dm_integrity_map()
1823 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ)) in dm_integrity_map()
1826 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in dm_integrity_map()
1827 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in dm_integrity_map()
1828 bio->bi_iter.bi_sector = get_data_sector(ic, area, offset); in dm_integrity_map()
1837 struct dm_integrity_c *ic = dio->ic; in __journal_read_write() local
1857 struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry); in __journal_read_write()
1868 __io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); in __journal_read_write()
1873 js = access_journal_data(ic, journal_section, journal_entry); in __journal_read_write()
1881 } while (++s < ic->sectors_per_block); in __journal_read_write()
1883 if (ic->internal_hash) { in __journal_read_write()
1886 integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack); in __journal_read_write()
1887 if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) { in __journal_read_write()
1895 if (!ic->internal_hash) { in __journal_read_write()
1897 unsigned tag_todo = ic->tag_size; in __journal_read_write()
1898 char *tag_ptr = journal_entry_tag(ic, je); in __journal_read_write()
1923 js = access_journal_data(ic, journal_section, journal_entry); in __journal_read_write()
1924 memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT); in __journal_read_write()
1929 } while (++s < ic->sectors_per_block); in __journal_read_write()
1931 if (ic->internal_hash) { in __journal_read_write()
1932 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash); in __journal_read_write()
1933 if (unlikely(digest_size > ic->tag_size)) { in __journal_read_write()
1935 integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack); in __journal_read_write()
1936 memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size); in __journal_read_write()
1938 integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je)); in __journal_read_write()
1943 logical_sector += ic->sectors_per_block; in __journal_read_write()
1946 if (unlikely(journal_entry == ic->journal_section_entries)) { in __journal_read_write()
1949 wraparound_section(ic, &journal_section); in __journal_read_write()
1952 bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT; in __journal_read_write()
1953 } while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT); in __journal_read_write()
1962 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait))) in __journal_read_write()
1963 wake_up(&ic->copy_to_journal_wait); in __journal_read_write()
1964 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) { in __journal_read_write()
1965 queue_work(ic->commit_wq, &ic->commit_work); in __journal_read_write()
1967 schedule_autocommit(ic); in __journal_read_write()
1970 remove_range(ic, &dio->range); in __journal_read_write()
1977 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in __journal_read_write()
1978 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in __journal_read_write()
1987 struct dm_integrity_c *ic = dio->ic; in dm_integrity_map_continue() local
1993 bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ; in dm_integrity_map_continue()
1994 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D') in dm_integrity_map_continue()
1999 queue_work(ic->offload_wq, &dio->work); in dm_integrity_map_continue()
2004 spin_lock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2006 if (unlikely(dm_integrity_failed(ic))) { in dm_integrity_map_continue()
2007 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2008 do_endio(ic, bio); in dm_integrity_map_continue()
2013 if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) { in dm_integrity_map_continue()
2019 (sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block); in dm_integrity_map_continue()
2023 sleep_on_endio_wait(ic); in dm_integrity_map_continue()
2026 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; in dm_integrity_map_continue()
2027 ic->free_sectors -= range_sectors; in dm_integrity_map_continue()
2028 journal_section = ic->free_section; in dm_integrity_map_continue()
2029 journal_entry = ic->free_section_entry; in dm_integrity_map_continue()
2031 next_entry = ic->free_section_entry + range_sectors; in dm_integrity_map_continue()
2032 ic->free_section_entry = next_entry % ic->journal_section_entries; in dm_integrity_map_continue()
2033 ic->free_section += next_entry / ic->journal_section_entries; in dm_integrity_map_continue()
2034 ic->n_uncommitted_sections += next_entry / ic->journal_section_entries; in dm_integrity_map_continue()
2035 wraparound_section(ic, &ic->free_section); in dm_integrity_map_continue()
2037 pos = journal_section * ic->journal_section_entries + journal_entry; in dm_integrity_map_continue()
2044 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i); in dm_integrity_map_continue()
2046 if (unlikely(pos >= ic->journal_entries)) in dm_integrity_map_continue()
2049 je = access_journal_entry(ic, ws, we); in dm_integrity_map_continue()
2053 if (unlikely(we == ic->journal_section_entries)) { in dm_integrity_map_continue()
2056 wraparound_section(ic, &ws); in dm_integrity_map_continue()
2058 } while ((i += ic->sectors_per_block) < dio->range.n_sectors); in dm_integrity_map_continue()
2060 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2064 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2071 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { in dm_integrity_map_continue()
2072 if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) in dm_integrity_map_continue()
2079 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_continue()
2087 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2089 queue_work(ic->wait_wq, &dio->work); in dm_integrity_map_continue()
2093 dio->range.n_sectors = ic->sectors_per_block; in dm_integrity_map_continue()
2094 wait_and_add_new_range(ic, &dio->range); in dm_integrity_map_continue()
2102 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2104 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
2109 if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) { in dm_integrity_map_continue()
2111 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2114 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
2115 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2116 queue_work(ic->commit_wq, &ic->commit_work); in dm_integrity_map_continue()
2117 flush_workqueue(ic->commit_wq); in dm_integrity_map_continue()
2118 queue_work(ic->writer_wq, &ic->writer_work); in dm_integrity_map_continue()
2119 flush_workqueue(ic->writer_wq); in dm_integrity_map_continue()
2124 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2127 journal_section = journal_read_pos / ic->journal_section_entries; in dm_integrity_map_continue()
2128 journal_entry = journal_read_pos % ic->journal_section_entries; in dm_integrity_map_continue()
2132 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) { in dm_integrity_map_continue()
2133 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2137 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector); in dm_integrity_map_continue()
2141 queue_work(ic->writer_wq, &bbs->work); in dm_integrity_map_continue()
2155 bio_set_dev(bio, ic->dev->bdev); in dm_integrity_map_continue()
2161 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) { in dm_integrity_map_continue()
2163 dm_integrity_flush_buffers(ic, false); in dm_integrity_map_continue()
2177 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && in dm_integrity_map_continue()
2178 dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) in dm_integrity_map_continue()
2180 if (ic->mode == 'B') { in dm_integrity_map_continue()
2181 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2194 queue_work(ic->metadata_wq, &dio->work); in dm_integrity_map_continue()
2203 do_endio_flush(ic, dio); in dm_integrity_map_continue()
2214 static void pad_uncommitted(struct dm_integrity_c *ic) in pad_uncommitted() argument
2216 if (ic->free_section_entry) { in pad_uncommitted()
2217 ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry; in pad_uncommitted()
2218 ic->free_section_entry = 0; in pad_uncommitted()
2219 ic->free_section++; in pad_uncommitted()
2220 wraparound_section(ic, &ic->free_section); in pad_uncommitted()
2221 ic->n_uncommitted_sections++; in pad_uncommitted()
2223 if (WARN_ON(ic->journal_sections * ic->journal_section_entries != in pad_uncommitted()
2224 (ic->n_uncommitted_sections + ic->n_committed_sections) * in pad_uncommitted()
2225 ic->journal_section_entries + ic->free_sectors)) { in pad_uncommitted()
2229 ic->journal_sections, ic->journal_section_entries, in pad_uncommitted()
2230 ic->n_uncommitted_sections, ic->n_committed_sections, in pad_uncommitted()
2231 ic->journal_section_entries, ic->free_sectors); in pad_uncommitted()
2237 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work); in integrity_commit() local
2242 del_timer(&ic->autocommit_timer); in integrity_commit()
2244 spin_lock_irq(&ic->endio_wait.lock); in integrity_commit()
2245 flushes = bio_list_get(&ic->flush_bio_list); in integrity_commit()
2246 if (unlikely(ic->mode != 'J')) { in integrity_commit()
2247 spin_unlock_irq(&ic->endio_wait.lock); in integrity_commit()
2248 dm_integrity_flush_buffers(ic, true); in integrity_commit()
2252 pad_uncommitted(ic); in integrity_commit()
2253 commit_start = ic->uncommitted_section; in integrity_commit()
2254 commit_sections = ic->n_uncommitted_sections; in integrity_commit()
2255 spin_unlock_irq(&ic->endio_wait.lock); in integrity_commit()
2260 ic->wrote_to_journal = true; in integrity_commit()
2264 for (j = 0; j < ic->journal_section_entries; j++) { in integrity_commit()
2266 je = access_journal_entry(ic, i, j); in integrity_commit()
2267 io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); in integrity_commit()
2269 for (j = 0; j < ic->journal_section_sectors; j++) { in integrity_commit()
2271 js = access_journal(ic, i, j); in integrity_commit()
2272 js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq); in integrity_commit()
2275 if (unlikely(i >= ic->journal_sections)) in integrity_commit()
2276 ic->commit_seq = next_commit_seq(ic->commit_seq); in integrity_commit()
2277 wraparound_section(ic, &i); in integrity_commit()
2281 write_journal(ic, commit_start, commit_sections); in integrity_commit()
2283 spin_lock_irq(&ic->endio_wait.lock); in integrity_commit()
2284 ic->uncommitted_section += commit_sections; in integrity_commit()
2285 wraparound_section(ic, &ic->uncommitted_section); in integrity_commit()
2286 ic->n_uncommitted_sections -= commit_sections; in integrity_commit()
2287 ic->n_committed_sections += commit_sections; in integrity_commit()
2288 spin_unlock_irq(&ic->endio_wait.lock); in integrity_commit()
2290 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) in integrity_commit()
2291 queue_work(ic->writer_wq, &ic->writer_work); in integrity_commit()
2297 do_endio(ic, flushes); in integrity_commit()
2306 struct dm_integrity_c *ic = comp->ic; in complete_copy_from_journal() local
2307 remove_range(ic, &io->range); in complete_copy_from_journal()
2308 mempool_free(io, &ic->journal_io_mempool); in complete_copy_from_journal()
2310 dm_integrity_io_error(ic, "copying from journal", -EIO); in complete_copy_from_journal()
2314 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js, in restore_last_bytes() argument
2321 } while (++s < ic->sectors_per_block); in restore_last_bytes()
2324 static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, in do_journal_write() argument
2333 comp.ic = ic; in do_journal_write()
2338 for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) { in do_journal_write()
2342 rw_section_mac(ic, i, false); in do_journal_write()
2343 for (j = 0; j < ic->journal_section_entries; j++) { in do_journal_write()
2344 struct journal_entry *je = access_journal_entry(ic, i, j); in do_journal_write()
2356 if (unlikely(sec & (unsigned)(ic->sectors_per_block - 1))) { in do_journal_write()
2357 dm_integrity_io_error(ic, "invalid sector in journal", -EIO); in do_journal_write()
2358 sec &= ~(sector_t)(ic->sectors_per_block - 1); in do_journal_write()
2360 if (unlikely(sec >= ic->provided_data_sectors)) { in do_journal_write()
2365 get_area_and_offset(ic, sec, &area, &offset); in do_journal_write()
2366 restore_last_bytes(ic, access_journal_data(ic, i, j), je); in do_journal_write()
2367 for (k = j + 1; k < ic->journal_section_entries; k++) { in do_journal_write()
2368 struct journal_entry *je2 = access_journal_entry(ic, i, k); in do_journal_write()
2374 if (unlikely(sec2 >= ic->provided_data_sectors)) in do_journal_write()
2376 get_area_and_offset(ic, sec2, &area2, &offset2); in do_journal_write()
2377 if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block)) in do_journal_write()
2379 restore_last_bytes(ic, access_journal_data(ic, i, k), je2); in do_journal_write()
2383 io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO); in do_journal_write()
2386 io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block; in do_journal_write()
2388 spin_lock_irq(&ic->endio_wait.lock); in do_journal_write()
2389 add_new_range_and_wait(ic, &io->range); in do_journal_write()
2392 struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries]; in do_journal_write()
2395 while (j < k && find_newer_committed_node(ic, §ion_node[j])) { in do_journal_write()
2396 struct journal_entry *je2 = access_journal_entry(ic, i, j); in do_journal_write()
2399 remove_journal_node(ic, §ion_node[j]); in do_journal_write()
2401 sec += ic->sectors_per_block; in do_journal_write()
2402 offset += ic->sectors_per_block; in do_journal_write()
2404 while (j < k && find_newer_committed_node(ic, §ion_node[k - 1])) { in do_journal_write()
2405 struct journal_entry *je2 = access_journal_entry(ic, i, k - 1); in do_journal_write()
2408 remove_journal_node(ic, §ion_node[k - 1]); in do_journal_write()
2412 remove_range_unlocked(ic, &io->range); in do_journal_write()
2413 spin_unlock_irq(&ic->endio_wait.lock); in do_journal_write()
2414 mempool_free(io, &ic->journal_io_mempool); in do_journal_write()
2418 remove_journal_node(ic, §ion_node[l]); in do_journal_write()
2421 spin_unlock_irq(&ic->endio_wait.lock); in do_journal_write()
2423 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); in do_journal_write()
2426 struct journal_entry *je2 = access_journal_entry(ic, i, l); in do_journal_write()
2432 ic->internal_hash) { in do_journal_write()
2435 integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), in do_journal_write()
2436 (char *)access_journal_data(ic, i, l), test_tag); in do_journal_write()
2437 if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) in do_journal_write()
2438 dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ); in do_journal_write()
2442 r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset, in do_journal_write()
2443 ic->tag_size, TAG_WRITE); in do_journal_write()
2445 dm_integrity_io_error(ic, "reading tags", r); in do_journal_write()
2450 copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block, in do_journal_write()
2451 (k - j) << ic->sb->log2_sectors_per_block, in do_journal_write()
2452 get_data_sector(ic, area, offset), in do_journal_write()
2459 dm_bufio_write_dirty_buffers_async(ic->bufio); in do_journal_write()
2466 dm_integrity_flush_buffers(ic, true); in do_journal_write()
2471 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work); in integrity_writer() local
2476 spin_lock_irq(&ic->endio_wait.lock); in integrity_writer()
2477 write_start = ic->committed_section; in integrity_writer()
2478 write_sections = ic->n_committed_sections; in integrity_writer()
2479 spin_unlock_irq(&ic->endio_wait.lock); in integrity_writer()
2484 do_journal_write(ic, write_start, write_sections, false); in integrity_writer()
2486 spin_lock_irq(&ic->endio_wait.lock); in integrity_writer()
2488 ic->committed_section += write_sections; in integrity_writer()
2489 wraparound_section(ic, &ic->committed_section); in integrity_writer()
2490 ic->n_committed_sections -= write_sections; in integrity_writer()
2492 prev_free_sectors = ic->free_sectors; in integrity_writer()
2493 ic->free_sectors += write_sections * ic->journal_section_entries; in integrity_writer()
2495 wake_up_locked(&ic->endio_wait); in integrity_writer()
2497 spin_unlock_irq(&ic->endio_wait.lock); in integrity_writer()
2500 static void recalc_write_super(struct dm_integrity_c *ic) in recalc_write_super() argument
2504 dm_integrity_flush_buffers(ic, false); in recalc_write_super()
2505 if (dm_integrity_failed(ic)) in recalc_write_super()
2508 r = sync_rw_sb(ic, REQ_OP_WRITE, 0); in recalc_write_super()
2510 dm_integrity_io_error(ic, "writing superblock", r); in recalc_write_super()
2515 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); in integrity_recalc() local
2528 DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector)); in integrity_recalc()
2530 spin_lock_irq(&ic->endio_wait.lock); in integrity_recalc()
2534 if (unlikely(dm_post_suspending(ic->ti))) in integrity_recalc()
2537 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); in integrity_recalc()
2538 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) { in integrity_recalc()
2539 if (ic->mode == 'B') { in integrity_recalc()
2540 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); in integrity_recalc()
2542 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); in integrity_recalc()
2547 get_area_and_offset(ic, range.logical_sector, &area, &offset); in integrity_recalc()
2548 range.n_sectors = min((sector_t)RECALC_SECTORS, ic->provided_data_sectors - range.logical_sector); in integrity_recalc()
2549 if (!ic->meta_dev) in integrity_recalc()
2550 …range.n_sectors = min(range.n_sectors, ((sector_t)1U << ic->sb->log2_interleave_sectors) - (unsign… in integrity_recalc()
2552 add_new_range_and_wait(ic, &range); in integrity_recalc()
2553 spin_unlock_irq(&ic->endio_wait.lock); in integrity_recalc()
2557 if (ic->mode == 'B') { in integrity_recalc()
2558 if (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) { in integrity_recalc()
2561 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, in integrity_recalc()
2562 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) { in integrity_recalc()
2563 logical_sector += ic->sectors_per_block; in integrity_recalc()
2564 n_sectors -= ic->sectors_per_block; in integrity_recalc()
2567 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector + n_sectors - ic->sectors_per_block, in integrity_recalc()
2568 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) { in integrity_recalc()
2569 n_sectors -= ic->sectors_per_block; in integrity_recalc()
2572 get_area_and_offset(ic, logical_sector, &area, &offset); in integrity_recalc()
2578 recalc_write_super(ic); in integrity_recalc()
2579 if (ic->mode == 'B') { in integrity_recalc()
2580 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval); in integrity_recalc()
2585 if (unlikely(dm_integrity_failed(ic))) in integrity_recalc()
2591 io_req.mem.ptr.addr = ic->recalc_buffer; in integrity_recalc()
2593 io_req.client = ic->io; in integrity_recalc()
2594 io_loc.bdev = ic->dev->bdev; in integrity_recalc()
2595 io_loc.sector = get_data_sector(ic, area, offset); in integrity_recalc()
2600 dm_integrity_io_error(ic, "reading data", r); in integrity_recalc()
2604 t = ic->recalc_tags; in integrity_recalc()
2605 for (i = 0; i < n_sectors; i += ic->sectors_per_block) { in integrity_recalc()
2606 integrity_sector_checksum(ic, logical_sector + i, ic->recalc_buffer + (i << SECTOR_SHIFT), t); in integrity_recalc()
2607 t += ic->tag_size; in integrity_recalc()
2610 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); in integrity_recalc()
2612 …r = dm_integrity_rw_tag(ic, ic->recalc_tags, &metadata_block, &metadata_offset, t - ic->recalc_tag… in integrity_recalc()
2614 dm_integrity_io_error(ic, "writing tags", r); in integrity_recalc()
2618 if (ic->mode == 'B') { in integrity_recalc()
2621 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) << in integrity_recalc()
2622 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in integrity_recalc()
2624 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) << in integrity_recalc()
2625 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in integrity_recalc()
2626 block_bitmap_op(ic, ic->recalc_bitmap, start, end - start, BITMAP_OP_CLEAR); in integrity_recalc()
2632 spin_lock_irq(&ic->endio_wait.lock); in integrity_recalc()
2633 remove_range_unlocked(ic, &range); in integrity_recalc()
2634 ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); in integrity_recalc()
2638 remove_range(ic, &range); in integrity_recalc()
2642 spin_unlock_irq(&ic->endio_wait.lock); in integrity_recalc()
2644 recalc_write_super(ic); in integrity_recalc()
2650 struct dm_integrity_c *ic = bbs->ic; in bitmap_block_work() local
2667 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2669 remove_range(ic, &dio->range); in bitmap_block_work()
2671 queue_work(ic->offload_wq, &dio->work); in bitmap_block_work()
2673 block_bitmap_op(ic, ic->journal, dio->range.logical_sector, in bitmap_block_work()
2682 rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, in bitmap_block_work()
2689 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2692 remove_range(ic, &dio->range); in bitmap_block_work()
2694 queue_work(ic->offload_wq, &dio->work); in bitmap_block_work()
2697 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval); in bitmap_block_work()
2702 struct dm_integrity_c *ic = container_of(work, struct dm_integrity_c, bitmap_flush_work.work); in bitmap_flush_work() local
2707 dm_integrity_flush_buffers(ic, false); in bitmap_flush_work()
2710 range.n_sectors = ic->provided_data_sectors; in bitmap_flush_work()
2712 spin_lock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2713 add_new_range_and_wait(ic, &range); in bitmap_flush_work()
2714 spin_unlock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2716 dm_integrity_flush_buffers(ic, true); in bitmap_flush_work()
2718 limit = ic->provided_data_sectors; in bitmap_flush_work()
2719 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { in bitmap_flush_work()
2720 limit = le64_to_cpu(ic->sb->recalc_sector) in bitmap_flush_work()
2721 >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit) in bitmap_flush_work()
2722 << (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in bitmap_flush_work()
2725 block_bitmap_op(ic, ic->journal, 0, limit, BITMAP_OP_CLEAR); in bitmap_flush_work()
2726 block_bitmap_op(ic, ic->may_write_bitmap, 0, limit, BITMAP_OP_CLEAR); in bitmap_flush_work()
2728 rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, 0, in bitmap_flush_work()
2729 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in bitmap_flush_work()
2731 spin_lock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2732 remove_range_unlocked(ic, &range); in bitmap_flush_work()
2733 while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) { in bitmap_flush_work()
2735 spin_unlock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2736 spin_lock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2738 spin_unlock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2742 static void init_journal(struct dm_integrity_c *ic, unsigned start_section, in init_journal() argument
2752 wraparound_section(ic, &i); in init_journal()
2753 for (j = 0; j < ic->journal_section_sectors; j++) { in init_journal()
2754 struct journal_sector *js = access_journal(ic, i, j); in init_journal()
2756 js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq); in init_journal()
2758 for (j = 0; j < ic->journal_section_entries; j++) { in init_journal()
2759 struct journal_entry *je = access_journal_entry(ic, i, j); in init_journal()
2764 write_journal(ic, start_section, n_sections); in init_journal()
2767 static int find_commit_seq(struct dm_integrity_c *ic, unsigned i, unsigned j, commit_id_t id) in find_commit_seq() argument
2771 if (dm_integrity_commit_id(ic, i, j, k) == id) in find_commit_seq()
2774 dm_integrity_io_error(ic, "journal commit id", -EIO); in find_commit_seq()
2778 static void replay_journal(struct dm_integrity_c *ic) in replay_journal() argument
2788 if (ic->mode == 'R') in replay_journal()
2791 if (ic->journal_uptodate) in replay_journal()
2797 if (!ic->just_formatted) { in replay_journal()
2799 rw_journal(ic, REQ_OP_READ, 0, 0, ic->journal_sections, NULL); in replay_journal()
2800 if (ic->journal_io) in replay_journal()
2801 DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal"); in replay_journal()
2802 if (ic->journal_io) { in replay_journal()
2804 crypt_comp.ic = ic; in replay_journal()
2807 encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp); in replay_journal()
2810 DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal"); in replay_journal()
2813 if (dm_integrity_failed(ic)) in replay_journal()
2819 for (i = 0; i < ic->journal_sections; i++) { in replay_journal()
2820 for (j = 0; j < ic->journal_section_sectors; j++) { in replay_journal()
2822 struct journal_sector *js = access_journal(ic, i, j); in replay_journal()
2823 k = find_commit_seq(ic, i, j, js->commit_id); in replay_journal()
2830 for (j = 0; j < ic->journal_section_entries; j++) { in replay_journal()
2831 struct journal_entry *je = access_journal_entry(ic, i, j); in replay_journal()
2849 dm_integrity_io_error(ic, "journal commit ids", -EIO); in replay_journal()
2864 if (unlikely(write_start >= ic->journal_sections)) in replay_journal()
2866 wraparound_section(ic, &write_start); in replay_journal()
2869 for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) { in replay_journal()
2870 for (j = 0; j < ic->journal_section_sectors; j++) { in replay_journal()
2871 struct journal_sector *js = access_journal(ic, i, j); in replay_journal()
2873 if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) { in replay_journal()
2880 i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq); in replay_journal()
2885 if (unlikely(i >= ic->journal_sections)) in replay_journal()
2887 wraparound_section(ic, &i); in replay_journal()
2894 do_journal_write(ic, write_start, write_sections, true); in replay_journal()
2897 if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) { in replay_journal()
2899 ic->commit_seq = want_commit_seq; in replay_journal()
2900 DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq); in replay_journal()
2909 init_journal(ic, s, 1, erase_seq); in replay_journal()
2911 wraparound_section(ic, &s); in replay_journal()
2912 if (ic->journal_sections >= 2) { in replay_journal()
2913 init_journal(ic, s, ic->journal_sections - 2, erase_seq); in replay_journal()
2914 s += ic->journal_sections - 2; in replay_journal()
2915 wraparound_section(ic, &s); in replay_journal()
2916 init_journal(ic, s, 1, erase_seq); in replay_journal()
2920 ic->commit_seq = next_commit_seq(erase_seq); in replay_journal()
2923 ic->committed_section = continue_section; in replay_journal()
2924 ic->n_committed_sections = 0; in replay_journal()
2926 ic->uncommitted_section = continue_section; in replay_journal()
2927 ic->n_uncommitted_sections = 0; in replay_journal()
2929 ic->free_section = continue_section; in replay_journal()
2930 ic->free_section_entry = 0; in replay_journal()
2931 ic->free_sectors = ic->journal_entries; in replay_journal()
2933 ic->journal_tree_root = RB_ROOT; in replay_journal()
2934 for (i = 0; i < ic->journal_entries; i++) in replay_journal()
2935 init_journal_node(&ic->journal_tree[i]); in replay_journal()
2938 static void dm_integrity_enter_synchronous_mode(struct dm_integrity_c *ic) in dm_integrity_enter_synchronous_mode() argument
2942 if (ic->mode == 'B') { in dm_integrity_enter_synchronous_mode()
2943 ic->bitmap_flush_interval = msecs_to_jiffies(10) + 1; in dm_integrity_enter_synchronous_mode()
2944 ic->synchronous_mode = 1; in dm_integrity_enter_synchronous_mode()
2946 cancel_delayed_work_sync(&ic->bitmap_flush_work); in dm_integrity_enter_synchronous_mode()
2947 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); in dm_integrity_enter_synchronous_mode()
2948 flush_workqueue(ic->commit_wq); in dm_integrity_enter_synchronous_mode()
2954 struct dm_integrity_c *ic = container_of(n, struct dm_integrity_c, reboot_notifier); in dm_integrity_reboot() local
2958 dm_integrity_enter_synchronous_mode(ic); in dm_integrity_reboot()
2965 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; in dm_integrity_postsuspend() local
2968 WARN_ON(unregister_reboot_notifier(&ic->reboot_notifier)); in dm_integrity_postsuspend()
2970 del_timer_sync(&ic->autocommit_timer); in dm_integrity_postsuspend()
2972 if (ic->recalc_wq) in dm_integrity_postsuspend()
2973 drain_workqueue(ic->recalc_wq); in dm_integrity_postsuspend()
2975 if (ic->mode == 'B') in dm_integrity_postsuspend()
2976 cancel_delayed_work_sync(&ic->bitmap_flush_work); in dm_integrity_postsuspend()
2978 queue_work(ic->commit_wq, &ic->commit_work); in dm_integrity_postsuspend()
2979 drain_workqueue(ic->commit_wq); in dm_integrity_postsuspend()
2981 if (ic->mode == 'J') { in dm_integrity_postsuspend()
2982 queue_work(ic->writer_wq, &ic->writer_work); in dm_integrity_postsuspend()
2983 drain_workqueue(ic->writer_wq); in dm_integrity_postsuspend()
2984 dm_integrity_flush_buffers(ic, true); in dm_integrity_postsuspend()
2985 if (ic->wrote_to_journal) { in dm_integrity_postsuspend()
2986 init_journal(ic, ic->free_section, in dm_integrity_postsuspend()
2987 ic->journal_sections - ic->free_section, ic->commit_seq); in dm_integrity_postsuspend()
2988 if (ic->free_section) { in dm_integrity_postsuspend()
2989 init_journal(ic, 0, ic->free_section, in dm_integrity_postsuspend()
2990 next_commit_seq(ic->commit_seq)); in dm_integrity_postsuspend()
2995 if (ic->mode == 'B') { in dm_integrity_postsuspend()
2996 dm_integrity_flush_buffers(ic, true); in dm_integrity_postsuspend()
2999 init_journal(ic, 0, ic->journal_sections, 0); in dm_integrity_postsuspend()
3000 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP); in dm_integrity_postsuspend()
3001 r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA); in dm_integrity_postsuspend()
3003 dm_integrity_io_error(ic, "writing superblock", r); in dm_integrity_postsuspend()
3007 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); in dm_integrity_postsuspend()
3009 ic->journal_uptodate = true; in dm_integrity_postsuspend()
3014 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; in dm_integrity_resume() local
3015 __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); in dm_integrity_resume()
3020 ic->wrote_to_journal = false; in dm_integrity_resume()
3022 if (ic->provided_data_sectors != old_provided_data_sectors) { in dm_integrity_resume()
3023 if (ic->provided_data_sectors > old_provided_data_sectors && in dm_integrity_resume()
3024 ic->mode == 'B' && in dm_integrity_resume()
3025 ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { in dm_integrity_resume()
3026 rw_journal_sectors(ic, REQ_OP_READ, 0, 0, in dm_integrity_resume()
3027 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3028 block_bitmap_op(ic, ic->journal, old_provided_data_sectors, in dm_integrity_resume()
3029 ic->provided_data_sectors - old_provided_data_sectors, BITMAP_OP_SET); in dm_integrity_resume()
3030 rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, 0, in dm_integrity_resume()
3031 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3034 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); in dm_integrity_resume()
3035 r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA); in dm_integrity_resume()
3037 dm_integrity_io_error(ic, "writing superblock", r); in dm_integrity_resume()
3040 if (ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) { in dm_integrity_resume()
3042 rw_journal_sectors(ic, REQ_OP_READ, 0, 0, in dm_integrity_resume()
3043 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3044 if (ic->mode == 'B') { in dm_integrity_resume()
3045 if (ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { in dm_integrity_resume()
3046 block_bitmap_copy(ic, ic->recalc_bitmap, ic->journal); in dm_integrity_resume()
3047 block_bitmap_copy(ic, ic->may_write_bitmap, ic->journal); in dm_integrity_resume()
3048 if (!block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, in dm_integrity_resume()
3050 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_resume()
3051 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_resume()
3055 ic->sb->log2_blocks_per_bitmap_bit, ic->log2_blocks_per_bitmap_bit); in dm_integrity_resume()
3056 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit; in dm_integrity_resume()
3057 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET); in dm_integrity_resume()
3058 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET); in dm_integrity_resume()
3059 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_SET); in dm_integrity_resume()
3060 rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, 0, in dm_integrity_resume()
3061 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3062 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_resume()
3063 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_resume()
3066 if (!(ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit && in dm_integrity_resume()
3067 … block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_TEST_ALL_CLEAR))) { in dm_integrity_resume()
3068 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_resume()
3069 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_resume()
3071 init_journal(ic, 0, ic->journal_sections, 0); in dm_integrity_resume()
3072 replay_journal(ic); in dm_integrity_resume()
3073 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP); in dm_integrity_resume()
3075 r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA); in dm_integrity_resume()
3077 dm_integrity_io_error(ic, "writing superblock", r); in dm_integrity_resume()
3079 replay_journal(ic); in dm_integrity_resume()
3080 if (ic->mode == 'B') { in dm_integrity_resume()
3081 ic->sb->flags |= cpu_to_le32(SB_FLAG_DIRTY_BITMAP); in dm_integrity_resume()
3082 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit; in dm_integrity_resume()
3083 r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA); in dm_integrity_resume()
3085 dm_integrity_io_error(ic, "writing superblock", r); in dm_integrity_resume()
3087 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); in dm_integrity_resume()
3088 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); in dm_integrity_resume()
3089 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); in dm_integrity_resume()
3090 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && in dm_integrity_resume()
3091 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors) { in dm_integrity_resume()
3092 block_bitmap_op(ic, ic->journal, le64_to_cpu(ic->sb->recalc_sector), in dm_integrity_resume()
3093 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); in dm_integrity_resume()
3094 block_bitmap_op(ic, ic->recalc_bitmap, le64_to_cpu(ic->sb->recalc_sector), in dm_integrity_resume()
3095 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); in dm_integrity_resume()
3096 block_bitmap_op(ic, ic->may_write_bitmap, le64_to_cpu(ic->sb->recalc_sector), in dm_integrity_resume()
3097 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); in dm_integrity_resume()
3099 rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, 0, in dm_integrity_resume()
3100 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3104 DEBUG_print("testing recalc: %x\n", ic->sb->flags); in dm_integrity_resume()
3105 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { in dm_integrity_resume()
3106 __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector); in dm_integrity_resume()
3107 DEBUG_print("recalc pos: %llx / %llx\n", recalc_pos, ic->provided_data_sectors); in dm_integrity_resume()
3108 if (recalc_pos < ic->provided_data_sectors) { in dm_integrity_resume()
3109 queue_work(ic->recalc_wq, &ic->recalc_work); in dm_integrity_resume()
3110 } else if (recalc_pos > ic->provided_data_sectors) { in dm_integrity_resume()
3111 ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors); in dm_integrity_resume()
3112 recalc_write_super(ic); in dm_integrity_resume()
3116 ic->reboot_notifier.notifier_call = dm_integrity_reboot; in dm_integrity_resume()
3117 ic->reboot_notifier.next = NULL; in dm_integrity_resume()
3118 ic->reboot_notifier.priority = INT_MAX - 1; /* be notified after md and before hardware drivers */ in dm_integrity_resume()
3119 WARN_ON(register_reboot_notifier(&ic->reboot_notifier)); in dm_integrity_resume()
3123 dm_integrity_enter_synchronous_mode(ic); in dm_integrity_resume()
3130 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; in dm_integrity_status() local
3137 (unsigned long long)atomic64_read(&ic->number_of_mismatches), in dm_integrity_status()
3138 ic->provided_data_sectors); in dm_integrity_status()
3139 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) in dm_integrity_status()
3140 DMEMIT(" %llu", le64_to_cpu(ic->sb->recalc_sector)); in dm_integrity_status()
3146 __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100; in dm_integrity_status()
3147 watermark_percentage += ic->journal_entries / 2; in dm_integrity_status()
3148 do_div(watermark_percentage, ic->journal_entries); in dm_integrity_status()
3150 arg_count += !!ic->meta_dev; in dm_integrity_status()
3151 arg_count += ic->sectors_per_block != 1; in dm_integrity_status()
3152 arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); in dm_integrity_status()
3153 arg_count += ic->discard; in dm_integrity_status()
3154 arg_count += ic->mode == 'J'; in dm_integrity_status()
3155 arg_count += ic->mode == 'J'; in dm_integrity_status()
3156 arg_count += ic->mode == 'B'; in dm_integrity_status()
3157 arg_count += ic->mode == 'B'; in dm_integrity_status()
3158 arg_count += !!ic->internal_hash_alg.alg_string; in dm_integrity_status()
3159 arg_count += !!ic->journal_crypt_alg.alg_string; in dm_integrity_status()
3160 arg_count += !!ic->journal_mac_alg.alg_string; in dm_integrity_status()
3161 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0; in dm_integrity_status()
3162 arg_count += ic->legacy_recalculate; in dm_integrity_status()
3163 DMEMIT("%s %llu %u %c %u", ic->dev->name, ic->start, in dm_integrity_status()
3164 ic->tag_size, ic->mode, arg_count); in dm_integrity_status()
3165 if (ic->meta_dev) in dm_integrity_status()
3166 DMEMIT(" meta_device:%s", ic->meta_dev->name); in dm_integrity_status()
3167 if (ic->sectors_per_block != 1) in dm_integrity_status()
3168 DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT); in dm_integrity_status()
3169 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) in dm_integrity_status()
3171 if (ic->discard) in dm_integrity_status()
3173 DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS); in dm_integrity_status()
3174 DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); in dm_integrity_status()
3175 DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); in dm_integrity_status()
3176 if (ic->mode == 'J') { in dm_integrity_status()
3178 DMEMIT(" commit_time:%u", ic->autocommit_msec); in dm_integrity_status()
3180 if (ic->mode == 'B') { in dm_integrity_status()
3181 …DMEMIT(" sectors_per_bit:%llu", (sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit); in dm_integrity_status()
3182 DMEMIT(" bitmap_flush_interval:%u", jiffies_to_msecs(ic->bitmap_flush_interval)); in dm_integrity_status()
3184 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) in dm_integrity_status()
3186 if (ic->legacy_recalculate) in dm_integrity_status()
3191 if (ic->a.alg_string) { \ in dm_integrity_status()
3192 DMEMIT(" %s:%s", n, ic->a.alg_string); \ in dm_integrity_status()
3193 if (ic->a.key_string) \ in dm_integrity_status()
3194 DMEMIT(":%s", ic->a.key_string);\ in dm_integrity_status()
3208 struct dm_integrity_c *ic = ti->private; in dm_integrity_iterate_devices() local
3210 if (!ic->meta_dev) in dm_integrity_iterate_devices()
3211 return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data); in dm_integrity_iterate_devices()
3213 return fn(ti, ic->dev, 0, ti->len, data); in dm_integrity_iterate_devices()
3218 struct dm_integrity_c *ic = ti->private; in dm_integrity_io_hints() local
3220 if (ic->sectors_per_block > 1) { in dm_integrity_io_hints()
3221 limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT; in dm_integrity_io_hints()
3222 limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT; in dm_integrity_io_hints()
3223 blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT); in dm_integrity_io_hints()
3227 static void calculate_journal_section_size(struct dm_integrity_c *ic) in calculate_journal_section_size() argument
3231 ic->journal_sections = le32_to_cpu(ic->sb->journal_sections); in calculate_journal_section_size()
3232 …ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block])… in calculate_journal_section_size()
3235 if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) in calculate_journal_section_size()
3237 ic->journal_entries_per_sector = sector_space / ic->journal_entry_size; in calculate_journal_section_size()
3238 ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS; in calculate_journal_section_size()
3239 …ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JO… in calculate_journal_section_size()
3240 ic->journal_entries = ic->journal_section_entries * ic->journal_sections; in calculate_journal_section_size()
3243 static int calculate_device_limits(struct dm_integrity_c *ic) in calculate_device_limits() argument
3247 calculate_journal_section_size(ic); in calculate_device_limits()
3248 initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections; in calculate_device_limits()
3249 …if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UIN… in calculate_device_limits()
3251 ic->initial_sectors = initial_sectors; in calculate_device_limits()
3253 if (!ic->meta_dev) { in calculate_device_limits()
3258 ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING) ? in calculate_device_limits()
3262 …ic->metadata_run = round_up((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2… in calculate_device_limits()
3264 if (!(ic->metadata_run & (ic->metadata_run - 1))) in calculate_device_limits()
3265 ic->log2_metadata_run = __ffs(ic->metadata_run); in calculate_device_limits()
3267 ic->log2_metadata_run = -1; in calculate_device_limits()
3269 get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset); in calculate_device_limits()
3270 last_sector = get_data_sector(ic, last_area, last_offset); in calculate_device_limits()
3271 if (last_sector < ic->start || last_sector >= ic->meta_device_sectors) in calculate_device_limits()
3274 __u64 meta_size = (ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size; in calculate_device_limits()
3275 meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1)) in calculate_device_limits()
3276 >> (ic->log2_buffer_sectors + SECTOR_SHIFT); in calculate_device_limits()
3277 meta_size <<= ic->log2_buffer_sectors; in calculate_device_limits()
3278 if (ic->initial_sectors + meta_size < ic->initial_sectors || in calculate_device_limits()
3279 ic->initial_sectors + meta_size > ic->meta_device_sectors) in calculate_device_limits()
3281 ic->metadata_run = 1; in calculate_device_limits()
3282 ic->log2_metadata_run = 0; in calculate_device_limits()
3288 static void get_provided_data_sectors(struct dm_integrity_c *ic) in get_provided_data_sectors() argument
3290 if (!ic->meta_dev) { in get_provided_data_sectors()
3292 ic->provided_data_sectors = 0; in get_provided_data_sectors()
3293 for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) { in get_provided_data_sectors()
3294 __u64 prev_data_sectors = ic->provided_data_sectors; in get_provided_data_sectors()
3296 ic->provided_data_sectors |= (sector_t)1 << test_bit; in get_provided_data_sectors()
3297 if (calculate_device_limits(ic)) in get_provided_data_sectors()
3298 ic->provided_data_sectors = prev_data_sectors; in get_provided_data_sectors()
3301 ic->provided_data_sectors = ic->data_device_sectors; in get_provided_data_sectors()
3302 ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1); in get_provided_data_sectors()
3306 static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sectors, unsigned inte… in initialize_superblock() argument
3311 memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT); in initialize_superblock()
3312 memcpy(ic->sb->magic, SB_MAGIC, 8); in initialize_superblock()
3313 ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size); in initialize_superblock()
3314 ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block); in initialize_superblock()
3315 if (ic->journal_mac_alg.alg_string) in initialize_superblock()
3316 ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC); in initialize_superblock()
3318 calculate_journal_section_size(ic); in initialize_superblock()
3319 journal_sections = journal_sectors / ic->journal_section_sectors; in initialize_superblock()
3323 if (!ic->meta_dev) { in initialize_superblock()
3324 if (ic->fix_padding) in initialize_superblock()
3325 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING); in initialize_superblock()
3326 ic->sb->journal_sections = cpu_to_le32(journal_sections); in initialize_superblock()
3329 ic->sb->log2_interleave_sectors = __fls(interleave_sectors); in initialize_superblock()
3330 …ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_s… in initialize_superblock()
3331 …ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_s… in initialize_superblock()
3333 get_provided_data_sectors(ic); in initialize_superblock()
3334 if (!ic->provided_data_sectors) in initialize_superblock()
3337 ic->sb->log2_interleave_sectors = 0; in initialize_superblock()
3339 get_provided_data_sectors(ic); in initialize_superblock()
3340 if (!ic->provided_data_sectors) in initialize_superblock()
3344 ic->sb->journal_sections = cpu_to_le32(0); in initialize_superblock()
3346 __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections); in initialize_superblock()
3350 ic->sb->journal_sections = cpu_to_le32(test_journal_sections); in initialize_superblock()
3351 if (calculate_device_limits(ic)) in initialize_superblock()
3352 ic->sb->journal_sections = cpu_to_le32(prev_journal_sections); in initialize_superblock()
3355 if (!le32_to_cpu(ic->sb->journal_sections)) { in initialize_superblock()
3356 if (ic->log2_buffer_sectors > 3) { in initialize_superblock()
3357 ic->log2_buffer_sectors--; in initialize_superblock()
3364 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); in initialize_superblock()
3366 sb_set_version(ic); in initialize_superblock()
3371 static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic) in dm_integrity_set() argument
3378 bi.tuple_size = ic->tag_size; in dm_integrity_set()
3380 bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT; in dm_integrity_set()
3421 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **s… in dm_integrity_free_journal_scatterlist() argument
3424 for (i = 0; i < ic->journal_sections; i++) in dm_integrity_free_journal_scatterlist()
3429 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, in dm_integrity_alloc_journal_scatterlist() argument
3435 sl = kvmalloc_array(ic->journal_sections, in dm_integrity_alloc_journal_scatterlist()
3441 for (i = 0; i < ic->journal_sections; i++) { in dm_integrity_alloc_journal_scatterlist()
3448 page_list_location(ic, i, 0, &start_index, &start_offset); in dm_integrity_alloc_journal_scatterlist()
3449 page_list_location(ic, i, ic->journal_section_sectors - 1, in dm_integrity_alloc_journal_scatterlist()
3457 dm_integrity_free_journal_scatterlist(ic, sl); in dm_integrity_alloc_journal_scatterlist()
3548 static int create_journal(struct dm_integrity_c *ic, char **error) in create_journal() argument
3556 ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL); in create_journal()
3557 ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL); in create_journal()
3558 ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL); in create_journal()
3559 ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL); in create_journal()
3561 journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors, in create_journal()
3569 ic->journal_pages = journal_pages; in create_journal()
3571 ic->journal = dm_integrity_alloc_page_list(ic->journal_pages); in create_journal()
3572 if (!ic->journal) { in create_journal()
3577 if (ic->journal_crypt_alg.alg_string) { in create_journal()
3581 comp.ic = ic; in create_journal()
3582 …ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, CRYPTO_ALG_ALLOCATE… in create_journal()
3583 if (IS_ERR(ic->journal_crypt)) { in create_journal()
3585 r = PTR_ERR(ic->journal_crypt); in create_journal()
3586 ic->journal_crypt = NULL; in create_journal()
3589 ivsize = crypto_skcipher_ivsize(ic->journal_crypt); in create_journal()
3590 blocksize = crypto_skcipher_blocksize(ic->journal_crypt); in create_journal()
3592 if (ic->journal_crypt_alg.key) { in create_journal()
3593 r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key, in create_journal()
3594 ic->journal_crypt_alg.key_size); in create_journal()
3601 ic->journal_crypt_alg.alg_string, blocksize, ivsize); in create_journal()
3603 ic->journal_io = dm_integrity_alloc_page_list(ic->journal_pages); in create_journal()
3604 if (!ic->journal_io) { in create_journal()
3613 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); in create_journal()
3627 ic->journal_xor = dm_integrity_alloc_page_list(ic->journal_pages); in create_journal()
3628 if (!ic->journal_xor) { in create_journal()
3634 sg = kvmalloc_array(ic->journal_pages + 1, in create_journal()
3642 sg_init_table(sg, ic->journal_pages + 1); in create_journal()
3643 for (i = 0; i < ic->journal_pages; i++) { in create_journal()
3644 char *va = lowmem_page_address(ic->journal_xor[i].page); in create_journal()
3648 sg_set_buf(&sg[i], &ic->commit_ids, sizeof ic->commit_ids); in create_journal()
3651 PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, crypt_iv); in create_journal()
3657 r = dm_integrity_failed(ic); in create_journal()
3662 DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data"); in create_journal()
3664 crypto_free_skcipher(ic->journal_crypt); in create_journal()
3665 ic->journal_crypt = NULL; in create_journal()
3669 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); in create_journal()
3690 ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal); in create_journal()
3691 if (!ic->journal_scatterlist) { in create_journal()
3696 ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io); in create_journal()
3697 if (!ic->journal_io_scatterlist) { in create_journal()
3702 ic->sk_requests = kvmalloc_array(ic->journal_sections, in create_journal()
3705 if (!ic->sk_requests) { in create_journal()
3710 for (i = 0; i < ic->journal_sections; i++) { in create_journal()
3726 r = dm_integrity_failed(ic); in create_journal()
3732 section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); in create_journal()
3747 section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT; in create_journal()
3748 ic->sk_requests[i] = section_req; in create_journal()
3758 if (ic->commit_ids[j] == ic->commit_ids[i]) { in create_journal()
3759 ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1); in create_journal()
3763 DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]); in create_journal()
3766 journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node); in create_journal()
3772 ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL); in create_journal()
3773 if (!ic->journal_tree) { in create_journal()
3811 struct dm_integrity_c *ic; in dm_integrity_ctr() local
3835 ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL); in dm_integrity_ctr()
3836 if (!ic) { in dm_integrity_ctr()
3840 ti->private = ic; in dm_integrity_ctr()
3842 ic->ti = ti; in dm_integrity_ctr()
3844 ic->in_progress = RB_ROOT; in dm_integrity_ctr()
3845 INIT_LIST_HEAD(&ic->wait_list); in dm_integrity_ctr()
3846 init_waitqueue_head(&ic->endio_wait); in dm_integrity_ctr()
3847 bio_list_init(&ic->flush_bio_list); in dm_integrity_ctr()
3848 init_waitqueue_head(&ic->copy_to_journal_wait); in dm_integrity_ctr()
3849 init_completion(&ic->crypto_backoff); in dm_integrity_ctr()
3850 atomic64_set(&ic->number_of_mismatches, 0); in dm_integrity_ctr()
3851 ic->bitmap_flush_interval = BITMAP_FLUSH_INTERVAL; in dm_integrity_ctr()
3853 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev); in dm_integrity_ctr()
3864 ic->start = start; in dm_integrity_ctr()
3867 if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) { in dm_integrity_ctr()
3876 ic->mode = argv[3][0]; in dm_integrity_ctr()
3888 ic->sectors_per_block = 1; in dm_integrity_ctr()
3917 if (ic->meta_dev) { in dm_integrity_ctr()
3918 dm_put_device(ti, ic->meta_dev); in dm_integrity_ctr()
3919 ic->meta_dev = NULL; in dm_integrity_ctr()
3922 dm_table_get_mode(ti->table), &ic->meta_dev); in dm_integrity_ctr()
3935 ic->sectors_per_block = val >> SECTOR_SHIFT; in dm_integrity_ctr()
3944 ic->bitmap_flush_interval = msecs_to_jiffies(val); in dm_integrity_ctr()
3946 r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error, in dm_integrity_ctr()
3951 r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error, in dm_integrity_ctr()
3956 r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error, in dm_integrity_ctr()
3961 ic->recalculate_flag = true; in dm_integrity_ctr()
3963 ic->discard = true; in dm_integrity_ctr()
3965 ic->fix_padding = true; in dm_integrity_ctr()
3967 ic->legacy_recalculate = true; in dm_integrity_ctr()
3975 ic->data_device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT; in dm_integrity_ctr()
3976 if (!ic->meta_dev) in dm_integrity_ctr()
3977 ic->meta_device_sectors = ic->data_device_sectors; in dm_integrity_ctr()
3979 ic->meta_device_sectors = i_size_read(ic->meta_dev->bdev->bd_inode) >> SECTOR_SHIFT; in dm_integrity_ctr()
3983 ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR); in dm_integrity_ctr()
3988 ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT); in dm_integrity_ctr()
3990 r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error, in dm_integrity_ctr()
3995 r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error, in dm_integrity_ctr()
4000 if (!ic->tag_size) { in dm_integrity_ctr()
4001 if (!ic->internal_hash) { in dm_integrity_ctr()
4006 ic->tag_size = crypto_shash_digestsize(ic->internal_hash); in dm_integrity_ctr()
4008 if (ic->tag_size > MAX_TAG_SIZE) { in dm_integrity_ctr()
4013 if (!(ic->tag_size & (ic->tag_size - 1))) in dm_integrity_ctr()
4014 ic->log2_tag_size = __ffs(ic->tag_size); in dm_integrity_ctr()
4016 ic->log2_tag_size = -1; in dm_integrity_ctr()
4018 if (ic->mode == 'B' && !ic->internal_hash) { in dm_integrity_ctr()
4024 if (ic->discard && !ic->internal_hash) { in dm_integrity_ctr()
4030 ic->autocommit_jiffies = msecs_to_jiffies(sync_msec); in dm_integrity_ctr()
4031 ic->autocommit_msec = sync_msec; in dm_integrity_ctr()
4032 timer_setup(&ic->autocommit_timer, autocommit_fn, 0); in dm_integrity_ctr()
4034 ic->io = dm_io_client_create(); in dm_integrity_ctr()
4035 if (IS_ERR(ic->io)) { in dm_integrity_ctr()
4036 r = PTR_ERR(ic->io); in dm_integrity_ctr()
4037 ic->io = NULL; in dm_integrity_ctr()
4042 r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache); in dm_integrity_ctr()
4048 ic->metadata_wq = alloc_workqueue("dm-integrity-metadata", in dm_integrity_ctr()
4050 if (!ic->metadata_wq) { in dm_integrity_ctr()
4060 ic->wait_wq = alloc_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM | WQ_UNBOUND, 1); in dm_integrity_ctr()
4061 if (!ic->wait_wq) { in dm_integrity_ctr()
4067 ic->offload_wq = alloc_workqueue("dm-integrity-offload", WQ_MEM_RECLAIM, in dm_integrity_ctr()
4069 if (!ic->offload_wq) { in dm_integrity_ctr()
4075 ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1); in dm_integrity_ctr()
4076 if (!ic->commit_wq) { in dm_integrity_ctr()
4081 INIT_WORK(&ic->commit_work, integrity_commit); in dm_integrity_ctr()
4083 if (ic->mode == 'J' || ic->mode == 'B') { in dm_integrity_ctr()
4084 ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1); in dm_integrity_ctr()
4085 if (!ic->writer_wq) { in dm_integrity_ctr()
4090 INIT_WORK(&ic->writer_work, integrity_writer); in dm_integrity_ctr()
4093 ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL); in dm_integrity_ctr()
4094 if (!ic->sb) { in dm_integrity_ctr()
4100 r = sync_rw_sb(ic, REQ_OP_READ, 0); in dm_integrity_ctr()
4106 if (memcmp(ic->sb->magic, SB_MAGIC, 8)) { in dm_integrity_ctr()
4107 if (ic->mode != 'R') { in dm_integrity_ctr()
4108 if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) { in dm_integrity_ctr()
4115 r = initialize_superblock(ic, journal_sectors, interleave_sectors); in dm_integrity_ctr()
4120 if (ic->mode != 'R') in dm_integrity_ctr()
4124 if (!ic->sb->version || ic->sb->version > SB_VERSION_4) { in dm_integrity_ctr()
4129 if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) { in dm_integrity_ctr()
4134 if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) { in dm_integrity_ctr()
4139 if (!le32_to_cpu(ic->sb->journal_sections)) { in dm_integrity_ctr()
4145 if (!ic->meta_dev) { in dm_integrity_ctr()
4146 if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS || in dm_integrity_ctr()
4147 ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) { in dm_integrity_ctr()
4153 if (ic->sb->log2_interleave_sectors) { in dm_integrity_ctr()
4159 …if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string)… in dm_integrity_ctr()
4165 get_provided_data_sectors(ic); in dm_integrity_ctr()
4166 if (!ic->provided_data_sectors) { in dm_integrity_ctr()
4173 r = calculate_device_limits(ic); in dm_integrity_ctr()
4175 if (ic->meta_dev) { in dm_integrity_ctr()
4176 if (ic->log2_buffer_sectors > 3) { in dm_integrity_ctr()
4177 ic->log2_buffer_sectors--; in dm_integrity_ctr()
4187 if (log2_sectors_per_bitmap_bit < ic->sb->log2_sectors_per_block) in dm_integrity_ctr()
4188 log2_sectors_per_bitmap_bit = ic->sb->log2_sectors_per_block; in dm_integrity_ctr()
4190 …bits_in_journal = ((__u64)ic->journal_section_sectors * ic->journal_sections) << (SECTOR_SHIFT + 3… in dm_integrity_ctr()
4193 …while (bits_in_journal < (ic->provided_data_sectors + ((sector_t)1 << log2_sectors_per_bitmap_bit)… in dm_integrity_ctr()
4196 log2_blocks_per_bitmap_bit = log2_sectors_per_bitmap_bit - ic->sb->log2_sectors_per_block; in dm_integrity_ctr()
4197 ic->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit; in dm_integrity_ctr()
4199 ic->sb->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit; in dm_integrity_ctr()
4201 n_bitmap_bits = ((ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) in dm_integrity_ctr()
4203 ic->n_bitmap_blocks = DIV_ROUND_UP(n_bitmap_bits, BITMAP_BLOCK_SIZE * 8); in dm_integrity_ctr()
4205 if (!ic->meta_dev) in dm_integrity_ctr()
4206 ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run)); in dm_integrity_ctr()
4208 if (ti->len > ic->provided_data_sectors) { in dm_integrity_ctr()
4215 threshold = (__u64)ic->journal_entries * (100 - journal_watermark); in dm_integrity_ctr()
4218 ic->free_sectors_threshold = threshold; in dm_integrity_ctr()
4221 DEBUG_print(" integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size)); in dm_integrity_ctr()
4222 DEBUG_print(" journal_entry_size %u\n", ic->journal_entry_size); in dm_integrity_ctr()
4223 DEBUG_print(" journal_entries_per_sector %u\n", ic->journal_entries_per_sector); in dm_integrity_ctr()
4224 DEBUG_print(" journal_section_entries %u\n", ic->journal_section_entries); in dm_integrity_ctr()
4225 DEBUG_print(" journal_section_sectors %u\n", ic->journal_section_sectors); in dm_integrity_ctr()
4226 DEBUG_print(" journal_sections %u\n", (unsigned)le32_to_cpu(ic->sb->journal_sections)); in dm_integrity_ctr()
4227 DEBUG_print(" journal_entries %u\n", ic->journal_entries); in dm_integrity_ctr()
4228 DEBUG_print(" log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors); in dm_integrity_ctr()
4229 DEBUG_print(" data_device_sectors 0x%llx\n", i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT); in dm_integrity_ctr()
4230 DEBUG_print(" initial_sectors 0x%x\n", ic->initial_sectors); in dm_integrity_ctr()
4231 DEBUG_print(" metadata_run 0x%x\n", ic->metadata_run); in dm_integrity_ctr()
4232 DEBUG_print(" log2_metadata_run %d\n", ic->log2_metadata_run); in dm_integrity_ctr()
4233 …DEBUG_print(" provided_data_sectors 0x%llx (%llu)\n", ic->provided_data_sectors, ic->provided_data… in dm_integrity_ctr()
4234 DEBUG_print(" log2_buffer_sectors %u\n", ic->log2_buffer_sectors); in dm_integrity_ctr()
4237 if (ic->recalculate_flag && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) { in dm_integrity_ctr()
4238 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_ctr()
4239 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_ctr()
4242 if (ic->internal_hash) { in dm_integrity_ctr()
4244 ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1); in dm_integrity_ctr()
4245 if (!ic->recalc_wq ) { in dm_integrity_ctr()
4250 INIT_WORK(&ic->recalc_work, integrity_recalc); in dm_integrity_ctr()
4251 ic->recalc_buffer = vmalloc(RECALC_SECTORS << SECTOR_SHIFT); in dm_integrity_ctr()
4252 if (!ic->recalc_buffer) { in dm_integrity_ctr()
4257 recalc_tags_size = (RECALC_SECTORS >> ic->sb->log2_sectors_per_block) * ic->tag_size; in dm_integrity_ctr()
4258 if (crypto_shash_digestsize(ic->internal_hash) > ic->tag_size) in dm_integrity_ctr()
4259 recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tag_size; in dm_integrity_ctr()
4260 ic->recalc_tags = kvmalloc(recalc_tags_size, GFP_KERNEL); in dm_integrity_ctr()
4261 if (!ic->recalc_tags) { in dm_integrity_ctr()
4267 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { in dm_integrity_ctr()
4274 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && in dm_integrity_ctr()
4275 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors && in dm_integrity_ctr()
4276 dm_integrity_disable_recalculate(ic)) { in dm_integrity_ctr()
4282 ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev, in dm_integrity_ctr()
4283 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL); in dm_integrity_ctr()
4284 if (IS_ERR(ic->bufio)) { in dm_integrity_ctr()
4285 r = PTR_ERR(ic->bufio); in dm_integrity_ctr()
4287 ic->bufio = NULL; in dm_integrity_ctr()
4290 dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors); in dm_integrity_ctr()
4292 if (ic->mode != 'R') { in dm_integrity_ctr()
4293 r = create_journal(ic, &ti->error); in dm_integrity_ctr()
4299 if (ic->mode == 'B') { in dm_integrity_ctr()
4301 unsigned n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE); in dm_integrity_ctr()
4303 ic->recalc_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages); in dm_integrity_ctr()
4304 if (!ic->recalc_bitmap) { in dm_integrity_ctr()
4308 ic->may_write_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages); in dm_integrity_ctr()
4309 if (!ic->may_write_bitmap) { in dm_integrity_ctr()
4313 ic->bbs = kvmalloc_array(ic->n_bitmap_blocks, sizeof(struct bitmap_block_status), GFP_KERNEL); in dm_integrity_ctr()
4314 if (!ic->bbs) { in dm_integrity_ctr()
4318 INIT_DELAYED_WORK(&ic->bitmap_flush_work, bitmap_flush_work); in dm_integrity_ctr()
4319 for (i = 0; i < ic->n_bitmap_blocks; i++) { in dm_integrity_ctr()
4320 struct bitmap_block_status *bbs = &ic->bbs[i]; in dm_integrity_ctr()
4324 bbs->ic = ic; in dm_integrity_ctr()
4333 bbs->bitmap = lowmem_page_address(ic->journal[pl_index].page) + pl_offset; in dm_integrity_ctr()
4338 init_journal(ic, 0, ic->journal_sections, 0); in dm_integrity_ctr()
4339 r = dm_integrity_failed(ic); in dm_integrity_ctr()
4344 r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA); in dm_integrity_ctr()
4349 ic->just_formatted = true; in dm_integrity_ctr()
4352 if (!ic->meta_dev) { in dm_integrity_ctr()
4353 r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors); in dm_integrity_ctr()
4357 if (ic->mode == 'B') { in dm_integrity_ctr()
4358 …unsigned max_io_len = ((sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit) * (BITMA… in dm_integrity_ctr()
4369 if (!ic->internal_hash) in dm_integrity_ctr()
4370 dm_integrity_set(ti, ic); in dm_integrity_ctr()
4374 if (ic->discard) in dm_integrity_ctr()
4386 struct dm_integrity_c *ic = ti->private; in dm_integrity_dtr() local
4388 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); in dm_integrity_dtr()
4389 BUG_ON(!list_empty(&ic->wait_list)); in dm_integrity_dtr()
4391 if (ic->metadata_wq) in dm_integrity_dtr()
4392 destroy_workqueue(ic->metadata_wq); in dm_integrity_dtr()
4393 if (ic->wait_wq) in dm_integrity_dtr()
4394 destroy_workqueue(ic->wait_wq); in dm_integrity_dtr()
4395 if (ic->offload_wq) in dm_integrity_dtr()
4396 destroy_workqueue(ic->offload_wq); in dm_integrity_dtr()
4397 if (ic->commit_wq) in dm_integrity_dtr()
4398 destroy_workqueue(ic->commit_wq); in dm_integrity_dtr()
4399 if (ic->writer_wq) in dm_integrity_dtr()
4400 destroy_workqueue(ic->writer_wq); in dm_integrity_dtr()
4401 if (ic->recalc_wq) in dm_integrity_dtr()
4402 destroy_workqueue(ic->recalc_wq); in dm_integrity_dtr()
4403 vfree(ic->recalc_buffer); in dm_integrity_dtr()
4404 kvfree(ic->recalc_tags); in dm_integrity_dtr()
4405 kvfree(ic->bbs); in dm_integrity_dtr()
4406 if (ic->bufio) in dm_integrity_dtr()
4407 dm_bufio_client_destroy(ic->bufio); in dm_integrity_dtr()
4408 mempool_exit(&ic->journal_io_mempool); in dm_integrity_dtr()
4409 if (ic->io) in dm_integrity_dtr()
4410 dm_io_client_destroy(ic->io); in dm_integrity_dtr()
4411 if (ic->dev) in dm_integrity_dtr()
4412 dm_put_device(ti, ic->dev); in dm_integrity_dtr()
4413 if (ic->meta_dev) in dm_integrity_dtr()
4414 dm_put_device(ti, ic->meta_dev); in dm_integrity_dtr()
4415 dm_integrity_free_page_list(ic->journal); in dm_integrity_dtr()
4416 dm_integrity_free_page_list(ic->journal_io); in dm_integrity_dtr()
4417 dm_integrity_free_page_list(ic->journal_xor); in dm_integrity_dtr()
4418 dm_integrity_free_page_list(ic->recalc_bitmap); in dm_integrity_dtr()
4419 dm_integrity_free_page_list(ic->may_write_bitmap); in dm_integrity_dtr()
4420 if (ic->journal_scatterlist) in dm_integrity_dtr()
4421 dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist); in dm_integrity_dtr()
4422 if (ic->journal_io_scatterlist) in dm_integrity_dtr()
4423 dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist); in dm_integrity_dtr()
4424 if (ic->sk_requests) { in dm_integrity_dtr()
4427 for (i = 0; i < ic->journal_sections; i++) { in dm_integrity_dtr()
4428 struct skcipher_request *req = ic->sk_requests[i]; in dm_integrity_dtr()
4434 kvfree(ic->sk_requests); in dm_integrity_dtr()
4436 kvfree(ic->journal_tree); in dm_integrity_dtr()
4437 if (ic->sb) in dm_integrity_dtr()
4438 free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT); in dm_integrity_dtr()
4440 if (ic->internal_hash) in dm_integrity_dtr()
4441 crypto_free_shash(ic->internal_hash); in dm_integrity_dtr()
4442 free_alg(&ic->internal_hash_alg); in dm_integrity_dtr()
4444 if (ic->journal_crypt) in dm_integrity_dtr()
4445 crypto_free_skcipher(ic->journal_crypt); in dm_integrity_dtr()
4446 free_alg(&ic->journal_crypt_alg); in dm_integrity_dtr()
4448 if (ic->journal_mac) in dm_integrity_dtr()
4449 crypto_free_shash(ic->journal_mac); in dm_integrity_dtr()
4450 free_alg(&ic->journal_mac_alg); in dm_integrity_dtr()
4452 kfree(ic); in dm_integrity_dtr()