374 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
1465 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)  in submit_flush_bio()  argument
1471 	bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in submit_flush_bio()
1494 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)  in do_endio_flush()  argument
1496 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in do_endio_flush()
1498 	if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))  in do_endio_flush()
1499 		submit_flush_bio(ic, dio);  in do_endio_flush()
1504 static void dec_in_flight(struct dm_integrity_io *dio)  in dec_in_flight()  argument
1506 	if (atomic_dec_and_test(&dio->in_flight)) {  in dec_in_flight()
1507 		struct dm_integrity_c *ic = dio->ic;  in dec_in_flight()
1510 		remove_range(ic, &dio->range);  in dec_in_flight()
1512 		if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))  in dec_in_flight()
1515 		bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in dec_in_flight()
1517 		if (unlikely(dio->bi_status) && !bio->bi_status)  in dec_in_flight()
1518 			bio->bi_status = dio->bi_status;  in dec_in_flight()
1519 		if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {  in dec_in_flight()
1520 			dio->range.logical_sector += dio->range.n_sectors;  in dec_in_flight()
1521 			bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);  in dec_in_flight()
1522 			INIT_WORK(&dio->work, integrity_bio_wait);  in dec_in_flight()
1523 			queue_work(ic->offload_wq, &dio->work);  in dec_in_flight()
1526 		do_endio_flush(ic, dio);  in dec_in_flight()
1532 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in integrity_end_io()  local
1534 	dm_bio_restore(&dio->bio_details, bio);  in integrity_end_io()
1538 	if (dio->completion)  in integrity_end_io()
1539 		complete(dio->completion);  in integrity_end_io()
1541 	dec_in_flight(dio);  in integrity_end_io()
1591 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);  in integrity_metadata()  local
1592 	struct dm_integrity_c *ic = dio->ic;  in integrity_metadata()
1600 		struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in integrity_metadata()
1610 		if (likely(dio->op != REQ_OP_DISCARD))  in integrity_metadata()
1624 		if (unlikely(dio->op == REQ_OP_DISCARD)) {  in integrity_metadata()
1625 			sector_t bi_sector = dio->bio_details.bi_iter.bi_sector;  in integrity_metadata()
1626 			unsigned bi_size = dio->bio_details.bi_iter.bi_size;  in integrity_metadata()
1634 				r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1656 		sector = dio->range.logical_sector;  in integrity_metadata()
1657 		sectors_to_process = dio->range.n_sectors;  in integrity_metadata()
1659 		__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {  in integrity_metadata()
1676 			r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1677 						checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);  in integrity_metadata()
1704 		struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;  in integrity_metadata()
1709 			unsigned data_to_process = dio->range.n_sectors;  in integrity_metadata()
1720 				r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1721 							this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);  in integrity_metadata()
1731 	dec_in_flight(dio);  in integrity_metadata()
1734 	dio->bi_status = errno_to_blk_status(r);  in integrity_metadata()
1735 	dec_in_flight(dio);  in integrity_metadata()
1741 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in dm_integrity_map()  local
1746 	dio->ic = ic;  in dm_integrity_map()
1747 	dio->bi_status = 0;  in dm_integrity_map()
1748 	dio->op = bio_op(bio);  in dm_integrity_map()
1750 	if (unlikely(dio->op == REQ_OP_DISCARD)) {  in dm_integrity_map()
1764 		submit_flush_bio(ic, dio);  in dm_integrity_map()
1768 	dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);  in dm_integrity_map()
1769 	dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;  in dm_integrity_map()
1770 	if (unlikely(dio->fua)) {  in dm_integrity_map()
1777 	if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {  in dm_integrity_map()
1779 		      dio->range.logical_sector, bio_sectors(bio),  in dm_integrity_map()
1783 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1…  in dm_integrity_map()
1786 		      dio->range.logical_sector, bio_sectors(bio));  in dm_integrity_map()
1790 	if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) {  in dm_integrity_map()
1823 	if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))  in dm_integrity_map()
1826 	get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);  in dm_integrity_map()
1827 	dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);  in dm_integrity_map()
1830 	dm_integrity_map_continue(dio, true);  in dm_integrity_map()
1834 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,  in __journal_read_write()  argument
1837 	struct dm_integrity_c *ic = dio->ic;  in __journal_read_write()
1841 	logical_sector = dio->range.logical_sector;  in __journal_read_write()
1842 	n_sectors = dio->range.n_sectors;  in __journal_read_write()
1853 		if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
1859 			if (unlikely(dio->op == REQ_OP_READ)) {  in __journal_read_write()
1906 					if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
1914 					if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
1919 			if (likely(dio->op == REQ_OP_WRITE)) {  in __journal_read_write()
1955 		if (unlikely(dio->op == REQ_OP_READ))  in __journal_read_write()
1960 	if (likely(dio->op == REQ_OP_WRITE)) {  in __journal_read_write()
1970 		remove_range(ic, &dio->range);  in __journal_read_write()
1976 		dio->range.logical_sector = logical_sector;  in __journal_read_write()
1977 		get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);  in __journal_read_write()
1978 		dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);  in __journal_read_write()
1985 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)  in dm_integrity_map_continue()  argument
1987 	struct dm_integrity_c *ic = dio->ic;  in dm_integrity_map_continue()
1988 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in dm_integrity_map_continue()
1993 	bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;  in dm_integrity_map_continue()
1994 	if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')  in dm_integrity_map_continue()
1998 		INIT_WORK(&dio->work, integrity_bio_wait);  in dm_integrity_map_continue()
1999 		queue_work(ic->offload_wq, &dio->work);  in dm_integrity_map_continue()
2011 	dio->range.n_sectors = bio_sectors(bio);  in dm_integrity_map_continue()
2013 	if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {  in dm_integrity_map_continue()
2014 		if (dio->op == REQ_OP_WRITE) {  in dm_integrity_map_continue()
2018 			dio->range.n_sectors = min(dio->range.n_sectors,  in dm_integrity_map_continue()
2020 			if (unlikely(!dio->range.n_sectors)) {  in dm_integrity_map_continue()
2026 			range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;  in dm_integrity_map_continue()
2044 				add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);  in dm_integrity_map_continue()
2058 			} while ((i += ic->sectors_per_block) < dio->range.n_sectors);  in dm_integrity_map_continue()
2064 			journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2066 				if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))  in dm_integrity_map_continue()
2067 					dio->range.n_sectors = next_sector - dio->range.logical_sector;  in dm_integrity_map_continue()
2071 				for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {  in dm_integrity_map_continue()
2072 					if (!test_journal_node(ic, jp, dio->range.logical_sector + i))  in dm_integrity_map_continue()
2075 				dio->range.n_sectors = i;  in dm_integrity_map_continue()
2079 	if (unlikely(!add_new_range(ic, &dio->range, true))) {  in dm_integrity_map_continue()
2088 			INIT_WORK(&dio->work, integrity_bio_wait);  in dm_integrity_map_continue()
2089 			queue_work(ic->wait_wq, &dio->work);  in dm_integrity_map_continue()
2093 			dio->range.n_sectors = ic->sectors_per_block;  in dm_integrity_map_continue()
2094 		wait_and_add_new_range(ic, &dio->range);  in dm_integrity_map_continue()
2102 			unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2104 				remove_range_unlocked(ic, &dio->range);  in dm_integrity_map_continue()
2109 	if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {  in dm_integrity_map_continue()
2111 		unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2113 		    unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {  in dm_integrity_map_continue()
2114 			remove_range_unlocked(ic, &dio->range);  in dm_integrity_map_continue()
2132 	if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {  in dm_integrity_map_continue()
2133 		if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in dm_integrity_map_continue()
2134 				     dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {  in dm_integrity_map_continue()
2137 			bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);  in dm_integrity_map_continue()
2146 	dio->in_flight = (atomic_t)ATOMIC_INIT(2);  in dm_integrity_map_continue()
2150 		dio->completion = &read_comp;  in dm_integrity_map_continue()
2152 		dio->completion = NULL;  in dm_integrity_map_continue()
2154 	dm_bio_record(&dio->bio_details, bio);  in dm_integrity_map_continue()
2159 	bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;  in dm_integrity_map_continue()
2161 	if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {  in dm_integrity_map_continue()
2162 		integrity_metadata(&dio->work);  in dm_integrity_map_continue()
2165 		dio->in_flight = (atomic_t)ATOMIC_INIT(1);  in dm_integrity_map_continue()
2166 		dio->completion = NULL;  in dm_integrity_map_continue()
2178 		    dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))  in dm_integrity_map_continue()
2181 			if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,  in dm_integrity_map_continue()
2182 					     dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))  in dm_integrity_map_continue()
2187 			integrity_metadata(&dio->work);  in dm_integrity_map_continue()
2190 			dec_in_flight(dio);  in dm_integrity_map_continue()
2193 		INIT_WORK(&dio->work, integrity_metadata);  in dm_integrity_map_continue()
2194 		queue_work(ic->metadata_wq, &dio->work);  in dm_integrity_map_continue()
2200 	if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))  in dm_integrity_map_continue()
2203 	do_endio_flush(ic, dio);  in dm_integrity_map_continue()
2209 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);  in integrity_bio_wait()  local
2211 	dm_integrity_map_continue(dio, false);  in integrity_bio_wait()
2663 		struct dm_integrity_io *dio;  in bitmap_block_work()  local
2665 		dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in bitmap_block_work()
2667 		if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in bitmap_block_work()
2668 				    dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {  in bitmap_block_work()
2669 			remove_range(ic, &dio->range);  in bitmap_block_work()
2670 			INIT_WORK(&dio->work, integrity_bio_wait);  in bitmap_block_work()
2671 			queue_work(ic->offload_wq, &dio->work);  in bitmap_block_work()
2673 			block_bitmap_op(ic, ic->journal, dio->range.logical_sector,  in bitmap_block_work()
2674 					dio->range.n_sectors, BITMAP_OP_SET);  in bitmap_block_work()
2687 		struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in bitmap_block_work()  local
2689 		block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in bitmap_block_work()
2690 				dio->range.n_sectors, BITMAP_OP_SET);  in bitmap_block_work()
2692 		remove_range(ic, &dio->range);  in bitmap_block_work()
2693 		INIT_WORK(&dio->work, integrity_bio_wait);  in bitmap_block_work()
2694 		queue_work(ic->offload_wq, &dio->work);  in bitmap_block_work()