64 	struct scrub_block	*sblock;  member
216 				struct scrub_block *sblock,
218 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
224 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
225 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
227 static int scrub_checksum_data(struct scrub_block *sblock);
228 static int scrub_checksum_tree_block(struct scrub_block *sblock);
229 static int scrub_checksum_super(struct scrub_block *sblock);
230 static void scrub_block_get(struct scrub_block *sblock);
231 static void scrub_block_put(struct scrub_block *sblock);
244 static void scrub_block_complete(struct scrub_block *sblock);
551 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
728 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)  in scrub_print_warning()  argument
745 	WARN_ON(sblock->page_count < 1);  in scrub_print_warning()
746 	dev = sblock->pagev[0]->dev;  in scrub_print_warning()
747 	fs_info = sblock->sctx->fs_info;  in scrub_print_warning()
753 	swarn.physical = sblock->pagev[0]->physical;  in scrub_print_warning()
754 	swarn.logical = sblock->pagev[0]->logical;  in scrub_print_warning()
1198 			struct scrub_block *sblock = sblocks_for_recheck +  in scrub_handle_errored_block()  local
1203 			for (page_index = 0; page_index < sblock->page_count;  in scrub_handle_errored_block()
1205 				sblock->pagev[page_index]->sblock = NULL;  in scrub_handle_errored_block()
1206 				recover = sblock->pagev[page_index]->recover;  in scrub_handle_errored_block()
1209 					sblock->pagev[page_index]->recover =  in scrub_handle_errored_block()
1212 				scrub_page_put(sblock->pagev[page_index]);  in scrub_handle_errored_block()
1327 			struct scrub_block *sblock;  in scrub_setup_recheck_block()  local
1330 			sblock = sblocks_for_recheck + mirror_index;  in scrub_setup_recheck_block()
1331 			sblock->sctx = sctx;  in scrub_setup_recheck_block()
1343 			sblock->pagev[page_index] = page;  in scrub_setup_recheck_block()
1344 			page->sblock = sblock;  in scrub_setup_recheck_block()
1373 			sblock->page_count++;  in scrub_setup_recheck_block()
1407 	mirror_num = page->sblock->pagev[0]->mirror_num;  in scrub_submit_raid56_bio_wait()
1419 					  struct scrub_block *sblock)  in scrub_recheck_block_on_raid56()  argument
1421 	struct scrub_page *first_page = sblock->pagev[0];  in scrub_recheck_block_on_raid56()
1433 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_recheck_block_on_raid56()
1434 		struct scrub_page *page = sblock->pagev[page_num];  in scrub_recheck_block_on_raid56()
1447 	scrub_recheck_block_checksum(sblock);  in scrub_recheck_block_on_raid56()
1451 	for (page_num = 0; page_num < sblock->page_count; page_num++)  in scrub_recheck_block_on_raid56()
1452 		sblock->pagev[page_num]->io_error = 1;  in scrub_recheck_block_on_raid56()
1454 	sblock->no_io_error_seen = 0;  in scrub_recheck_block_on_raid56()
1465 				struct scrub_block *sblock,  in scrub_recheck_block()  argument
1470 	sblock->no_io_error_seen = 1;  in scrub_recheck_block()
1473 	if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))  in scrub_recheck_block()
1474 		return scrub_recheck_block_on_raid56(fs_info, sblock);  in scrub_recheck_block()
1476 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_recheck_block()
1478 		struct scrub_page *page = sblock->pagev[page_num];  in scrub_recheck_block()
1482 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1496 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1502 	if (sblock->no_io_error_seen)  in scrub_recheck_block()
1503 		scrub_recheck_block_checksum(sblock);  in scrub_recheck_block()
1516 static void scrub_recheck_block_checksum(struct scrub_block *sblock)  in scrub_recheck_block_checksum()  argument
1518 	sblock->header_error = 0;  in scrub_recheck_block_checksum()
1519 	sblock->checksum_error = 0;  in scrub_recheck_block_checksum()
1520 	sblock->generation_error = 0;  in scrub_recheck_block_checksum()
1522 	if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)  in scrub_recheck_block_checksum()
1523 		scrub_checksum_data(sblock);  in scrub_recheck_block_checksum()
1525 		scrub_checksum_tree_block(sblock);  in scrub_recheck_block_checksum()
1592 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)  in scrub_write_block_to_dev_replace()  argument
1594 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_write_block_to_dev_replace()
1601 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
1604 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_write_block_to_dev_replace()
1607 		ret = scrub_write_page_to_dev_replace(sblock, page_num);  in scrub_write_block_to_dev_replace()
1613 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,  in scrub_write_page_to_dev_replace()  argument
1616 	struct scrub_page *spage = sblock->pagev[page_num];  in scrub_write_page_to_dev_replace()
1622 	return scrub_add_page_to_wr_bio(sblock->sctx, spage);  in scrub_write_page_to_dev_replace()
1749 static int scrub_checksum(struct scrub_block *sblock)  in scrub_checksum()  argument
1762 	sblock->header_error = 0;  in scrub_checksum()
1763 	sblock->generation_error = 0;  in scrub_checksum()
1764 	sblock->checksum_error = 0;  in scrub_checksum()
1766 	WARN_ON(sblock->page_count < 1);  in scrub_checksum()
1767 	flags = sblock->pagev[0]->flags;  in scrub_checksum()
1770 		ret = scrub_checksum_data(sblock);  in scrub_checksum()
1772 		ret = scrub_checksum_tree_block(sblock);  in scrub_checksum()
1774 		(void)scrub_checksum_super(sblock);  in scrub_checksum()
1778 		scrub_handle_errored_block(sblock);  in scrub_checksum()
1783 static int scrub_checksum_data(struct scrub_block *sblock)  in scrub_checksum_data()  argument
1785 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_data()
1792 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_data()
1793 	spage = sblock->pagev[0];  in scrub_checksum_data()
1804 		sblock->checksum_error = 1;  in scrub_checksum_data()
1806 	return sblock->checksum_error;  in scrub_checksum_data()
1809 static int scrub_checksum_tree_block(struct scrub_block *sblock)  in scrub_checksum_tree_block()  argument
1811 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_tree_block()
1822 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_tree_block()
1823 	spage = sblock->pagev[0];  in scrub_checksum_tree_block()
1834 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1837 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1838 		sblock->generation_error = 1;  in scrub_checksum_tree_block()
1842 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1846 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1854 		kaddr = page_address(sblock->pagev[i]->page);  in scrub_checksum_tree_block()
1860 		sblock->checksum_error = 1;  in scrub_checksum_tree_block()
1862 	return sblock->header_error || sblock->checksum_error;  in scrub_checksum_tree_block()
1865 static int scrub_checksum_super(struct scrub_block *sblock)  in scrub_checksum_super()  argument
1868 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_super()
1877 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_super()
1878 	spage = sblock->pagev[0];  in scrub_checksum_super()
1919 static void scrub_block_get(struct scrub_block *sblock)  in scrub_block_get()  argument
1921 	refcount_inc(&sblock->refs);  in scrub_block_get()
1924 static void scrub_block_put(struct scrub_block *sblock)  in scrub_block_put()  argument
1926 	if (refcount_dec_and_test(&sblock->refs)) {  in scrub_block_put()
1929 		if (sblock->sparity)  in scrub_block_put()
1930 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
1932 		for (i = 0; i < sblock->page_count; i++)  in scrub_block_put()
1933 			scrub_page_put(sblock->pagev[i]);  in scrub_block_put()
1934 		kfree(sblock);  in scrub_block_put()
1968 	struct scrub_block *sblock = spage->sblock;  in scrub_add_page_to_rd_bio()  local
2029 	scrub_block_get(sblock); /* one for the page added to the bio */  in scrub_add_page_to_rd_bio()
2030 	atomic_inc(&sblock->outstanding_pages);  in scrub_add_page_to_rd_bio()
2040 	struct scrub_block *sblock = bio->bi_private;  in scrub_missing_raid56_end_io()  local
2041 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_missing_raid56_end_io()
2044 		sblock->no_io_error_seen = 0;  in scrub_missing_raid56_end_io()
2048 	btrfs_queue_work(fs_info->scrub_workers, &sblock->work);  in scrub_missing_raid56_end_io()
2053 	struct scrub_block *sblock = container_of(work, struct scrub_block, work);  in scrub_missing_raid56_worker()  local
2054 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_worker()
2059 	logical = sblock->pagev[0]->logical;  in scrub_missing_raid56_worker()
2060 	dev = sblock->pagev[0]->dev;  in scrub_missing_raid56_worker()
2062 	if (sblock->no_io_error_seen)  in scrub_missing_raid56_worker()
2063 		scrub_recheck_block_checksum(sblock);  in scrub_missing_raid56_worker()
2065 	if (!sblock->no_io_error_seen) {  in scrub_missing_raid56_worker()
2072 	} else if (sblock->header_error || sblock->checksum_error) {  in scrub_missing_raid56_worker()
2080 		scrub_write_block_to_dev_replace(sblock);  in scrub_missing_raid56_worker()
2089 	scrub_block_put(sblock);  in scrub_missing_raid56_worker()
2093 static void scrub_missing_raid56_pages(struct scrub_block *sblock)  in scrub_missing_raid56_pages()  argument
2095 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_pages()
2097 	u64 length = sblock->page_count * PAGE_SIZE;  in scrub_missing_raid56_pages()
2098 	u64 logical = sblock->pagev[0]->logical;  in scrub_missing_raid56_pages()
2124 	bio->bi_private = sblock;  in scrub_missing_raid56_pages()
2131 	for (i = 0; i < sblock->page_count; i++) {  in scrub_missing_raid56_pages()
2132 		struct scrub_page *spage = sblock->pagev[i];  in scrub_missing_raid56_pages()
2137 	btrfs_init_work(&sblock->work, scrub_missing_raid56_worker, NULL, NULL);  in scrub_missing_raid56_pages()
2138 	scrub_block_get(sblock);  in scrub_missing_raid56_pages()
2158 	struct scrub_block *sblock;  in scrub_pages()  local
2161 	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);  in scrub_pages()
2162 	if (!sblock) {  in scrub_pages()
2171 	refcount_set(&sblock->refs, 1);  in scrub_pages()
2172 	sblock->sctx = sctx;  in scrub_pages()
2173 	sblock->no_io_error_seen = 1;  in scrub_pages()
2185 			scrub_block_put(sblock);  in scrub_pages()
2190 		sblock->pagev[index] = spage;  in scrub_pages()
2191 		spage->sblock = sblock;  in scrub_pages()
2205 		sblock->page_count++;  in scrub_pages()
2215 	WARN_ON(sblock->page_count == 0);  in scrub_pages()
2221 		scrub_missing_raid56_pages(sblock);  in scrub_pages()
2223 		for (index = 0; index < sblock->page_count; index++) {  in scrub_pages()
2224 			struct scrub_page *spage = sblock->pagev[index];  in scrub_pages()
2229 				scrub_block_put(sblock);  in scrub_pages()
2239 	scrub_block_put(sblock);  in scrub_pages()
2266 			spage->sblock->no_io_error_seen = 0;  in scrub_bio_end_io_worker()
2273 		struct scrub_block *sblock = spage->sblock;  in scrub_bio_end_io_worker()  local
2275 		if (atomic_dec_and_test(&sblock->outstanding_pages))  in scrub_bio_end_io_worker()
2276 			scrub_block_complete(sblock);  in scrub_bio_end_io_worker()
2277 		scrub_block_put(sblock);  in scrub_bio_end_io_worker()
2339 static void scrub_block_complete(struct scrub_block *sblock)  in scrub_block_complete()  argument
2343 	if (!sblock->no_io_error_seen) {  in scrub_block_complete()
2345 		scrub_handle_errored_block(sblock);  in scrub_block_complete()
2352 		corrupted = scrub_checksum(sblock);  in scrub_block_complete()
2353 		if (!corrupted && sblock->sctx->is_dev_replace)  in scrub_block_complete()
2354 			scrub_write_block_to_dev_replace(sblock);  in scrub_block_complete()
2357 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2358 		u64 start = sblock->pagev[0]->logical;  in scrub_block_complete()
2359 		u64 end = sblock->pagev[sblock->page_count - 1]->logical +  in scrub_block_complete()
2362 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2463 	struct scrub_block *sblock;  in scrub_pages_for_parity()  local
2466 	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);  in scrub_pages_for_parity()
2467 	if (!sblock) {  in scrub_pages_for_parity()
2476 	refcount_set(&sblock->refs, 1);  in scrub_pages_for_parity()
2477 	sblock->sctx = sctx;  in scrub_pages_for_parity()
2478 	sblock->no_io_error_seen = 1;  in scrub_pages_for_parity()
2479 	sblock->sparity = sparity;  in scrub_pages_for_parity()
2492 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2498 		sblock->pagev[index] = spage;  in scrub_pages_for_parity()
2502 		spage->sblock = sblock;  in scrub_pages_for_parity()
2515 		sblock->page_count++;  in scrub_pages_for_parity()
2524 	WARN_ON(sblock->page_count == 0);  in scrub_pages_for_parity()
2525 	for (index = 0; index < sblock->page_count; index++) {  in scrub_pages_for_parity()
2526 		struct scrub_page *spage = sblock->pagev[index];  in scrub_pages_for_parity()
2531 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2537 	scrub_block_put(sblock);  in scrub_pages_for_parity()