547 		struct scrub_bio *sbio = sctx->bios[sctx->curr];  in scrub_free_ctx()  local
549 		for (i = 0; i < sbio->page_count; i++) {  in scrub_free_ctx()
550 			WARN_ON(!sbio->pagev[i]->page);  in scrub_free_ctx()
551 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
553 		bio_put(sbio->bio);  in scrub_free_ctx()
557 		struct scrub_bio *sbio = sctx->bios[i];  in scrub_free_ctx()  local
559 		if (!sbio)  in scrub_free_ctx()
561 		kfree(sbio);  in scrub_free_ctx()
591 		struct scrub_bio *sbio;  in scrub_setup_ctx()  local
593 		sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);  in scrub_setup_ctx()
594 		if (!sbio)  in scrub_setup_ctx()
596 		sctx->bios[i] = sbio;  in scrub_setup_ctx()
598 		sbio->index = i;  in scrub_setup_ctx()
599 		sbio->sctx = sctx;  in scrub_setup_ctx()
600 		sbio->page_count = 0;  in scrub_setup_ctx()
601 		btrfs_init_work(&sbio->work, scrub_bio_end_io_worker, NULL,  in scrub_setup_ctx()
1628 	struct scrub_bio *sbio;  in scrub_add_page_to_wr_bio()  local
1643 	sbio = sctx->wr_curr_bio;  in scrub_add_page_to_wr_bio()
1644 	if (sbio->page_count == 0) {  in scrub_add_page_to_wr_bio()
1647 		sbio->physical = spage->physical_for_dev_replace;  in scrub_add_page_to_wr_bio()
1648 		sbio->logical = spage->logical;  in scrub_add_page_to_wr_bio()
1649 		sbio->dev = sctx->wr_tgtdev;  in scrub_add_page_to_wr_bio()
1650 		bio = sbio->bio;  in scrub_add_page_to_wr_bio()
1653 			sbio->bio = bio;  in scrub_add_page_to_wr_bio()
1656 		bio->bi_private = sbio;  in scrub_add_page_to_wr_bio()
1658 		bio_set_dev(bio, sbio->dev->bdev);  in scrub_add_page_to_wr_bio()
1659 		bio->bi_iter.bi_sector = sbio->physical >> 9;  in scrub_add_page_to_wr_bio()
1661 		sbio->status = 0;  in scrub_add_page_to_wr_bio()
1662 	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_wr_bio()
1664 		   sbio->logical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_wr_bio()
1670 	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);  in scrub_add_page_to_wr_bio()
1672 		if (sbio->page_count < 1) {  in scrub_add_page_to_wr_bio()
1673 			bio_put(sbio->bio);  in scrub_add_page_to_wr_bio()
1674 			sbio->bio = NULL;  in scrub_add_page_to_wr_bio()
1682 	sbio->pagev[sbio->page_count] = spage;  in scrub_add_page_to_wr_bio()
1684 	sbio->page_count++;  in scrub_add_page_to_wr_bio()
1685 	if (sbio->page_count == sctx->pages_per_wr_bio)  in scrub_add_page_to_wr_bio()
1694 	struct scrub_bio *sbio;  in scrub_wr_submit()  local
1699 	sbio = sctx->wr_curr_bio;  in scrub_wr_submit()
1701 	WARN_ON(!sbio->bio->bi_disk);  in scrub_wr_submit()
1707 	btrfsic_submit_bio(sbio->bio);  in scrub_wr_submit()
1712 	struct scrub_bio *sbio = bio->bi_private;  in scrub_wr_bio_end_io()  local
1713 	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;  in scrub_wr_bio_end_io()
1715 	sbio->status = bio->bi_status;  in scrub_wr_bio_end_io()
1716 	sbio->bio = bio;  in scrub_wr_bio_end_io()
1718 	btrfs_init_work(&sbio->work, scrub_wr_bio_end_io_worker, NULL, NULL);  in scrub_wr_bio_end_io()
1719 	btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);  in scrub_wr_bio_end_io()
1724 	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);  in scrub_wr_bio_end_io_worker()  local
1725 	struct scrub_ctx *sctx = sbio->sctx;  in scrub_wr_bio_end_io_worker()
1728 	WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);  in scrub_wr_bio_end_io_worker()
1729 	if (sbio->status) {  in scrub_wr_bio_end_io_worker()
1731 			&sbio->sctx->fs_info->dev_replace;  in scrub_wr_bio_end_io_worker()
1733 		for (i = 0; i < sbio->page_count; i++) {  in scrub_wr_bio_end_io_worker()
1734 			struct scrub_page *spage = sbio->pagev[i];  in scrub_wr_bio_end_io_worker()
1741 	for (i = 0; i < sbio->page_count; i++)  in scrub_wr_bio_end_io_worker()
1742 		scrub_page_put(sbio->pagev[i]);  in scrub_wr_bio_end_io_worker()
1744 	bio_put(sbio->bio);  in scrub_wr_bio_end_io_worker()
1745 	kfree(sbio);  in scrub_wr_bio_end_io_worker()
1954 	struct scrub_bio *sbio;  in scrub_submit()  local
1959 	sbio = sctx->bios[sctx->curr];  in scrub_submit()
1962 	btrfsic_submit_bio(sbio->bio);  in scrub_submit()
1969 	struct scrub_bio *sbio;  in scrub_add_page_to_rd_bio()  local
1989 	sbio = sctx->bios[sctx->curr];  in scrub_add_page_to_rd_bio()
1990 	if (sbio->page_count == 0) {  in scrub_add_page_to_rd_bio()
1993 		sbio->physical = spage->physical;  in scrub_add_page_to_rd_bio()
1994 		sbio->logical = spage->logical;  in scrub_add_page_to_rd_bio()
1995 		sbio->dev = spage->dev;  in scrub_add_page_to_rd_bio()
1996 		bio = sbio->bio;  in scrub_add_page_to_rd_bio()
1999 			sbio->bio = bio;  in scrub_add_page_to_rd_bio()
2002 		bio->bi_private = sbio;  in scrub_add_page_to_rd_bio()
2004 		bio_set_dev(bio, sbio->dev->bdev);  in scrub_add_page_to_rd_bio()
2005 		bio->bi_iter.bi_sector = sbio->physical >> 9;  in scrub_add_page_to_rd_bio()
2007 		sbio->status = 0;  in scrub_add_page_to_rd_bio()
2008 	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_rd_bio()
2010 		   sbio->logical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_rd_bio()
2012 		   sbio->dev != spage->dev) {  in scrub_add_page_to_rd_bio()
2017 	sbio->pagev[sbio->page_count] = spage;  in scrub_add_page_to_rd_bio()
2018 	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);  in scrub_add_page_to_rd_bio()
2020 		if (sbio->page_count < 1) {  in scrub_add_page_to_rd_bio()
2021 			bio_put(sbio->bio);  in scrub_add_page_to_rd_bio()
2022 			sbio->bio = NULL;  in scrub_add_page_to_rd_bio()
2031 	sbio->page_count++;  in scrub_add_page_to_rd_bio()
2032 	if (sbio->page_count == sctx->pages_per_rd_bio)  in scrub_add_page_to_rd_bio()
2245 	struct scrub_bio *sbio = bio->bi_private;  in scrub_bio_end_io()  local
2246 	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;  in scrub_bio_end_io()
2248 	sbio->status = bio->bi_status;  in scrub_bio_end_io()
2249 	sbio->bio = bio;  in scrub_bio_end_io()
2251 	btrfs_queue_work(fs_info->scrub_workers, &sbio->work);  in scrub_bio_end_io()
2256 	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);  in scrub_bio_end_io_worker()  local
2257 	struct scrub_ctx *sctx = sbio->sctx;  in scrub_bio_end_io_worker()
2260 	BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);  in scrub_bio_end_io_worker()
2261 	if (sbio->status) {  in scrub_bio_end_io_worker()
2262 		for (i = 0; i < sbio->page_count; i++) {  in scrub_bio_end_io_worker()
2263 			struct scrub_page *spage = sbio->pagev[i];  in scrub_bio_end_io_worker()
2271 	for (i = 0; i < sbio->page_count; i++) {  in scrub_bio_end_io_worker()
2272 		struct scrub_page *spage = sbio->pagev[i];  in scrub_bio_end_io_worker()
2280 	bio_put(sbio->bio);  in scrub_bio_end_io_worker()
2281 	sbio->bio = NULL;  in scrub_bio_end_io_worker()
2283 	sbio->next_free = sctx->first_free;  in scrub_bio_end_io_worker()
2284 	sctx->first_free = sbio->index;  in scrub_bio_end_io_worker()