120 static void bio_put_slab(struct bio_set *bs)  in bio_put_slab()  argument
128 		if (bs->bio_slab == bio_slabs[i].slab) {  in bio_put_slab()
252 	struct bio_set *bs = bio->bi_pool;  in bio_free()  local
257 	if (bs) {  in bio_free()
258 		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));  in bio_free()
264 		p -= bs->front_pad;  in bio_free()
266 		mempool_free(p, &bs->bio_pool);  in bio_free()
350 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);  in bio_alloc_rescue()  local
354 		spin_lock(&bs->rescue_lock);  in bio_alloc_rescue()
355 		bio = bio_list_pop(&bs->rescue_list);  in bio_alloc_rescue()
356 		spin_unlock(&bs->rescue_lock);  in bio_alloc_rescue()
365 static void punt_bios_to_rescuer(struct bio_set *bs)  in punt_bios_to_rescuer()  argument
370 	if (WARN_ON_ONCE(!bs->rescue_workqueue))  in punt_bios_to_rescuer()
387 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
392 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
395 	spin_lock(&bs->rescue_lock);  in punt_bios_to_rescuer()
396 	bio_list_merge(&bs->rescue_list, &punt);  in punt_bios_to_rescuer()
397 	spin_unlock(&bs->rescue_lock);  in punt_bios_to_rescuer()
399 	queue_work(bs->rescue_workqueue, &bs->rescue_work);  in punt_bios_to_rescuer()
438 			     struct bio_set *bs)  in bio_alloc_bioset()  argument
447 	if (!bs) {  in bio_alloc_bioset()
456 		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&  in bio_alloc_bioset()
483 		    bs->rescue_workqueue)  in bio_alloc_bioset()
486 		p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
488 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
490 			p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
493 		front_pad = bs->front_pad;  in bio_alloc_bioset()
506 		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);  in bio_alloc_bioset()
508 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
510 			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);  in bio_alloc_bioset()
521 	bio->bi_pool = bs;  in bio_alloc_bioset()
527 	mempool_free(p, &bs->bio_pool);  in bio_alloc_bioset()
707 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)  in bio_clone_fast()  argument
711 	b = bio_alloc_bioset(gfp_mask, 0, bs);  in bio_clone_fast()
1478 		      gfp_t gfp, struct bio_set *bs)  in bio_split()  argument
1489 	split = bio_clone_fast(bio, gfp, bs);  in bio_split()
1549 void bioset_exit(struct bio_set *bs)  in bioset_exit()  argument
1551 	if (bs->rescue_workqueue)  in bioset_exit()
1552 		destroy_workqueue(bs->rescue_workqueue);  in bioset_exit()
1553 	bs->rescue_workqueue = NULL;  in bioset_exit()
1555 	mempool_exit(&bs->bio_pool);  in bioset_exit()
1556 	mempool_exit(&bs->bvec_pool);  in bioset_exit()
1558 	bioset_integrity_free(bs);  in bioset_exit()
1559 	if (bs->bio_slab)  in bioset_exit()
1560 		bio_put_slab(bs);  in bioset_exit()
1561 	bs->bio_slab = NULL;  in bioset_exit()
1586 int bioset_init(struct bio_set *bs,  in bioset_init()  argument
1593 	bs->front_pad = front_pad;  in bioset_init()
1595 	spin_lock_init(&bs->rescue_lock);  in bioset_init()
1596 	bio_list_init(&bs->rescue_list);  in bioset_init()
1597 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);  in bioset_init()
1599 	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);  in bioset_init()
1600 	if (!bs->bio_slab)  in bioset_init()
1603 	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))  in bioset_init()
1607 	    biovec_init_pool(&bs->bvec_pool, pool_size))  in bioset_init()
1613 	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);  in bioset_init()
1614 	if (!bs->rescue_workqueue)  in bioset_init()
1619 	bioset_exit(bs);  in bioset_init()
1628 int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)  in bioset_init_from_src()  argument
1638 	return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags);  in bioset_init_from_src()