83 	struct dm_rq_target_io *tio = info->tio;  in end_clone_bio()  local
90 	if (tio->error)  in end_clone_bio()
103 		tio->error = error;  in end_clone_bio()
111 	tio->completed += nr_bytes;  in end_clone_bio()
120 		blk_update_request(tio->orig, BLK_STS_OK, tio->completed);  in end_clone_bio()
131 		struct dm_rq_target_io *tio = tio_from_request(orig);  in rq_end_stats()  local
132 		tio->duration_jiffies = jiffies - tio->duration_jiffies;  in rq_end_stats()
134 				    blk_rq_pos(orig), tio->n_sectors, true,  in rq_end_stats()
135 				    tio->duration_jiffies, &tio->stats_aux);  in rq_end_stats()
159 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_end_request()  local
160 	struct mapped_device *md = tio->md;  in dm_end_request()
161 	struct request *rq = tio->orig;  in dm_end_request()
164 	tio->ti->type->release_clone_rq(clone, NULL);  in dm_end_request()
188 static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_requeue)  in dm_requeue_original_request()  argument
190 	struct mapped_device *md = tio->md;  in dm_requeue_original_request()
191 	struct request *rq = tio->orig;  in dm_requeue_original_request()
195 	if (tio->clone) {  in dm_requeue_original_request()
196 		blk_rq_unprep_clone(tio->clone);  in dm_requeue_original_request()
197 		tio->ti->type->release_clone_rq(tio->clone, NULL);  in dm_requeue_original_request()
207 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_done()  local
210 	if (tio->ti) {  in dm_done()
211 		rq_end_io = tio->ti->type->rq_end_io;  in dm_done()
214 			r = rq_end_io(tio->ti, clone, error, &tio->info);  in dm_done()
220 			disable_discard(tio->md);  in dm_done()
223 			disable_write_same(tio->md);  in dm_done()
226 			disable_write_zeroes(tio->md);  in dm_done()
239 		dm_requeue_original_request(tio, false);  in dm_done()
243 		dm_requeue_original_request(tio, true);  in dm_done()
257 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_softirq_done()  local
258 	struct request *clone = tio->clone;  in dm_softirq_done()
261 		struct mapped_device *md = tio->md;  in dm_softirq_done()
264 		blk_mq_end_request(rq, tio->error);  in dm_softirq_done()
272 	dm_done(clone, tio->error, mapped);  in dm_softirq_done()
281 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_complete_request()  local
283 	tio->error = error;  in dm_complete_request()
302 	struct dm_rq_target_io *tio = clone->end_io_data;  in end_clone_request()  local
304 	dm_complete_request(tio->orig, error);  in end_clone_request()
325 	struct dm_rq_target_io *tio = data;  in dm_rq_bio_constructor()  local
330 	info->tio = tio;  in dm_rq_bio_constructor()
337 		       struct dm_rq_target_io *tio, gfp_t gfp_mask)  in setup_clone()  argument
341 	r = blk_rq_prep_clone(clone, rq, &tio->md->bs, gfp_mask,  in setup_clone()
342 			      dm_rq_bio_constructor, tio);  in setup_clone()
347 	clone->end_io_data = tio;  in setup_clone()
349 	tio->clone = clone;  in setup_clone()
354 static void init_tio(struct dm_rq_target_io *tio, struct request *rq,  in init_tio()  argument
357 	tio->md = md;  in init_tio()
358 	tio->ti = NULL;  in init_tio()
359 	tio->clone = NULL;  in init_tio()
360 	tio->orig = rq;  in init_tio()
361 	tio->error = 0;  in init_tio()
362 	tio->completed = 0;  in init_tio()
369 		memset(&tio->info, 0, sizeof(tio->info));  in init_tio()
378 static int map_request(struct dm_rq_target_io *tio)  in map_request()  argument
381 	struct dm_target *ti = tio->ti;  in map_request()
382 	struct mapped_device *md = tio->md;  in map_request()
383 	struct request *rq = tio->orig;  in map_request()
387 	r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);  in map_request()
393 		if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {  in map_request()
395 			ti->type->release_clone_rq(clone, &tio->info);  in map_request()
406 			tio->ti->type->release_clone_rq(clone, &tio->info);  in map_request()
407 			tio->clone = NULL;  in map_request()
416 		dm_requeue_original_request(tio, true);  in map_request()
447 		struct dm_rq_target_io *tio = tio_from_request(orig);  in dm_start_request()  local
448 		tio->duration_jiffies = jiffies;  in dm_start_request()
449 		tio->n_sectors = blk_rq_sectors(orig);  in dm_start_request()
451 				    blk_rq_pos(orig), tio->n_sectors, false, 0,  in dm_start_request()
452 				    &tio->stats_aux);  in dm_start_request()
469 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);  in dm_mq_init_request()  local
475 	tio->md = md;  in dm_mq_init_request()
479 		tio->info.ptr = tio + 1;  in dm_mq_init_request()
489 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);  in dm_mq_queue_rq()  local
490 	struct mapped_device *md = tio->md;  in dm_mq_queue_rq()
512 	init_tio(tio, rq, md);  in dm_mq_queue_rq()
517 	tio->ti = ti;  in dm_mq_queue_rq()
520 	if (map_request(tio) == DM_MAPIO_REQUEUE) {  in dm_mq_queue_rq()