116 static int dmz_submit_bio(struct dmz_target *dmz, struct dm_zone *zone,  in dmz_submit_bio()  argument
128 	clone = bio_clone_fast(bio, GFP_NOIO, &dmz->bio_set);  in dmz_submit_bio()
135 		dmz_start_sect(dmz->metadata, zone) + dmz_blk2sect(chunk_block);  in dmz_submit_bio()
154 static void dmz_handle_read_zero(struct dmz_target *dmz, struct bio *bio,  in dmz_handle_read_zero()  argument
170 static int dmz_handle_read(struct dmz_target *dmz, struct dm_zone *zone,  in dmz_handle_read()  argument
173 	struct dmz_metadata *zmd = dmz->metadata;  in dmz_handle_read()
230 			ret = dmz_submit_bio(dmz, rzone, bio,  in dmz_handle_read()
237 			dmz_handle_read_zero(dmz, bio, chunk_block, 1);  in dmz_handle_read()
250 static int dmz_handle_direct_write(struct dmz_target *dmz,  in dmz_handle_direct_write()  argument
255 	struct dmz_metadata *zmd = dmz->metadata;  in dmz_handle_direct_write()
263 	ret = dmz_submit_bio(dmz, zone, bio, chunk_block, nr_blocks);  in dmz_handle_direct_write()
283 static int dmz_handle_buffered_write(struct dmz_target *dmz,  in dmz_handle_buffered_write()  argument
288 	struct dmz_metadata *zmd = dmz->metadata;  in dmz_handle_buffered_write()
301 	ret = dmz_submit_bio(dmz, bzone, bio, chunk_block, nr_blocks);  in dmz_handle_buffered_write()
319 static int dmz_handle_write(struct dmz_target *dmz, struct dm_zone *zone,  in dmz_handle_write()  argument
322 	struct dmz_metadata *zmd = dmz->metadata;  in dmz_handle_write()
344 		return dmz_handle_direct_write(dmz, zone, bio,  in dmz_handle_write()
352 	return dmz_handle_buffered_write(dmz, zone, bio, chunk_block, nr_blocks);  in dmz_handle_write()
358 static int dmz_handle_discard(struct dmz_target *dmz, struct dm_zone *zone,  in dmz_handle_discard()  argument
361 	struct dmz_metadata *zmd = dmz->metadata;  in dmz_handle_discard()
375 		dmz_metadata_label(dmz->metadata),  in dmz_handle_discard()
396 static void dmz_handle_bio(struct dmz_target *dmz, struct dm_chunk_work *cw,  in dmz_handle_bio()  argument
401 	struct dmz_metadata *zmd = dmz->metadata;  in dmz_handle_bio()
428 		ret = dmz_handle_read(dmz, zone, bio);  in dmz_handle_bio()
431 		ret = dmz_handle_write(dmz, zone, bio);  in dmz_handle_bio()
435 		ret = dmz_handle_discard(dmz, zone, bio);  in dmz_handle_bio()
439 		      dmz_metadata_label(dmz->metadata), bio_op(bio));  in dmz_handle_bio()
482 	struct dmz_target *dmz = cw->target;  in dmz_chunk_work()  local
485 	mutex_lock(&dmz->chunk_lock);  in dmz_chunk_work()
489 		mutex_unlock(&dmz->chunk_lock);  in dmz_chunk_work()
490 		dmz_handle_bio(dmz, cw, bio);  in dmz_chunk_work()
491 		mutex_lock(&dmz->chunk_lock);  in dmz_chunk_work()
498 	mutex_unlock(&dmz->chunk_lock);  in dmz_chunk_work()
506 	struct dmz_target *dmz = container_of(work, struct dmz_target, flush_work.work);  in dmz_flush_work()  local
511 	ret = dmz_flush_metadata(dmz->metadata);  in dmz_flush_work()
514 			dmz_metadata_label(dmz->metadata), ret);  in dmz_flush_work()
518 		spin_lock(&dmz->flush_lock);  in dmz_flush_work()
519 		bio = bio_list_pop(&dmz->flush_list);  in dmz_flush_work()
520 		spin_unlock(&dmz->flush_lock);  in dmz_flush_work()
528 	queue_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);  in dmz_flush_work()
535 static int dmz_queue_chunk_work(struct dmz_target *dmz, struct bio *bio)  in dmz_queue_chunk_work()  argument
537 	unsigned int chunk = dmz_bio_chunk(dmz->metadata, bio);  in dmz_queue_chunk_work()
541 	mutex_lock(&dmz->chunk_lock);  in dmz_queue_chunk_work()
544 	cw = radix_tree_lookup(&dmz->chunk_rxtree, chunk);  in dmz_queue_chunk_work()
557 		cw->target = dmz;  in dmz_queue_chunk_work()
561 		ret = radix_tree_insert(&dmz->chunk_rxtree, chunk, cw);  in dmz_queue_chunk_work()
570 	if (queue_work(dmz->chunk_wq, &cw->work))  in dmz_queue_chunk_work()
573 	mutex_unlock(&dmz->chunk_lock);  in dmz_queue_chunk_work()
628 	struct dmz_target *dmz = ti->private;  in dmz_map()  local
629 	struct dmz_metadata *zmd = dmz->metadata;  in dmz_map()
661 		spin_lock(&dmz->flush_lock);  in dmz_map()
662 		bio_list_add(&dmz->flush_list, bio);  in dmz_map()
663 		spin_unlock(&dmz->flush_lock);  in dmz_map()
664 		mod_delayed_work(dmz->flush_wq, &dmz->flush_work, 0);  in dmz_map()
674 	ret = dmz_queue_chunk_work(dmz, bio);  in dmz_map()
692 	struct dmz_target *dmz = ti->private;  in dmz_get_zoned_device()  local
715 		if (dmz->ddev[0]) {  in dmz_get_zoned_device()
719 		dev = &dmz->dev[idx];  in dmz_get_zoned_device()
722 		if (dmz->ddev[idx]) {  in dmz_get_zoned_device()
730 		dev = &dmz->dev[idx];  in dmz_get_zoned_device()
742 	dmz->ddev[idx] = ddev;  in dmz_get_zoned_device()
755 	struct dmz_target *dmz = ti->private;  in dmz_put_zoned_device()  local
758 	for (i = 0; i < dmz->nr_ddevs; i++) {  in dmz_put_zoned_device()
759 		if (dmz->ddev[i]) {  in dmz_put_zoned_device()
760 			dm_put_device(ti, dmz->ddev[i]);  in dmz_put_zoned_device()
761 			dmz->ddev[i] = NULL;  in dmz_put_zoned_device()
768 	struct dmz_target *dmz = ti->private;  in dmz_fixup_devices()  local
778 	if (dmz->nr_ddevs > 1) {  in dmz_fixup_devices()
779 		reg_dev = &dmz->dev[0];  in dmz_fixup_devices()
784 		for (i = 1; i < dmz->nr_ddevs; i++) {  in dmz_fixup_devices()
785 			zoned_dev = &dmz->dev[i];  in dmz_fixup_devices()
803 		zoned_dev = &dmz->dev[0];  in dmz_fixup_devices()
822 		for (i = 1; i < dmz->nr_ddevs; i++) {  in dmz_fixup_devices()
823 			dmz->dev[i].zone_offset = zone_offset;  in dmz_fixup_devices()
824 			zone_offset += dmz->dev[i].nr_zones;  in dmz_fixup_devices()
835 	struct dmz_target *dmz;  in dmz_ctr()  local
845 	dmz = kzalloc(sizeof(struct dmz_target), GFP_KERNEL);  in dmz_ctr()
846 	if (!dmz) {  in dmz_ctr()
850 	dmz->dev = kcalloc(argc, sizeof(struct dmz_dev), GFP_KERNEL);  in dmz_ctr()
851 	if (!dmz->dev) {  in dmz_ctr()
853 		kfree(dmz);  in dmz_ctr()
856 	dmz->ddev = kcalloc(argc, sizeof(struct dm_dev *), GFP_KERNEL);  in dmz_ctr()
857 	if (!dmz->ddev) {  in dmz_ctr()
862 	dmz->nr_ddevs = argc;  in dmz_ctr()
864 	ti->private = dmz;  in dmz_ctr()
877 	ret = dmz_ctr_metadata(dmz->dev, argc, &dmz->metadata,  in dmz_ctr()
885 	ti->max_io_len = dmz_zone_nr_sectors(dmz->metadata);  in dmz_ctr()
894 	ti->len = (sector_t)dmz_nr_chunks(dmz->metadata) <<  in dmz_ctr()
895 		dmz_zone_nr_sectors_shift(dmz->metadata);  in dmz_ctr()
898 	ret = bioset_init(&dmz->bio_set, DMZ_MIN_BIOS, 0, 0);  in dmz_ctr()
905 	mutex_init(&dmz->chunk_lock);  in dmz_ctr()
906 	INIT_RADIX_TREE(&dmz->chunk_rxtree, GFP_NOIO);  in dmz_ctr()
907 	dmz->chunk_wq = alloc_workqueue("dmz_cwq_%s",  in dmz_ctr()
909 					dmz_metadata_label(dmz->metadata));  in dmz_ctr()
910 	if (!dmz->chunk_wq) {  in dmz_ctr()
917 	spin_lock_init(&dmz->flush_lock);  in dmz_ctr()
918 	bio_list_init(&dmz->flush_list);  in dmz_ctr()
919 	INIT_DELAYED_WORK(&dmz->flush_work, dmz_flush_work);  in dmz_ctr()
920 	dmz->flush_wq = alloc_ordered_workqueue("dmz_fwq_%s", WQ_MEM_RECLAIM,  in dmz_ctr()
921 						dmz_metadata_label(dmz->metadata));  in dmz_ctr()
922 	if (!dmz->flush_wq) {  in dmz_ctr()
927 	mod_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);  in dmz_ctr()
930 	for (i = 0; i < dmz->nr_ddevs; i++) {  in dmz_ctr()
931 		ret = dmz_ctr_reclaim(dmz->metadata, &dmz->dev[i].reclaim, i);  in dmz_ctr()
939 	       dmz_metadata_label(dmz->metadata),  in dmz_ctr()
945 	destroy_workqueue(dmz->flush_wq);  in dmz_ctr()
947 	destroy_workqueue(dmz->chunk_wq);  in dmz_ctr()
949 	mutex_destroy(&dmz->chunk_lock);  in dmz_ctr()
950 	bioset_exit(&dmz->bio_set);  in dmz_ctr()
952 	dmz_dtr_metadata(dmz->metadata);  in dmz_ctr()
956 	kfree(dmz->dev);  in dmz_ctr()
957 	kfree(dmz);  in dmz_ctr()
967 	struct dmz_target *dmz = ti->private;  in dmz_dtr()  local
970 	flush_workqueue(dmz->chunk_wq);  in dmz_dtr()
971 	destroy_workqueue(dmz->chunk_wq);  in dmz_dtr()
973 	for (i = 0; i < dmz->nr_ddevs; i++)  in dmz_dtr()
974 		dmz_dtr_reclaim(dmz->dev[i].reclaim);  in dmz_dtr()
976 	cancel_delayed_work_sync(&dmz->flush_work);  in dmz_dtr()
977 	destroy_workqueue(dmz->flush_wq);  in dmz_dtr()
979 	(void) dmz_flush_metadata(dmz->metadata);  in dmz_dtr()
981 	dmz_dtr_metadata(dmz->metadata);  in dmz_dtr()
983 	bioset_exit(&dmz->bio_set);  in dmz_dtr()
987 	mutex_destroy(&dmz->chunk_lock);  in dmz_dtr()
989 	kfree(dmz->dev);  in dmz_dtr()
990 	kfree(dmz);  in dmz_dtr()
998 	struct dmz_target *dmz = ti->private;  in dmz_io_hints()  local
999 	unsigned int chunk_sectors = dmz_zone_nr_sectors(dmz->metadata);  in dmz_io_hints()
1026 	struct dmz_target *dmz = ti->private;  in dmz_prepare_ioctl()  local
1027 	struct dmz_dev *dev = &dmz->dev[0];  in dmz_prepare_ioctl()
1042 	struct dmz_target *dmz = ti->private;  in dmz_suspend()  local
1045 	flush_workqueue(dmz->chunk_wq);  in dmz_suspend()
1046 	for (i = 0; i < dmz->nr_ddevs; i++)  in dmz_suspend()
1047 		dmz_suspend_reclaim(dmz->dev[i].reclaim);  in dmz_suspend()
1048 	cancel_delayed_work_sync(&dmz->flush_work);  in dmz_suspend()
1056 	struct dmz_target *dmz = ti->private;  in dmz_resume()  local
1059 	queue_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);  in dmz_resume()
1060 	for (i = 0; i < dmz->nr_ddevs; i++)  in dmz_resume()
1061 		dmz_resume_reclaim(dmz->dev[i].reclaim);  in dmz_resume()
1067 	struct dmz_target *dmz = ti->private;  in dmz_iterate_devices()  local
1068 	unsigned int zone_nr_sectors = dmz_zone_nr_sectors(dmz->metadata);  in dmz_iterate_devices()
1072 	for (i = 0; i < dmz->nr_ddevs; i++) {  in dmz_iterate_devices()
1073 		capacity = dmz->dev[i].capacity & ~(zone_nr_sectors - 1);  in dmz_iterate_devices()
1074 		r = fn(ti, dmz->ddev[i], 0, capacity, data);  in dmz_iterate_devices()
1085 	struct dmz_target *dmz = ti->private;  in dmz_status()  local
1094 		       dmz_nr_zones(dmz->metadata),  in dmz_status()
1095 		       dmz_nr_unmap_cache_zones(dmz->metadata),  in dmz_status()
1096 		       dmz_nr_cache_zones(dmz->metadata));  in dmz_status()
1097 		for (i = 0; i < dmz->nr_ddevs; i++) {  in dmz_status()
1103 			    (dmz_nr_cache_zones(dmz->metadata) > 0))  in dmz_status()
1106 			       dmz_nr_unmap_rnd_zones(dmz->metadata, i),  in dmz_status()
1107 			       dmz_nr_rnd_zones(dmz->metadata, i),  in dmz_status()
1108 			       dmz_nr_unmap_seq_zones(dmz->metadata, i),  in dmz_status()
1109 			       dmz_nr_seq_zones(dmz->metadata, i));  in dmz_status()
1113 		dev = &dmz->dev[0];  in dmz_status()
1116 		for (i = 1; i < dmz->nr_ddevs; i++) {  in dmz_status()
1117 			dev = &dmz->dev[i];  in dmz_status()
1129 	struct dmz_target *dmz = ti->private;  in dmz_message()  local
1135 		for (i = 0; i < dmz->nr_ddevs; i++)  in dmz_message()
1136 			dmz_schedule_reclaim(dmz->dev[i].reclaim);  in dmz_message()