57 	struct dmirror			*dmirror;  member
64 struct dmirror {  struct
120 	struct dmirror *dmirror;  in dmirror_fops_open()  local
124 	dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);  in dmirror_fops_open()
125 	if (dmirror == NULL)  in dmirror_fops_open()
128 	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);  in dmirror_fops_open()
129 	mutex_init(&dmirror->mutex);  in dmirror_fops_open()
130 	xa_init(&dmirror->pt);  in dmirror_fops_open()
132 	ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,  in dmirror_fops_open()
135 		kfree(dmirror);  in dmirror_fops_open()
139 	filp->private_data = dmirror;  in dmirror_fops_open()
145 	struct dmirror *dmirror = filp->private_data;  in dmirror_fops_release()  local
147 	mmu_interval_notifier_remove(&dmirror->notifier);  in dmirror_fops_release()
148 	xa_destroy(&dmirror->pt);  in dmirror_fops_release()
149 	kfree(dmirror);  in dmirror_fops_release()
160 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)  in dmirror_do_fault()  argument
186 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);  in dmirror_do_fault()
194 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,  in dmirror_do_update()  argument
205 	xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,  in dmirror_do_update()
207 		xa_erase(&dmirror->pt, pfn);  in dmirror_do_update()
214 	struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);  in dmirror_interval_invalidate()  local
221 	    range->migrate_pgmap_owner == dmirror->mdevice)  in dmirror_interval_invalidate()
225 		mutex_lock(&dmirror->mutex);  in dmirror_interval_invalidate()
226 	else if (!mutex_trylock(&dmirror->mutex))  in dmirror_interval_invalidate()
230 	dmirror_do_update(dmirror, range->start, range->end);  in dmirror_interval_invalidate()
232 	mutex_unlock(&dmirror->mutex);  in dmirror_interval_invalidate()
240 static int dmirror_range_fault(struct dmirror *dmirror,  in dmirror_range_fault()  argument
243 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_range_fault()
264 		mutex_lock(&dmirror->mutex);  in dmirror_range_fault()
267 			mutex_unlock(&dmirror->mutex);  in dmirror_range_fault()
273 	ret = dmirror_do_fault(dmirror, range);  in dmirror_range_fault()
275 	mutex_unlock(&dmirror->mutex);  in dmirror_range_fault()
280 static int dmirror_fault(struct dmirror *dmirror, unsigned long start,  in dmirror_fault()  argument
283 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_fault()
287 		.notifier = &dmirror->notifier,  in dmirror_fault()
292 		.dev_private_owner = dmirror->mdevice,  in dmirror_fault()
304 		ret = dmirror_range_fault(dmirror, &range);  in dmirror_fault()
313 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,  in dmirror_do_read()  argument
326 		entry = xa_load(&dmirror->pt, pfn);  in dmirror_do_read()
342 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)  in dmirror_read()  argument
359 		mutex_lock(&dmirror->mutex);  in dmirror_read()
360 		ret = dmirror_do_read(dmirror, start, end, &bounce);  in dmirror_read()
361 		mutex_unlock(&dmirror->mutex);  in dmirror_read()
366 		ret = dmirror_fault(dmirror, start, end, false);  in dmirror_read()
382 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,  in dmirror_do_write()  argument
395 		entry = xa_load(&dmirror->pt, pfn);  in dmirror_do_write()
411 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)  in dmirror_write()  argument
433 		mutex_lock(&dmirror->mutex);  in dmirror_write()
434 		ret = dmirror_do_write(dmirror, start, end, &bounce);  in dmirror_write()
435 		mutex_unlock(&dmirror->mutex);  in dmirror_write()
440 		ret = dmirror_fault(dmirror, start, end, true);  in dmirror_write()
574 					   struct dmirror *dmirror)  in dmirror_migrate_alloc_and_copy()  argument
576 	struct dmirror_device *mdevice = dmirror->mdevice;  in dmirror_migrate_alloc_and_copy()
612 		rpage->zone_device_data = dmirror;  in dmirror_migrate_alloc_and_copy()
623 					    struct dmirror *dmirror)  in dmirror_migrate_finalize_and_map()  argument
632 	mutex_lock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
653 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);  in dmirror_migrate_finalize_and_map()
655 			mutex_unlock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
660 	mutex_unlock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
664 static int dmirror_migrate(struct dmirror *dmirror,  in dmirror_migrate()  argument
669 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_migrate()
704 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate()
710 		dmirror_migrate_alloc_and_copy(&args, dmirror);  in dmirror_migrate()
712 		dmirror_migrate_finalize_and_map(&args, dmirror);  in dmirror_migrate()
722 	mutex_lock(&dmirror->mutex);  in dmirror_migrate()
723 	ret = dmirror_do_read(dmirror, start, end, &bounce);  in dmirror_migrate()
724 	mutex_unlock(&dmirror->mutex);  in dmirror_migrate()
740 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,  in dmirror_mkentry()  argument
757 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
781 	struct dmirror *dmirror = dmi->dmirror;  in dmirror_snapshot_invalidate()  local
784 		mutex_lock(&dmirror->mutex);  in dmirror_snapshot_invalidate()
785 	else if (!mutex_trylock(&dmirror->mutex))  in dmirror_snapshot_invalidate()
794 	mutex_unlock(&dmirror->mutex);  in dmirror_snapshot_invalidate()
802 static int dmirror_range_snapshot(struct dmirror *dmirror,  in dmirror_range_snapshot()  argument
806 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_range_snapshot()
814 	notifier.dmirror = dmirror;  in dmirror_range_snapshot()
840 		mutex_lock(&dmirror->mutex);  in dmirror_range_snapshot()
843 			mutex_unlock(&dmirror->mutex);  in dmirror_range_snapshot()
851 		dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);  in dmirror_range_snapshot()
853 	mutex_unlock(&dmirror->mutex);  in dmirror_range_snapshot()
859 static int dmirror_snapshot(struct dmirror *dmirror,  in dmirror_snapshot()  argument
862 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_snapshot()
872 		.dev_private_owner = dmirror->mdevice,  in dmirror_snapshot()
897 		ret = dmirror_range_snapshot(dmirror, &range, perm);  in dmirror_snapshot()
921 	struct dmirror *dmirror;  in dmirror_fops_unlocked_ioctl()  local
924 	dmirror = filp->private_data;  in dmirror_fops_unlocked_ioctl()
925 	if (!dmirror)  in dmirror_fops_unlocked_ioctl()
941 		ret = dmirror_read(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
945 		ret = dmirror_write(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
949 		ret = dmirror_migrate(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
953 		ret = dmirror_snapshot(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1018 						      struct dmirror *dmirror)  in dmirror_devmem_fault_alloc_and_copy()  argument
1040 		xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);  in dmirror_devmem_fault_alloc_and_copy()
1055 	struct dmirror *dmirror;  in dmirror_devmem_fault()  local
1064 	dmirror = rpage->zone_device_data;  in dmirror_devmem_fault()
1072 	args.pgmap_owner = dmirror->mdevice;  in dmirror_devmem_fault()
1078 	ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);  in dmirror_devmem_fault()