121 static void __flush_svm_range_dev(struct intel_svm *svm,  in __flush_svm_range_dev()  argument
131 	qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih);  in __flush_svm_range_dev()
134 					 svm->pasid, sdev->qdep, address,  in __flush_svm_range_dev()
138 static void intel_flush_svm_range_dev(struct intel_svm *svm,  in intel_flush_svm_range_dev()  argument
149 		__flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih);  in intel_flush_svm_range_dev()
154 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address,  in intel_flush_svm_range()  argument
160 	list_for_each_entry_rcu(sdev, &svm->devs, list)  in intel_flush_svm_range()
161 		intel_flush_svm_range_dev(svm, sdev, address, pages, ih);  in intel_flush_svm_range()
170 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_invalidate_range()  local
172 	intel_flush_svm_range(svm, start,  in intel_invalidate_range()
178 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_mm_release()  local
194 	list_for_each_entry_rcu(sdev, &svm->devs, list)  in intel_mm_release()
196 					    svm->pasid, true);  in intel_mm_release()
209 #define for_each_svm_dev(sdev, svm, d)			\  argument
210 	list_for_each_entry((sdev), &(svm)->devs, list)	\
218 	struct intel_svm *svm;  in pasid_to_svm_sdev()  local
227 	svm = ioasid_find(NULL, pasid, NULL);  in pasid_to_svm_sdev()
228 	if (IS_ERR(svm))  in pasid_to_svm_sdev()
229 		return PTR_ERR(svm);  in pasid_to_svm_sdev()
231 	if (!svm)  in pasid_to_svm_sdev()
238 	if (WARN_ON(list_empty(&svm->devs)))  in pasid_to_svm_sdev()
242 	list_for_each_entry_rcu(d, &svm->devs, list) {  in pasid_to_svm_sdev()
251 	*rsvm = svm;  in pasid_to_svm_sdev()
264 	struct intel_svm *svm = NULL;  in intel_svm_bind_gpasid()  local
303 	ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev);  in intel_svm_bind_gpasid()
314 				     svm->pasid);  in intel_svm_bind_gpasid()
319 	if (!svm) {  in intel_svm_bind_gpasid()
321 		svm = kzalloc(sizeof(*svm), GFP_KERNEL);  in intel_svm_bind_gpasid()
322 		if (!svm) {  in intel_svm_bind_gpasid()
331 		svm->mm = get_task_mm(current);  in intel_svm_bind_gpasid()
332 		svm->pasid = data->hpasid;  in intel_svm_bind_gpasid()
334 			svm->gpasid = data->gpasid;  in intel_svm_bind_gpasid()
335 			svm->flags |= SVM_FLAG_GUEST_PASID;  in intel_svm_bind_gpasid()
337 		ioasid_set_data(data->hpasid, svm);  in intel_svm_bind_gpasid()
338 		INIT_LIST_HEAD_RCU(&svm->devs);  in intel_svm_bind_gpasid()
339 		mmput(svm->mm);  in intel_svm_bind_gpasid()
385 	svm->flags |= SVM_FLAG_GUEST_MODE;  in intel_svm_bind_gpasid()
388 	list_add_rcu(&sdev->list, &svm->devs);  in intel_svm_bind_gpasid()
390 	if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) {  in intel_svm_bind_gpasid()
392 		kfree(svm);  in intel_svm_bind_gpasid()
403 	struct intel_svm *svm;  in intel_svm_unbind_gpasid()  local
410 	ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);  in intel_svm_unbind_gpasid()
420 						    svm->pasid, false);  in intel_svm_unbind_gpasid()
421 			intel_svm_drain_prq(dev, svm->pasid);  in intel_svm_unbind_gpasid()
424 			if (list_empty(&svm->devs)) {  in intel_svm_unbind_gpasid()
435 				kfree(svm);  in intel_svm_unbind_gpasid()
471 	struct intel_svm *svm = NULL;  in intel_svm_bind_mm()  local
504 			svm = t;  in intel_svm_bind_mm()
505 			if (svm->pasid >= pasid_max) {  in intel_svm_bind_mm()
508 					 svm->pasid);  in intel_svm_bind_mm()
514 			for_each_svm_dev(sdev, svm, dev) {  in intel_svm_bind_mm()
556 	if (!svm) {  in intel_svm_bind_mm()
557 		svm = kzalloc(sizeof(*svm), GFP_KERNEL);  in intel_svm_bind_mm()
558 		if (!svm) {  in intel_svm_bind_mm()
568 		svm->pasid = ioasid_alloc(NULL, PASID_MIN,  in intel_svm_bind_mm()
569 					  pasid_max - 1, svm);  in intel_svm_bind_mm()
570 		if (svm->pasid == INVALID_IOASID) {  in intel_svm_bind_mm()
571 			kfree(svm);  in intel_svm_bind_mm()
576 		svm->notifier.ops = &intel_mmuops;  in intel_svm_bind_mm()
577 		svm->mm = mm;  in intel_svm_bind_mm()
578 		svm->flags = flags;  in intel_svm_bind_mm()
579 		INIT_LIST_HEAD_RCU(&svm->devs);  in intel_svm_bind_mm()
580 		INIT_LIST_HEAD(&svm->list);  in intel_svm_bind_mm()
583 			ret = mmu_notifier_register(&svm->notifier, mm);  in intel_svm_bind_mm()
585 				ioasid_free(svm->pasid);  in intel_svm_bind_mm()
586 				kfree(svm);  in intel_svm_bind_mm()
595 				svm->pasid, FLPT_DEFAULT_DID,  in intel_svm_bind_mm()
602 				mmu_notifier_unregister(&svm->notifier, mm);  in intel_svm_bind_mm()
603 			ioasid_free(svm->pasid);  in intel_svm_bind_mm()
604 			kfree(svm);  in intel_svm_bind_mm()
609 		list_add_tail(&svm->list, &global_svm_list);  in intel_svm_bind_mm()
612 			load_pasid(mm, svm->pasid);  in intel_svm_bind_mm()
622 						svm->pasid, FLPT_DEFAULT_DID,  in intel_svm_bind_mm()
632 	list_add_rcu(&sdev->list, &svm->devs);  in intel_svm_bind_mm()
634 	sdev->pasid = svm->pasid;  in intel_svm_bind_mm()
648 	struct intel_svm *svm;  in intel_svm_unbind_mm()  local
655 	ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);  in intel_svm_unbind_mm()
671 						    svm->pasid, false);  in intel_svm_unbind_mm()
672 			intel_svm_drain_prq(dev, svm->pasid);  in intel_svm_unbind_mm()
675 			if (list_empty(&svm->devs)) {  in intel_svm_unbind_mm()
676 				ioasid_free(svm->pasid);  in intel_svm_unbind_mm()
677 				if (svm->mm) {  in intel_svm_unbind_mm()
678 					mmu_notifier_unregister(&svm->notifier, svm->mm);  in intel_svm_unbind_mm()
680 					load_pasid(svm->mm, PASID_DISABLED);  in intel_svm_unbind_mm()
682 				list_del(&svm->list);  in intel_svm_unbind_mm()
687 				memset(svm, 0x6b, sizeof(*svm));  in intel_svm_unbind_mm()
688 				kfree(svm);  in intel_svm_unbind_mm()
896 	struct intel_svm *svm = NULL;  in prq_event_thread()  local
936 		if (!svm || svm->pasid != req->pasid) {  in prq_event_thread()
938 			svm = ioasid_find(NULL, req->pasid, NULL);  in prq_event_thread()
943 			if (IS_ERR_OR_NULL(svm)) {  in prq_event_thread()
956 			list_for_each_entry_rcu(t, &svm->devs, list) {  in prq_event_thread()
968 		if (!svm->mm)  in prq_event_thread()
979 		if (svm->flags & SVM_FLAG_GUEST_MODE) {  in prq_event_thread()
987 		if (!mmget_not_zero(svm->mm))  in prq_event_thread()
990 		mmap_read_lock(svm->mm);  in prq_event_thread()
991 		vma = find_extend_vma(svm->mm, address);  in prq_event_thread()
1006 		mmap_read_unlock(svm->mm);  in prq_event_thread()
1007 		mmput(svm->mm);  in prq_event_thread()
1019 		svm = NULL;  in prq_event_thread()
1132 	struct intel_svm *svm = NULL;  in intel_svm_page_response()  local
1169 	ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev);  in intel_svm_page_response()
1179 	if (svm->flags & SVM_FLAG_GUEST_MODE) {  in intel_svm_page_response()
1188 		if (mm != svm->mm) {  in intel_svm_page_response()