64 			struct nouveau_svmm *svmm;  member
80 	struct nouveau_svmm *svmm;  member
164 	if (!cli->svm.svmm) {  in nouveau_svmm_bind()
181 		nouveau_dmem_migrate_vma(cli->drm, cli->svm.svmm, vma, addr,  in nouveau_svmm_bind()
201 nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst)  in nouveau_svmm_part()  argument
204 	if (svmm) {  in nouveau_svmm_part()
205 		mutex_lock(&svmm->vmm->cli->drm->svm->mutex);  in nouveau_svmm_part()
206 		ivmm = nouveau_ivmm_find(svmm->vmm->cli->drm->svm, inst);  in nouveau_svmm_part()
211 		mutex_unlock(&svmm->vmm->cli->drm->svm->mutex);  in nouveau_svmm_part()
217 nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst)  in nouveau_svmm_join()  argument
220 	if (svmm) {  in nouveau_svmm_join()
223 		ivmm->svmm = svmm;  in nouveau_svmm_join()
226 		mutex_lock(&svmm->vmm->cli->drm->svm->mutex);  in nouveau_svmm_join()
227 		list_add(&ivmm->head, &svmm->vmm->cli->drm->svm->inst);  in nouveau_svmm_join()
228 		mutex_unlock(&svmm->vmm->cli->drm->svm->mutex);  in nouveau_svmm_join()
235 nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit)  in nouveau_svmm_invalidate()  argument
238 		bool super = svmm->vmm->vmm.object.client->super;  in nouveau_svmm_invalidate()
239 		svmm->vmm->vmm.object.client->super = true;  in nouveau_svmm_invalidate()
240 		nvif_object_mthd(&svmm->vmm->vmm.object, NVIF_VMM_V0_PFNCLR,  in nouveau_svmm_invalidate()
245 		svmm->vmm->vmm.object.client->super = super;  in nouveau_svmm_invalidate()
253 	struct nouveau_svmm *svmm =  in nouveau_svmm_invalidate_range_start()  local
261 	SVMM_DBG(svmm, "invalidate %016lx-%016lx", start, limit);  in nouveau_svmm_invalidate_range_start()
263 	mutex_lock(&svmm->mutex);  in nouveau_svmm_invalidate_range_start()
264 	if (unlikely(!svmm->vmm))  in nouveau_svmm_invalidate_range_start()
272 	    update->migrate_pgmap_owner == svmm->vmm->cli->drm->dev)  in nouveau_svmm_invalidate_range_start()
275 	if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) {  in nouveau_svmm_invalidate_range_start()
276 		if (start < svmm->unmanaged.start) {  in nouveau_svmm_invalidate_range_start()
277 			nouveau_svmm_invalidate(svmm, start,  in nouveau_svmm_invalidate_range_start()
278 						svmm->unmanaged.limit);  in nouveau_svmm_invalidate_range_start()
280 		start = svmm->unmanaged.limit;  in nouveau_svmm_invalidate_range_start()
283 	nouveau_svmm_invalidate(svmm, start, limit);  in nouveau_svmm_invalidate_range_start()
286 	mutex_unlock(&svmm->mutex);  in nouveau_svmm_invalidate_range_start()
303 	struct nouveau_svmm *svmm = *psvmm;  in nouveau_svmm_fini()  local
304 	if (svmm) {  in nouveau_svmm_fini()
305 		mutex_lock(&svmm->mutex);  in nouveau_svmm_fini()
306 		svmm->vmm = NULL;  in nouveau_svmm_fini()
307 		mutex_unlock(&svmm->mutex);  in nouveau_svmm_fini()
308 		mmu_notifier_put(&svmm->notifier);  in nouveau_svmm_fini()
318 	struct nouveau_svmm *svmm;  in nouveau_svmm_init()  local
327 	if (!(svmm = kzalloc(sizeof(*svmm), GFP_KERNEL)))  in nouveau_svmm_init()
329 	svmm->vmm = &cli->svm;  in nouveau_svmm_init()
330 	svmm->unmanaged.start = args->unmanaged_addr;  in nouveau_svmm_init()
331 	svmm->unmanaged.limit = args->unmanaged_addr + args->unmanaged_size;  in nouveau_svmm_init()
332 	mutex_init(&svmm->mutex);  in nouveau_svmm_init()
357 	svmm->notifier.ops = &nouveau_mn_ops;  in nouveau_svmm_init()
358 	ret = __mmu_notifier_register(&svmm->notifier, current->mm);  in nouveau_svmm_init()
363 	cli->svm.svmm = svmm;  in nouveau_svmm_init()
373 	kfree(svmm);  in nouveau_svmm_init()
484 	struct nouveau_svmm *svmm;  member
502 		mutex_lock(&sn->svmm->mutex);  in nouveau_svm_range_invalidate()
503 	else if (!mutex_trylock(&sn->svmm->mutex))  in nouveau_svm_range_invalidate()
506 	mutex_unlock(&sn->svmm->mutex);  in nouveau_svm_range_invalidate()
562 static int nouveau_range_fault(struct nouveau_svmm *svmm,  in nouveau_range_fault()  argument
597 		mutex_lock(&svmm->mutex);  in nouveau_range_fault()
600 			mutex_unlock(&svmm->mutex);  in nouveau_range_fault()
608 	svmm->vmm->vmm.object.client->super = true;  in nouveau_range_fault()
609 	ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL);  in nouveau_range_fault()
610 	svmm->vmm->vmm.object.client->super = false;  in nouveau_range_fault()
611 	mutex_unlock(&svmm->mutex);  in nouveau_range_fault()
624 	struct nouveau_svmm *svmm;  in nouveau_svm_fault()  local
664 	for (fi = 0, svmm = NULL; fi < buffer->fault_nr; fi++) {  in nouveau_svm_fault()
665 		if (!svmm || buffer->fault[fi]->inst != inst) {  in nouveau_svm_fault()
668 			svmm = ivmm ? ivmm->svmm : NULL;  in nouveau_svm_fault()
670 			SVM_DBG(svm, "inst %016llx -> svm-%p", inst, svmm);  in nouveau_svm_fault()
672 		buffer->fault[fi]->svmm = svmm;  in nouveau_svm_fault()
688 		if (!(svmm = buffer->fault[fi]->svmm)) {  in nouveau_svm_fault()
692 		SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr);  in nouveau_svm_fault()
699 		if (start < svmm->unmanaged.limit)  in nouveau_svm_fault()
700 			limit = min_t(u64, limit, svmm->unmanaged.start);  in nouveau_svm_fault()
727 		mm = svmm->notifier.mm;  in nouveau_svm_fault()
733 		notifier.svmm = svmm;  in nouveau_svm_fault()
738 			ret = nouveau_range_fault(svmm, svm->drm, &args.i,  in nouveau_svm_fault()
755 			if (buffer->fault[fn]->svmm != svmm ||  in nouveau_svm_fault()
814 nouveau_pfns_map(struct nouveau_svmm *svmm, struct mm_struct *mm,  in nouveau_pfns_map()  argument
823 	mutex_lock(&svmm->mutex);  in nouveau_pfns_map()
825 	svmm->vmm->vmm.object.client->super = true;  in nouveau_pfns_map()
826 	ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, sizeof(*args) +  in nouveau_pfns_map()
828 	svmm->vmm->vmm.object.client->super = false;  in nouveau_pfns_map()
830 	mutex_unlock(&svmm->mutex);  in nouveau_pfns_map()