34 struct pasid_state {  struct
56 	struct pasid_state **states;  argument
69 	struct pasid_state *state;
152 static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,  in __get_pasid_state_ptr()
155 	struct pasid_state **root, **ptr;  in __get_pasid_state_ptr()
178 		root   = (struct pasid_state **)*ptr;  in __get_pasid_state_ptr()
186 			   struct pasid_state *pasid_state,  in set_pasid_state()  argument
189 	struct pasid_state **ptr;  in set_pasid_state()
204 	*ptr = pasid_state;  in set_pasid_state()
216 	struct pasid_state **ptr;  in clear_pasid_state()
231 static struct pasid_state *get_pasid_state(struct device_state *dev_state,  in get_pasid_state()
234 	struct pasid_state **ptr, *ret = NULL;  in get_pasid_state()
253 static void free_pasid_state(struct pasid_state *pasid_state)  in free_pasid_state()  argument
255 	kfree(pasid_state);  in free_pasid_state()
258 static void put_pasid_state(struct pasid_state *pasid_state)  in put_pasid_state()  argument
260 	if (atomic_dec_and_test(&pasid_state->count))  in put_pasid_state()
261 		wake_up(&pasid_state->wq);  in put_pasid_state()
264 static void put_pasid_state_wait(struct pasid_state *pasid_state)  in put_pasid_state_wait()  argument
266 	atomic_dec(&pasid_state->count);  in put_pasid_state_wait()
267 	wait_event(pasid_state->wq, !atomic_read(&pasid_state->count));  in put_pasid_state_wait()
268 	free_pasid_state(pasid_state);  in put_pasid_state_wait()
271 static void unbind_pasid(struct pasid_state *pasid_state)  in unbind_pasid()  argument
275 	domain = pasid_state->device_state->domain;  in unbind_pasid()
281 	pasid_state->invalid = true;  in unbind_pasid()
287 	amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);  in unbind_pasid()
293 static void free_pasid_states_level1(struct pasid_state **tbl)  in free_pasid_states_level1()
305 static void free_pasid_states_level2(struct pasid_state **tbl)  in free_pasid_states_level2()
307 	struct pasid_state **ptr;  in free_pasid_states_level2()
314 		ptr = (struct pasid_state **)tbl[i];  in free_pasid_states_level2()
321 	struct pasid_state *pasid_state;  in free_pasid_states()  local
325 		pasid_state = get_pasid_state(dev_state, i);  in free_pasid_states()
326 		if (pasid_state == NULL)  in free_pasid_states()
329 		put_pasid_state(pasid_state);  in free_pasid_states()
335 		mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in free_pasid_states()
337 		put_pasid_state_wait(pasid_state); /* Reference taken in  in free_pasid_states()
354 static struct pasid_state *mn_to_state(struct mmu_notifier *mn)  in mn_to_state()
356 	return container_of(mn, struct pasid_state, mn);  in mn_to_state()
363 	struct pasid_state *pasid_state;  in mn_invalidate_range()  local
366 	pasid_state = mn_to_state(mn);  in mn_invalidate_range()
367 	dev_state   = pasid_state->device_state;  in mn_invalidate_range()
370 		amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,  in mn_invalidate_range()
373 		amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);  in mn_invalidate_range()
378 	struct pasid_state *pasid_state;  in mn_release()  local
384 	pasid_state    = mn_to_state(mn);  in mn_release()
385 	dev_state      = pasid_state->device_state;  in mn_release()
386 	run_inv_ctx_cb = !pasid_state->invalid;  in mn_release()
389 		dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);  in mn_release()
391 	unbind_pasid(pasid_state);  in mn_release()
399 static void set_pri_tag_status(struct pasid_state *pasid_state,  in set_pri_tag_status()  argument
404 	spin_lock_irqsave(&pasid_state->lock, flags);  in set_pri_tag_status()
405 	pasid_state->pri[tag].status = status;  in set_pri_tag_status()
406 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in set_pri_tag_status()
410 			   struct pasid_state *pasid_state,  in finish_pri_tag()  argument
415 	spin_lock_irqsave(&pasid_state->lock, flags);  in finish_pri_tag()
416 	if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&  in finish_pri_tag()
417 	    pasid_state->pri[tag].finish) {  in finish_pri_tag()
418 		amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,  in finish_pri_tag()
419 				       pasid_state->pri[tag].status, tag);  in finish_pri_tag()
420 		pasid_state->pri[tag].finish = false;  in finish_pri_tag()
421 		pasid_state->pri[tag].status = PPR_SUCCESS;  in finish_pri_tag()
423 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in finish_pri_tag()
516 	struct pasid_state *pasid_state;  in ppr_notifier()  local
548 	pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);  in ppr_notifier()
549 	if (pasid_state == NULL || pasid_state->invalid) {  in ppr_notifier()
556 	spin_lock_irqsave(&pasid_state->lock, flags);  in ppr_notifier()
557 	atomic_inc(&pasid_state->pri[tag].inflight);  in ppr_notifier()
559 		pasid_state->pri[tag].finish = true;  in ppr_notifier()
560 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in ppr_notifier()
565 		finish_pri_tag(dev_state, pasid_state, tag);  in ppr_notifier()
571 	fault->state     = pasid_state;  in ppr_notifier()
584 	if (ret != NOTIFY_OK && pasid_state)  in ppr_notifier()
585 		put_pasid_state(pasid_state);  in ppr_notifier()
600 	struct pasid_state *pasid_state;  in amd_iommu_bind_pasid()  local
622 	pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);  in amd_iommu_bind_pasid()
623 	if (pasid_state == NULL)  in amd_iommu_bind_pasid()
627 	atomic_set(&pasid_state->count, 1);  in amd_iommu_bind_pasid()
628 	init_waitqueue_head(&pasid_state->wq);  in amd_iommu_bind_pasid()
629 	spin_lock_init(&pasid_state->lock);  in amd_iommu_bind_pasid()
632 	pasid_state->mm           = mm;  in amd_iommu_bind_pasid()
633 	pasid_state->device_state = dev_state;  in amd_iommu_bind_pasid()
634 	pasid_state->pasid        = pasid;  in amd_iommu_bind_pasid()
635 	pasid_state->invalid      = true; /* Mark as valid only if we are  in amd_iommu_bind_pasid()
637 	pasid_state->mn.ops       = &iommu_mn;  in amd_iommu_bind_pasid()
639 	if (pasid_state->mm == NULL)  in amd_iommu_bind_pasid()
642 	mmu_notifier_register(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
644 	ret = set_pasid_state(dev_state, pasid_state, pasid);  in amd_iommu_bind_pasid()
649 					__pa(pasid_state->mm->pgd));  in amd_iommu_bind_pasid()
654 	pasid_state->invalid = false;  in amd_iommu_bind_pasid()
669 	mmu_notifier_unregister(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
673 	free_pasid_state(pasid_state);  in amd_iommu_bind_pasid()
684 	struct pasid_state *pasid_state;  in amd_iommu_unbind_pasid()  local
701 	pasid_state = get_pasid_state(dev_state, pasid);  in amd_iommu_unbind_pasid()
702 	if (pasid_state == NULL)  in amd_iommu_unbind_pasid()
708 	put_pasid_state(pasid_state);  in amd_iommu_unbind_pasid()
711 	clear_pasid_state(dev_state, pasid_state->pasid);  in amd_iommu_unbind_pasid()
717 	mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in amd_iommu_unbind_pasid()
719 	put_pasid_state_wait(pasid_state); /* Reference taken in  in amd_iommu_unbind_pasid()