188 	struct iommu_dev_data *dev_data;  in alloc_dev_data()  local
190 	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);  in alloc_dev_data()
191 	if (!dev_data)  in alloc_dev_data()
194 	spin_lock_init(&dev_data->lock);  in alloc_dev_data()
195 	dev_data->devid = devid;  in alloc_dev_data()
196 	ratelimit_default_init(&dev_data->rs);  in alloc_dev_data()
198 	llist_add(&dev_data->dev_data_list, &dev_data_list);  in alloc_dev_data()
199 	return dev_data;  in alloc_dev_data()
204 	struct iommu_dev_data *dev_data;  in search_dev_data()  local
211 	llist_for_each_entry(dev_data, node, dev_data_list) {  in search_dev_data()
212 		if (dev_data->devid == devid)  in search_dev_data()
213 			return dev_data;  in search_dev_data()
275 	struct iommu_dev_data *dev_data;  in find_dev_data()  local
278 	dev_data = search_dev_data(devid);  in find_dev_data()
280 	if (dev_data == NULL) {  in find_dev_data()
281 		dev_data = alloc_dev_data(devid);  in find_dev_data()
282 		if (!dev_data)  in find_dev_data()
286 			dev_data->defer_attach = true;  in find_dev_data()
289 	return dev_data;  in find_dev_data()
339 	struct iommu_dev_data *dev_data;  in pdev_pri_erratum()  local
341 	dev_data = dev_iommu_priv_get(&pdev->dev);  in pdev_pri_erratum()
343 	return dev_data->errata & (1 << erratum) ? true : false;  in pdev_pri_erratum()
373 	struct iommu_dev_data *dev_data;  in iommu_init_device()  local
383 	dev_data = find_dev_data(devid);  in iommu_init_device()
384 	if (!dev_data)  in iommu_init_device()
387 	dev_data->pdev = setup_aliases(dev);  in iommu_init_device()
399 		iommu = amd_iommu_rlookup_table[dev_data->devid];  in iommu_init_device()
400 		dev_data->iommu_v2 = iommu->is_iommu_v2;  in iommu_init_device()
403 	dev_iommu_priv_set(dev, dev_data);  in iommu_init_device()
424 	struct iommu_dev_data *dev_data;  in amd_iommu_uninit_device()  local
426 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_uninit_device()
427 	if (!dev_data)  in amd_iommu_uninit_device()
430 	if (dev_data->domain)  in amd_iommu_uninit_device()
490 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_rmp_hw_error()  local
503 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_rmp_hw_error()
505 	if (dev_data && __ratelimit(&dev_data->rs)) {  in amd_iommu_report_rmp_hw_error()
520 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_rmp_fault()  local
534 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_rmp_fault()
536 	if (dev_data && __ratelimit(&dev_data->rs)) {  in amd_iommu_report_rmp_fault()
552 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_page_fault()  local
558 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_page_fault()
560 	if (dev_data && __ratelimit(&dev_data->rs)) {  in amd_iommu_report_page_fault()
1249 static int device_flush_iotlb(struct iommu_dev_data *dev_data,  in device_flush_iotlb()  argument
1256 	qdep     = dev_data->ats.qdep;  in device_flush_iotlb()
1257 	iommu    = amd_iommu_rlookup_table[dev_data->devid];  in device_flush_iotlb()
1259 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);  in device_flush_iotlb()
1274 static int device_flush_dte(struct iommu_dev_data *dev_data)  in device_flush_dte()  argument
1280 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in device_flush_dte()
1282 	if (dev_data->pdev)  in device_flush_dte()
1283 		ret = pci_for_each_dma_alias(dev_data->pdev,  in device_flush_dte()
1286 		ret = iommu_flush_dte(iommu, dev_data->devid);  in device_flush_dte()
1290 	alias = amd_iommu_alias_table[dev_data->devid];  in device_flush_dte()
1291 	if (alias != dev_data->devid) {  in device_flush_dte()
1297 	if (dev_data->ats.enabled)  in device_flush_dte()
1298 		ret = device_flush_iotlb(dev_data, 0, ~0UL);  in device_flush_dte()
1311 	struct iommu_dev_data *dev_data;  in __domain_flush_pages()  local
1328 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in __domain_flush_pages()
1330 		if (!dev_data->ats.enabled)  in __domain_flush_pages()
1333 		ret |= device_flush_iotlb(dev_data, address, size);  in __domain_flush_pages()
1387 	struct iommu_dev_data *dev_data;  in domain_flush_devices()  local
1389 	list_for_each_entry(dev_data, &domain->dev_list, list)  in domain_flush_devices()
1390 		device_flush_dte(dev_data);  in domain_flush_devices()
1989 static void do_attach(struct iommu_dev_data *dev_data,  in do_attach()  argument
1996 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in do_attach()
1997 	ats   = dev_data->ats.enabled;  in do_attach()
2000 	dev_data->domain = domain;  in do_attach()
2001 	list_add(&dev_data->list, &domain->dev_list);  in do_attach()
2009 	set_dte_entry(dev_data->devid, domain, &pgtable,  in do_attach()
2010 		      ats, dev_data->iommu_v2);  in do_attach()
2011 	clone_aliases(dev_data->pdev);  in do_attach()
2013 	device_flush_dte(dev_data);  in do_attach()
2016 static void do_detach(struct iommu_dev_data *dev_data)  in do_detach()  argument
2018 	struct protection_domain *domain = dev_data->domain;  in do_detach()
2021 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in do_detach()
2024 	dev_data->domain = NULL;  in do_detach()
2025 	list_del(&dev_data->list);  in do_detach()
2026 	clear_dte_entry(dev_data->devid);  in do_detach()
2027 	clone_aliases(dev_data->pdev);  in do_detach()
2030 	device_flush_dte(dev_data);  in do_detach()
2119 	struct iommu_dev_data *dev_data;  in attach_device()  local
2126 	dev_data = dev_iommu_priv_get(dev);  in attach_device()
2128 	spin_lock(&dev_data->lock);  in attach_device()
2131 	if (dev_data->domain != NULL)  in attach_device()
2145 		if (dev_data->iommu_v2) {  in attach_device()
2149 			dev_data->ats.enabled = true;  in attach_device()
2150 			dev_data->ats.qdep    = pci_ats_queue_depth(pdev);  in attach_device()
2151 			dev_data->pri_tlp     = pci_prg_resp_pasid_required(pdev);  in attach_device()
2155 		dev_data->ats.enabled = true;  in attach_device()
2156 		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);  in attach_device()
2162 	do_attach(dev_data, domain);  in attach_device()
2174 	spin_unlock(&dev_data->lock);  in attach_device()
2187 	struct iommu_dev_data *dev_data;  in detach_device()  local
2190 	dev_data = dev_iommu_priv_get(dev);  in detach_device()
2191 	domain   = dev_data->domain;  in detach_device()
2195 	spin_lock(&dev_data->lock);  in detach_device()
2203 	if (WARN_ON(!dev_data->domain))  in detach_device()
2206 	do_detach(dev_data);  in detach_device()
2211 	if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)  in detach_device()
2213 	else if (dev_data->ats.enabled)  in detach_device()
2216 	dev_data->ats.enabled = false;  in detach_device()
2219 	spin_unlock(&dev_data->lock);  in detach_device()
2319 	struct iommu_dev_data *dev_data;  in update_device_table()  local
2321 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in update_device_table()
2322 		set_dte_entry(dev_data->devid, domain, pgtable,  in update_device_table()
2323 			      dev_data->ats.enabled, dev_data->iommu_v2);  in update_device_table()
2324 		clone_aliases(dev_data->pdev);  in update_device_table()
2524 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in amd_iommu_detach_device()  local
2535 	if (dev_data->domain != NULL)  in amd_iommu_detach_device()
2545 		dev_data->use_vapic = 0;  in amd_iommu_detach_device()
2555 	struct iommu_dev_data *dev_data;  in amd_iommu_attach_device()  local
2562 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_attach_device()
2563 	dev_data->defer_attach = false;  in amd_iommu_attach_device()
2565 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in amd_iommu_attach_device()
2569 	if (dev_data->domain)  in amd_iommu_attach_device()
2577 			dev_data->use_vapic = 1;  in amd_iommu_attach_device()
2579 			dev_data->use_vapic = 0;  in amd_iommu_attach_device()
2721 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in amd_iommu_is_attach_deferred()  local
2723 	return dev_data->defer_attach;  in amd_iommu_is_attach_deferred()
2746 	struct iommu_dev_data *dev_data;  in amd_iommu_def_domain_type()  local
2748 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_def_domain_type()
2749 	if (!dev_data)  in amd_iommu_def_domain_type()
2757 	if (!mem_encrypt_active() && dev_data->iommu_v2)  in amd_iommu_def_domain_type()
2882 	struct iommu_dev_data *dev_data;  in __flush_pasid()  local
2908 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in __flush_pasid()
2916 		if (!dev_data->ats.enabled)  in __flush_pasid()
2919 		qdep  = dev_data->ats.qdep;  in __flush_pasid()
2920 		iommu = amd_iommu_rlookup_table[dev_data->devid];  in __flush_pasid()
2922 		build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,  in __flush_pasid()
3082 	struct iommu_dev_data *dev_data;  in amd_iommu_complete_ppr()  local
3086 	dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_complete_ppr()
3087 	iommu    = amd_iommu_rlookup_table[dev_data->devid];  in amd_iommu_complete_ppr()
3089 	build_complete_ppr(&cmd, dev_data->devid, pasid, status,  in amd_iommu_complete_ppr()
3090 			   tag, dev_data->pri_tlp);  in amd_iommu_complete_ppr()
3099 	struct iommu_dev_data *dev_data;  in amd_iommu_get_v2_domain()  local
3106 	dev_data  = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_get_v2_domain()
3107 	pdomain   = dev_data->domain;  in amd_iommu_get_v2_domain()
3110 	if (pdomain == NULL && dev_data->defer_attach) {  in amd_iommu_get_v2_domain()
3111 		dev_data->defer_attach = false;  in amd_iommu_get_v2_domain()
3132 	struct iommu_dev_data *dev_data;  in amd_iommu_enable_device_erratum()  local
3137 	dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_enable_device_erratum()
3138 	dev_data->errata |= (1 << erratum);  in amd_iommu_enable_device_erratum()
3976 	struct iommu_dev_data *dev_data = search_dev_data(irte_info->devid);  in amd_ir_set_vcpu_affinity()  local
3982 	if (!dev_data || !dev_data->use_vapic)  in amd_ir_set_vcpu_affinity()