296 static struct dmar_domain *si_domain;
329 static void domain_exit(struct dmar_domain *domain);
330 static void domain_remove_dev_info(struct dmar_domain *domain);
479 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)  in get_iommu_domain()
481 	struct dmar_domain **domains;  in get_iommu_domain()
492 			     struct dmar_domain *domain)  in set_iommu_domain()
494 	struct dmar_domain **domains;  in set_iommu_domain()
498 		size_t size = 256 * sizeof(struct dmar_domain *);  in set_iommu_domain()
545 static inline int domain_type_is_si(struct dmar_domain *domain)  in domain_type_is_si()
550 static inline bool domain_use_first_level(struct dmar_domain *domain)  in domain_use_first_level()
555 static inline int domain_pfn_supported(struct dmar_domain *domain,  in domain_pfn_supported()
619 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)  in domain_get_iommu()
642 static void domain_update_iommu_coherency(struct dmar_domain *domain)  in domain_update_iommu_coherency()
699 static int domain_update_iommu_superpage(struct dmar_domain *domain,  in domain_update_iommu_superpage()
730 static int domain_update_device_node(struct dmar_domain *domain)  in domain_update_device_node()
759 static void domain_update_iommu_cap(struct dmar_domain *domain)  in domain_update_iommu_cap()
968 static void domain_flush_cache(struct dmar_domain *domain,  in domain_flush_cache()
1018 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,  in pfn_to_dma_pte()
1078 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,  in dma_pfn_level_pte()
1110 static void dma_pte_clear_range(struct dmar_domain *domain,  in dma_pte_clear_range()
1141 static void dma_pte_free_level(struct dmar_domain *domain, int level,  in dma_pte_free_level()
1184 static void dma_pte_free_pagetable(struct dmar_domain *domain,  in dma_pte_free_pagetable()
1212 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,  in dma_pte_list_pagetables()
1236 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,  in dma_pte_clear_level()
1288 static struct page *domain_unmap(struct dmar_domain *domain,  in domain_unmap()
1490 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,  in iommu_support_dev_iotlb()
1511 static void domain_update_iotlb(struct dmar_domain *domain)  in domain_update_iotlb()
1609 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,  in iommu_flush_dev_iotlb()
1633 				struct dmar_domain *domain,  in domain_flush_piotlb()
1647 				  struct dmar_domain *domain,  in iommu_flush_iotlb_psi()
1708 					struct dmar_domain *domain,  in __mapping_notify_one()
1723 	struct dmar_domain *domain;  in iommu_flush_iova()
1726 	domain = container_of(iovad, struct dmar_domain, iovad);  in iommu_flush_iova()
1819 	size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);  in iommu_init_domains()
1823 		size = 256 * sizeof(struct dmar_domain *);  in iommu_init_domains()
1939 static struct dmar_domain *alloc_domain(int flags)  in alloc_domain()
1941 	struct dmar_domain *domain;  in alloc_domain()
1959 static int domain_attach_iommu(struct dmar_domain *domain,  in domain_attach_iommu()
1993 static int domain_detach_iommu(struct dmar_domain *domain,  in domain_detach_iommu()
2071 static void domain_exit(struct dmar_domain *domain)  in domain_exit()
2140 static int domain_context_mapping_one(struct dmar_domain *domain,  in domain_context_mapping_one()
2290 	struct dmar_domain *domain;
2306 domain_context_mapping(struct dmar_domain *domain, struct device *dev)  in domain_context_mapping()
2364 static inline int hardware_largepage_caps(struct dmar_domain *domain,  in hardware_largepage_caps()
2391 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in __domain_mapping()
2517 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in domain_mapping()
2537 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in domain_sg_mapping()
2544 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in domain_pfn_mapping()
2595 static void domain_remove_dev_info(struct dmar_domain *domain)  in domain_remove_dev_info()
2606 struct dmar_domain *find_domain(struct device *dev)  in find_domain()
2648 				    struct dmar_domain *domain,  in domain_setup_first_level()
2689 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,  in dmar_insert_one_dev_info()
2692 						    struct dmar_domain *domain)  in dmar_insert_one_dev_info()
2694 	struct dmar_domain *found = NULL;  in dmar_insert_one_dev_info()
2820 static int iommu_domain_identity_map(struct dmar_domain *domain,  in iommu_domain_identity_map()
2835 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2894 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)  in domain_add_dev_info()
2896 	struct dmar_domain *ndomain;  in domain_add_dev_info()
3522 				     struct dmar_domain *domain,  in intel_alloc_iova()
3570 	struct dmar_domain *domain;  in __intel_map_single()
3647 	struct dmar_domain *domain;  in intel_unmap()
3775 	struct dmar_domain *domain;  in intel_map_sg()
3858 	struct dmar_domain *domain;  in bounce_sync_single()
3876 	struct dmar_domain *domain;  in bounce_map_single()
3965 	struct dmar_domain *domain;  in bounce_unmap_single()
4113 					 sizeof(struct dmar_domain),  in iommu_domain_cache_init()
4755 		struct dmar_domain *domain;  in free_all_cpu_cached_iovas()
5132 	struct dmar_domain *domain;  in __dmar_remove_one_dev_info()
5176 static int md_domain_init(struct dmar_domain *domain, int guest_width)  in md_domain_init()
5198 static void intel_init_iova_domain(struct dmar_domain *dmar_domain)  in intel_init_iova_domain()  argument
5200 	init_iova_domain(&dmar_domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);  in intel_init_iova_domain()
5201 	copy_reserved_iova(&reserved_iova_list, &dmar_domain->iovad);  in intel_init_iova_domain()
5204 	    init_iova_flush_queue(&dmar_domain->iovad,  in intel_init_iova_domain()
5211 	struct dmar_domain *dmar_domain;  in intel_iommu_domain_alloc()  local
5217 		dmar_domain = alloc_domain(0);  in intel_iommu_domain_alloc()
5218 		if (!dmar_domain) {  in intel_iommu_domain_alloc()
5222 		if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {  in intel_iommu_domain_alloc()
5224 			domain_exit(dmar_domain);  in intel_iommu_domain_alloc()
5229 			intel_init_iova_domain(dmar_domain);  in intel_iommu_domain_alloc()
5231 		domain = &dmar_domain->domain;  in intel_iommu_domain_alloc()
5234 				__DOMAIN_MAX_ADDR(dmar_domain->gaw);  in intel_iommu_domain_alloc()
5266 static void auxiliary_link_device(struct dmar_domain *domain,  in auxiliary_link_device()
5279 static void auxiliary_unlink_device(struct dmar_domain *domain,  in auxiliary_unlink_device()
5295 static int aux_domain_add_dev(struct dmar_domain *domain,  in aux_domain_add_dev()
5358 static void aux_domain_remove_dev(struct dmar_domain *domain,  in aux_domain_remove_dev()
5385 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in prepare_domain_attach_device()  local
5398 	if (dmar_domain->max_addr > (1LL << addr_width)) {  in prepare_domain_attach_device()
5401 		        __func__, addr_width, dmar_domain->max_addr);  in prepare_domain_attach_device()
5404 	dmar_domain->gaw = addr_width;  in prepare_domain_attach_device()
5409 	while (iommu->agaw < dmar_domain->agaw) {  in prepare_domain_attach_device()
5412 		pte = dmar_domain->pgd;  in prepare_domain_attach_device()
5414 			dmar_domain->pgd = (struct dma_pte *)  in prepare_domain_attach_device()
5418 		dmar_domain->agaw--;  in prepare_domain_attach_device()
5440 		struct dmar_domain *old_domain;  in intel_iommu_attach_device()
5532 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_sva_invalidate()  local
5542 	if (!inv_info || !dmar_domain)  in intel_iommu_sva_invalidate()
5552 	if (!(dmar_domain->flags & DOMAIN_FLAG_NESTING_MODE))  in intel_iommu_sva_invalidate()
5562 	did = dmar_domain->iommu_did[iommu->seq_id];  in intel_iommu_sva_invalidate()
5664 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_map()  local
5673 	if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)  in intel_iommu_map()
5677 	if (dmar_domain->max_addr < max_addr) {  in intel_iommu_map()
5681 		end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;  in intel_iommu_map()
5685 			       __func__, dmar_domain->gaw, max_addr);  in intel_iommu_map()
5688 		dmar_domain->max_addr = max_addr;  in intel_iommu_map()
5693 	ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,  in intel_iommu_map()
5702 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_unmap()  local
5710 	BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));  in intel_iommu_unmap()
5718 	freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);  in intel_iommu_unmap()
5722 	for_each_domain_iommu(iommu_id, dmar_domain)  in intel_iommu_unmap()
5723 		iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,  in intel_iommu_unmap()
5728 	if (dmar_domain->max_addr == iova + size)  in intel_iommu_unmap()
5729 		dmar_domain->max_addr = iova;  in intel_iommu_unmap()
5737 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_iova_to_phys()  local
5742 	pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);  in intel_iommu_iova_to_phys()
5916 	struct dmar_domain *domain;  in intel_iommu_enable_pasid()
5967 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_apply_resv_region()  local
5973 	WARN_ON_ONCE(!reserve_iova(&dmar_domain->iovad, start, end));  in intel_iommu_apply_resv_region()
6121 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_aux_get_pasid()  local
6123 	return dmar_domain->default_pasid > 0 ?  in intel_iommu_aux_get_pasid()
6124 			dmar_domain->default_pasid : -EINVAL;  in intel_iommu_aux_get_pasid()
6137 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_domain_set_attr()  local
6148 		    list_empty(&dmar_domain->devices)) {  in intel_iommu_domain_set_attr()
6149 			dmar_domain->flags |= DOMAIN_FLAG_NESTING_MODE;  in intel_iommu_domain_set_attr()
6150 			dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL;  in intel_iommu_domain_set_attr()