54 	struct iommu_dma_cookie		cookie;  member
58 static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)  in cookie_msi_granule()  argument
60 	if (cookie->type == IOMMU_DMA_IOVA_COOKIE)  in cookie_msi_granule()
61 		return cookie->iovad.granule;  in cookie_msi_granule()
67 	struct iommu_dma_cookie_ext *cookie;  in cookie_alloc()  local
69 	cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);  in cookie_alloc()
70 	if (cookie) {  in cookie_alloc()
71 		INIT_LIST_HEAD(&cookie->cookie.msi_page_list);  in cookie_alloc()
72 		cookie->cookie.type = type;  in cookie_alloc()
73 		mutex_init(&cookie->mutex);  in cookie_alloc()
75 	return &cookie->cookie;  in cookie_alloc()
112 	struct iommu_dma_cookie *cookie;  in iommu_get_msi_cookie()  local
120 	cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);  in iommu_get_msi_cookie()
121 	if (!cookie)  in iommu_get_msi_cookie()
124 	cookie->msi_iova = base;  in iommu_get_msi_cookie()
125 	domain->iova_cookie = cookie;  in iommu_get_msi_cookie()
139 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_put_dma_cookie()  local
142 	if (!cookie)  in iommu_put_dma_cookie()
145 	if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule)  in iommu_put_dma_cookie()
146 		put_iova_domain(&cookie->iovad);  in iommu_put_dma_cookie()
148 	list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {  in iommu_put_dma_cookie()
152 	kfree(cookie);  in iommu_put_dma_cookie()
176 static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,  in cookie_init_hw_msi_region()  argument
179 	struct iova_domain *iovad = &cookie->iovad;  in cookie_init_hw_msi_region()
194 		list_add(&msi_page->list, &cookie->msi_page_list);  in cookie_init_hw_msi_region()
249 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iova_reserve_iommu_regions()  local
250 	struct iova_domain *iovad = &cookie->iovad;  in iova_reserve_iommu_regions()
274 			ret = cookie_init_hw_msi_region(cookie, region->start,  in iova_reserve_iommu_regions()
286 	struct iommu_dma_cookie *cookie;  in iommu_dma_flush_iotlb_all()  local
289 	cookie = container_of(iovad, struct iommu_dma_cookie, iovad);  in iommu_dma_flush_iotlb_all()
290 	domain = cookie->fq_domain;  in iommu_dma_flush_iotlb_all()
313 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_init_domain()  local
320 	if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)  in iommu_dma_init_domain()
323 	iovad = &cookie->iovad;  in iommu_dma_init_domain()
342 	cookie_ext = container_of(cookie, struct iommu_dma_cookie_ext, cookie);  in iommu_dma_init_domain()
358 	if (!cookie->fq_domain && !iommu_domain_get_attr(domain,  in iommu_dma_init_domain()
364 			cookie->fq_domain = domain;  in iommu_dma_init_domain()
401 	struct iommu_dma_cookie *cookie;  in iommu_dma_reserve_iova()  local
409 	cookie = domain->iova_cookie;  in iommu_dma_reserve_iova()
410 	iovad = &cookie->iovad;  in iommu_dma_reserve_iova()
476 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_alloc_iova()  local
477 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_iova()
480 	if (cookie->type == IOMMU_DMA_MSI_COOKIE) {  in iommu_dma_alloc_iova()
481 		cookie->msi_iova += size;  in iommu_dma_alloc_iova()
482 		return cookie->msi_iova - size;  in iommu_dma_alloc_iova()
516 static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,  in iommu_dma_free_iova()  argument
519 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_free_iova()
522 	if (cookie->type == IOMMU_DMA_MSI_COOKIE)  in iommu_dma_free_iova()
523 		cookie->msi_iova -= size;  in iommu_dma_free_iova()
524 	else if (cookie->fq_domain)	/* non-strict mode */  in iommu_dma_free_iova()
539 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in __iommu_dma_unmap()  local
540 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_unmap()
552 	if (!cookie->fq_domain)  in __iommu_dma_unmap()
554 	iommu_dma_free_iova(cookie, dma_addr, size);  in __iommu_dma_unmap()
561 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in __iommu_dma_map()  local
562 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_map()
576 		iommu_dma_free_iova(cookie, iova, size);  in __iommu_dma_map()
664 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_alloc_remap()  local
665 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_remap()
729 	iommu_dma_free_iova(cookie, iova, size);  in iommu_dma_alloc_remap()
912 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_map_sg()  local
913 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_map_sg()
980 	iommu_dma_free_iova(cookie, iova, iova_len);  in iommu_dma_map_sg()
1279 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_get_msi_page()  local
1283 	size_t size = cookie_msi_granule(cookie);  in iommu_dma_get_msi_page()
1286 	list_for_each_entry(msi_page, &cookie->msi_page_list, list)  in iommu_dma_get_msi_page()
1304 	list_add(&msi_page->list, &cookie->msi_page_list);  in iommu_dma_get_msi_page()
1308 	iommu_dma_free_iova(cookie, iova, size);  in iommu_dma_get_msi_page()