43 		struct iova_domain	iovad;  member
61 		return cookie->iovad.granule;  in cookie_msi_granule()
145 	if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule)  in iommu_put_dma_cookie()
146 		put_iova_domain(&cookie->iovad);  in iommu_put_dma_cookie()
179 	struct iova_domain *iovad = &cookie->iovad;  in cookie_init_hw_msi_region()  local
183 	start -= iova_offset(iovad, start);  in cookie_init_hw_msi_region()
184 	num_pages = iova_align(iovad, end - start) >> iova_shift(iovad);  in cookie_init_hw_msi_region()
195 		start += iovad->granule;  in cookie_init_hw_msi_region()
202 		struct iova_domain *iovad)  in iova_reserve_pci_windows()  argument
213 		lo = iova_pfn(iovad, window->res->start - window->offset);  in iova_reserve_pci_windows()
214 		hi = iova_pfn(iovad, window->res->end - window->offset);  in iova_reserve_pci_windows()
215 		reserve_iova(iovad, lo, hi);  in iova_reserve_pci_windows()
223 			lo = iova_pfn(iovad, start);  in iova_reserve_pci_windows()
224 			hi = iova_pfn(iovad, end);  in iova_reserve_pci_windows()
225 			reserve_iova(iovad, lo, hi);  in iova_reserve_pci_windows()
250 	struct iova_domain *iovad = &cookie->iovad;  in iova_reserve_iommu_regions()  local
256 		ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad);  in iova_reserve_iommu_regions()
269 		lo = iova_pfn(iovad, region->start);  in iova_reserve_iommu_regions()
270 		hi = iova_pfn(iovad, region->start + region->length - 1);  in iova_reserve_iommu_regions()
271 		reserve_iova(iovad, lo, hi);  in iova_reserve_iommu_regions()
284 static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad)  in iommu_dma_flush_iotlb_all()  argument
289 	cookie = container_of(iovad, struct iommu_dma_cookie, iovad);  in iommu_dma_flush_iotlb_all()
316 	struct iova_domain *iovad;  in iommu_dma_init_domain()  local
323 	iovad = &cookie->iovad;  in iommu_dma_init_domain()
344 	if (iovad->start_pfn) {  in iommu_dma_init_domain()
345 		if (1UL << order != iovad->granule ||  in iommu_dma_init_domain()
346 		    base_pfn != iovad->start_pfn) {  in iommu_dma_init_domain()
356 	init_iova_domain(iovad, 1UL << order, base_pfn);  in iommu_dma_init_domain()
360 		if (init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all,  in iommu_dma_init_domain()
402 	struct iova_domain *iovad;  in iommu_dma_reserve_iova()  local
410 	iovad = &cookie->iovad;  in iommu_dma_reserve_iova()
413 	pfn_lo = iova_pfn(iovad, base);  in iommu_dma_reserve_iova()
414 	pfn_hi = iova_pfn(iovad, base + size - 1);  in iommu_dma_reserve_iova()
415 	if (!reserve_iova(iovad, pfn_lo, pfn_hi))  in iommu_dma_reserve_iova()
428 	struct iova_domain *iovad;  in iommu_dma_enable_best_fit_algo()  local
434 	iovad = &((struct iommu_dma_cookie *)domain->iova_cookie)->iovad;  in iommu_dma_enable_best_fit_algo()
435 	iovad->best_fit = true;  in iommu_dma_enable_best_fit_algo()
477 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_iova()  local
485 	shift = iova_shift(iovad);  in iommu_dma_alloc_iova()
503 		iova = alloc_iova_fast(iovad, iova_len,  in iommu_dma_alloc_iova()
507 		iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,  in iommu_dma_alloc_iova()
511 	trace_android_vh_iommu_iovad_alloc_iova(dev, iovad, (dma_addr_t)iova << shift, size);  in iommu_dma_alloc_iova()
519 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_free_iova()  local
525 		queue_iova(iovad, iova_pfn(iovad, iova),  in iommu_dma_free_iova()
526 				size >> iova_shift(iovad), 0);  in iommu_dma_free_iova()
528 		free_iova_fast(iovad, iova_pfn(iovad, iova),  in iommu_dma_free_iova()
529 				size >> iova_shift(iovad));  in iommu_dma_free_iova()
532 	trace_android_vh_iommu_iovad_free_iova(iovad, iova, size);  in iommu_dma_free_iova()
540 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_unmap()  local
541 	size_t iova_off = iova_offset(iovad, dma_addr);  in __iommu_dma_unmap()
546 	size = iova_align(iovad, size + iova_off);  in __iommu_dma_unmap()
562 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_map()  local
563 	size_t iova_off = iova_offset(iovad, phys);  in __iommu_dma_map()
569 	size = iova_align(iovad, size + iova_off);  in __iommu_dma_map()
665 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_remap()  local
695 	size = iova_align(iovad, size);  in iommu_dma_alloc_remap()
913 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_map_sg()  local
934 		size_t s_iova_off = iova_offset(iovad, s->offset);  in iommu_dma_map_sg()
941 		s_length = iova_align(iovad, s_length + s_iova_off);  in iommu_dma_map_sg()