260 			       unsigned long iova, size_t size, size_t pgcount,
285 			     unsigned long iova, phys_addr_t paddr,  in arm_lpae_init_pte()  argument
304 			tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_init_pte()
305 			if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1,  in arm_lpae_init_pte()
348 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,  in __arm_lpae_map()  argument
360 	map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_map()
367 		ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);  in __arm_lpae_map()
401 	return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1,  in __arm_lpae_map()
479 static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,  in arm_lpae_map_pages()  argument
488 	long iaext = (s64)iova >> cfg->ias;  in arm_lpae_map_pages()
503 	ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl,  in arm_lpae_map_pages()
515 static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,  in arm_lpae_map()  argument
518 	return arm_lpae_map_pages(ops, iova, paddr, size, 1, iommu_prot, gfp,  in arm_lpae_map()
563 				       unsigned long iova, size_t size,  in arm_lpae_split_blk_unmap()  argument
583 		unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_split_blk_unmap()
613 			io_pgtable_tlb_add_page(&data->iop, gather, iova + i * size, size);  in arm_lpae_split_blk_unmap()
618 	return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl, tablep);  in arm_lpae_split_blk_unmap()
623 			       unsigned long iova, size_t size, size_t pgcount,  in __arm_lpae_unmap()  argument
634 	unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_unmap()
654 				io_pgtable_tlb_flush_walk(iop, iova + i * size, size,  in __arm_lpae_unmap()
665 				io_pgtable_tlb_add_page(iop, gather, iova + i * size, size);  in __arm_lpae_unmap()
678 		return arm_lpae_split_blk_unmap(data, gather, iova, size, pte,  in __arm_lpae_unmap()
684 	return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep);  in __arm_lpae_unmap()
687 static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova,  in arm_lpae_unmap_pages()  argument
694 	long iaext = (s64)iova >> cfg->ias;  in arm_lpae_unmap_pages()
704 	return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount,  in arm_lpae_unmap_pages()
708 static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,  in arm_lpae_unmap()  argument
711 	return arm_lpae_unmap_pages(ops, iova, size, 1, gather);  in arm_lpae_unmap()
715 					 unsigned long iova)  in arm_lpae_iova_to_phys()  argument
727 		ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_iova_to_phys()
746 	iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1);  in arm_lpae_iova_to_phys()
747 	return iopte_to_paddr(pte, data) | iova;  in arm_lpae_iova_to_phys()
1153 static void __init dummy_tlb_flush(unsigned long iova, size_t size,  in dummy_tlb_flush()  argument
1161 				      unsigned long iova, size_t granule,  in dummy_tlb_add_page()  argument
1164 	dummy_tlb_flush(iova, granule, granule, cookie);  in dummy_tlb_add_page()
1200 	unsigned long iova;  in arm_lpae_run_tests()  local
1230 		iova = 0;  in arm_lpae_run_tests()
1234 			if (ops->map(ops, iova, iova, size, IOMMU_READ |  in arm_lpae_run_tests()
1241 			if (!ops->map(ops, iova, iova + size, size,  in arm_lpae_run_tests()
1245 			if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_lpae_run_tests()
1248 			iova += SZ_1G;  in arm_lpae_run_tests()
1264 		iova = 0;  in arm_lpae_run_tests()
1268 			if (ops->unmap(ops, iova, size, NULL) != size)  in arm_lpae_run_tests()
1271 			if (ops->iova_to_phys(ops, iova + 42))  in arm_lpae_run_tests()
1275 			if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL))  in arm_lpae_run_tests()
1278 			if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_lpae_run_tests()
1281 			iova += SZ_1G;  in arm_lpae_run_tests()