195 				    struct io_pgtable_cfg *cfg)  in __arm_lpae_alloc_pages()  argument
197 	struct device *dev = cfg->iommu_dev;  in __arm_lpae_alloc_pages()
210 	if (!cfg->coherent_walk) {  in __arm_lpae_alloc_pages()
234 				  struct io_pgtable_cfg *cfg)  in __arm_lpae_free_pages()  argument
236 	if (!cfg->coherent_walk)  in __arm_lpae_free_pages()
237 		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),  in __arm_lpae_free_pages()
243 				struct io_pgtable_cfg *cfg)  in __arm_lpae_sync_pte()  argument
245 	dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),  in __arm_lpae_sync_pte()
249 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg)  in __arm_lpae_clear_pte()  argument
254 	if (!cfg->coherent_walk)  in __arm_lpae_clear_pte()
255 		__arm_lpae_sync_pte(ptep, 1, cfg);  in __arm_lpae_clear_pte()
268 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_init_pte()  local
280 	if (!cfg->coherent_walk)  in __arm_lpae_init_pte()
281 		__arm_lpae_sync_pte(ptep, num_entries, cfg);  in __arm_lpae_init_pte()
322 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_install_table()  local
325 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_install_table()
337 	if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC))  in arm_lpae_install_table()
341 	__arm_lpae_sync_pte(ptep, 1, cfg);  in arm_lpae_install_table()
356 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_map()  local
381 		cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg);  in __arm_lpae_map()
387 			__arm_lpae_free_pages(cptep, tblsz, cfg);  in __arm_lpae_map()
388 	} else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {  in __arm_lpae_map()
389 		__arm_lpae_sync_pte(ptep, 1, cfg);  in __arm_lpae_map()
470 	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_prot_to_pte()
484 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_map_pages()  local
488 	long iaext = (s64)iova >> cfg->ias;  in arm_lpae_map_pages()
494 	if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize))  in arm_lpae_map_pages()
497 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)  in arm_lpae_map_pages()
499 	if (WARN_ON(iaext || paddr >> cfg->oas))  in arm_lpae_map_pages()
550 	__arm_lpae_free_pages(start, table_size, &data->iop.cfg);  in __arm_lpae_free_pgtable()
567 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_split_blk_unmap()  local
578 	tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg);  in arm_lpae_split_blk_unmap()
601 		__arm_lpae_free_pages(tablep, tablesz, cfg);  in arm_lpae_split_blk_unmap()
650 			__arm_lpae_clear_pte(ptep, &iop->cfg);  in __arm_lpae_unmap()
657 			} else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {  in __arm_lpae_unmap()
692 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_unmap_pages()  local
694 	long iaext = (s64)iova >> cfg->ias;  in arm_lpae_unmap_pages()
696 	if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount))  in arm_lpae_unmap_pages()
699 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)  in arm_lpae_unmap_pages()
750 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)  in arm_lpae_restrict_pgsizes()  argument
762 	if (cfg->pgsize_bitmap & PAGE_SIZE)  in arm_lpae_restrict_pgsizes()
764 	else if (cfg->pgsize_bitmap & ~PAGE_MASK)  in arm_lpae_restrict_pgsizes()
765 		granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);  in arm_lpae_restrict_pgsizes()
766 	else if (cfg->pgsize_bitmap & PAGE_MASK)  in arm_lpae_restrict_pgsizes()
767 		granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);  in arm_lpae_restrict_pgsizes()
781 		if (cfg->oas > 48)  in arm_lpae_restrict_pgsizes()
788 	cfg->pgsize_bitmap &= page_sizes;  in arm_lpae_restrict_pgsizes()
789 	cfg->ias = min(cfg->ias, max_addr_bits);  in arm_lpae_restrict_pgsizes()
790 	cfg->oas = min(cfg->oas, max_addr_bits);  in arm_lpae_restrict_pgsizes()
794 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)  in arm_lpae_alloc_pgtable()  argument
799 	arm_lpae_restrict_pgsizes(cfg);  in arm_lpae_alloc_pgtable()
801 	if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))  in arm_lpae_alloc_pgtable()
804 	if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
807 	if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
814 	pg_shift = __ffs(cfg->pgsize_bitmap);  in arm_lpae_alloc_pgtable()
817 	va_bits = cfg->ias - pg_shift;  in arm_lpae_alloc_pgtable()
836 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s1()  argument
840 	typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;  in arm_64_lpae_alloc_pgtable_s1()
843 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |  in arm_64_lpae_alloc_pgtable_s1()
848 	data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s1()
853 	if (cfg->coherent_walk) {  in arm_64_lpae_alloc_pgtable_s1()
863 	tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;  in arm_64_lpae_alloc_pgtable_s1()
876 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s1()
902 	tcr->tsz = 64ULL - cfg->ias;  in arm_64_lpae_alloc_pgtable_s1()
918 	cfg->arm_lpae_s1_cfg.mair = reg;  in arm_64_lpae_alloc_pgtable_s1()
922 					   GFP_KERNEL, cfg);  in arm_64_lpae_alloc_pgtable_s1()
930 	cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s1()
939 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s2()  argument
943 	typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;  in arm_64_lpae_alloc_pgtable_s2()
946 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NON_STRICT))  in arm_64_lpae_alloc_pgtable_s2()
949 	data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s2()
968 	if (cfg->coherent_walk) {  in arm_64_lpae_alloc_pgtable_s2()
993 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s2()
1019 	vtcr->tsz = 64ULL - cfg->ias;  in arm_64_lpae_alloc_pgtable_s2()
1024 					   GFP_KERNEL, cfg);  in arm_64_lpae_alloc_pgtable_s2()
1032 	cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s2()
1041 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s1()  argument
1043 	if (cfg->ias > 32 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s1()
1046 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s1()
1047 	return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s1()
1051 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s2()  argument
1053 	if (cfg->ias > 40 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s2()
1056 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s2()
1057 	return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s2()
1061 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)  in arm_mali_lpae_alloc_pgtable()  argument
1066 	if (cfg->quirks)  in arm_mali_lpae_alloc_pgtable()
1069 	if (cfg->ias > 48 || cfg->oas > 40)  in arm_mali_lpae_alloc_pgtable()
1072 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_mali_lpae_alloc_pgtable()
1074 	data = arm_lpae_alloc_pgtable(cfg);  in arm_mali_lpae_alloc_pgtable()
1090 	cfg->arm_mali_lpae_cfg.memattr =  in arm_mali_lpae_alloc_pgtable()
1099 					   cfg);  in arm_mali_lpae_alloc_pgtable()
1106 	cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |  in arm_mali_lpae_alloc_pgtable()
1109 	if (cfg->coherent_walk)  in arm_mali_lpae_alloc_pgtable()
1110 		cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;  in arm_mali_lpae_alloc_pgtable()
1176 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_dump_ops()  local
1179 		cfg->pgsize_bitmap, cfg->ias);  in arm_lpae_dump_ops()
1192 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)  in arm_lpae_run_tests()  argument
1207 		cfg_cookie = cfg;  in arm_lpae_run_tests()
1208 		ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);  in arm_lpae_run_tests()
1231 		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {  in arm_lpae_run_tests()
1252 		size = 1UL << __ffs(cfg->pgsize_bitmap);  in arm_lpae_run_tests()
1265 		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {  in arm_lpae_run_tests()
1304 	struct io_pgtable_cfg cfg = {  in arm_lpae_do_selftests()  local
1312 			cfg.pgsize_bitmap = pgsize[i];  in arm_lpae_do_selftests()
1313 			cfg.ias = ias[j];  in arm_lpae_do_selftests()
1316 			if (arm_lpae_run_tests(&cfg))  in arm_lpae_do_selftests()