51 #define _ARM_V7S_LVL_BITS(lvl, cfg)	((lvl) == 1 ? ((cfg)->ias - 20) : 8)  argument
52 #define ARM_V7S_LVL_SHIFT(lvl)		((lvl) == 1 ? 20 : 12)  argument
55 #define ARM_V7S_PTES_PER_LVL(lvl, cfg)	(1 << _ARM_V7S_LVL_BITS(lvl, cfg))  argument
56 #define ARM_V7S_TABLE_SIZE(lvl, cfg)						\  argument
57 	(ARM_V7S_PTES_PER_LVL(lvl, cfg) * sizeof(arm_v7s_iopte))
59 #define ARM_V7S_BLOCK_SIZE(lvl)		(1UL << ARM_V7S_LVL_SHIFT(lvl))  argument
60 #define ARM_V7S_LVL_MASK(lvl)		((u32)(~0U << ARM_V7S_LVL_SHIFT(lvl)))  argument
62 #define _ARM_V7S_IDX_MASK(lvl, cfg)	(ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1)  argument
63 #define ARM_V7S_LVL_IDX(addr, lvl, cfg)	({				\  argument
64 	int _l = lvl;							\
84 #define ARM_V7S_PTE_IS_TABLE(pte, lvl) \  argument
85 	((lvl) == 1 && (((pte) & 0x3) == ARM_V7S_PTE_TYPE_TABLE))
88 #define ARM_V7S_ATTR_XN(lvl)		BIT(4 * (2 - (lvl)))  argument
102 #define ARM_V7S_ATTR_SHIFT(lvl)		(16 - (lvl) * 6)  argument
172 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl);
185 static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl,  in paddr_to_iopte()  argument
188 	arm_v7s_iopte pte = paddr & ARM_V7S_LVL_MASK(lvl);  in paddr_to_iopte()
202 static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl,  in iopte_to_paddr()  argument
208 	if (ARM_V7S_PTE_IS_TABLE(pte, lvl))  in iopte_to_paddr()
210 	else if (arm_v7s_pte_is_cont(pte, lvl))  in iopte_to_paddr()
211 		mask = ARM_V7S_LVL_MASK(lvl) * ARM_V7S_CONT_PAGES;  in iopte_to_paddr()
213 		mask = ARM_V7S_LVL_MASK(lvl);  in iopte_to_paddr()
228 static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl,  in iopte_deref()  argument
231 	return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg));  in iopte_deref()
234 static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp,  in __arm_v7s_alloc_table()  argument
241 	size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg);  in __arm_v7s_alloc_table()
244 	if (lvl == 1)  in __arm_v7s_alloc_table()
247 	else if (lvl == 2)  in __arm_v7s_alloc_table()
271 	if (lvl == 2)  in __arm_v7s_alloc_table()
279 	if (lvl == 1)  in __arm_v7s_alloc_table()
286 static void __arm_v7s_free_table(void *table, int lvl,  in __arm_v7s_free_table()  argument
291 	size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg);  in __arm_v7s_free_table()
296 	if (lvl == 1)  in __arm_v7s_free_table()
322 static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl,  in arm_v7s_prot_to_pte()  argument
337 	pte <<= ARM_V7S_ATTR_SHIFT(lvl);  in arm_v7s_prot_to_pte()
340 		pte |= ARM_V7S_ATTR_XN(lvl);  in arm_v7s_prot_to_pte()
347 	if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS))  in arm_v7s_prot_to_pte()
353 static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl)  in arm_v7s_pte_to_prot()  argument
356 	arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl);  in arm_v7s_pte_to_prot()
366 	if (pte & ARM_V7S_ATTR_XN(lvl))  in arm_v7s_pte_to_prot()
372 static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl)  in arm_v7s_pte_to_cont()  argument
374 	if (lvl == 1) {  in arm_v7s_pte_to_cont()
376 	} else if (lvl == 2) {  in arm_v7s_pte_to_cont()
377 		arm_v7s_iopte xn = pte & ARM_V7S_ATTR_XN(lvl);  in arm_v7s_pte_to_cont()
388 static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl)  in arm_v7s_cont_to_pte()  argument
390 	if (lvl == 1) {  in arm_v7s_cont_to_pte()
392 	} else if (lvl == 2) {  in arm_v7s_cont_to_pte()
405 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl)  in arm_v7s_pte_is_cont()  argument
407 	if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl))  in arm_v7s_pte_is_cont()
409 	else if (lvl == 2)  in arm_v7s_pte_is_cont()
420 			    int lvl, int num_entries, arm_v7s_iopte *ptep)  in arm_v7s_init_pte()  argument
427 		if (ARM_V7S_PTE_IS_TABLE(ptep[i], lvl)) {  in arm_v7s_init_pte()
433 			size_t sz = ARM_V7S_BLOCK_SIZE(lvl);  in arm_v7s_init_pte()
435 			tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg);  in arm_v7s_init_pte()
437 						    sz, lvl, tblp) != sz))  in arm_v7s_init_pte()
445 	pte = arm_v7s_prot_to_pte(prot, lvl, cfg);  in arm_v7s_init_pte()
447 		pte = arm_v7s_pte_to_cont(pte, lvl);  in arm_v7s_init_pte()
449 	pte |= paddr_to_iopte(paddr, lvl, cfg);  in arm_v7s_init_pte()
481 			 int lvl, arm_v7s_iopte *ptep, gfp_t gfp)  in __arm_v7s_map()  argument
485 	int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl);  in __arm_v7s_map()
488 	ptep += ARM_V7S_LVL_IDX(iova, lvl, cfg);  in __arm_v7s_map()
493 					lvl, num_entries, ptep);  in __arm_v7s_map()
496 	if (WARN_ON(lvl == 2))  in __arm_v7s_map()
502 		cptep = __arm_v7s_alloc_table(lvl + 1, gfp, data);  in __arm_v7s_map()
508 			__arm_v7s_free_table(cptep, lvl + 1, data);  in __arm_v7s_map()
514 	if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) {  in __arm_v7s_map()
515 		cptep = iopte_deref(pte, lvl, data);  in __arm_v7s_map()
523 	return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp);  in __arm_v7s_map()
585 					unsigned long iova, int idx, int lvl,  in arm_v7s_split_cont()  argument
590 	size_t size = ARM_V7S_BLOCK_SIZE(lvl);  in arm_v7s_split_cont()
595 	if (!arm_v7s_pte_is_cont(pte, lvl))  in arm_v7s_split_cont()
599 	pte = arm_v7s_cont_to_pte(pte, lvl);  in arm_v7s_split_cont()
657 			      unsigned long iova, size_t size, int lvl,  in __arm_v7s_unmap()  argument
662 	int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl);  in __arm_v7s_unmap()
665 	if (WARN_ON(lvl > 2))  in __arm_v7s_unmap()
668 	idx = ARM_V7S_LVL_IDX(iova, lvl, &iop->cfg);  in __arm_v7s_unmap()
686 	if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) {  in __arm_v7s_unmap()
690 		pte[0] = arm_v7s_split_cont(data, iova, idx, lvl, ptep);  in __arm_v7s_unmap()
696 		size_t blk_size = ARM_V7S_BLOCK_SIZE(lvl);  in __arm_v7s_unmap()
701 			if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) {  in __arm_v7s_unmap()
704 						ARM_V7S_BLOCK_SIZE(lvl + 1));  in __arm_v7s_unmap()
705 				ptep = iopte_deref(pte[i], lvl, data);  in __arm_v7s_unmap()
706 				__arm_v7s_free_table(ptep, lvl + 1, data);  in __arm_v7s_unmap()
720 	} else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) {  in __arm_v7s_unmap()
730 	ptep = iopte_deref(pte[0], lvl, data);  in __arm_v7s_unmap()
731 	return __arm_v7s_unmap(data, gather, iova, size, lvl + 1, ptep);  in __arm_v7s_unmap()
767 	int lvl = 0;  in arm_v7s_iova_to_phys()  local
771 		ptep += ARM_V7S_LVL_IDX(iova, ++lvl, &data->iop.cfg);  in arm_v7s_iova_to_phys()
773 		ptep = iopte_deref(pte, lvl, data);  in arm_v7s_iova_to_phys()
774 	} while (ARM_V7S_PTE_IS_TABLE(pte, lvl));  in arm_v7s_iova_to_phys()
779 	mask = ARM_V7S_LVL_MASK(lvl);  in arm_v7s_iova_to_phys()
780 	if (arm_v7s_pte_is_cont(pte, lvl))  in arm_v7s_iova_to_phys()
782 	return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask);  in arm_v7s_iova_to_phys()