83 #define ARM_V7S_PTE_IS_VALID(pte)	(((pte) & 0x3) != 0)  argument
84 #define ARM_V7S_PTE_IS_TABLE(pte, lvl) \  argument
85 	((lvl) == 1 && (((pte) & 0x3) == ARM_V7S_PTE_TYPE_TABLE))
172 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl);
188 	arm_v7s_iopte pte = paddr & ARM_V7S_LVL_MASK(lvl);  in paddr_to_iopte()  local
191 		return pte;  in paddr_to_iopte()
194 		pte |= ARM_V7S_ATTR_MTK_PA_BIT32;  in paddr_to_iopte()
196 		pte |= ARM_V7S_ATTR_MTK_PA_BIT33;  in paddr_to_iopte()
198 		pte |= ARM_V7S_ATTR_MTK_PA_BIT34;  in paddr_to_iopte()
199 	return pte;  in paddr_to_iopte()
202 static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl,  in iopte_to_paddr()  argument
208 	if (ARM_V7S_PTE_IS_TABLE(pte, lvl))  in iopte_to_paddr()
210 	else if (arm_v7s_pte_is_cont(pte, lvl))  in iopte_to_paddr()
215 	paddr = pte & mask;  in iopte_to_paddr()
219 	if (pte & ARM_V7S_ATTR_MTK_PA_BIT32)  in iopte_to_paddr()
221 	if (pte & ARM_V7S_ATTR_MTK_PA_BIT33)  in iopte_to_paddr()
223 	if (pte & ARM_V7S_ATTR_MTK_PA_BIT34)  in iopte_to_paddr()
228 static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl,  in iopte_deref()  argument
231 	return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg));  in iopte_deref()
311 static void __arm_v7s_set_pte(arm_v7s_iopte *ptep, arm_v7s_iopte pte,  in __arm_v7s_set_pte()  argument
317 		ptep[i] = pte;  in __arm_v7s_set_pte()
326 	arm_v7s_iopte pte = ARM_V7S_ATTR_NG | ARM_V7S_ATTR_S;  in arm_v7s_prot_to_pte()  local
329 		pte |= ARM_V7S_ATTR_TEX(1);  in arm_v7s_prot_to_pte()
331 		pte |= ARM_V7S_PTE_AF;  in arm_v7s_prot_to_pte()
333 			pte |= ARM_V7S_PTE_AP_UNPRIV;  in arm_v7s_prot_to_pte()
335 			pte |= ARM_V7S_PTE_AP_RDONLY;  in arm_v7s_prot_to_pte()
337 	pte <<= ARM_V7S_ATTR_SHIFT(lvl);  in arm_v7s_prot_to_pte()
340 		pte |= ARM_V7S_ATTR_XN(lvl);  in arm_v7s_prot_to_pte()
342 		pte |= ARM_V7S_ATTR_B;  in arm_v7s_prot_to_pte()
344 		pte |= ARM_V7S_ATTR_B | ARM_V7S_ATTR_C;  in arm_v7s_prot_to_pte()
346 	pte |= ARM_V7S_PTE_TYPE_PAGE;  in arm_v7s_prot_to_pte()
348 		pte |= ARM_V7S_ATTR_NS_SECTION;  in arm_v7s_prot_to_pte()
350 	return pte;  in arm_v7s_prot_to_pte()
353 static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl)  in arm_v7s_pte_to_prot()  argument
356 	arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl);  in arm_v7s_pte_to_prot()
364 	else if (pte & ARM_V7S_ATTR_C)  in arm_v7s_pte_to_prot()
366 	if (pte & ARM_V7S_ATTR_XN(lvl))  in arm_v7s_pte_to_prot()
372 static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl)  in arm_v7s_pte_to_cont()  argument
375 		pte |= ARM_V7S_CONT_SECTION;  in arm_v7s_pte_to_cont()
377 		arm_v7s_iopte xn = pte & ARM_V7S_ATTR_XN(lvl);  in arm_v7s_pte_to_cont()
378 		arm_v7s_iopte tex = pte & ARM_V7S_CONT_PAGE_TEX_MASK;  in arm_v7s_pte_to_cont()
380 		pte ^= xn | tex | ARM_V7S_PTE_TYPE_PAGE;  in arm_v7s_pte_to_cont()
381 		pte |= (xn << ARM_V7S_CONT_PAGE_XN_SHIFT) |  in arm_v7s_pte_to_cont()
385 	return pte;  in arm_v7s_pte_to_cont()
388 static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl)  in arm_v7s_cont_to_pte()  argument
391 		pte &= ~ARM_V7S_CONT_SECTION;  in arm_v7s_cont_to_pte()
393 		arm_v7s_iopte xn = pte & BIT(ARM_V7S_CONT_PAGE_XN_SHIFT);  in arm_v7s_cont_to_pte()
394 		arm_v7s_iopte tex = pte & (ARM_V7S_CONT_PAGE_TEX_MASK <<  in arm_v7s_cont_to_pte()
397 		pte ^= xn | tex | ARM_V7S_PTE_TYPE_CONT_PAGE;  in arm_v7s_cont_to_pte()
398 		pte |= (xn >> ARM_V7S_CONT_PAGE_XN_SHIFT) |  in arm_v7s_cont_to_pte()
402 	return pte;  in arm_v7s_cont_to_pte()
405 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl)  in arm_v7s_pte_is_cont()  argument
407 	if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl))  in arm_v7s_pte_is_cont()
408 		return pte & ARM_V7S_CONT_SECTION;  in arm_v7s_pte_is_cont()
410 		return !(pte & ARM_V7S_PTE_TYPE_PAGE);  in arm_v7s_pte_is_cont()
423 	arm_v7s_iopte pte;  in arm_v7s_init_pte()  local
445 	pte = arm_v7s_prot_to_pte(prot, lvl, cfg);  in arm_v7s_init_pte()
447 		pte = arm_v7s_pte_to_cont(pte, lvl);  in arm_v7s_init_pte()
449 	pte |= paddr_to_iopte(paddr, lvl, cfg);  in arm_v7s_init_pte()
451 	__arm_v7s_set_pte(ptep, pte, num_entries, cfg);  in arm_v7s_init_pte()
484 	arm_v7s_iopte pte, *cptep;  in __arm_v7s_map()  local
500 	pte = READ_ONCE(*ptep);  in __arm_v7s_map()
501 	if (!pte) {  in __arm_v7s_map()
506 		pte = arm_v7s_install_table(cptep, ptep, 0, cfg);  in __arm_v7s_map()
507 		if (pte)  in __arm_v7s_map()
514 	if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) {  in __arm_v7s_map()
515 		cptep = iopte_deref(pte, lvl, data);  in __arm_v7s_map()
516 	} else if (pte) {  in __arm_v7s_map()
573 		arm_v7s_iopte pte = data->pgd[i];  in arm_v7s_free_pgtable()  local
575 		if (ARM_V7S_PTE_IS_TABLE(pte, 1))  in arm_v7s_free_pgtable()
576 			__arm_v7s_free_table(iopte_deref(pte, 1, data),  in arm_v7s_free_pgtable()
589 	arm_v7s_iopte pte;  in arm_v7s_split_cont()  local
594 	pte = *ptep;  in arm_v7s_split_cont()
595 	if (!arm_v7s_pte_is_cont(pte, lvl))  in arm_v7s_split_cont()
596 		return pte;  in arm_v7s_split_cont()
599 	pte = arm_v7s_cont_to_pte(pte, lvl);  in arm_v7s_split_cont()
601 		ptep[i] = pte + i * size;  in arm_v7s_split_cont()
607 	return pte;  in arm_v7s_split_cont()
617 	arm_v7s_iopte pte, *tablep;  in arm_v7s_split_blk_unmap()  local
628 	pte = arm_v7s_prot_to_pte(arm_v7s_pte_to_prot(blk_pte, 1), 2, cfg);  in arm_v7s_split_blk_unmap()
630 		pte = arm_v7s_pte_to_cont(pte, 2);  in arm_v7s_split_blk_unmap()
632 	for (i = 0; i < num_ptes; i += num_entries, pte += size) {  in arm_v7s_split_blk_unmap()
637 		__arm_v7s_set_pte(&tablep[i], pte, num_entries, cfg);  in arm_v7s_split_blk_unmap()
640 	pte = arm_v7s_install_table(tablep, ptep, blk_pte, cfg);  in arm_v7s_split_blk_unmap()
641 	if (pte != blk_pte) {  in arm_v7s_split_blk_unmap()
644 		if (!ARM_V7S_PTE_IS_TABLE(pte, 1))  in arm_v7s_split_blk_unmap()
647 		tablep = iopte_deref(pte, 1, data);  in arm_v7s_split_blk_unmap()
660 	arm_v7s_iopte pte[ARM_V7S_CONT_PAGES];  in __arm_v7s_unmap()  local
671 		pte[i] = READ_ONCE(ptep[i]);  in __arm_v7s_unmap()
672 		if (WARN_ON(!ARM_V7S_PTE_IS_VALID(pte[i])))  in __arm_v7s_unmap()
686 	if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) {  in __arm_v7s_unmap()
690 		pte[0] = arm_v7s_split_cont(data, iova, idx, lvl, ptep);  in __arm_v7s_unmap()
701 			if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) {  in __arm_v7s_unmap()
705 				ptep = iopte_deref(pte[i], lvl, data);  in __arm_v7s_unmap()
720 	} else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) {  in __arm_v7s_unmap()
725 		return arm_v7s_split_blk_unmap(data, gather, iova, size, pte[0],  in __arm_v7s_unmap()
730 	ptep = iopte_deref(pte[0], lvl, data);  in __arm_v7s_unmap()
766 	arm_v7s_iopte *ptep = data->pgd, pte;  in arm_v7s_iova_to_phys()  local
772 		pte = READ_ONCE(*ptep);  in arm_v7s_iova_to_phys()
773 		ptep = iopte_deref(pte, lvl, data);  in arm_v7s_iova_to_phys()
774 	} while (ARM_V7S_PTE_IS_TABLE(pte, lvl));  in arm_v7s_iova_to_phys()
776 	if (!ARM_V7S_PTE_IS_VALID(pte))  in arm_v7s_iova_to_phys()
780 	if (arm_v7s_pte_is_cont(pte, lvl))  in arm_v7s_iova_to_phys()
782 	return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask);  in arm_v7s_iova_to_phys()