157 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,  in iopte_leaf()  argument
160 	if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)  in iopte_leaf()
161 		return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_PAGE;  in iopte_leaf()
163 	return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK;  in iopte_leaf()
261 			       int lvl, arm_lpae_iopte *ptep);
265 				int lvl, int num_entries, arm_lpae_iopte *ptep)  in __arm_lpae_init_pte()  argument
269 	size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);  in __arm_lpae_init_pte()
272 	if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)  in __arm_lpae_init_pte()
286 			     arm_lpae_iopte prot, int lvl, int num_entries,  in arm_lpae_init_pte()  argument
292 		if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {  in arm_lpae_init_pte()
296 		} else if (iopte_type(ptep[i], lvl) == ARM_LPAE_PTE_TYPE_TABLE) {  in arm_lpae_init_pte()
302 			size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);  in arm_lpae_init_pte()
304 			tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_init_pte()
306 					     lvl, tblp) != sz) {  in arm_lpae_init_pte()
312 	__arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep);  in arm_lpae_init_pte()
350 			  arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep,  in __arm_lpae_map()  argument
354 	size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);  in __arm_lpae_map()
360 	map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_map()
367 		ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);  in __arm_lpae_map()
375 	if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))  in __arm_lpae_map()
392 	if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {  in __arm_lpae_map()
401 	return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1,  in __arm_lpae_map()
486 	int ret, lvl = data->start_level;  in arm_lpae_map_pages()  local
503 	ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl,  in arm_lpae_map_pages()
522 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,  in __arm_lpae_free_pgtable()  argument
528 	if (lvl == data->start_level)  in __arm_lpae_free_pgtable()
536 	if (lvl == ARM_LPAE_MAX_LEVELS - 1)  in __arm_lpae_free_pgtable()
544 		if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))  in __arm_lpae_free_pgtable()
547 		__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));  in __arm_lpae_free_pgtable()
564 				       arm_lpae_iopte blk_pte, int lvl,  in arm_lpae_split_blk_unmap()  argument
571 	size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data);  in arm_lpae_split_blk_unmap()
575 	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))  in arm_lpae_split_blk_unmap()
583 		unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_split_blk_unmap()
596 		__arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]);  in arm_lpae_split_blk_unmap()
607 		if (iopte_type(pte, lvl - 1) != ARM_LPAE_PTE_TYPE_TABLE)  in arm_lpae_split_blk_unmap()
618 	return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl, tablep);  in arm_lpae_split_blk_unmap()
624 			       int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_unmap()  argument
631 	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))  in __arm_lpae_unmap()
634 	unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_unmap()
641 	if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {  in __arm_lpae_unmap()
652 			if (!iopte_leaf(pte, lvl, iop->fmt)) {  in __arm_lpae_unmap()
656 				__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));  in __arm_lpae_unmap()
673 	} else if (iopte_leaf(pte, lvl, iop->fmt)) {  in __arm_lpae_unmap()
679 						lvl + 1, ptep, pgcount);  in __arm_lpae_unmap()
684 	return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep);  in __arm_lpae_unmap()
719 	int lvl = data->start_level;  in arm_lpae_iova_to_phys()  local
727 		ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_iova_to_phys()
735 		if (iopte_leaf(pte, lvl, data->iop.fmt))  in arm_lpae_iova_to_phys()
740 	} while (++lvl < ARM_LPAE_MAX_LEVELS);  in arm_lpae_iova_to_phys()
746 	iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1);  in arm_lpae_iova_to_phys()