242 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,  in __arm_lpae_sync_pte()  argument
245 	dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),  in __arm_lpae_sync_pte()
246 				   sizeof(*ptep) * num_entries, DMA_TO_DEVICE);  in __arm_lpae_sync_pte()
249 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg)  in __arm_lpae_clear_pte()  argument
252 	*ptep = 0;  in __arm_lpae_clear_pte()
255 		__arm_lpae_sync_pte(ptep, 1, cfg);  in __arm_lpae_clear_pte()
261 			       int lvl, arm_lpae_iopte *ptep);
265 				int lvl, int num_entries, arm_lpae_iopte *ptep)  in __arm_lpae_init_pte()  argument
278 		ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data);  in __arm_lpae_init_pte()
281 		__arm_lpae_sync_pte(ptep, num_entries, cfg);  in __arm_lpae_init_pte()
287 			     arm_lpae_iopte *ptep)  in arm_lpae_init_pte()  argument
292 		if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {  in arm_lpae_init_pte()
296 		} else if (iopte_type(ptep[i], lvl) == ARM_LPAE_PTE_TYPE_TABLE) {  in arm_lpae_init_pte()
304 			tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_init_pte()
312 	__arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep);  in arm_lpae_init_pte()
317 					     arm_lpae_iopte *ptep,  in arm_lpae_install_table()  argument
335 	old = cmpxchg64_relaxed(ptep, curr, new);  in arm_lpae_install_table()
341 	__arm_lpae_sync_pte(ptep, 1, cfg);  in arm_lpae_install_table()
343 		WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC);  in arm_lpae_install_table()
350 			  arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep,  in __arm_lpae_map()  argument
361 	ptep += map_idx_start;  in __arm_lpae_map()
367 		ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);  in __arm_lpae_map()
379 	pte = READ_ONCE(*ptep);  in __arm_lpae_map()
385 		pte = arm_lpae_install_table(cptep, ptep, 0, data);  in __arm_lpae_map()
389 		__arm_lpae_sync_pte(ptep, 1, cfg);  in __arm_lpae_map()
485 	arm_lpae_iopte *ptep = data->pgd;  in arm_lpae_map_pages()  local
504 			     ptep, gfp, mapped);  in arm_lpae_map_pages()
523 				    arm_lpae_iopte *ptep)  in __arm_lpae_free_pgtable()  argument
533 	start = ptep;  in __arm_lpae_free_pgtable()
537 		end = ptep;  in __arm_lpae_free_pgtable()
539 		end = (void *)ptep + table_size;  in __arm_lpae_free_pgtable()
541 	while (ptep != end) {  in __arm_lpae_free_pgtable()
542 		arm_lpae_iopte pte = *ptep++;  in __arm_lpae_free_pgtable()
565 				       arm_lpae_iopte *ptep, size_t pgcount)  in arm_lpae_split_blk_unmap()  argument
599 	pte = arm_lpae_install_table(tablep, ptep, blk_pte, data);  in arm_lpae_split_blk_unmap()
624 			       int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_unmap()  argument
635 	ptep += unmap_idx_start;  in __arm_lpae_unmap()
636 	pte = READ_ONCE(*ptep);  in __arm_lpae_unmap()
646 			pte = READ_ONCE(*ptep);  in __arm_lpae_unmap()
650 			__arm_lpae_clear_pte(ptep, &iop->cfg);  in __arm_lpae_unmap()
668 			ptep++;  in __arm_lpae_unmap()
679 						lvl + 1, ptep, pgcount);  in __arm_lpae_unmap()
683 	ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
684 	return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep);  in __arm_lpae_unmap()
693 	arm_lpae_iopte *ptep = data->pgd;  in arm_lpae_unmap_pages()  local
705 				data->start_level, ptep);  in arm_lpae_unmap_pages()
718 	arm_lpae_iopte pte, *ptep = data->pgd;  in arm_lpae_iova_to_phys()  local
723 		if (!ptep)  in arm_lpae_iova_to_phys()
727 		ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_iova_to_phys()
728 		pte = READ_ONCE(*ptep);  in arm_lpae_iova_to_phys()
739 		ptep = iopte_deref(pte, data);  in arm_lpae_iova_to_phys()