232 static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)  in iommu_report_fault()  argument
239 		*da = 0;  in iommu_report_fault()
244 	*da = fault_addr;  in iommu_report_fault()
309 	if (e->da & ~(get_cam_va_mask(e->pgsz))) {  in iotlb_alloc_cr()
311 			e->da);  in iotlb_alloc_cr()
319 	cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid;  in iotlb_alloc_cr()
408 static void flush_iotlb_page(struct omap_iommu *obj, u32 da)  in flush_iotlb_page()  argument
425 		if ((start <= da) && (da < start + bytes)) {  in flush_iotlb_page()
427 				__func__, start, da, bytes);  in flush_iotlb_page()
436 		dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);  in flush_iotlb_page()
486 			dma_addr_t *pt_dma, u32 da)  in iopte_alloc()  argument
489 	unsigned long offset = iopgd_index(da) * sizeof(da);  in iopte_alloc()
536 	iopte = iopte_offset(iopgd, da);  in iopte_alloc()
540 		 __func__, da, iopgd, *iopgd, iopte, *iopte);  in iopte_alloc()
545 static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopgd_alloc_section()  argument
547 	u32 *iopgd = iopgd_offset(obj, da);  in iopgd_alloc_section()
548 	unsigned long offset = iopgd_index(da) * sizeof(da);  in iopgd_alloc_section()
550 	if ((da | pa) & ~IOSECTION_MASK) {  in iopgd_alloc_section()
552 			__func__, da, pa, IOSECTION_SIZE);  in iopgd_alloc_section()
561 static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopgd_alloc_super()  argument
563 	u32 *iopgd = iopgd_offset(obj, da);  in iopgd_alloc_super()
564 	unsigned long offset = iopgd_index(da) * sizeof(da);  in iopgd_alloc_super()
567 	if ((da | pa) & ~IOSUPER_MASK) {  in iopgd_alloc_super()
569 			__func__, da, pa, IOSUPER_SIZE);  in iopgd_alloc_super()
579 static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopte_alloc_page()  argument
581 	u32 *iopgd = iopgd_offset(obj, da);  in iopte_alloc_page()
583 	u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);  in iopte_alloc_page()
584 	unsigned long offset = iopte_index(da) * sizeof(da);  in iopte_alloc_page()
593 		 __func__, da, pa, iopte, *iopte);  in iopte_alloc_page()
598 static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopte_alloc_large()  argument
600 	u32 *iopgd = iopgd_offset(obj, da);  in iopte_alloc_large()
602 	u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);  in iopte_alloc_large()
603 	unsigned long offset = iopte_index(da) * sizeof(da);  in iopte_alloc_large()
606 	if ((da | pa) & ~IOLARGE_MASK) {  in iopte_alloc_large()
608 			__func__, da, pa, IOLARGE_SIZE);  in iopte_alloc_large()
655 	err = fn(obj, e->da, e->pa, prot);  in iopgtable_store_entry_core()
671 	flush_iotlb_page(obj, e->da);  in omap_iopgtable_store_entry()
686 iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)  in iopgtable_lookup_entry()  argument
690 	iopgd = iopgd_offset(obj, da);  in iopgtable_lookup_entry()
695 		iopte = iopte_offset(iopgd, da);  in iopgtable_lookup_entry()
701 static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)  in iopgtable_clear_entry_core()  argument
704 	u32 *iopgd = iopgd_offset(obj, da);  in iopgtable_clear_entry_core()
707 	unsigned long pd_offset = iopgd_index(da) * sizeof(da);  in iopgtable_clear_entry_core()
708 	unsigned long pt_offset = iopte_index(da) * sizeof(da);  in iopgtable_clear_entry_core()
715 		u32 *iopte = iopte_offset(iopgd, da);  in iopgtable_clear_entry_core()
721 			iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));  in iopgtable_clear_entry_core()
743 			iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));  in iopgtable_clear_entry_core()
758 static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)  in iopgtable_clear_entry()  argument
764 	bytes = iopgtable_clear_entry_core(obj, da);  in iopgtable_clear_entry()
765 	flush_iotlb_page(obj, da);  in iopgtable_clear_entry()
780 		u32 da;  in iopgtable_clear_entry_all()  local
783 		da = i << IOPGD_SHIFT;  in iopgtable_clear_entry_all()
784 		iopgd = iopgd_offset(obj, da);  in iopgtable_clear_entry_all()
785 		offset = iopgd_index(da) * sizeof(da);  in iopgtable_clear_entry_all()
807 	u32 da, errs;  in iommu_fault_handler()  local
816 	errs = iommu_report_fault(obj, &da);  in iommu_fault_handler()
821 	if (!report_iommu_fault(domain, obj->dev, da, 0))  in iommu_fault_handler()
826 	iopgd = iopgd_offset(obj, da);  in iommu_fault_handler()
830 			obj->name, errs, da, iopgd, *iopgd);  in iommu_fault_handler()
834 	iopte = iopte_offset(iopgd, da);  in iommu_fault_handler()
837 		obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);  in iommu_fault_handler()
1310 static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)  in iotlb_init_entry()  argument
1314 	e->da		= da;  in iotlb_init_entry()
1325 static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,  in omap_iommu_map()  argument
1343 	dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%zx\n", da, &pa, bytes);  in omap_iommu_map()
1345 	iotlb_init_entry(&e, da, pa, omap_pgsz);  in omap_iommu_map()
1362 			iopgtable_clear_entry(oiommu, da);  in omap_iommu_map()
1369 static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,  in omap_iommu_unmap()  argument
1380 	dev_dbg(dev, "unmapping da 0x%lx size %zu\n", da, size);  in omap_iommu_unmap()
1385 		bytes = iopgtable_clear_entry(oiommu, da);  in omap_iommu_unmap()
1608 					   dma_addr_t da)  in omap_iommu_iova_to_phys()  argument
1621 	iopgtable_lookup_entry(oiommu, da, &pgd, &pte);  in omap_iommu_iova_to_phys()
1625 			ret = omap_iommu_translate(*pte, da, IOPTE_MASK);  in omap_iommu_iova_to_phys()
1627 			ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);  in omap_iommu_iova_to_phys()
1630 				(unsigned long long)da);  in omap_iommu_iova_to_phys()
1633 			ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);  in omap_iommu_iova_to_phys()
1635 			ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);  in omap_iommu_iova_to_phys()
1638 				(unsigned long long)da);  in omap_iommu_iova_to_phys()