896 static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,  in arm_smmu_sync_cd()  argument
903 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_sync_cd()
912 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);  in arm_smmu_sync_cd()
913 	list_for_each_entry(master, &smmu_domain->devices, domain_head) {  in arm_smmu_sync_cd()
919 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);  in arm_smmu_sync_cd()
949 static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain,  in arm_smmu_get_cd_ptr()  argument
955 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_get_cd_ptr()
956 	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;  in arm_smmu_get_cd_ptr()
958 	if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)  in arm_smmu_get_cd_ptr()
970 		arm_smmu_sync_cd(smmu_domain, ssid, false);  in arm_smmu_get_cd_ptr()
976 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,  in arm_smmu_write_ctx_desc()  argument
991 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_write_ctx_desc()
993 	if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))  in arm_smmu_write_ctx_desc()
996 	cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);  in arm_smmu_write_ctx_desc()
1022 		arm_smmu_sync_cd(smmu_domain, ssid, true);  in arm_smmu_write_ctx_desc()
1049 	arm_smmu_sync_cd(smmu_domain, ssid, true);  in arm_smmu_write_ctx_desc()
1053 static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)  in arm_smmu_alloc_cd_tables()  argument
1058 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_alloc_cd_tables()
1059 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;  in arm_smmu_alloc_cd_tables()
1102 static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)  in arm_smmu_free_cd_tables()  argument
1106 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_free_cd_tables()
1107 	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;  in arm_smmu_free_cd_tables()
1200 	struct arm_smmu_domain *smmu_domain = NULL;  in arm_smmu_write_strtab_ent()  local
1209 		smmu_domain = master->domain;  in arm_smmu_write_strtab_ent()
1213 	if (smmu_domain) {  in arm_smmu_write_strtab_ent()
1214 		switch (smmu_domain->stage) {  in arm_smmu_write_strtab_ent()
1216 			s1_cfg = &smmu_domain->s1_cfg;  in arm_smmu_write_strtab_ent()
1220 			s2_cfg = &smmu_domain->s2_cfg;  in arm_smmu_write_strtab_ent()
1247 	if (!smmu_domain || !(s1_cfg || s2_cfg)) {  in arm_smmu_write_strtab_ent()
1248 		if (!smmu_domain && disable_bypass)  in arm_smmu_write_strtab_ent()
1581 static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,  in arm_smmu_atc_inv_domain()  argument
1590 	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))  in arm_smmu_atc_inv_domain()
1607 	if (!atomic_read(&smmu_domain->nr_ats_masters))  in arm_smmu_atc_inv_domain()
1612 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);  in arm_smmu_atc_inv_domain()
1613 	list_for_each_entry(master, &smmu_domain->devices, domain_head) {  in arm_smmu_atc_inv_domain()
1619 			arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);  in arm_smmu_atc_inv_domain()
1622 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);  in arm_smmu_atc_inv_domain()
1624 	return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);  in arm_smmu_atc_inv_domain()
1630 	struct arm_smmu_domain *smmu_domain = cookie;  in arm_smmu_tlb_inv_context()  local
1631 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_context()
1641 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_tlb_inv_context()
1642 		arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);  in arm_smmu_tlb_inv_context()
1645 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;  in arm_smmu_tlb_inv_context()
1649 	arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);  in arm_smmu_tlb_inv_context()
1654 				   struct arm_smmu_domain *smmu_domain)  in arm_smmu_tlb_inv_range()  argument
1656 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_range()
1669 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_tlb_inv_range()
1671 		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;  in arm_smmu_tlb_inv_range()
1674 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;  in arm_smmu_tlb_inv_range()
1679 		tg = __ffs(smmu_domain->domain.pgsize_bitmap);  in arm_smmu_tlb_inv_range()
1726 	arm_smmu_atc_inv_domain(smmu_domain, 0, start, size);  in arm_smmu_tlb_inv_range()
1733 	struct arm_smmu_domain *smmu_domain = cookie;  in arm_smmu_tlb_inv_page_nosync()  local
1734 	struct iommu_domain *domain = &smmu_domain->domain;  in arm_smmu_tlb_inv_page_nosync()
1766 	struct arm_smmu_domain *smmu_domain;  in arm_smmu_domain_alloc()  local
1778 	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);  in arm_smmu_domain_alloc()
1779 	if (!smmu_domain)  in arm_smmu_domain_alloc()
1783 	    iommu_get_dma_cookie(&smmu_domain->domain)) {  in arm_smmu_domain_alloc()
1784 		kfree(smmu_domain);  in arm_smmu_domain_alloc()
1788 	mutex_init(&smmu_domain->init_mutex);  in arm_smmu_domain_alloc()
1789 	INIT_LIST_HEAD(&smmu_domain->devices);  in arm_smmu_domain_alloc()
1790 	spin_lock_init(&smmu_domain->devices_lock);  in arm_smmu_domain_alloc()
1792 	return &smmu_domain->domain;  in arm_smmu_domain_alloc()
1815 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_free()  local
1816 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_free()
1819 	free_io_pgtable_ops(smmu_domain->pgtbl_ops);  in arm_smmu_domain_free()
1822 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_domain_free()
1823 		struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;  in arm_smmu_domain_free()
1828 			arm_smmu_free_cd_tables(smmu_domain);  in arm_smmu_domain_free()
1832 		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;  in arm_smmu_domain_free()
1837 	kfree(smmu_domain);  in arm_smmu_domain_free()
1840 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,  in arm_smmu_domain_finalise_s1()  argument
1846 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise_s1()
1847 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;  in arm_smmu_domain_finalise_s1()
1861 	ret = arm_smmu_alloc_cd_tables(smmu_domain);  in arm_smmu_domain_finalise_s1()
1881 	ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd);  in arm_smmu_domain_finalise_s1()
1889 	arm_smmu_free_cd_tables(smmu_domain);  in arm_smmu_domain_finalise_s1()
1897 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,  in arm_smmu_domain_finalise_s2()  argument
1902 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise_s2()
1903 	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;  in arm_smmu_domain_finalise_s2()
1934 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_finalise()  local
1935 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise()
1938 		smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;  in arm_smmu_domain_finalise()
1944 		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;  in arm_smmu_domain_finalise()
1946 		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;  in arm_smmu_domain_finalise()
1948 	switch (smmu_domain->stage) {  in arm_smmu_domain_finalise()
1976 	if (smmu_domain->non_strict)  in arm_smmu_domain_finalise()
1979 	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);  in arm_smmu_domain_finalise()
1987 	ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);  in arm_smmu_domain_finalise()
1993 	smmu_domain->pgtbl_ops = pgtbl_ops;  in arm_smmu_domain_finalise()
2059 	struct arm_smmu_domain *smmu_domain = master->domain;  in arm_smmu_enable_ats()  local
2069 	atomic_inc(&smmu_domain->nr_ats_masters);  in arm_smmu_enable_ats()
2070 	arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);  in arm_smmu_enable_ats()
2077 	struct arm_smmu_domain *smmu_domain = master->domain;  in arm_smmu_disable_ats()  local
2089 	atomic_dec(&smmu_domain->nr_ats_masters);  in arm_smmu_disable_ats()
2142 	struct arm_smmu_domain *smmu_domain = master->domain;  in arm_smmu_detach_dev()  local
2144 	if (!smmu_domain)  in arm_smmu_detach_dev()
2149 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);  in arm_smmu_detach_dev()
2151 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);  in arm_smmu_detach_dev()
2164 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_attach_dev()  local
2185 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_attach_dev()
2187 	if (!smmu_domain->smmu) {  in arm_smmu_attach_dev()
2188 		smmu_domain->smmu = smmu;  in arm_smmu_attach_dev()
2191 			smmu_domain->smmu = NULL;  in arm_smmu_attach_dev()
2194 	} else if (smmu_domain->smmu != smmu) {  in arm_smmu_attach_dev()
2197 			dev_name(smmu_domain->smmu->dev),  in arm_smmu_attach_dev()
2201 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&  in arm_smmu_attach_dev()
2202 		   master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {  in arm_smmu_attach_dev()
2205 			smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);  in arm_smmu_attach_dev()
2210 	master->domain = smmu_domain;  in arm_smmu_attach_dev()
2212 	if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)  in arm_smmu_attach_dev()
2217 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);  in arm_smmu_attach_dev()
2218 	list_add(&master->domain_head, &smmu_domain->devices);  in arm_smmu_attach_dev()
2219 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);  in arm_smmu_attach_dev()
2224 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_attach_dev()
2242 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_unmap()  local
2243 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;  in arm_smmu_unmap()
2253 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_flush_iotlb_all()  local
2255 	if (smmu_domain->smmu)  in arm_smmu_flush_iotlb_all()
2256 		arm_smmu_tlb_inv_context(smmu_domain);  in arm_smmu_flush_iotlb_all()
2262 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_iotlb_sync()  local
2265 			       gather->pgsize, true, smmu_domain);  in arm_smmu_iotlb_sync()
2410 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_get_attr()  local
2416 			*(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);  in arm_smmu_domain_get_attr()
2425 			*(int *)data = smmu_domain->non_strict;  in arm_smmu_domain_get_attr()
2440 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_set_attr()  local
2442 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_domain_set_attr()
2448 			if (smmu_domain->smmu) {  in arm_smmu_domain_set_attr()
2454 				smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;  in arm_smmu_domain_set_attr()
2456 				smmu_domain->stage = ARM_SMMU_DOMAIN_S1;  in arm_smmu_domain_set_attr()
2465 			smmu_domain->non_strict = *(int *)data;  in arm_smmu_domain_set_attr()
2476 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_domain_set_attr()