Lines Matching full:smmu
34 #include "arm-smmu-v3.h"
39 …domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
86 struct arm_smmu_device *smmu) in arm_smmu_page1_fixup() argument
89 return smmu->page1 + offset - SZ_64K; in arm_smmu_page1_fixup()
91 return smmu->base + offset; in arm_smmu_page1_fixup()
99 static void parse_driver_options(struct arm_smmu_device *smmu) in parse_driver_options() argument
104 if (of_property_read_bool(smmu->dev->of_node, in parse_driver_options()
106 smmu->options |= arm_smmu_options[i].opt; in parse_driver_options()
107 dev_notice(smmu->dev, "option %s\n", in parse_driver_options()
190 static void queue_poll_init(struct arm_smmu_device *smmu, in queue_poll_init() argument
195 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); in queue_poll_init()
338 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu, in arm_smmu_cmdq_build_sync_cmd() argument
341 struct arm_smmu_queue *q = &smmu->cmdq.q; in arm_smmu_cmdq_build_sync_cmd()
350 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) { in arm_smmu_cmdq_build_sync_cmd()
358 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) in arm_smmu_cmdq_skip_err() argument
369 struct arm_smmu_queue *q = &smmu->cmdq.q; in arm_smmu_cmdq_skip_err()
376 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons, in arm_smmu_cmdq_skip_err()
381 dev_err(smmu->dev, "retrying command fetch\n"); in arm_smmu_cmdq_skip_err()
402 dev_err(smmu->dev, "skipping command in error state:\n"); in arm_smmu_cmdq_skip_err()
404 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); in arm_smmu_cmdq_skip_err()
408 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n"); in arm_smmu_cmdq_skip_err()
506 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
576 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu, in arm_smmu_cmdq_poll_until_not_full() argument
581 struct arm_smmu_cmdq *cmdq = &smmu->cmdq; in arm_smmu_cmdq_poll_until_not_full()
595 queue_poll_init(smmu, &qp); in arm_smmu_cmdq_poll_until_not_full()
597 llq->val = READ_ONCE(smmu->cmdq.q.llq.val); in arm_smmu_cmdq_poll_until_not_full()
608 * Wait until the SMMU signals a CMD_SYNC completion MSI.
611 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu, in __arm_smmu_cmdq_poll_until_msi() argument
616 struct arm_smmu_cmdq *cmdq = &smmu->cmdq; in __arm_smmu_cmdq_poll_until_msi()
619 queue_poll_init(smmu, &qp); in __arm_smmu_cmdq_poll_until_msi()
632 * Wait until the SMMU cons index passes llq->prod.
635 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu, in __arm_smmu_cmdq_poll_until_consumed() argument
639 struct arm_smmu_cmdq *cmdq = &smmu->cmdq; in __arm_smmu_cmdq_poll_until_consumed()
643 queue_poll_init(smmu, &qp); in __arm_smmu_cmdq_poll_until_consumed()
644 llq->val = READ_ONCE(smmu->cmdq.q.llq.val); in __arm_smmu_cmdq_poll_until_consumed()
685 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu, in arm_smmu_cmdq_poll_until_sync() argument
688 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) in arm_smmu_cmdq_poll_until_sync()
689 return __arm_smmu_cmdq_poll_until_msi(smmu, llq); in arm_smmu_cmdq_poll_until_sync()
691 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq); in arm_smmu_cmdq_poll_until_sync()
727 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, in arm_smmu_cmdq_issue_cmdlist() argument
734 struct arm_smmu_cmdq *cmdq = &smmu->cmdq; in arm_smmu_cmdq_issue_cmdlist()
748 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq)) in arm_smmu_cmdq_issue_cmdlist()
749 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); in arm_smmu_cmdq_issue_cmdlist()
774 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod); in arm_smmu_cmdq_issue_cmdlist()
790 /* 4. If we are the owner, take control of the SMMU hardware */ in arm_smmu_cmdq_issue_cmdlist()
824 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq); in arm_smmu_cmdq_issue_cmdlist()
826 dev_err_ratelimited(smmu->dev, in arm_smmu_cmdq_issue_cmdlist()
847 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, in arm_smmu_cmdq_issue_cmd() argument
853 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", in arm_smmu_cmdq_issue_cmd()
858 return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false); in arm_smmu_cmdq_issue_cmd()
861 static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu) in arm_smmu_cmdq_issue_sync() argument
863 return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true); in arm_smmu_cmdq_issue_sync()
866 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, in arm_smmu_cmdq_batch_add() argument
871 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false); in arm_smmu_cmdq_batch_add()
878 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, in arm_smmu_cmdq_batch_submit() argument
881 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true); in arm_smmu_cmdq_batch_submit()
885 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) in arm_smmu_tlb_inv_asid() argument
892 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_tlb_inv_asid()
893 arm_smmu_cmdq_issue_sync(smmu); in arm_smmu_tlb_inv_asid()
903 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_sync_cd() local
916 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); in arm_smmu_sync_cd()
921 arm_smmu_cmdq_batch_submit(smmu, &cmds); in arm_smmu_sync_cd()
924 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu, in arm_smmu_alloc_cd_leaf_table() argument
929 l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, in arm_smmu_alloc_cd_leaf_table()
932 dev_warn(smmu->dev, in arm_smmu_alloc_cd_leaf_table()
955 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_get_cd_ptr() local
964 if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) in arm_smmu_get_cd_ptr()
991 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_write_ctx_desc() local
1018 * STE is live, and the SMMU might read dwords of this CD in any in arm_smmu_write_ctx_desc()
1035 if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE) in arm_smmu_write_ctx_desc()
1040 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3 in arm_smmu_write_ctx_desc()
1043 * The size of single-copy atomic reads made by the SMMU is in arm_smmu_write_ctx_desc()
1058 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_alloc_cd_tables() local
1064 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) || in arm_smmu_alloc_cd_tables()
1075 cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents, in arm_smmu_alloc_cd_tables()
1084 cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma, in arm_smmu_alloc_cd_tables()
1087 dev_warn(smmu->dev, "failed to allocate context descriptor\n"); in arm_smmu_alloc_cd_tables()
1096 devm_kfree(smmu->dev, cdcfg->l1_desc); in arm_smmu_alloc_cd_tables()
1106 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_free_cd_tables() local
1116 dmam_free_coherent(smmu->dev, size, in arm_smmu_free_cd_tables()
1120 devm_kfree(smmu->dev, cdcfg->l1_desc); in arm_smmu_free_cd_tables()
1128 dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma); in arm_smmu_free_cd_tables()
1162 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_sync_ste_for_sid() argument
1172 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_sync_ste_for_sid()
1173 arm_smmu_cmdq_issue_sync(smmu); in arm_smmu_sync_ste_for_sid()
1187 * Given that we can't update the STE atomically and the SMMU in arm_smmu_write_strtab_ent()
1197 struct arm_smmu_device *smmu = NULL; in arm_smmu_write_strtab_ent() local
1210 smmu = master->smmu; in arm_smmu_write_strtab_ent()
1258 * The SMMU can perform negative caching, so we must sync in arm_smmu_write_strtab_ent()
1261 if (smmu) in arm_smmu_write_strtab_ent()
1262 arm_smmu_sync_ste_for_sid(smmu, sid); in arm_smmu_write_strtab_ent()
1275 if (smmu->features & ARM_SMMU_FEAT_STALLS && in arm_smmu_write_strtab_ent()
1276 !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) in arm_smmu_write_strtab_ent()
1305 arm_smmu_sync_ste_for_sid(smmu, sid); in arm_smmu_write_strtab_ent()
1308 arm_smmu_sync_ste_for_sid(smmu, sid); in arm_smmu_write_strtab_ent()
1311 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) in arm_smmu_write_strtab_ent()
1312 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); in arm_smmu_write_strtab_ent()
1325 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_init_l2_strtab() argument
1329 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_l2_strtab()
1339 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma, in arm_smmu_init_l2_strtab()
1342 dev_err(smmu->dev, in arm_smmu_init_l2_strtab()
1357 struct arm_smmu_device *smmu = dev; in arm_smmu_evtq_thread() local
1358 struct arm_smmu_queue *q = &smmu->evtq.q; in arm_smmu_evtq_thread()
1366 dev_info(smmu->dev, "event 0x%02x received:\n", id); in arm_smmu_evtq_thread()
1368 dev_info(smmu->dev, "\t0x%016llx\n", in arm_smmu_evtq_thread()
1379 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n"); in arm_smmu_evtq_thread()
1388 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) in arm_smmu_handle_ppr() argument
1400 dev_info(smmu->dev, "unexpected PRI request received:\n"); in arm_smmu_handle_ppr()
1401 dev_info(smmu->dev, in arm_smmu_handle_ppr()
1422 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_handle_ppr()
1428 struct arm_smmu_device *smmu = dev; in arm_smmu_priq_thread() local
1429 struct arm_smmu_queue *q = &smmu->priq.q; in arm_smmu_priq_thread()
1435 arm_smmu_handle_ppr(smmu, evt); in arm_smmu_priq_thread()
1438 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n"); in arm_smmu_priq_thread()
1448 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1453 struct arm_smmu_device *smmu = dev; in arm_smmu_gerror_handler() local
1455 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); in arm_smmu_gerror_handler()
1456 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); in arm_smmu_gerror_handler()
1462 dev_warn(smmu->dev, in arm_smmu_gerror_handler()
1467 dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); in arm_smmu_gerror_handler()
1468 arm_smmu_device_disable(smmu); in arm_smmu_gerror_handler()
1472 dev_warn(smmu->dev, "GERROR MSI write aborted\n"); in arm_smmu_gerror_handler()
1475 dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); in arm_smmu_gerror_handler()
1478 dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); in arm_smmu_gerror_handler()
1481 dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); in arm_smmu_gerror_handler()
1484 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); in arm_smmu_gerror_handler()
1487 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); in arm_smmu_gerror_handler()
1490 arm_smmu_cmdq_skip_err(smmu); in arm_smmu_gerror_handler()
1492 writel(gerror, smmu->base + ARM_SMMU_GERRORN); in arm_smmu_gerror_handler()
1498 struct arm_smmu_device *smmu = dev; in arm_smmu_combined_irq_thread() local
1501 if (smmu->features & ARM_SMMU_FEAT_PRI) in arm_smmu_combined_irq_thread()
1575 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); in arm_smmu_atc_inv_master()
1578 return arm_smmu_cmdq_issue_sync(master->smmu); in arm_smmu_atc_inv_master()
1590 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS)) in arm_smmu_atc_inv_domain()
1619 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd); in arm_smmu_atc_inv_domain()
1624 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds); in arm_smmu_atc_inv_domain()
1631 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_inv_context() local
1637 * to the SMMU. We are relying on the dma_wmb() implicit during cmd in arm_smmu_tlb_inv_context()
1642 arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid); in arm_smmu_tlb_inv_context()
1646 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_tlb_inv_context()
1647 arm_smmu_cmdq_issue_sync(smmu); in arm_smmu_tlb_inv_context()
1656 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_inv_range() local
1677 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { in arm_smmu_tlb_inv_range()
1691 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { in arm_smmu_tlb_inv_range()
1717 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); in arm_smmu_tlb_inv_range()
1720 arm_smmu_cmdq_batch_submit(smmu, &cmds); in arm_smmu_tlb_inv_range()
1816 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_free() local
1834 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid); in arm_smmu_domain_free()
1846 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_finalise_s1() local
1855 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); in arm_smmu_domain_finalise_s1()
1902 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_finalise_s2() local
1906 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits); in arm_smmu_domain_finalise_s2()
1935 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_finalise() local
1943 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) in arm_smmu_domain_finalise()
1945 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) in arm_smmu_domain_finalise()
1950 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; in arm_smmu_domain_finalise()
1952 oas = smmu->ias; in arm_smmu_domain_finalise()
1958 ias = smmu->ias; in arm_smmu_domain_finalise()
1959 oas = smmu->oas; in arm_smmu_domain_finalise()
1968 .pgsize_bitmap = smmu->pgsize_bitmap, in arm_smmu_domain_finalise()
1971 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, in arm_smmu_domain_finalise()
1973 .iommu_dev = smmu->dev, in arm_smmu_domain_finalise()
1997 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_get_step_for_sid() argument
2000 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_get_step_for_sid()
2002 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { in arm_smmu_get_step_for_sid()
2022 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_install_ste_for_dev() local
2026 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); in arm_smmu_install_ste_for_dev()
2042 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_ats_supported() local
2045 if (!(smmu->features & ARM_SMMU_FEAT_ATS)) in arm_smmu_ats_supported()
2058 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_enable_ats() local
2066 stu = __ffs(smmu->pgsize_bitmap); in arm_smmu_enable_ats()
2085 * ATC invalidation via the SMMU. in arm_smmu_disable_ats()
2119 master->smmu->ssid_bits); in arm_smmu_enable_pasid()
2163 struct arm_smmu_device *smmu; in arm_smmu_attach_dev() local
2171 smmu = master->smmu; in arm_smmu_attach_dev()
2187 if (!smmu_domain->smmu) { in arm_smmu_attach_dev()
2188 smmu_domain->smmu = smmu; in arm_smmu_attach_dev()
2191 smmu_domain->smmu = NULL; in arm_smmu_attach_dev()
2194 } else if (smmu_domain->smmu != smmu) { in arm_smmu_attach_dev()
2196 "cannot attach to SMMU %s (upstream of %s)\n", in arm_smmu_attach_dev()
2197 dev_name(smmu_domain->smmu->dev), in arm_smmu_attach_dev()
2198 dev_name(smmu->dev)); in arm_smmu_attach_dev()
2255 if (smmu_domain->smmu) in arm_smmu_flush_iotlb_all()
2293 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_sid_in_range() argument
2295 unsigned long limit = smmu->strtab_cfg.num_l1_ents; in arm_smmu_sid_in_range()
2297 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) in arm_smmu_sid_in_range()
2308 struct arm_smmu_device *smmu; in arm_smmu_probe_device() local
2318 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); in arm_smmu_probe_device()
2319 if (!smmu) in arm_smmu_probe_device()
2327 master->smmu = smmu; in arm_smmu_probe_device()
2333 /* Check the SIDs are in range of the SMMU and our stream table */ in arm_smmu_probe_device()
2337 if (!arm_smmu_sid_in_range(smmu, sid)) { in arm_smmu_probe_device()
2343 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { in arm_smmu_probe_device()
2344 ret = arm_smmu_init_l2_strtab(smmu, sid); in arm_smmu_probe_device()
2350 master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits); in arm_smmu_probe_device()
2362 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB)) in arm_smmu_probe_device()
2366 return &smmu->iommu; in arm_smmu_probe_device()
2448 if (smmu_domain->smmu) { in arm_smmu_domain_set_attr()
2590 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, in arm_smmu_init_one_queue() argument
2600 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, in arm_smmu_init_one_queue()
2609 dev_err(smmu->dev, in arm_smmu_init_one_queue()
2616 dev_info(smmu->dev, "allocated %u entries for %s\n", in arm_smmu_init_one_queue()
2620 q->prod_reg = arm_smmu_page1_fixup(prod_off, smmu); in arm_smmu_init_one_queue()
2621 q->cons_reg = arm_smmu_page1_fixup(cons_off, smmu); in arm_smmu_init_one_queue()
2638 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu) in arm_smmu_cmdq_init() argument
2641 struct arm_smmu_cmdq *cmdq = &smmu->cmdq; in arm_smmu_cmdq_init()
2650 dev_err(smmu->dev, "failed to allocate cmdq bitmap\n"); in arm_smmu_cmdq_init()
2654 devm_add_action(smmu->dev, arm_smmu_cmdq_free_bitmap, bitmap); in arm_smmu_cmdq_init()
2660 static int arm_smmu_init_queues(struct arm_smmu_device *smmu) in arm_smmu_init_queues() argument
2665 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD, in arm_smmu_init_queues()
2671 ret = arm_smmu_cmdq_init(smmu); in arm_smmu_init_queues()
2676 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD, in arm_smmu_init_queues()
2683 if (!(smmu->features & ARM_SMMU_FEAT_PRI)) in arm_smmu_init_queues()
2686 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD, in arm_smmu_init_queues()
2691 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) in arm_smmu_init_l1_strtab() argument
2694 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_l1_strtab()
2696 void *strtab = smmu->strtab_cfg.strtab; in arm_smmu_init_l1_strtab()
2698 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL); in arm_smmu_init_l1_strtab()
2700 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n"); in arm_smmu_init_l1_strtab()
2712 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) in arm_smmu_init_strtab_2lvl() argument
2717 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_strtab_2lvl()
2721 size = min(size, smmu->sid_bits - STRTAB_SPLIT); in arm_smmu_init_strtab_2lvl()
2725 if (size < smmu->sid_bits) in arm_smmu_init_strtab_2lvl()
2726 dev_warn(smmu->dev, in arm_smmu_init_strtab_2lvl()
2728 size, smmu->sid_bits); in arm_smmu_init_strtab_2lvl()
2731 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, in arm_smmu_init_strtab_2lvl()
2734 dev_err(smmu->dev, in arm_smmu_init_strtab_2lvl()
2747 return arm_smmu_init_l1_strtab(smmu); in arm_smmu_init_strtab_2lvl()
2750 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) in arm_smmu_init_strtab_linear() argument
2755 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_strtab_linear()
2757 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3); in arm_smmu_init_strtab_linear()
2758 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma, in arm_smmu_init_strtab_linear()
2761 dev_err(smmu->dev, in arm_smmu_init_strtab_linear()
2767 cfg->num_l1_ents = 1 << smmu->sid_bits; in arm_smmu_init_strtab_linear()
2771 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); in arm_smmu_init_strtab_linear()
2778 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) in arm_smmu_init_strtab() argument
2783 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) in arm_smmu_init_strtab()
2784 ret = arm_smmu_init_strtab_2lvl(smmu); in arm_smmu_init_strtab()
2786 ret = arm_smmu_init_strtab_linear(smmu); in arm_smmu_init_strtab()
2792 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK; in arm_smmu_init_strtab()
2794 smmu->strtab_cfg.strtab_base = reg; in arm_smmu_init_strtab()
2797 set_bit(0, smmu->vmid_map); in arm_smmu_init_strtab()
2801 static int arm_smmu_init_structures(struct arm_smmu_device *smmu) in arm_smmu_init_structures() argument
2805 ret = arm_smmu_init_queues(smmu); in arm_smmu_init_structures()
2809 return arm_smmu_init_strtab(smmu); in arm_smmu_init_structures()
2812 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, in arm_smmu_write_reg_sync() argument
2817 writel_relaxed(val, smmu->base + reg_off); in arm_smmu_write_reg_sync()
2818 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, in arm_smmu_write_reg_sync()
2823 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr) in arm_smmu_update_gbpa() argument
2826 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA; in arm_smmu_update_gbpa()
2840 dev_err(smmu->dev, "GBPA not responding to update\n"); in arm_smmu_update_gbpa()
2854 struct arm_smmu_device *smmu = dev_get_drvdata(dev); in arm_smmu_write_msi_msg() local
2860 writeq_relaxed(doorbell, smmu->base + cfg[0]); in arm_smmu_write_msi_msg()
2861 writel_relaxed(msg->data, smmu->base + cfg[1]); in arm_smmu_write_msi_msg()
2862 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); in arm_smmu_write_msi_msg()
2865 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) in arm_smmu_setup_msis() argument
2869 struct device *dev = smmu->dev; in arm_smmu_setup_msis()
2872 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0); in arm_smmu_setup_msis()
2873 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0); in arm_smmu_setup_msis()
2875 if (smmu->features & ARM_SMMU_FEAT_PRI) in arm_smmu_setup_msis()
2876 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0); in arm_smmu_setup_msis()
2880 if (!(smmu->features & ARM_SMMU_FEAT_MSI)) in arm_smmu_setup_msis()
2884 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n"); in arm_smmu_setup_msis()
2898 smmu->evtq.q.irq = desc->irq; in arm_smmu_setup_msis()
2901 smmu->gerr_irq = desc->irq; in arm_smmu_setup_msis()
2904 smmu->priq.q.irq = desc->irq; in arm_smmu_setup_msis()
2915 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) in arm_smmu_setup_unique_irqs() argument
2919 arm_smmu_setup_msis(smmu); in arm_smmu_setup_unique_irqs()
2922 irq = smmu->evtq.q.irq; in arm_smmu_setup_unique_irqs()
2924 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, in arm_smmu_setup_unique_irqs()
2927 "arm-smmu-v3-evtq", smmu); in arm_smmu_setup_unique_irqs()
2929 dev_warn(smmu->dev, "failed to enable evtq irq\n"); in arm_smmu_setup_unique_irqs()
2931 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n"); in arm_smmu_setup_unique_irqs()
2934 irq = smmu->gerr_irq; in arm_smmu_setup_unique_irqs()
2936 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, in arm_smmu_setup_unique_irqs()
2937 0, "arm-smmu-v3-gerror", smmu); in arm_smmu_setup_unique_irqs()
2939 dev_warn(smmu->dev, "failed to enable gerror irq\n"); in arm_smmu_setup_unique_irqs()
2941 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n"); in arm_smmu_setup_unique_irqs()
2944 if (smmu->features & ARM_SMMU_FEAT_PRI) { in arm_smmu_setup_unique_irqs()
2945 irq = smmu->priq.q.irq; in arm_smmu_setup_unique_irqs()
2947 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, in arm_smmu_setup_unique_irqs()
2950 "arm-smmu-v3-priq", in arm_smmu_setup_unique_irqs()
2951 smmu); in arm_smmu_setup_unique_irqs()
2953 dev_warn(smmu->dev, in arm_smmu_setup_unique_irqs()
2956 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n"); in arm_smmu_setup_unique_irqs()
2961 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) in arm_smmu_setup_irqs() argument
2967 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, in arm_smmu_setup_irqs()
2970 dev_err(smmu->dev, "failed to disable irqs\n"); in arm_smmu_setup_irqs()
2974 irq = smmu->combined_irq; in arm_smmu_setup_irqs()
2980 ret = devm_request_threaded_irq(smmu->dev, irq, in arm_smmu_setup_irqs()
2984 "arm-smmu-v3-combined-irq", smmu); in arm_smmu_setup_irqs()
2986 dev_warn(smmu->dev, "failed to enable combined irq\n"); in arm_smmu_setup_irqs()
2988 arm_smmu_setup_unique_irqs(smmu); in arm_smmu_setup_irqs()
2990 if (smmu->features & ARM_SMMU_FEAT_PRI) in arm_smmu_setup_irqs()
2993 /* Enable interrupt generation on the SMMU */ in arm_smmu_setup_irqs()
2994 ret = arm_smmu_write_reg_sync(smmu, irqen_flags, in arm_smmu_setup_irqs()
2997 dev_warn(smmu->dev, "failed to enable irqs\n"); in arm_smmu_setup_irqs()
3002 static int arm_smmu_device_disable(struct arm_smmu_device *smmu) in arm_smmu_device_disable() argument
3006 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); in arm_smmu_device_disable()
3008 dev_err(smmu->dev, "failed to clear cr0\n"); in arm_smmu_device_disable()
3013 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) in arm_smmu_device_reset() argument
3019 /* Clear CR0 and sync (disables SMMU and queue processing) */ in arm_smmu_device_reset()
3020 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); in arm_smmu_device_reset()
3022 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); in arm_smmu_device_reset()
3024 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); in arm_smmu_device_reset()
3027 ret = arm_smmu_device_disable(smmu); in arm_smmu_device_reset()
3038 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); in arm_smmu_device_reset()
3042 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2); in arm_smmu_device_reset()
3045 writeq_relaxed(smmu->strtab_cfg.strtab_base, in arm_smmu_device_reset()
3046 smmu->base + ARM_SMMU_STRTAB_BASE); in arm_smmu_device_reset()
3047 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg, in arm_smmu_device_reset()
3048 smmu->base + ARM_SMMU_STRTAB_BASE_CFG); in arm_smmu_device_reset()
3051 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); in arm_smmu_device_reset()
3052 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD); in arm_smmu_device_reset()
3053 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS); in arm_smmu_device_reset()
3056 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
3059 dev_err(smmu->dev, "failed to enable command queue\n"); in arm_smmu_device_reset()
3065 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_device_reset()
3066 arm_smmu_cmdq_issue_sync(smmu); in arm_smmu_device_reset()
3069 if (smmu->features & ARM_SMMU_FEAT_HYP) { in arm_smmu_device_reset()
3071 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_device_reset()
3075 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_device_reset()
3076 arm_smmu_cmdq_issue_sync(smmu); in arm_smmu_device_reset()
3079 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); in arm_smmu_device_reset()
3080 writel_relaxed(smmu->evtq.q.llq.prod, in arm_smmu_device_reset()
3081 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu)); in arm_smmu_device_reset()
3082 writel_relaxed(smmu->evtq.q.llq.cons, in arm_smmu_device_reset()
3083 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu)); in arm_smmu_device_reset()
3086 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
3089 dev_err(smmu->dev, "failed to enable event queue\n"); in arm_smmu_device_reset()
3094 if (smmu->features & ARM_SMMU_FEAT_PRI) { in arm_smmu_device_reset()
3095 writeq_relaxed(smmu->priq.q.q_base, in arm_smmu_device_reset()
3096 smmu->base + ARM_SMMU_PRIQ_BASE); in arm_smmu_device_reset()
3097 writel_relaxed(smmu->priq.q.llq.prod, in arm_smmu_device_reset()
3098 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu)); in arm_smmu_device_reset()
3099 writel_relaxed(smmu->priq.q.llq.cons, in arm_smmu_device_reset()
3100 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu)); in arm_smmu_device_reset()
3103 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
3106 dev_err(smmu->dev, "failed to enable PRI queue\n"); in arm_smmu_device_reset()
3111 if (smmu->features & ARM_SMMU_FEAT_ATS) { in arm_smmu_device_reset()
3113 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
3116 dev_err(smmu->dev, "failed to enable ATS check\n"); in arm_smmu_device_reset()
3121 ret = arm_smmu_setup_irqs(smmu); in arm_smmu_device_reset()
3123 dev_err(smmu->dev, "failed to setup irqs\n"); in arm_smmu_device_reset()
3130 /* Enable the SMMU interface, or ensure bypass */ in arm_smmu_device_reset()
3134 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT); in arm_smmu_device_reset()
3138 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
3141 dev_err(smmu->dev, "failed to enable SMMU interface\n"); in arm_smmu_device_reset()
3148 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) in arm_smmu_device_hw_probe() argument
3151 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY; in arm_smmu_device_hw_probe()
3154 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); in arm_smmu_device_hw_probe()
3158 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB; in arm_smmu_device_hw_probe()
3161 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB; in arm_smmu_device_hw_probe()
3170 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE; in arm_smmu_device_hw_probe()
3174 smmu->features |= ARM_SMMU_FEAT_TT_BE; in arm_smmu_device_hw_probe()
3178 smmu->features |= ARM_SMMU_FEAT_TT_LE; in arm_smmu_device_hw_probe()
3182 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n"); in arm_smmu_device_hw_probe()
3188 smmu->features |= ARM_SMMU_FEAT_PRI; in arm_smmu_device_hw_probe()
3191 smmu->features |= ARM_SMMU_FEAT_ATS; in arm_smmu_device_hw_probe()
3194 smmu->features |= ARM_SMMU_FEAT_SEV; in arm_smmu_device_hw_probe()
3197 smmu->features |= ARM_SMMU_FEAT_MSI; in arm_smmu_device_hw_probe()
3199 smmu->options |= ARM_SMMU_OPT_MSIPOLL; in arm_smmu_device_hw_probe()
3203 smmu->features |= ARM_SMMU_FEAT_HYP; in arm_smmu_device_hw_probe()
3210 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n", in arm_smmu_device_hw_probe()
3215 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE; in arm_smmu_device_hw_probe()
3218 smmu->features |= ARM_SMMU_FEAT_STALLS; in arm_smmu_device_hw_probe()
3222 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; in arm_smmu_device_hw_probe()
3225 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; in arm_smmu_device_hw_probe()
3228 dev_err(smmu->dev, "no translation support!\n"); in arm_smmu_device_hw_probe()
3235 smmu->ias = 40; in arm_smmu_device_hw_probe()
3240 dev_err(smmu->dev, "AArch64 table format not supported!\n"); in arm_smmu_device_hw_probe()
3245 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8; in arm_smmu_device_hw_probe()
3246 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8; in arm_smmu_device_hw_probe()
3249 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1); in arm_smmu_device_hw_probe()
3251 dev_err(smmu->dev, "embedded implementation not supported\n"); in arm_smmu_device_hw_probe()
3256 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()
3258 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) { in arm_smmu_device_hw_probe()
3265 dev_err(smmu->dev, "command queue size <= %d entries not supported\n", in arm_smmu_device_hw_probe()
3270 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()
3272 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()
3276 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg); in arm_smmu_device_hw_probe()
3277 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg); in arm_smmu_device_hw_probe()
3280 * If the SMMU supports fewer bits than would fill a single L2 stream in arm_smmu_device_hw_probe()
3283 if (smmu->sid_bits <= STRTAB_SPLIT) in arm_smmu_device_hw_probe()
3284 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; in arm_smmu_device_hw_probe()
3287 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3); in arm_smmu_device_hw_probe()
3289 smmu->features |= ARM_SMMU_FEAT_RANGE_INV; in arm_smmu_device_hw_probe()
3292 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); in arm_smmu_device_hw_probe()
3295 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg); in arm_smmu_device_hw_probe()
3299 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; in arm_smmu_device_hw_probe()
3301 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; in arm_smmu_device_hw_probe()
3303 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; in arm_smmu_device_hw_probe()
3307 smmu->features |= ARM_SMMU_FEAT_VAX; in arm_smmu_device_hw_probe()
3312 smmu->oas = 32; in arm_smmu_device_hw_probe()
3315 smmu->oas = 36; in arm_smmu_device_hw_probe()
3318 smmu->oas = 40; in arm_smmu_device_hw_probe()
3321 smmu->oas = 42; in arm_smmu_device_hw_probe()
3324 smmu->oas = 44; in arm_smmu_device_hw_probe()
3327 smmu->oas = 52; in arm_smmu_device_hw_probe()
3328 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */ in arm_smmu_device_hw_probe()
3331 dev_info(smmu->dev, in arm_smmu_device_hw_probe()
3335 smmu->oas = 48; in arm_smmu_device_hw_probe()
3339 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; in arm_smmu_device_hw_probe()
3341 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; in arm_smmu_device_hw_probe()
3344 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas))) in arm_smmu_device_hw_probe()
3345 dev_warn(smmu->dev, in arm_smmu_device_hw_probe()
3348 smmu->ias = max(smmu->ias, smmu->oas); in arm_smmu_device_hw_probe()
3350 if (arm_smmu_sva_supported(smmu)) in arm_smmu_device_hw_probe()
3351 smmu->features |= ARM_SMMU_FEAT_SVA; in arm_smmu_device_hw_probe()
3353 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n", in arm_smmu_device_hw_probe()
3354 smmu->ias, smmu->oas, smmu->features); in arm_smmu_device_hw_probe()
3359 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu) in acpi_smmu_get_options() argument
3363 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY; in acpi_smmu_get_options()
3366 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH; in acpi_smmu_get_options()
3370 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options); in acpi_smmu_get_options()
3374 struct arm_smmu_device *smmu) in arm_smmu_device_acpi_probe() argument
3377 struct device *dev = smmu->dev; in arm_smmu_device_acpi_probe()
3385 acpi_smmu_get_options(iort_smmu->model, smmu); in arm_smmu_device_acpi_probe()
3388 smmu->features |= ARM_SMMU_FEAT_COHERENCY; in arm_smmu_device_acpi_probe()
3394 struct arm_smmu_device *smmu) in arm_smmu_device_acpi_probe() argument
3401 struct arm_smmu_device *smmu) in arm_smmu_device_dt_probe() argument
3414 parse_driver_options(smmu); in arm_smmu_device_dt_probe()
3417 smmu->features |= ARM_SMMU_FEAT_COHERENCY; in arm_smmu_device_dt_probe()
3422 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) in arm_smmu_resource_size() argument
3424 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY) in arm_smmu_resource_size()
3484 struct arm_smmu_device *smmu; in arm_smmu_device_probe() local
3488 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); in arm_smmu_device_probe()
3489 if (!smmu) { in arm_smmu_device_probe()
3493 smmu->dev = dev; in arm_smmu_device_probe()
3496 ret = arm_smmu_device_dt_probe(pdev, smmu); in arm_smmu_device_probe()
3498 ret = arm_smmu_device_acpi_probe(pdev, smmu); in arm_smmu_device_probe()
3510 if (resource_size(res) < arm_smmu_resource_size(smmu)) { in arm_smmu_device_probe()
3520 smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ); in arm_smmu_device_probe()
3521 if (IS_ERR(smmu->base)) in arm_smmu_device_probe()
3522 return PTR_ERR(smmu->base); in arm_smmu_device_probe()
3524 if (arm_smmu_resource_size(smmu) > SZ_64K) { in arm_smmu_device_probe()
3525 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K, in arm_smmu_device_probe()
3527 if (IS_ERR(smmu->page1)) in arm_smmu_device_probe()
3528 return PTR_ERR(smmu->page1); in arm_smmu_device_probe()
3530 smmu->page1 = smmu->base; in arm_smmu_device_probe()
3537 smmu->combined_irq = irq; in arm_smmu_device_probe()
3541 smmu->evtq.q.irq = irq; in arm_smmu_device_probe()
3545 smmu->priq.q.irq = irq; in arm_smmu_device_probe()
3549 smmu->gerr_irq = irq; in arm_smmu_device_probe()
3552 ret = arm_smmu_device_hw_probe(smmu); in arm_smmu_device_probe()
3557 ret = arm_smmu_init_structures(smmu); in arm_smmu_device_probe()
3562 platform_set_drvdata(pdev, smmu); in arm_smmu_device_probe()
3565 ret = arm_smmu_device_reset(smmu, bypass); in arm_smmu_device_probe()
3570 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, in arm_smmu_device_probe()
3575 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops); in arm_smmu_device_probe()
3576 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); in arm_smmu_device_probe()
3578 ret = iommu_device_register(&smmu->iommu); in arm_smmu_device_probe()
3589 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); in arm_smmu_device_remove() local
3592 iommu_device_unregister(&smmu->iommu); in arm_smmu_device_remove()
3593 iommu_device_sysfs_remove(&smmu->iommu); in arm_smmu_device_remove()
3594 arm_smmu_device_disable(smmu); in arm_smmu_device_remove()
3605 { .compatible = "arm,smmu-v3", },
3612 .name = "arm-smmu-v3",
3624 MODULE_ALIAS("platform:arm-smmu-v3");