1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-iommu.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/err.h>
27 #include <linux/interrupt.h>
28 #include <linux/io.h>
29 #include <linux/iopoll.h>
30 #include <linux/module.h>
31 #include <linux/of.h>
32 #include <linux/of_address.h>
33 #include <linux/of_device.h>
34 #include <linux/of_iommu.h>
35 #include <linux/pci.h>
36 #include <linux/platform_device.h>
37 #include <linux/pm_runtime.h>
38 #include <linux/ratelimit.h>
39 #include <linux/slab.h>
40
41 #include <linux/amba/bus.h>
42 #include <linux/fsl/mc.h>
43
44 #include "arm-smmu.h"
45
46 /*
47 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
48 * global register space are still, in fact, using a hypervisor to mediate it
49 * by trapping and emulating register accesses. Sadly, some deployed versions
50 * of said trapping code have bugs wherein they go horribly wrong for stores
51 * using r31 (i.e. XZR/WZR) as the source register.
52 */
53 #define QCOM_DUMMY_VAL -1
54
55 #define MSI_IOVA_BASE 0x8000000
56 #define MSI_IOVA_LENGTH 0x100000
57
58 static int force_stage;
59 module_param(force_stage, int, S_IRUGO);
60 MODULE_PARM_DESC(force_stage,
61 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
62 static bool disable_bypass =
63 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
64 module_param(disable_bypass, bool, S_IRUGO);
65 MODULE_PARM_DESC(disable_bypass,
66 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
67
68 #define s2cr_init_val (struct arm_smmu_s2cr){ \
69 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
70 }
71
72 static bool using_legacy_binding, using_generic_binding;
73
arm_smmu_rpm_get(struct arm_smmu_device * smmu)74 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
75 {
76 if (pm_runtime_enabled(smmu->dev))
77 return pm_runtime_resume_and_get(smmu->dev);
78
79 return 0;
80 }
81
arm_smmu_rpm_put(struct arm_smmu_device * smmu)82 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
83 {
84 if (pm_runtime_enabled(smmu->dev))
85 pm_runtime_put_autosuspend(smmu->dev);
86 }
87
to_smmu_domain(struct iommu_domain * dom)88 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
89 {
90 return container_of(dom, struct arm_smmu_domain, domain);
91 }
92
93 static struct platform_driver arm_smmu_driver;
94 static struct iommu_ops arm_smmu_ops;
95
96 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
97 static int arm_smmu_bus_init(struct iommu_ops *ops);
98
dev_get_dev_node(struct device * dev)99 static struct device_node *dev_get_dev_node(struct device *dev)
100 {
101 if (dev_is_pci(dev)) {
102 struct pci_bus *bus = to_pci_dev(dev)->bus;
103
104 while (!pci_is_root_bus(bus))
105 bus = bus->parent;
106 return of_node_get(bus->bridge->parent->of_node);
107 }
108
109 return of_node_get(dev->of_node);
110 }
111
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)112 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
113 {
114 *((__be32 *)data) = cpu_to_be32(alias);
115 return 0; /* Continue walking */
116 }
117
__find_legacy_master_phandle(struct device * dev,void * data)118 static int __find_legacy_master_phandle(struct device *dev, void *data)
119 {
120 struct of_phandle_iterator *it = *(void **)data;
121 struct device_node *np = it->node;
122 int err;
123
124 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
125 "#stream-id-cells", -1)
126 if (it->node == np) {
127 *(void **)data = dev;
128 return 1;
129 }
130 it->node = np;
131 return err == -ENOENT ? 0 : err;
132 }
133
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)134 static int arm_smmu_register_legacy_master(struct device *dev,
135 struct arm_smmu_device **smmu)
136 {
137 struct device *smmu_dev;
138 struct device_node *np;
139 struct of_phandle_iterator it;
140 void *data = ⁢
141 u32 *sids;
142 __be32 pci_sid;
143 int err;
144
145 np = dev_get_dev_node(dev);
146 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
147 of_node_put(np);
148 return -ENODEV;
149 }
150
151 it.node = np;
152 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
153 __find_legacy_master_phandle);
154 smmu_dev = data;
155 of_node_put(np);
156 if (err == 0)
157 return -ENODEV;
158 if (err < 0)
159 return err;
160
161 if (dev_is_pci(dev)) {
162 /* "mmu-masters" assumes Stream ID == Requester ID */
163 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
164 &pci_sid);
165 it.cur = &pci_sid;
166 it.cur_count = 1;
167 }
168
169 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
170 &arm_smmu_ops);
171 if (err)
172 return err;
173
174 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
175 if (!sids)
176 return -ENOMEM;
177
178 *smmu = dev_get_drvdata(smmu_dev);
179 of_phandle_iterator_args(&it, sids, it.cur_count);
180 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
181 kfree(sids);
182 return err;
183 }
184
185 /*
186 * With the legacy DT binding in play, we have no guarantees about
187 * probe order, but then we're also not doing default domains, so we can
188 * delay setting bus ops until we're sure every possible SMMU is ready,
189 * and that way ensure that no probe_device() calls get missed.
190 */
arm_smmu_legacy_bus_init(void)191 static int arm_smmu_legacy_bus_init(void)
192 {
193 if (using_legacy_binding)
194 return arm_smmu_bus_init(&arm_smmu_ops);
195 return 0;
196 }
197 device_initcall_sync(arm_smmu_legacy_bus_init);
198 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)199 static int arm_smmu_register_legacy_master(struct device *dev,
200 struct arm_smmu_device **smmu)
201 {
202 return -ENODEV;
203 }
204 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
205
__arm_smmu_free_bitmap(unsigned long * map,int idx)206 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
207 {
208 clear_bit(idx, map);
209 }
210
211 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)212 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
213 int sync, int status)
214 {
215 unsigned int spin_cnt, delay;
216 u32 reg;
217
218 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
219 return smmu->impl->tlb_sync(smmu, page, sync, status);
220
221 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
222 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
223 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
224 reg = arm_smmu_readl(smmu, page, status);
225 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
226 return;
227 cpu_relax();
228 }
229 udelay(delay);
230 }
231 dev_err_ratelimited(smmu->dev,
232 "TLB sync timed out -- SMMU may be deadlocked\n");
233 }
234
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)235 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
236 {
237 unsigned long flags;
238
239 spin_lock_irqsave(&smmu->global_sync_lock, flags);
240 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
241 ARM_SMMU_GR0_sTLBGSTATUS);
242 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
243 }
244
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)245 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
246 {
247 struct arm_smmu_device *smmu = smmu_domain->smmu;
248 unsigned long flags;
249
250 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
251 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
252 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
253 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
254 }
255
arm_smmu_tlb_inv_context_s1(void * cookie)256 static void arm_smmu_tlb_inv_context_s1(void *cookie)
257 {
258 struct arm_smmu_domain *smmu_domain = cookie;
259 /*
260 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
261 * current CPU are visible beforehand.
262 */
263 wmb();
264 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
265 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
266 arm_smmu_tlb_sync_context(smmu_domain);
267 }
268
arm_smmu_tlb_inv_context_s2(void * cookie)269 static void arm_smmu_tlb_inv_context_s2(void *cookie)
270 {
271 struct arm_smmu_domain *smmu_domain = cookie;
272 struct arm_smmu_device *smmu = smmu_domain->smmu;
273
274 /* See above */
275 wmb();
276 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
277 arm_smmu_tlb_sync_global(smmu);
278 }
279
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)280 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
281 size_t granule, void *cookie, int reg)
282 {
283 struct arm_smmu_domain *smmu_domain = cookie;
284 struct arm_smmu_device *smmu = smmu_domain->smmu;
285 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
286 int idx = cfg->cbndx;
287
288 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
289 wmb();
290
291 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
292 iova = (iova >> 12) << 12;
293 iova |= cfg->asid;
294 do {
295 arm_smmu_cb_write(smmu, idx, reg, iova);
296 iova += granule;
297 } while (size -= granule);
298 } else {
299 iova >>= 12;
300 iova |= (u64)cfg->asid << 48;
301 do {
302 arm_smmu_cb_writeq(smmu, idx, reg, iova);
303 iova += granule >> 12;
304 } while (size -= granule);
305 }
306 }
307
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)308 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
309 size_t granule, void *cookie, int reg)
310 {
311 struct arm_smmu_domain *smmu_domain = cookie;
312 struct arm_smmu_device *smmu = smmu_domain->smmu;
313 int idx = smmu_domain->cfg.cbndx;
314
315 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
316 wmb();
317
318 iova >>= 12;
319 do {
320 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
321 arm_smmu_cb_writeq(smmu, idx, reg, iova);
322 else
323 arm_smmu_cb_write(smmu, idx, reg, iova);
324 iova += granule >> 12;
325 } while (size -= granule);
326 }
327
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)328 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
329 size_t granule, void *cookie)
330 {
331 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
332 ARM_SMMU_CB_S1_TLBIVA);
333 arm_smmu_tlb_sync_context(cookie);
334 }
335
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)336 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
337 unsigned long iova, size_t granule,
338 void *cookie)
339 {
340 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
341 ARM_SMMU_CB_S1_TLBIVAL);
342 }
343
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)344 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
345 size_t granule, void *cookie)
346 {
347 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
348 ARM_SMMU_CB_S2_TLBIIPAS2);
349 arm_smmu_tlb_sync_context(cookie);
350 }
351
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)352 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
353 unsigned long iova, size_t granule,
354 void *cookie)
355 {
356 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
357 ARM_SMMU_CB_S2_TLBIIPAS2L);
358 }
359
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)360 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
361 size_t granule, void *cookie)
362 {
363 arm_smmu_tlb_inv_context_s2(cookie);
364 }
365 /*
366 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
367 * almost negligible, but the benefit of getting the first one in as far ahead
368 * of the sync as possible is significant, hence we don't just make this a
369 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
370 * think.
371 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)372 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
373 unsigned long iova, size_t granule,
374 void *cookie)
375 {
376 struct arm_smmu_domain *smmu_domain = cookie;
377 struct arm_smmu_device *smmu = smmu_domain->smmu;
378
379 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
380 wmb();
381
382 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
383 }
384
385 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
386 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
387 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
388 .tlb_add_page = arm_smmu_tlb_add_page_s1,
389 };
390
391 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
392 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
393 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
394 .tlb_add_page = arm_smmu_tlb_add_page_s2,
395 };
396
397 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
398 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
399 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
400 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
401 };
402
arm_smmu_context_fault(int irq,void * dev)403 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
404 {
405 u32 fsr, fsynr, cbfrsynra;
406 unsigned long iova;
407 struct iommu_domain *domain = dev;
408 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
409 struct arm_smmu_device *smmu = smmu_domain->smmu;
410 int idx = smmu_domain->cfg.cbndx;
411
412 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
413 if (!(fsr & ARM_SMMU_FSR_FAULT))
414 return IRQ_NONE;
415
416 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
417 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
418 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
419
420 dev_err_ratelimited(smmu->dev,
421 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
422 fsr, iova, fsynr, cbfrsynra, idx);
423
424 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
425 return IRQ_HANDLED;
426 }
427
arm_smmu_global_fault(int irq,void * dev)428 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
429 {
430 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
431 struct arm_smmu_device *smmu = dev;
432 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
433 DEFAULT_RATELIMIT_BURST);
434
435 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
436 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
437 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
438 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
439
440 if (!gfsr)
441 return IRQ_NONE;
442
443 if (__ratelimit(&rs)) {
444 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
445 (gfsr & ARM_SMMU_sGFSR_USF))
446 dev_err(smmu->dev,
447 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
448 (u16)gfsynr1);
449 else
450 dev_err(smmu->dev,
451 "Unexpected global fault, this could be serious\n");
452 dev_err(smmu->dev,
453 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
454 gfsr, gfsynr0, gfsynr1, gfsynr2);
455 }
456
457 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
458 return IRQ_HANDLED;
459 }
460
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)461 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
462 struct io_pgtable_cfg *pgtbl_cfg)
463 {
464 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
465 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
466 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
467
468 cb->cfg = cfg;
469
470 /* TCR */
471 if (stage1) {
472 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
473 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
474 } else {
475 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
476 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
477 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
478 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
479 else
480 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
481 }
482 } else {
483 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
484 }
485
486 /* TTBRs */
487 if (stage1) {
488 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
489 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
490 cb->ttbr[1] = 0;
491 } else {
492 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
493 cfg->asid);
494 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
495 cfg->asid);
496
497 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
498 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
499 else
500 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
501 }
502 } else {
503 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
504 }
505
506 /* MAIRs (stage-1 only) */
507 if (stage1) {
508 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
509 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
510 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
511 } else {
512 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
513 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
514 }
515 }
516 }
517
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)518 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
519 {
520 u32 reg;
521 bool stage1;
522 struct arm_smmu_cb *cb = &smmu->cbs[idx];
523 struct arm_smmu_cfg *cfg = cb->cfg;
524
525 /* Unassigned context banks only need disabling */
526 if (!cfg) {
527 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
528 return;
529 }
530
531 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
532
533 /* CBA2R */
534 if (smmu->version > ARM_SMMU_V1) {
535 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
536 reg = ARM_SMMU_CBA2R_VA64;
537 else
538 reg = 0;
539 /* 16-bit VMIDs live in CBA2R */
540 if (smmu->features & ARM_SMMU_FEAT_VMID16)
541 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
542
543 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
544 }
545
546 /* CBAR */
547 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
548 if (smmu->version < ARM_SMMU_V2)
549 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
550
551 /*
552 * Use the weakest shareability/memory types, so they are
553 * overridden by the ttbcr/pte.
554 */
555 if (stage1) {
556 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
557 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
558 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
559 ARM_SMMU_CBAR_S1_MEMATTR_WB);
560 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
561 /* 8-bit VMIDs live in CBAR */
562 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
563 }
564 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
565
566 /*
567 * TCR
568 * We must write this before the TTBRs, since it determines the
569 * access behaviour of some fields (in particular, ASID[15:8]).
570 */
571 if (stage1 && smmu->version > ARM_SMMU_V1)
572 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
573 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
574
575 /* TTBRs */
576 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
577 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
578 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
579 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
580 } else {
581 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
582 if (stage1)
583 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
584 cb->ttbr[1]);
585 }
586
587 /* MAIRs (stage-1 only) */
588 if (stage1) {
589 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
590 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
591 }
592
593 /* SCTLR */
594 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
595 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
596 if (stage1)
597 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
598 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
599 reg |= ARM_SMMU_SCTLR_E;
600
601 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
602 }
603
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)604 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
605 struct arm_smmu_device *smmu,
606 struct device *dev, unsigned int start)
607 {
608 if (smmu->impl && smmu->impl->alloc_context_bank)
609 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
610
611 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
612 }
613
arm_smmu_init_domain_context(struct iommu_domain * domain,struct arm_smmu_device * smmu,struct device * dev)614 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
615 struct arm_smmu_device *smmu,
616 struct device *dev)
617 {
618 int irq, start, ret = 0;
619 unsigned long ias, oas;
620 struct io_pgtable_ops *pgtbl_ops;
621 struct io_pgtable_cfg pgtbl_cfg;
622 enum io_pgtable_fmt fmt;
623 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
624 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
625 irqreturn_t (*context_fault)(int irq, void *dev);
626
627 mutex_lock(&smmu_domain->init_mutex);
628 if (smmu_domain->smmu)
629 goto out_unlock;
630
631 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
632 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
633 smmu_domain->smmu = smmu;
634 goto out_unlock;
635 }
636
637 /*
638 * Mapping the requested stage onto what we support is surprisingly
639 * complicated, mainly because the spec allows S1+S2 SMMUs without
640 * support for nested translation. That means we end up with the
641 * following table:
642 *
643 * Requested Supported Actual
644 * S1 N S1
645 * S1 S1+S2 S1
646 * S1 S2 S2
647 * S1 S1 S1
648 * N N N
649 * N S1+S2 S2
650 * N S2 S2
651 * N S1 S1
652 *
653 * Note that you can't actually request stage-2 mappings.
654 */
655 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
656 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
657 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
658 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
659
660 /*
661 * Choosing a suitable context format is even more fiddly. Until we
662 * grow some way for the caller to express a preference, and/or move
663 * the decision into the io-pgtable code where it arguably belongs,
664 * just aim for the closest thing to the rest of the system, and hope
665 * that the hardware isn't esoteric enough that we can't assume AArch64
666 * support to be a superset of AArch32 support...
667 */
668 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
669 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
670 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
671 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
672 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
673 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
674 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
675 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
676 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
677 ARM_SMMU_FEAT_FMT_AARCH64_16K |
678 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
679 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
680
681 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
682 ret = -EINVAL;
683 goto out_unlock;
684 }
685
686 switch (smmu_domain->stage) {
687 case ARM_SMMU_DOMAIN_S1:
688 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
689 start = smmu->num_s2_context_banks;
690 ias = smmu->va_size;
691 oas = smmu->ipa_size;
692 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
693 fmt = ARM_64_LPAE_S1;
694 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
695 fmt = ARM_32_LPAE_S1;
696 ias = min(ias, 32UL);
697 oas = min(oas, 40UL);
698 } else {
699 fmt = ARM_V7S;
700 ias = min(ias, 32UL);
701 oas = min(oas, 32UL);
702 }
703 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
704 break;
705 case ARM_SMMU_DOMAIN_NESTED:
706 /*
707 * We will likely want to change this if/when KVM gets
708 * involved.
709 */
710 case ARM_SMMU_DOMAIN_S2:
711 cfg->cbar = CBAR_TYPE_S2_TRANS;
712 start = 0;
713 ias = smmu->ipa_size;
714 oas = smmu->pa_size;
715 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
716 fmt = ARM_64_LPAE_S2;
717 } else {
718 fmt = ARM_32_LPAE_S2;
719 ias = min(ias, 40UL);
720 oas = min(oas, 40UL);
721 }
722 if (smmu->version == ARM_SMMU_V2)
723 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
724 else
725 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
726 break;
727 default:
728 ret = -EINVAL;
729 goto out_unlock;
730 }
731
732 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
733 if (ret < 0) {
734 goto out_unlock;
735 }
736
737 smmu_domain->smmu = smmu;
738
739 cfg->cbndx = ret;
740 if (smmu->version < ARM_SMMU_V2) {
741 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
742 cfg->irptndx %= smmu->num_context_irqs;
743 } else {
744 cfg->irptndx = cfg->cbndx;
745 }
746
747 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
748 cfg->vmid = cfg->cbndx + 1;
749 else
750 cfg->asid = cfg->cbndx;
751
752 pgtbl_cfg = (struct io_pgtable_cfg) {
753 .pgsize_bitmap = smmu->pgsize_bitmap,
754 .ias = ias,
755 .oas = oas,
756 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
757 .tlb = smmu_domain->flush_ops,
758 .iommu_dev = smmu->dev,
759 };
760
761 if (smmu->impl && smmu->impl->init_context) {
762 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
763 if (ret)
764 goto out_clear_smmu;
765 }
766
767 if (smmu_domain->non_strict)
768 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
769
770 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
771 if (!pgtbl_ops) {
772 ret = -ENOMEM;
773 goto out_clear_smmu;
774 }
775
776 /* Update the domain's page sizes to reflect the page table format */
777 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
778
779 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
780 domain->geometry.aperture_start = ~0UL << ias;
781 domain->geometry.aperture_end = ~0UL;
782 } else {
783 domain->geometry.aperture_end = (1UL << ias) - 1;
784 }
785
786 domain->geometry.force_aperture = true;
787
788 /* Initialise the context bank with our page table cfg */
789 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
790 arm_smmu_write_context_bank(smmu, cfg->cbndx);
791
792 /*
793 * Request context fault interrupt. Do this last to avoid the
794 * handler seeing a half-initialised domain state.
795 */
796 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
797
798 if (smmu->impl && smmu->impl->context_fault)
799 context_fault = smmu->impl->context_fault;
800 else
801 context_fault = arm_smmu_context_fault;
802
803 ret = devm_request_irq(smmu->dev, irq, context_fault,
804 IRQF_SHARED, "arm-smmu-context-fault", domain);
805 if (ret < 0) {
806 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
807 cfg->irptndx, irq);
808 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
809 }
810
811 mutex_unlock(&smmu_domain->init_mutex);
812
813 /* Publish page table ops for map/unmap */
814 smmu_domain->pgtbl_ops = pgtbl_ops;
815 return 0;
816
817 out_clear_smmu:
818 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
819 smmu_domain->smmu = NULL;
820 out_unlock:
821 mutex_unlock(&smmu_domain->init_mutex);
822 return ret;
823 }
824
arm_smmu_destroy_domain_context(struct iommu_domain * domain)825 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
826 {
827 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
828 struct arm_smmu_device *smmu = smmu_domain->smmu;
829 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
830 int ret, irq;
831
832 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
833 return;
834
835 ret = arm_smmu_rpm_get(smmu);
836 if (ret < 0)
837 return;
838
839 /*
840 * Disable the context bank and free the page tables before freeing
841 * it.
842 */
843 smmu->cbs[cfg->cbndx].cfg = NULL;
844 arm_smmu_write_context_bank(smmu, cfg->cbndx);
845
846 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
847 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
848 devm_free_irq(smmu->dev, irq, domain);
849 }
850
851 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
852 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
853
854 arm_smmu_rpm_put(smmu);
855 }
856
arm_smmu_domain_alloc(unsigned type)857 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
858 {
859 struct arm_smmu_domain *smmu_domain;
860
861 if (type != IOMMU_DOMAIN_UNMANAGED &&
862 type != IOMMU_DOMAIN_DMA &&
863 type != IOMMU_DOMAIN_IDENTITY)
864 return NULL;
865 /*
866 * Allocate the domain and initialise some of its data structures.
867 * We can't really do anything meaningful until we've added a
868 * master.
869 */
870 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
871 if (!smmu_domain)
872 return NULL;
873
874 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
875 iommu_get_dma_cookie(&smmu_domain->domain))) {
876 kfree(smmu_domain);
877 return NULL;
878 }
879
880 mutex_init(&smmu_domain->init_mutex);
881 spin_lock_init(&smmu_domain->cb_lock);
882
883 return &smmu_domain->domain;
884 }
885
arm_smmu_domain_free(struct iommu_domain * domain)886 static void arm_smmu_domain_free(struct iommu_domain *domain)
887 {
888 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
889
890 /*
891 * Free the domain resources. We assume that all devices have
892 * already been detached.
893 */
894 iommu_put_dma_cookie(domain);
895 arm_smmu_destroy_domain_context(domain);
896 kfree(smmu_domain);
897 }
898
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)899 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
900 {
901 struct arm_smmu_smr *smr = smmu->smrs + idx;
902 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
903 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
904
905 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
906 reg |= ARM_SMMU_SMR_VALID;
907 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
908 }
909
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)910 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
911 {
912 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
913 u32 reg;
914
915 if (smmu->impl && smmu->impl->write_s2cr) {
916 smmu->impl->write_s2cr(smmu, idx);
917 return;
918 }
919
920 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
921 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
922 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
923
924 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
925 smmu->smrs[idx].valid)
926 reg |= ARM_SMMU_S2CR_EXIDVALID;
927 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
928 }
929
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)930 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
931 {
932 arm_smmu_write_s2cr(smmu, idx);
933 if (smmu->smrs)
934 arm_smmu_write_smr(smmu, idx);
935 }
936
937 /*
938 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
939 * should be called after sCR0 is written.
940 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)941 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
942 {
943 u32 smr;
944 int i;
945
946 if (!smmu->smrs)
947 return;
948 /*
949 * If we've had to accommodate firmware memory regions, we may
950 * have live SMRs by now; tread carefully...
951 *
952 * Somewhat perversely, not having a free SMR for this test implies we
953 * can get away without it anyway, as we'll only be able to 'allocate'
954 * these SMRs for the ID/mask values we're already trusting to be OK.
955 */
956 for (i = 0; i < smmu->num_mapping_groups; i++)
957 if (!smmu->smrs[i].valid)
958 goto smr_ok;
959 return;
960 smr_ok:
961 /*
962 * SMR.ID bits may not be preserved if the corresponding MASK
963 * bits are set, so check each one separately. We can reject
964 * masters later if they try to claim IDs outside these masks.
965 */
966 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
967 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
968 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
969 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
970
971 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
972 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
973 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
974 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
975 }
976
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)977 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
978 {
979 struct arm_smmu_smr *smrs = smmu->smrs;
980 int i, free_idx = -ENOSPC;
981
982 /* Stream indexing is blissfully easy */
983 if (!smrs)
984 return id;
985
986 /* Validating SMRs is... less so */
987 for (i = 0; i < smmu->num_mapping_groups; ++i) {
988 if (!smrs[i].valid) {
989 /*
990 * Note the first free entry we come across, which
991 * we'll claim in the end if nothing else matches.
992 */
993 if (free_idx < 0)
994 free_idx = i;
995 continue;
996 }
997 /*
998 * If the new entry is _entirely_ matched by an existing entry,
999 * then reuse that, with the guarantee that there also cannot
1000 * be any subsequent conflicting entries. In normal use we'd
1001 * expect simply identical entries for this case, but there's
1002 * no harm in accommodating the generalisation.
1003 */
1004 if ((mask & smrs[i].mask) == mask &&
1005 !((id ^ smrs[i].id) & ~smrs[i].mask))
1006 return i;
1007 /*
1008 * If the new entry has any other overlap with an existing one,
1009 * though, then there always exists at least one stream ID
1010 * which would cause a conflict, and we can't allow that risk.
1011 */
1012 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1013 return -EINVAL;
1014 }
1015
1016 return free_idx;
1017 }
1018
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1019 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1020 {
1021 bool pinned = smmu->s2crs[idx].pinned;
1022 u8 cbndx = smmu->s2crs[idx].cbndx;
1023
1024 if (--smmu->s2crs[idx].count)
1025 return false;
1026
1027 smmu->s2crs[idx] = s2cr_init_val;
1028 if (pinned) {
1029 smmu->s2crs[idx].pinned = true;
1030 smmu->s2crs[idx].cbndx = cbndx;
1031 } else if (smmu->smrs) {
1032 smmu->smrs[idx].valid = false;
1033 }
1034
1035 return true;
1036 }
1037
arm_smmu_master_alloc_smes(struct device * dev)1038 static int arm_smmu_master_alloc_smes(struct device *dev)
1039 {
1040 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1041 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1042 struct arm_smmu_device *smmu = cfg->smmu;
1043 struct arm_smmu_smr *smrs = smmu->smrs;
1044 int i, idx, ret;
1045
1046 mutex_lock(&smmu->stream_map_mutex);
1047 /* Figure out a viable stream map entry allocation */
1048 for_each_cfg_sme(cfg, fwspec, i, idx) {
1049 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1050 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1051
1052 if (idx != INVALID_SMENDX) {
1053 ret = -EEXIST;
1054 goto out_err;
1055 }
1056
1057 ret = arm_smmu_find_sme(smmu, sid, mask);
1058 if (ret < 0)
1059 goto out_err;
1060
1061 idx = ret;
1062 if (smrs && smmu->s2crs[idx].count == 0) {
1063 smrs[idx].id = sid;
1064 smrs[idx].mask = mask;
1065 smrs[idx].valid = true;
1066 }
1067 smmu->s2crs[idx].count++;
1068 cfg->smendx[i] = (s16)idx;
1069 }
1070
1071 /* It worked! Now, poke the actual hardware */
1072 for_each_cfg_sme(cfg, fwspec, i, idx)
1073 arm_smmu_write_sme(smmu, idx);
1074
1075 mutex_unlock(&smmu->stream_map_mutex);
1076 return 0;
1077
1078 out_err:
1079 while (i--) {
1080 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1081 cfg->smendx[i] = INVALID_SMENDX;
1082 }
1083 mutex_unlock(&smmu->stream_map_mutex);
1084 return ret;
1085 }
1086
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1087 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1088 struct iommu_fwspec *fwspec)
1089 {
1090 struct arm_smmu_device *smmu = cfg->smmu;
1091 int i, idx;
1092
1093 mutex_lock(&smmu->stream_map_mutex);
1094 for_each_cfg_sme(cfg, fwspec, i, idx) {
1095 if (arm_smmu_free_sme(smmu, idx))
1096 arm_smmu_write_sme(smmu, idx);
1097 cfg->smendx[i] = INVALID_SMENDX;
1098 }
1099 mutex_unlock(&smmu->stream_map_mutex);
1100 }
1101
arm_smmu_domain_add_master(struct arm_smmu_domain * smmu_domain,struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1102 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1103 struct arm_smmu_master_cfg *cfg,
1104 struct iommu_fwspec *fwspec)
1105 {
1106 struct arm_smmu_device *smmu = smmu_domain->smmu;
1107 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1108 u8 cbndx = smmu_domain->cfg.cbndx;
1109 enum arm_smmu_s2cr_type type;
1110 int i, idx;
1111
1112 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1113 type = S2CR_TYPE_BYPASS;
1114 else
1115 type = S2CR_TYPE_TRANS;
1116
1117 for_each_cfg_sme(cfg, fwspec, i, idx) {
1118 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1119 continue;
1120
1121 /* Don't bypasss pinned streams; leave them as they are */
1122 if (type == S2CR_TYPE_BYPASS && s2cr[idx].pinned)
1123 continue;
1124
1125 s2cr[idx].type = type;
1126 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1127 s2cr[idx].cbndx = cbndx;
1128 arm_smmu_write_s2cr(smmu, idx);
1129 }
1130 return 0;
1131 }
1132
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1133 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1134 {
1135 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1136 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1137 struct arm_smmu_master_cfg *cfg;
1138 struct arm_smmu_device *smmu;
1139 int ret;
1140
1141 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1142 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1143 return -ENXIO;
1144 }
1145
1146 /*
1147 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1148 * domains between of_xlate() and probe_device() - we have no way to cope
1149 * with that, so until ARM gets converted to rely on groups and default
1150 * domains, just say no (but more politely than by dereferencing NULL).
1151 * This should be at least a WARN_ON once that's sorted.
1152 */
1153 cfg = dev_iommu_priv_get(dev);
1154 if (!cfg)
1155 return -ENODEV;
1156
1157 smmu = cfg->smmu;
1158
1159 ret = arm_smmu_rpm_get(smmu);
1160 if (ret < 0)
1161 return ret;
1162
1163 /* Ensure that the domain is finalised */
1164 ret = arm_smmu_init_domain_context(domain, smmu, dev);
1165 if (ret < 0)
1166 goto rpm_put;
1167
1168 /*
1169 * Sanity check the domain. We don't support domains across
1170 * different SMMUs.
1171 */
1172 if (smmu_domain->smmu != smmu) {
1173 dev_err(dev,
1174 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1175 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1176 ret = -EINVAL;
1177 goto rpm_put;
1178 }
1179
1180 /* Looks ok, so add the device to the domain */
1181 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1182
1183 /*
1184 * Setup an autosuspend delay to avoid bouncing runpm state.
1185 * Otherwise, if a driver for a suspended consumer device
1186 * unmaps buffers, it will runpm resume/suspend for each one.
1187 *
1188 * For example, when used by a GPU device, when an application
1189 * or game exits, it can trigger unmapping 100s or 1000s of
1190 * buffers. With a runpm cycle for each buffer, that adds up
1191 * to 5-10sec worth of reprogramming the context bank, while
1192 * the system appears to be locked up to the user.
1193 */
1194 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1195 pm_runtime_use_autosuspend(smmu->dev);
1196
1197 rpm_put:
1198 arm_smmu_rpm_put(smmu);
1199 return ret;
1200 }
1201
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1202 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1203 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1204 int prot, gfp_t gfp, size_t *mapped)
1205 {
1206 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1207 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1208 int ret;
1209
1210 if (!ops)
1211 return -ENODEV;
1212
1213 arm_smmu_rpm_get(smmu);
1214 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1215 arm_smmu_rpm_put(smmu);
1216
1217 return ret;
1218 }
1219
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1220 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1221 size_t pgsize, size_t pgcount,
1222 struct iommu_iotlb_gather *iotlb_gather)
1223 {
1224 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1225 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1226 size_t ret;
1227
1228 if (!ops)
1229 return 0;
1230
1231 arm_smmu_rpm_get(smmu);
1232 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1233 arm_smmu_rpm_put(smmu);
1234
1235 return ret;
1236 }
1237
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1238 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1239 {
1240 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1241 struct arm_smmu_device *smmu = smmu_domain->smmu;
1242
1243 if (smmu_domain->flush_ops) {
1244 arm_smmu_rpm_get(smmu);
1245 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1246 arm_smmu_rpm_put(smmu);
1247 }
1248 }
1249
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1250 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1251 struct iommu_iotlb_gather *gather)
1252 {
1253 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1254 struct arm_smmu_device *smmu = smmu_domain->smmu;
1255
1256 if (!smmu)
1257 return;
1258
1259 arm_smmu_rpm_get(smmu);
1260 if (smmu->version == ARM_SMMU_V2 ||
1261 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1262 arm_smmu_tlb_sync_context(smmu_domain);
1263 else
1264 arm_smmu_tlb_sync_global(smmu);
1265 arm_smmu_rpm_put(smmu);
1266 }
1267
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1268 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1269 dma_addr_t iova)
1270 {
1271 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1272 struct arm_smmu_device *smmu = smmu_domain->smmu;
1273 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1274 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1275 struct device *dev = smmu->dev;
1276 void __iomem *reg;
1277 u32 tmp;
1278 u64 phys;
1279 unsigned long va, flags;
1280 int ret, idx = cfg->cbndx;
1281 phys_addr_t addr = 0;
1282
1283 ret = arm_smmu_rpm_get(smmu);
1284 if (ret < 0)
1285 return 0;
1286
1287 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1288 va = iova & ~0xfffUL;
1289 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1290 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1291 else
1292 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1293
1294 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1295 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1296 5, 50)) {
1297 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1298 dev_err(dev,
1299 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1300 &iova);
1301 arm_smmu_rpm_put(smmu);
1302 return ops->iova_to_phys(ops, iova);
1303 }
1304
1305 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1306 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1307 if (phys & ARM_SMMU_CB_PAR_F) {
1308 dev_err(dev, "translation fault!\n");
1309 dev_err(dev, "PAR = 0x%llx\n", phys);
1310 goto out;
1311 }
1312
1313 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1314 out:
1315 arm_smmu_rpm_put(smmu);
1316
1317 return addr;
1318 }
1319
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1320 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1321 dma_addr_t iova)
1322 {
1323 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1324 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1325
1326 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1327 return iova;
1328
1329 if (!ops)
1330 return 0;
1331
1332 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1333 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1334 return arm_smmu_iova_to_phys_hard(domain, iova);
1335
1336 return ops->iova_to_phys(ops, iova);
1337 }
1338
arm_smmu_capable(enum iommu_cap cap)1339 static bool arm_smmu_capable(enum iommu_cap cap)
1340 {
1341 switch (cap) {
1342 case IOMMU_CAP_CACHE_COHERENCY:
1343 /*
1344 * Return true here as the SMMU can always send out coherent
1345 * requests.
1346 */
1347 return true;
1348 case IOMMU_CAP_NOEXEC:
1349 return true;
1350 default:
1351 return false;
1352 }
1353 }
1354
1355 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1356 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1357 {
1358 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1359 fwnode);
1360 put_device(dev);
1361 return dev ? dev_get_drvdata(dev) : NULL;
1362 }
1363
arm_smmu_probe_device(struct device * dev)1364 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1365 {
1366 struct arm_smmu_device *smmu = NULL;
1367 struct arm_smmu_master_cfg *cfg;
1368 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1369 int i, ret;
1370
1371 if (using_legacy_binding) {
1372 ret = arm_smmu_register_legacy_master(dev, &smmu);
1373
1374 /*
1375 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1376 * will allocate/initialise a new one. Thus we need to update fwspec for
1377 * later use.
1378 */
1379 fwspec = dev_iommu_fwspec_get(dev);
1380 if (ret)
1381 goto out_free;
1382 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1383 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1384 } else {
1385 return ERR_PTR(-ENODEV);
1386 }
1387
1388 ret = -EINVAL;
1389 for (i = 0; i < fwspec->num_ids; i++) {
1390 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1391 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1392
1393 if (sid & ~smmu->streamid_mask) {
1394 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1395 sid, smmu->streamid_mask);
1396 goto out_free;
1397 }
1398 if (mask & ~smmu->smr_mask_mask) {
1399 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1400 mask, smmu->smr_mask_mask);
1401 goto out_free;
1402 }
1403 }
1404
1405 ret = -ENOMEM;
1406 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1407 GFP_KERNEL);
1408 if (!cfg)
1409 goto out_free;
1410
1411 cfg->smmu = smmu;
1412 dev_iommu_priv_set(dev, cfg);
1413 while (i--)
1414 cfg->smendx[i] = INVALID_SMENDX;
1415
1416 ret = arm_smmu_rpm_get(smmu);
1417 if (ret < 0)
1418 goto out_cfg_free;
1419
1420 ret = arm_smmu_master_alloc_smes(dev);
1421 arm_smmu_rpm_put(smmu);
1422
1423 if (ret)
1424 goto out_cfg_free;
1425
1426 device_link_add(dev, smmu->dev,
1427 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1428
1429 return &smmu->iommu;
1430
1431 out_cfg_free:
1432 kfree(cfg);
1433 out_free:
1434 iommu_fwspec_free(dev);
1435 return ERR_PTR(ret);
1436 }
1437
arm_smmu_release_device(struct device * dev)1438 static void arm_smmu_release_device(struct device *dev)
1439 {
1440 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1441 struct arm_smmu_master_cfg *cfg;
1442 struct arm_smmu_device *smmu;
1443 int ret;
1444
1445 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1446 return;
1447
1448 cfg = dev_iommu_priv_get(dev);
1449 smmu = cfg->smmu;
1450
1451 ret = arm_smmu_rpm_get(smmu);
1452 if (ret < 0)
1453 return;
1454
1455 arm_smmu_master_free_smes(cfg, fwspec);
1456
1457 arm_smmu_rpm_put(smmu);
1458
1459 dev_iommu_priv_set(dev, NULL);
1460 kfree(cfg);
1461 iommu_fwspec_free(dev);
1462 }
1463
arm_smmu_device_group(struct device * dev)1464 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1465 {
1466 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1467 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1468 struct arm_smmu_device *smmu = cfg->smmu;
1469 struct iommu_group *group = NULL;
1470 int i, idx;
1471
1472 for_each_cfg_sme(cfg, fwspec, i, idx) {
1473 if (group && smmu->s2crs[idx].group &&
1474 group != smmu->s2crs[idx].group)
1475 return ERR_PTR(-EINVAL);
1476
1477 group = smmu->s2crs[idx].group;
1478 }
1479
1480 if (group)
1481 return iommu_group_ref_get(group);
1482
1483 if (dev_is_pci(dev))
1484 group = pci_device_group(dev);
1485 else if (dev_is_fsl_mc(dev))
1486 group = fsl_mc_device_group(dev);
1487 else
1488 group = generic_device_group(dev);
1489
1490 /* Remember group for faster lookups */
1491 if (!IS_ERR(group))
1492 for_each_cfg_sme(cfg, fwspec, i, idx)
1493 smmu->s2crs[idx].group = group;
1494
1495 return group;
1496 }
1497
arm_smmu_domain_get_attr(struct iommu_domain * domain,enum iommu_attr attr,void * data)1498 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1499 enum iommu_attr attr, void *data)
1500 {
1501 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1502
1503 switch(domain->type) {
1504 case IOMMU_DOMAIN_UNMANAGED:
1505 switch (attr) {
1506 case DOMAIN_ATTR_NESTING:
1507 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1508 return 0;
1509 default:
1510 return -ENODEV;
1511 }
1512 break;
1513 case IOMMU_DOMAIN_DMA:
1514 switch (attr) {
1515 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1516 *(int *)data = smmu_domain->non_strict;
1517 return 0;
1518 default:
1519 return -ENODEV;
1520 }
1521 break;
1522 default:
1523 return -EINVAL;
1524 }
1525 }
1526
arm_smmu_domain_set_attr(struct iommu_domain * domain,enum iommu_attr attr,void * data)1527 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1528 enum iommu_attr attr, void *data)
1529 {
1530 int ret = 0;
1531 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1532
1533 mutex_lock(&smmu_domain->init_mutex);
1534
1535 switch(domain->type) {
1536 case IOMMU_DOMAIN_UNMANAGED:
1537 switch (attr) {
1538 case DOMAIN_ATTR_NESTING:
1539 if (smmu_domain->smmu) {
1540 ret = -EPERM;
1541 goto out_unlock;
1542 }
1543
1544 if (*(int *)data)
1545 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1546 else
1547 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1548 break;
1549 default:
1550 ret = -ENODEV;
1551 }
1552 break;
1553 case IOMMU_DOMAIN_DMA:
1554 switch (attr) {
1555 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1556 smmu_domain->non_strict = *(int *)data;
1557 break;
1558 default:
1559 ret = -ENODEV;
1560 }
1561 break;
1562 default:
1563 ret = -EINVAL;
1564 }
1565 out_unlock:
1566 mutex_unlock(&smmu_domain->init_mutex);
1567 return ret;
1568 }
1569
arm_smmu_of_xlate(struct device * dev,struct of_phandle_args * args)1570 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1571 {
1572 u32 mask, fwid = 0;
1573
1574 if (args->args_count > 0)
1575 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1576
1577 if (args->args_count > 1)
1578 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1579 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1580 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1581
1582 return iommu_fwspec_add_ids(dev, &fwid, 1);
1583 }
1584
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1585 static void arm_smmu_get_resv_regions(struct device *dev,
1586 struct list_head *head)
1587 {
1588 struct iommu_resv_region *region;
1589 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1590
1591 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1592 prot, IOMMU_RESV_SW_MSI);
1593 if (!region)
1594 return;
1595
1596 list_add_tail(®ion->list, head);
1597
1598 iommu_dma_get_resv_regions(dev, head);
1599 }
1600
arm_smmu_def_domain_type(struct device * dev)1601 static int arm_smmu_def_domain_type(struct device *dev)
1602 {
1603 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1604 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1605
1606 if (impl && impl->def_domain_type)
1607 return impl->def_domain_type(dev);
1608
1609 return 0;
1610 }
1611
1612 static struct iommu_ops arm_smmu_ops = {
1613 .capable = arm_smmu_capable,
1614 .domain_alloc = arm_smmu_domain_alloc,
1615 .domain_free = arm_smmu_domain_free,
1616 .attach_dev = arm_smmu_attach_dev,
1617 .map_pages = arm_smmu_map_pages,
1618 .unmap_pages = arm_smmu_unmap_pages,
1619 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1620 .iotlb_sync = arm_smmu_iotlb_sync,
1621 .iova_to_phys = arm_smmu_iova_to_phys,
1622 .probe_device = arm_smmu_probe_device,
1623 .release_device = arm_smmu_release_device,
1624 .device_group = arm_smmu_device_group,
1625 .domain_get_attr = arm_smmu_domain_get_attr,
1626 .domain_set_attr = arm_smmu_domain_set_attr,
1627 .of_xlate = arm_smmu_of_xlate,
1628 .get_resv_regions = arm_smmu_get_resv_regions,
1629 .put_resv_regions = generic_iommu_put_resv_regions,
1630 .def_domain_type = arm_smmu_def_domain_type,
1631 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1632 };
1633
arm_smmu_device_reset(struct arm_smmu_device * smmu)1634 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1635 {
1636 int i;
1637 u32 reg;
1638
1639 /* clear global FSR */
1640 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1641 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1642
1643 /*
1644 * Reset stream mapping groups: Initial values mark all SMRn as
1645 * invalid and all S2CRn as bypass unless overridden.
1646 */
1647 for (i = 0; i < smmu->num_mapping_groups; ++i)
1648 arm_smmu_write_sme(smmu, i);
1649
1650 /* Make sure all context banks are disabled and clear CB_FSR */
1651 for (i = 0; i < smmu->num_context_banks; ++i) {
1652 arm_smmu_write_context_bank(smmu, i);
1653 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1654 }
1655
1656 /* Invalidate the TLB, just in case */
1657 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1658 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1659
1660 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1661
1662 /* Enable fault reporting */
1663 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1664 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1665
1666 /* Disable TLB broadcasting. */
1667 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1668
1669 /* Enable client access, handling unmatched streams as appropriate */
1670 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1671 if (disable_bypass)
1672 reg |= ARM_SMMU_sCR0_USFCFG;
1673 else
1674 reg &= ~ARM_SMMU_sCR0_USFCFG;
1675
1676 /* Disable forced broadcasting */
1677 reg &= ~ARM_SMMU_sCR0_FB;
1678
1679 /* Don't upgrade barriers */
1680 reg &= ~(ARM_SMMU_sCR0_BSU);
1681
1682 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1683 reg |= ARM_SMMU_sCR0_VMID16EN;
1684
1685 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1686 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1687
1688 if (smmu->impl && smmu->impl->reset)
1689 smmu->impl->reset(smmu);
1690
1691 /* Push the button */
1692 arm_smmu_tlb_sync_global(smmu);
1693 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1694 }
1695
arm_smmu_id_size_to_bits(int size)1696 static int arm_smmu_id_size_to_bits(int size)
1697 {
1698 switch (size) {
1699 case 0:
1700 return 32;
1701 case 1:
1702 return 36;
1703 case 2:
1704 return 40;
1705 case 3:
1706 return 42;
1707 case 4:
1708 return 44;
1709 case 5:
1710 default:
1711 return 48;
1712 }
1713 }
1714
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1715 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1716 {
1717 unsigned int size;
1718 u32 id;
1719 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1720 int i, ret;
1721
1722 dev_notice(smmu->dev, "probing hardware configuration...\n");
1723 dev_notice(smmu->dev, "SMMUv%d with:\n",
1724 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1725
1726 /* ID0 */
1727 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1728
1729 /* Restrict available stages based on module parameter */
1730 if (force_stage == 1)
1731 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1732 else if (force_stage == 2)
1733 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1734
1735 if (id & ARM_SMMU_ID0_S1TS) {
1736 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1737 dev_notice(smmu->dev, "\tstage 1 translation\n");
1738 }
1739
1740 if (id & ARM_SMMU_ID0_S2TS) {
1741 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1742 dev_notice(smmu->dev, "\tstage 2 translation\n");
1743 }
1744
1745 if (id & ARM_SMMU_ID0_NTS) {
1746 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1747 dev_notice(smmu->dev, "\tnested translation\n");
1748 }
1749
1750 if (!(smmu->features &
1751 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1752 dev_err(smmu->dev, "\tno translation support!\n");
1753 return -ENODEV;
1754 }
1755
1756 if ((id & ARM_SMMU_ID0_S1TS) &&
1757 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1758 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1759 dev_notice(smmu->dev, "\taddress translation ops\n");
1760 }
1761
1762 /*
1763 * In order for DMA API calls to work properly, we must defer to what
1764 * the FW says about coherency, regardless of what the hardware claims.
1765 * Fortunately, this also opens up a workaround for systems where the
1766 * ID register value has ended up configured incorrectly.
1767 */
1768 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1769 if (cttw_fw || cttw_reg)
1770 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1771 cttw_fw ? "" : "non-");
1772 if (cttw_fw != cttw_reg)
1773 dev_notice(smmu->dev,
1774 "\t(IDR0.CTTW overridden by FW configuration)\n");
1775
1776 /* Max. number of entries we have for stream matching/indexing */
1777 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1778 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1779 size = 1 << 16;
1780 } else {
1781 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1782 }
1783 smmu->streamid_mask = size - 1;
1784 if (id & ARM_SMMU_ID0_SMS) {
1785 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1786 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1787 if (size == 0) {
1788 dev_err(smmu->dev,
1789 "stream-matching supported, but no SMRs present!\n");
1790 return -ENODEV;
1791 }
1792
1793 /* Zero-initialised to mark as invalid */
1794 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1795 GFP_KERNEL);
1796 if (!smmu->smrs)
1797 return -ENOMEM;
1798
1799 dev_notice(smmu->dev,
1800 "\tstream matching with %u register groups", size);
1801 }
1802 /* s2cr->type == 0 means translation, so initialise explicitly */
1803 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1804 GFP_KERNEL);
1805 if (!smmu->s2crs)
1806 return -ENOMEM;
1807 for (i = 0; i < size; i++)
1808 smmu->s2crs[i] = s2cr_init_val;
1809
1810 smmu->num_mapping_groups = size;
1811 mutex_init(&smmu->stream_map_mutex);
1812 spin_lock_init(&smmu->global_sync_lock);
1813
1814 if (smmu->version < ARM_SMMU_V2 ||
1815 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1816 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1817 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1818 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1819 }
1820
1821 /* ID1 */
1822 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1823 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1824
1825 /* Check for size mismatch of SMMU address space from mapped region */
1826 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1827 if (smmu->numpage != 2 * size << smmu->pgshift)
1828 dev_warn(smmu->dev,
1829 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1830 2 * size << smmu->pgshift, smmu->numpage);
1831 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1832 smmu->numpage = size;
1833
1834 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1835 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1836 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1837 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1838 return -ENODEV;
1839 }
1840 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1841 smmu->num_context_banks, smmu->num_s2_context_banks);
1842 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1843 sizeof(*smmu->cbs), GFP_KERNEL);
1844 if (!smmu->cbs)
1845 return -ENOMEM;
1846
1847 /* ID2 */
1848 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1849 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1850 smmu->ipa_size = size;
1851
1852 /* The output mask is also applied for bypass */
1853 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1854 smmu->pa_size = size;
1855
1856 if (id & ARM_SMMU_ID2_VMID16)
1857 smmu->features |= ARM_SMMU_FEAT_VMID16;
1858
1859 /*
1860 * What the page table walker can address actually depends on which
1861 * descriptor format is in use, but since a) we don't know that yet,
1862 * and b) it can vary per context bank, this will have to do...
1863 */
1864 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1865 dev_warn(smmu->dev,
1866 "failed to set DMA mask for table walker\n");
1867
1868 if (smmu->version < ARM_SMMU_V2) {
1869 smmu->va_size = smmu->ipa_size;
1870 if (smmu->version == ARM_SMMU_V1_64K)
1871 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1872 } else {
1873 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1874 smmu->va_size = arm_smmu_id_size_to_bits(size);
1875 if (id & ARM_SMMU_ID2_PTFS_4K)
1876 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1877 if (id & ARM_SMMU_ID2_PTFS_16K)
1878 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1879 if (id & ARM_SMMU_ID2_PTFS_64K)
1880 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1881 }
1882
1883 if (smmu->impl && smmu->impl->cfg_probe) {
1884 ret = smmu->impl->cfg_probe(smmu);
1885 if (ret)
1886 return ret;
1887 }
1888
1889 /* Now we've corralled the various formats, what'll it do? */
1890 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1891 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1892 if (smmu->features &
1893 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1894 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1895 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1896 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1897 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1898 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1899
1900 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1901 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1902 else
1903 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1904 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1905 smmu->pgsize_bitmap);
1906
1907
1908 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1909 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1910 smmu->va_size, smmu->ipa_size);
1911
1912 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1913 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1914 smmu->ipa_size, smmu->pa_size);
1915
1916 return 0;
1917 }
1918
1919 struct arm_smmu_match_data {
1920 enum arm_smmu_arch_version version;
1921 enum arm_smmu_implementation model;
1922 };
1923
1924 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1925 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1926
1927 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1928 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1929 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1930 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1931 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1932 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1933
1934 static const struct of_device_id arm_smmu_of_match[] = {
1935 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1936 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1937 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1938 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1939 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1940 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1941 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1942 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1943 { },
1944 };
1945 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1946
1947 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1948 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1949 {
1950 int ret = 0;
1951
1952 switch (model) {
1953 case ACPI_IORT_SMMU_V1:
1954 case ACPI_IORT_SMMU_CORELINK_MMU400:
1955 smmu->version = ARM_SMMU_V1;
1956 smmu->model = GENERIC_SMMU;
1957 break;
1958 case ACPI_IORT_SMMU_CORELINK_MMU401:
1959 smmu->version = ARM_SMMU_V1_64K;
1960 smmu->model = GENERIC_SMMU;
1961 break;
1962 case ACPI_IORT_SMMU_V2:
1963 smmu->version = ARM_SMMU_V2;
1964 smmu->model = GENERIC_SMMU;
1965 break;
1966 case ACPI_IORT_SMMU_CORELINK_MMU500:
1967 smmu->version = ARM_SMMU_V2;
1968 smmu->model = ARM_MMU500;
1969 break;
1970 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1971 smmu->version = ARM_SMMU_V2;
1972 smmu->model = CAVIUM_SMMUV2;
1973 break;
1974 default:
1975 ret = -ENODEV;
1976 }
1977
1978 return ret;
1979 }
1980
arm_smmu_device_acpi_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)1981 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1982 struct arm_smmu_device *smmu)
1983 {
1984 struct device *dev = smmu->dev;
1985 struct acpi_iort_node *node =
1986 *(struct acpi_iort_node **)dev_get_platdata(dev);
1987 struct acpi_iort_smmu *iort_smmu;
1988 int ret;
1989
1990 /* Retrieve SMMU1/2 specific data */
1991 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1992
1993 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1994 if (ret < 0)
1995 return ret;
1996
1997 /* Ignore the configuration access interrupt */
1998 smmu->num_global_irqs = 1;
1999
2000 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2001 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2002
2003 return 0;
2004 }
2005 #else
arm_smmu_device_acpi_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)2006 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2007 struct arm_smmu_device *smmu)
2008 {
2009 return -ENODEV;
2010 }
2011 #endif
2012
arm_smmu_device_dt_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)2013 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2014 struct arm_smmu_device *smmu)
2015 {
2016 const struct arm_smmu_match_data *data;
2017 struct device *dev = &pdev->dev;
2018 bool legacy_binding;
2019
2020 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2021 &smmu->num_global_irqs)) {
2022 dev_err(dev, "missing #global-interrupts property\n");
2023 return -ENODEV;
2024 }
2025
2026 data = of_device_get_match_data(dev);
2027 smmu->version = data->version;
2028 smmu->model = data->model;
2029
2030 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2031 if (legacy_binding && !using_generic_binding) {
2032 if (!using_legacy_binding) {
2033 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2034 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2035 }
2036 using_legacy_binding = true;
2037 } else if (!legacy_binding && !using_legacy_binding) {
2038 using_generic_binding = true;
2039 } else {
2040 dev_err(dev, "not probing due to mismatched DT properties\n");
2041 return -ENODEV;
2042 }
2043
2044 if (of_dma_is_coherent(dev->of_node))
2045 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2046
2047 return 0;
2048 }
2049
arm_smmu_bus_init(struct iommu_ops * ops)2050 static int arm_smmu_bus_init(struct iommu_ops *ops)
2051 {
2052 int err;
2053
2054 /* Oh, for a proper bus abstraction */
2055 if (!iommu_present(&platform_bus_type)) {
2056 err = bus_set_iommu(&platform_bus_type, ops);
2057 if (err)
2058 return err;
2059 }
2060 #ifdef CONFIG_ARM_AMBA
2061 if (!iommu_present(&amba_bustype)) {
2062 err = bus_set_iommu(&amba_bustype, ops);
2063 if (err)
2064 goto err_reset_platform_ops;
2065 }
2066 #endif
2067 #ifdef CONFIG_PCI
2068 if (!iommu_present(&pci_bus_type)) {
2069 err = bus_set_iommu(&pci_bus_type, ops);
2070 if (err)
2071 goto err_reset_amba_ops;
2072 }
2073 #endif
2074 #ifdef CONFIG_FSL_MC_BUS
2075 if (!iommu_present(&fsl_mc_bus_type)) {
2076 err = bus_set_iommu(&fsl_mc_bus_type, ops);
2077 if (err)
2078 goto err_reset_pci_ops;
2079 }
2080 #endif
2081 return 0;
2082
2083 err_reset_pci_ops: __maybe_unused;
2084 #ifdef CONFIG_PCI
2085 bus_set_iommu(&pci_bus_type, NULL);
2086 #endif
2087 err_reset_amba_ops: __maybe_unused;
2088 #ifdef CONFIG_ARM_AMBA
2089 bus_set_iommu(&amba_bustype, NULL);
2090 #endif
2091 err_reset_platform_ops: __maybe_unused;
2092 bus_set_iommu(&platform_bus_type, NULL);
2093 return err;
2094 }
2095
arm_smmu_device_probe(struct platform_device * pdev)2096 static int arm_smmu_device_probe(struct platform_device *pdev)
2097 {
2098 struct resource *res;
2099 resource_size_t ioaddr;
2100 struct arm_smmu_device *smmu;
2101 struct device *dev = &pdev->dev;
2102 int num_irqs, i, err;
2103 irqreturn_t (*global_fault)(int irq, void *dev);
2104
2105 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2106 if (!smmu) {
2107 dev_err(dev, "failed to allocate arm_smmu_device\n");
2108 return -ENOMEM;
2109 }
2110 smmu->dev = dev;
2111
2112 if (dev->of_node)
2113 err = arm_smmu_device_dt_probe(pdev, smmu);
2114 else
2115 err = arm_smmu_device_acpi_probe(pdev, smmu);
2116
2117 if (err)
2118 return err;
2119
2120 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2121 if (IS_ERR(smmu->base))
2122 return PTR_ERR(smmu->base);
2123 ioaddr = res->start;
2124 /*
2125 * The resource size should effectively match the value of SMMU_TOP;
2126 * stash that temporarily until we know PAGESIZE to validate it with.
2127 */
2128 smmu->numpage = resource_size(res);
2129
2130 smmu = arm_smmu_impl_init(smmu);
2131 if (IS_ERR(smmu))
2132 return PTR_ERR(smmu);
2133
2134 num_irqs = 0;
2135 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2136 num_irqs++;
2137 if (num_irqs > smmu->num_global_irqs)
2138 smmu->num_context_irqs++;
2139 }
2140
2141 if (!smmu->num_context_irqs) {
2142 dev_err(dev, "found %d interrupts but expected at least %d\n",
2143 num_irqs, smmu->num_global_irqs + 1);
2144 return -ENODEV;
2145 }
2146
2147 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2148 GFP_KERNEL);
2149 if (!smmu->irqs) {
2150 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2151 return -ENOMEM;
2152 }
2153
2154 for (i = 0; i < num_irqs; ++i) {
2155 int irq = platform_get_irq(pdev, i);
2156
2157 if (irq < 0)
2158 return -ENODEV;
2159 smmu->irqs[i] = irq;
2160 }
2161
2162 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2163 if (err < 0) {
2164 dev_err(dev, "failed to get clocks %d\n", err);
2165 return err;
2166 }
2167 smmu->num_clks = err;
2168
2169 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2170 if (err)
2171 return err;
2172
2173 err = arm_smmu_device_cfg_probe(smmu);
2174 if (err)
2175 return err;
2176
2177 if (smmu->version == ARM_SMMU_V2) {
2178 if (smmu->num_context_banks > smmu->num_context_irqs) {
2179 dev_err(dev,
2180 "found only %d context irq(s) but %d required\n",
2181 smmu->num_context_irqs, smmu->num_context_banks);
2182 return -ENODEV;
2183 }
2184
2185 /* Ignore superfluous interrupts */
2186 smmu->num_context_irqs = smmu->num_context_banks;
2187 }
2188
2189 if (smmu->impl && smmu->impl->global_fault)
2190 global_fault = smmu->impl->global_fault;
2191 else
2192 global_fault = arm_smmu_global_fault;
2193
2194 for (i = 0; i < smmu->num_global_irqs; ++i) {
2195 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2196 global_fault,
2197 IRQF_SHARED,
2198 "arm-smmu global fault",
2199 smmu);
2200 if (err) {
2201 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2202 i, smmu->irqs[i]);
2203 return err;
2204 }
2205 }
2206
2207 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2208 "smmu.%pa", &ioaddr);
2209 if (err) {
2210 dev_err(dev, "Failed to register iommu in sysfs\n");
2211 return err;
2212 }
2213
2214 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2215 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2216
2217 err = iommu_device_register(&smmu->iommu);
2218 if (err) {
2219 dev_err(dev, "Failed to register iommu\n");
2220 return err;
2221 }
2222
2223 platform_set_drvdata(pdev, smmu);
2224 arm_smmu_device_reset(smmu);
2225 arm_smmu_test_smr_masks(smmu);
2226
2227 /*
2228 * We want to avoid touching dev->power.lock in fastpaths unless
2229 * it's really going to do something useful - pm_runtime_enabled()
2230 * can serve as an ideal proxy for that decision. So, conditionally
2231 * enable pm_runtime.
2232 */
2233 if (dev->pm_domain) {
2234 pm_runtime_set_active(dev);
2235 pm_runtime_enable(dev);
2236 }
2237
2238 /*
2239 * For ACPI and generic DT bindings, an SMMU will be probed before
2240 * any device which might need it, so we want the bus ops in place
2241 * ready to handle default domain setup as soon as any SMMU exists.
2242 */
2243 if (!using_legacy_binding)
2244 return arm_smmu_bus_init(&arm_smmu_ops);
2245
2246 return 0;
2247 }
2248
arm_smmu_device_remove(struct platform_device * pdev)2249 static int arm_smmu_device_remove(struct platform_device *pdev)
2250 {
2251 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2252
2253 if (!smmu)
2254 return -ENODEV;
2255
2256 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2257 dev_notice(&pdev->dev, "disabling translation\n");
2258
2259 arm_smmu_bus_init(NULL);
2260 iommu_device_unregister(&smmu->iommu);
2261 iommu_device_sysfs_remove(&smmu->iommu);
2262
2263 arm_smmu_rpm_get(smmu);
2264 /* Turn the thing off */
2265 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2266 arm_smmu_rpm_put(smmu);
2267
2268 if (pm_runtime_enabled(smmu->dev))
2269 pm_runtime_force_suspend(smmu->dev);
2270 else
2271 clk_bulk_disable(smmu->num_clks, smmu->clks);
2272
2273 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2274 return 0;
2275 }
2276
arm_smmu_device_shutdown(struct platform_device * pdev)2277 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2278 {
2279 arm_smmu_device_remove(pdev);
2280 }
2281
arm_smmu_runtime_resume(struct device * dev)2282 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2283 {
2284 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2285 int ret;
2286
2287 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2288 if (ret)
2289 return ret;
2290
2291 arm_smmu_device_reset(smmu);
2292
2293 return 0;
2294 }
2295
arm_smmu_runtime_suspend(struct device * dev)2296 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2297 {
2298 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2299
2300 clk_bulk_disable(smmu->num_clks, smmu->clks);
2301
2302 return 0;
2303 }
2304
arm_smmu_pm_resume(struct device * dev)2305 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2306 {
2307 if (pm_runtime_suspended(dev))
2308 return 0;
2309
2310 return arm_smmu_runtime_resume(dev);
2311 }
2312
arm_smmu_pm_suspend(struct device * dev)2313 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2314 {
2315 if (pm_runtime_suspended(dev))
2316 return 0;
2317
2318 return arm_smmu_runtime_suspend(dev);
2319 }
2320
2321 static const struct dev_pm_ops arm_smmu_pm_ops = {
2322 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2323 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2324 arm_smmu_runtime_resume, NULL)
2325 };
2326
2327 static struct platform_driver arm_smmu_driver = {
2328 .driver = {
2329 .name = "arm-smmu",
2330 .of_match_table = arm_smmu_of_match,
2331 .pm = &arm_smmu_pm_ops,
2332 .suppress_bind_attrs = true,
2333 },
2334 .probe = arm_smmu_device_probe,
2335 .remove = arm_smmu_device_remove,
2336 .shutdown = arm_smmu_device_shutdown,
2337 };
2338 module_platform_driver(arm_smmu_driver);
2339
2340 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2341 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2342 MODULE_ALIAS("platform:arm-smmu");
2343 MODULE_LICENSE("GPL v2");
2344