1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2019, The Linux Foundation. All rights reserved.
4 */
5
6 #include <linux/adreno-smmu-priv.h>
7 #include <linux/bitfield.h>
8 #include <linux/of_device.h>
9 #include <linux/qcom_scm.h>
10
11 #include "arm-smmu.h"
12
13 struct qcom_smmu {
14 struct arm_smmu_device smmu;
15 bool bypass_quirk;
16 u8 bypass_cbndx;
17 };
18
qcom_sdm845_smmu500_cfg_probe(struct arm_smmu_device * smmu)19 static int qcom_sdm845_smmu500_cfg_probe(struct arm_smmu_device *smmu)
20 {
21 u32 s2cr;
22 u32 smr;
23 int i;
24
25 for (i = 0; i < smmu->num_mapping_groups; i++) {
26 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
27 s2cr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_S2CR(i));
28
29 smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
30 smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr);
31 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
32 smmu->smrs[i].valid = FIELD_GET(
33 ARM_SMMU_S2CR_EXIDVALID,
34 s2cr);
35 else
36 smmu->smrs[i].valid = FIELD_GET(
37 ARM_SMMU_SMR_VALID,
38 smr);
39
40 smmu->s2crs[i].group = NULL;
41 smmu->s2crs[i].count = 0;
42 smmu->s2crs[i].type = FIELD_GET(ARM_SMMU_S2CR_TYPE, s2cr);
43 smmu->s2crs[i].privcfg = FIELD_GET(ARM_SMMU_S2CR_PRIVCFG, s2cr);
44 smmu->s2crs[i].cbndx = FIELD_GET(ARM_SMMU_S2CR_CBNDX, s2cr);
45
46 if (!smmu->smrs[i].valid)
47 continue;
48
49 smmu->s2crs[i].pinned = true;
50 bitmap_set(smmu->context_map, smmu->s2crs[i].cbndx, 1);
51 }
52
53 return 0;
54 }
55
56 #define QCOM_ADRENO_SMMU_GPU_SID 0
57
qcom_adreno_smmu_is_gpu_device(struct device * dev)58 static bool qcom_adreno_smmu_is_gpu_device(struct device *dev)
59 {
60 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
61 int i;
62
63 /*
64 * The GPU will always use SID 0 so that is a handy way to uniquely
65 * identify it and configure it for per-instance pagetables
66 */
67 for (i = 0; i < fwspec->num_ids; i++) {
68 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
69
70 if (sid == QCOM_ADRENO_SMMU_GPU_SID)
71 return true;
72 }
73
74 return false;
75 }
76
qcom_adreno_smmu_get_ttbr1_cfg(const void * cookie)77 static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg(
78 const void *cookie)
79 {
80 struct arm_smmu_domain *smmu_domain = (void *)cookie;
81 struct io_pgtable *pgtable =
82 io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
83 return &pgtable->cfg;
84 }
85
86 /*
87 * Local implementation to configure TTBR0 with the specified pagetable config.
88 * The GPU driver will call this to enable TTBR0 when per-instance pagetables
89 * are active
90 */
91
qcom_adreno_smmu_set_ttbr0_cfg(const void * cookie,const struct io_pgtable_cfg * pgtbl_cfg)92 static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
93 const struct io_pgtable_cfg *pgtbl_cfg)
94 {
95 struct arm_smmu_domain *smmu_domain = (void *)cookie;
96 struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
97 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
98 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
99
100 /* The domain must have split pagetables already enabled */
101 if (cb->tcr[0] & ARM_SMMU_TCR_EPD1)
102 return -EINVAL;
103
104 /* If the pagetable config is NULL, disable TTBR0 */
105 if (!pgtbl_cfg) {
106 /* Do nothing if it is already disabled */
107 if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0))
108 return -EINVAL;
109
110 /* Set TCR to the original configuration */
111 cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg);
112 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
113 } else {
114 u32 tcr = cb->tcr[0];
115
116 /* Don't call this again if TTBR0 is already enabled */
117 if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0))
118 return -EINVAL;
119
120 tcr |= arm_smmu_lpae_tcr(pgtbl_cfg);
121 tcr &= ~(ARM_SMMU_TCR_EPD0 | ARM_SMMU_TCR_EPD1);
122
123 cb->tcr[0] = tcr;
124 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
125 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
126 }
127
128 arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx);
129
130 return 0;
131 }
132
qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,int start)133 static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
134 struct arm_smmu_device *smmu,
135 struct device *dev, int start)
136 {
137 int count;
138
139 /*
140 * Assign context bank 0 to the GPU device so the GPU hardware can
141 * switch pagetables
142 */
143 if (qcom_adreno_smmu_is_gpu_device(dev)) {
144 start = 0;
145 count = 1;
146 } else {
147 start = 1;
148 count = smmu->num_context_banks;
149 }
150
151 return __arm_smmu_alloc_bitmap(smmu->context_map, start, count);
152 }
153
qcom_adreno_smmu_init_context(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg,struct device * dev)154 static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
155 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
156 {
157 struct adreno_smmu_priv *priv;
158
159 /* Only enable split pagetables for the GPU device (SID 0) */
160 if (!qcom_adreno_smmu_is_gpu_device(dev))
161 return 0;
162
163 /*
164 * All targets that use the qcom,adreno-smmu compatible string *should*
165 * be AARCH64 stage 1 but double check because the arm-smmu code assumes
166 * that is the case when the TTBR1 quirk is enabled
167 */
168 if ((smmu_domain->stage == ARM_SMMU_DOMAIN_S1) &&
169 (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64))
170 pgtbl_cfg->quirks |= IO_PGTABLE_QUIRK_ARM_TTBR1;
171
172 /*
173 * Initialize private interface with GPU:
174 */
175
176 priv = dev_get_drvdata(dev);
177 priv->cookie = smmu_domain;
178 priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg;
179 priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg;
180
181 return 0;
182 }
183
to_qcom_smmu(struct arm_smmu_device * smmu)184 static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu)
185 {
186 return container_of(smmu, struct qcom_smmu, smmu);
187 }
188
189 static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
190 { .compatible = "qcom,adreno" },
191 { .compatible = "qcom,mdp4" },
192 { .compatible = "qcom,mdss" },
193 { .compatible = "qcom,sc7180-mdss" },
194 { .compatible = "qcom,sc7180-mss-pil" },
195 { .compatible = "qcom,sdm845-mdss" },
196 { .compatible = "qcom,sdm845-mss-pil" },
197 { }
198 };
199
qcom_smmu_cfg_probe(struct arm_smmu_device * smmu)200 static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu)
201 {
202 unsigned int last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1);
203 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
204 u32 reg;
205 u32 smr;
206 int i;
207
208 /*
209 * With some firmware versions writes to S2CR of type FAULT are
210 * ignored, and writing BYPASS will end up written as FAULT in the
211 * register. Perform a write to S2CR to detect if this is the case and
212 * if so reserve a context bank to emulate bypass streams.
213 */
214 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) |
215 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) |
216 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT);
217 arm_smmu_gr0_write(smmu, last_s2cr, reg);
218 reg = arm_smmu_gr0_read(smmu, last_s2cr);
219 if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) {
220 qsmmu->bypass_quirk = true;
221 qsmmu->bypass_cbndx = smmu->num_context_banks - 1;
222
223 set_bit(qsmmu->bypass_cbndx, smmu->context_map);
224
225 arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0);
226
227 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS);
228 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg);
229 }
230
231 for (i = 0; i < smmu->num_mapping_groups; i++) {
232 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
233
234 if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) {
235 /* Ignore valid bit for SMR mask extraction. */
236 smr &= ~ARM_SMMU_SMR_VALID;
237 smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr);
238 smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
239 smmu->smrs[i].valid = true;
240
241 smmu->s2crs[i].type = S2CR_TYPE_BYPASS;
242 smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT;
243 smmu->s2crs[i].cbndx = 0xff;
244 }
245 }
246
247 return 0;
248 }
249
qcom_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)250 static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
251 {
252 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
253 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
254 u32 cbndx = s2cr->cbndx;
255 u32 type = s2cr->type;
256 u32 reg;
257
258 if (qsmmu->bypass_quirk) {
259 if (type == S2CR_TYPE_BYPASS) {
260 /*
261 * Firmware with quirky S2CR handling will substitute
262 * BYPASS writes with FAULT, so point the stream to the
263 * reserved context bank and ask for translation on the
264 * stream
265 */
266 type = S2CR_TYPE_TRANS;
267 cbndx = qsmmu->bypass_cbndx;
268 } else if (type == S2CR_TYPE_FAULT) {
269 /*
270 * Firmware with quirky S2CR handling will ignore FAULT
271 * writes, so trick it to write FAULT by asking for a
272 * BYPASS.
273 */
274 type = S2CR_TYPE_BYPASS;
275 cbndx = 0xff;
276 }
277 }
278
279 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) |
280 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) |
281 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
282 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
283 }
284
qcom_smmu_def_domain_type(struct device * dev)285 static int qcom_smmu_def_domain_type(struct device *dev)
286 {
287 const struct of_device_id *match =
288 of_match_device(qcom_smmu_client_of_match, dev);
289
290 return match ? IOMMU_DOMAIN_IDENTITY : 0;
291 }
292
qcom_sdm845_smmu500_reset(struct arm_smmu_device * smmu)293 static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
294 {
295 int ret;
296
297 /*
298 * To address performance degradation in non-real time clients,
299 * such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
300 * such as MTP and db845, whose firmwares implement secure monitor
301 * call handlers to turn on/off the wait-for-safe logic.
302 */
303 ret = qcom_scm_qsmmu500_wait_safe_toggle(0);
304 if (ret)
305 dev_warn(smmu->dev, "Failed to turn off SAFE logic\n");
306
307 return ret;
308 }
309
qcom_smmu500_reset(struct arm_smmu_device * smmu)310 static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
311 {
312 const struct device_node *np = smmu->dev->of_node;
313
314 arm_mmu500_reset(smmu);
315
316 if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
317 return qcom_sdm845_smmu500_reset(smmu);
318
319 return 0;
320 }
321
322 static const struct arm_smmu_impl qcom_smmu_impl = {
323 .cfg_probe = qcom_smmu_cfg_probe,
324 .def_domain_type = qcom_smmu_def_domain_type,
325 .cfg_probe = qcom_sdm845_smmu500_cfg_probe,
326 .reset = qcom_smmu500_reset,
327 .write_s2cr = qcom_smmu_write_s2cr,
328 };
329
330 static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
331 .init_context = qcom_adreno_smmu_init_context,
332 .def_domain_type = qcom_smmu_def_domain_type,
333 .reset = qcom_smmu500_reset,
334 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
335 };
336
qcom_smmu_create(struct arm_smmu_device * smmu,const struct arm_smmu_impl * impl)337 static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
338 const struct arm_smmu_impl *impl)
339 {
340 struct qcom_smmu *qsmmu;
341
342 /* Check to make sure qcom_scm has finished probing */
343 if (!qcom_scm_is_available())
344 return ERR_PTR(-EPROBE_DEFER);
345
346 qsmmu = devm_kzalloc(smmu->dev, sizeof(*qsmmu), GFP_KERNEL);
347 if (!qsmmu)
348 return ERR_PTR(-ENOMEM);
349
350 qsmmu->smmu = *smmu;
351
352 qsmmu->smmu.impl = impl;
353 devm_kfree(smmu->dev, smmu);
354
355 return &qsmmu->smmu;
356 }
357
qcom_smmu_impl_init(struct arm_smmu_device * smmu)358 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
359 {
360 return qcom_smmu_create(smmu, &qcom_smmu_impl);
361 }
362
qcom_adreno_smmu_impl_init(struct arm_smmu_device * smmu)363 struct arm_smmu_device *qcom_adreno_smmu_impl_init(struct arm_smmu_device *smmu)
364 {
365 return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);
366 }
367