1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Compatible with the IOMMU of av1 decode
4 *
5 * Module Authors: Yandong Lin <yandong.lin@rock-chips.com>
6 * Simon Xue <xxm@rock-chips.com>
7 */
8
9 #include <linux/clk.h>
10 #include <linux/compiler.h>
11 #include <linux/delay.h>
12 #include <linux/device.h>
13 #include <linux/dma-iommu.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dma-map-ops.h>
16 #include <linux/errno.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/iommu.h>
20 #include <linux/iopoll.h>
21 #include <linux/list.h>
22 #include <linux/mm.h>
23 #include <linux/module.h>
24 #include <linux/init.h>
25 #include <linux/of.h>
26 #include <linux/of_iommu.h>
27 #include <linux/of_platform.h>
28 #include <linux/platform_device.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/slab.h>
31 #include <linux/spinlock.h>
32 #include "mpp_debug.h"
33 #include "mpp_common.h"
34 #include "mpp_iommu.h"
35
36 struct av1_iommu_domain {
37 struct list_head iommus;
38 u32 *dt; /* page directory table */
39 dma_addr_t dt_dma;
40 spinlock_t iommus_lock; /* lock for iommus list */
41 spinlock_t dt_lock; /* lock for modifying page directory table */
42 struct iommu_domain domain;
43 /* for av1 iommu */
44 u64 *pta; /* page directory table */
45 dma_addr_t pta_dma;
46 };
47
48 struct av1_iommu {
49 struct device *dev;
50 void __iomem **bases;
51 int num_mmu;
52 int num_irq;
53 struct clk_bulk_data *clocks;
54 int num_clocks;
55 struct iommu_device iommu;
56 struct list_head node; /* entry in rk_iommu_domain.iommus */
57 struct iommu_domain *domain; /* domain to which iommu is attached */
58 struct iommu_group *group;
59 };
60
61 struct av1_iommudata {
62 struct device_link *link; /* runtime PM link from IOMMU to master */
63 struct av1_iommu *iommu;
64 bool defer_attach;
65 };
66
67 #define RK_IOMMU_AV1 0xa
68 #define NUM_DT_ENTRIES 1024
69 #define NUM_PT_ENTRIES 1024
70
71 #define SPAGE_ORDER 12
72 #define SPAGE_SIZE (1 << SPAGE_ORDER)
73
74 /* av1 iommu regs address */
75 #define AV1_CLOCK_CTRL_BASE 0x0
76 #define AV1_IDLE_ST_BASE 0x4
77 #define AV1_MMU_CONFIG0_BASE 0x184
78 #define AV1_MMU_CONFIG1_BASE 0x1ac
79 #define AV1_MMU_AHB_EXCEPTION_BASE 0x380
80 #define AV1_MMU_AHB_STATUS_BASE 0x384
81 #define AV1_MMU_AHB_CONTROL_BASE 0x388
82 #define AV1_MMU_AHB_TBL_ARRAY_BASE_L_BASE 0x38C
83 #define AV1_MMU_AHB_TBL_ARRAY_BASE_H_BASE 0x390
84 #define AV1_MMU_AHB_CTX_PD_BASE 0x3b4
85 #define AV1_MMU_BUTT_BASE 0xffff
86
87 /* MMU register offsets */
88 #define AV1_MMU_FLUSH_BASE 0x184
89 #define AV1_MMU_BIT_FLUSH BIT(4)
90
91 #define AV1_MMU_PAGE_FAULT_ADDR 0x380
92 #define AV1_MMU_STATUS_BASE 0x384 /* IRQ status */
93
94 #define AV1_MMU_EN_BASE 0x388
95 #define AV1_MMU_BIT_ENABLE BIT(0)
96
97 #define AV1_MMU_OUT_OF_BOUND BIT(28)
98 /* Irq mask */
99 #define AV1_MMU_IRQ_MASK 0x7
100
101 #define AV1_DTE_PT_ADDRESS_MASK 0xffffffc0
102 #define AV1_DTE_PT_VALID BIT(0)
103
104 #define AV1_PAGE_DESC_LO_MASK 0xfffff000
105 #define AV1_PAGE_DESC_HI_MASK GENMASK_ULL(39, 32)
106 #define AV1_PAGE_DESC_HI_SHIFT (32-4)
107
108 #define AV1_IOMMU_PGSIZE_BITMAP 0x007ff000
109
av1_dte_pt_address(u32 dte)110 static inline phys_addr_t av1_dte_pt_address(u32 dte)
111 {
112 return (phys_addr_t)dte & AV1_DTE_PT_ADDRESS_MASK;
113 }
114
av1_mk_dte(dma_addr_t pt_dma)115 static inline u32 av1_mk_dte(dma_addr_t pt_dma)
116 {
117 return (pt_dma) | AV1_DTE_PT_VALID;
118 }
119
120 #define AV1_PTE_PAGE_ADDRESS_MASK 0xfffffff0
121 #define AV1_PTE_PAGE_WRITABLE BIT(2)
122 #define AV1_PTE_PAGE_VALID BIT(0)
123
124 static struct device *dma_dev;
125
av1_pte_page_address(u32 pte)126 static inline phys_addr_t av1_pte_page_address(u32 pte)
127 {
128 u64 pte_av1 = pte;
129
130 pte_av1 = ((pte_av1 & AV1_PAGE_DESC_HI_MASK) << AV1_PAGE_DESC_HI_SHIFT) |
131 (pte_av1 & AV1_PAGE_DESC_LO_MASK);
132
133 return (phys_addr_t)pte_av1;
134 }
135
av1_mk_pte(phys_addr_t page,int prot)136 static u32 av1_mk_pte(phys_addr_t page, int prot)
137 {
138 u32 flags = 0;
139
140 flags |= (prot & IOMMU_WRITE) ? AV1_PTE_PAGE_WRITABLE : 0;
141 page = (page & AV1_PAGE_DESC_LO_MASK) |
142 ((page & AV1_PAGE_DESC_HI_MASK) >> AV1_PAGE_DESC_HI_SHIFT);
143 page &= AV1_PTE_PAGE_ADDRESS_MASK;
144
145 return page | flags | AV1_PTE_PAGE_VALID;
146 }
147
148 #define AV1_DTE_PT_VALID BIT(0)
149
av1_dte_is_pt_valid(u32 dte)150 static inline bool av1_dte_is_pt_valid(u32 dte)
151 {
152 return dte & AV1_DTE_PT_VALID;
153 }
154
av1_pte_is_page_valid(u32 pte)155 static inline bool av1_pte_is_page_valid(u32 pte)
156 {
157 return pte & AV1_PTE_PAGE_VALID;
158 }
159
av1_mk_pte_invalid(u32 pte)160 static u32 av1_mk_pte_invalid(u32 pte)
161 {
162 return pte & ~AV1_PTE_PAGE_VALID;
163 }
164
165 #define AV1_MASTER_TLB_MASK GENMASK_ULL(31, 10)
166 /* mode 0 : 4k */
167 #define AV1_PTA_4K_MODE 0
168
av1_iommu_from_dev(struct device * dev)169 static struct av1_iommu *av1_iommu_from_dev(struct device *dev)
170 {
171 struct av1_iommudata *data = dev_iommu_priv_get(dev);
172
173 return data ? data->iommu : NULL;
174 }
175
av1_mk_pta(dma_addr_t dt_dma)176 static u64 av1_mk_pta(dma_addr_t dt_dma)
177 {
178 u64 val = (dt_dma & AV1_MASTER_TLB_MASK) | AV1_PTA_4K_MODE;
179
180 return val;
181 }
182
to_av1_domain(struct iommu_domain * dom)183 static struct av1_iommu_domain *to_av1_domain(struct iommu_domain *dom)
184 {
185 return container_of(dom, struct av1_iommu_domain, domain);
186 }
187
av1_iommu_disable(struct av1_iommu * iommu)188 static void av1_iommu_disable(struct av1_iommu *iommu)
189 {
190 int i;
191
192 /* Ignore error while disabling, just keep going */
193 WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks));
194 for (i = 0; i < iommu->num_mmu; i++)
195 writel(0, iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE);
196
197 clk_bulk_disable(iommu->num_clocks, iommu->clocks);
198 }
199
mpp_av1_iommu_disable(struct device * dev)200 int mpp_av1_iommu_disable(struct device *dev)
201 {
202 struct av1_iommu *iommu = av1_iommu_from_dev(dev);
203
204 if (!iommu->domain)
205 return 0;
206
207 av1_iommu_disable(iommu);
208
209 return 0;
210 }
211
av1_iommu_enable(struct av1_iommu * iommu)212 static int av1_iommu_enable(struct av1_iommu *iommu)
213 {
214 struct iommu_domain *domain = iommu->domain;
215 struct av1_iommu_domain *av1_domain = to_av1_domain(domain);
216 int ret, i;
217
218 ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);
219 if (ret)
220 return ret;
221
222 for (i = 0; i < iommu->num_mmu; i++) {
223 u32 val = readl(iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE);
224
225 if (!(val & AV1_MMU_BIT_ENABLE)) {
226 writel(av1_domain->pta_dma,
227 iommu->bases[i] + AV1_MMU_AHB_TBL_ARRAY_BASE_L_BASE);
228 writel(AV1_MMU_OUT_OF_BOUND, iommu->bases[i] + AV1_MMU_CONFIG1_BASE);
229 writel(AV1_MMU_BIT_ENABLE, iommu->bases[i] + AV1_MMU_AHB_EXCEPTION_BASE);
230 writel(AV1_MMU_BIT_ENABLE, iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE);
231 }
232 }
233 clk_bulk_disable(iommu->num_clocks, iommu->clocks);
234 return ret;
235 }
236
mpp_av1_iommu_enable(struct device * dev)237 int mpp_av1_iommu_enable(struct device *dev)
238 {
239 struct av1_iommu *iommu = av1_iommu_from_dev(dev);
240
241 if (!iommu->domain)
242 return 0;
243
244 return av1_iommu_enable(iommu);
245 }
246
av1_table_flush(struct av1_iommu_domain * dom,dma_addr_t dma,unsigned int count)247 static inline void av1_table_flush(struct av1_iommu_domain *dom, dma_addr_t dma,
248 unsigned int count)
249 {
250 size_t size = count * sizeof(u32); /* count of u32 entry */
251
252 dma_sync_single_for_device(dma_dev, dma, size, DMA_TO_DEVICE);
253 }
254
255 #define AV1_IOVA_DTE_MASK 0xffc00000
256 #define AV1_IOVA_DTE_SHIFT 22
257 #define AV1_IOVA_PTE_MASK 0x003ff000
258 #define AV1_IOVA_PTE_SHIFT 12
259 #define AV1_IOVA_PAGE_MASK 0x00000fff
260 #define AV1_IOVA_PAGE_SHIFT 0
261
av1_iova_dte_index(dma_addr_t iova)262 static u32 av1_iova_dte_index(dma_addr_t iova)
263 {
264 return (u32)(iova & AV1_IOVA_DTE_MASK) >> AV1_IOVA_DTE_SHIFT;
265 }
266
av1_iova_pte_index(dma_addr_t iova)267 static u32 av1_iova_pte_index(dma_addr_t iova)
268 {
269 return (u32)(iova & AV1_IOVA_PTE_MASK) >> AV1_IOVA_PTE_SHIFT;
270 }
271
av1_iova_page_offset(dma_addr_t iova)272 static u32 av1_iova_page_offset(dma_addr_t iova)
273 {
274 return (u32)(iova & AV1_IOVA_PAGE_MASK) >> AV1_IOVA_PAGE_SHIFT;
275 }
276
av1_iommu_read(void __iomem * base,u32 offset)277 static u32 av1_iommu_read(void __iomem *base, u32 offset)
278 {
279 return readl(base + offset);
280 }
281
av1_iommu_write(void __iomem * base,u32 offset,u32 value)282 static void av1_iommu_write(void __iomem *base, u32 offset, u32 value)
283 {
284 writel(value, base + offset);
285 }
286
287
av1_iommu_flush_tlb_all(struct iommu_domain * domain)288 static void av1_iommu_flush_tlb_all(struct iommu_domain *domain)
289 {
290 struct av1_iommu_domain *av1_domain = to_av1_domain(domain);
291 struct list_head *pos;
292 unsigned long flags;
293 int i;
294
295 spin_lock_irqsave(&av1_domain->iommus_lock, flags);
296 list_for_each(pos, &av1_domain->iommus) {
297 struct av1_iommu *iommu;
298 int ret;
299
300 iommu = list_entry(pos, struct av1_iommu, node);
301 ret = pm_runtime_get_if_in_use(iommu->dev);
302 if (WARN_ON_ONCE(ret < 0))
303 continue;
304 if (ret) {
305 WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks));
306 for (i = 0; i < iommu->num_mmu; i++) {
307 writel(AV1_MMU_BIT_FLUSH,
308 iommu->bases[i] + AV1_MMU_FLUSH_BASE);
309 writel(0, iommu->bases[i] + AV1_MMU_FLUSH_BASE);
310 }
311 clk_bulk_disable(iommu->num_clocks, iommu->clocks);
312 pm_runtime_put(iommu->dev);
313 }
314 }
315 spin_unlock_irqrestore(&av1_domain->iommus_lock, flags);
316 }
317
av1_iommu_irq(int irq,void * dev_id)318 static irqreturn_t av1_iommu_irq(int irq, void *dev_id)
319 {
320 struct av1_iommu *iommu = dev_id;
321 u32 int_status;
322 dma_addr_t iova;
323 irqreturn_t ret = IRQ_NONE;
324 int i, err;
325
326 err = pm_runtime_get_if_in_use(iommu->dev);
327 if (!err || WARN_ON_ONCE(err < 0))
328 return ret;
329
330 if (WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)))
331 goto out;
332
333 for (i = 0; i < iommu->num_mmu; i++) {
334 int_status = av1_iommu_read(iommu->bases[i], AV1_MMU_STATUS_BASE);
335 if (int_status & AV1_MMU_IRQ_MASK) {
336 dev_err(iommu->dev, "unexpected int_status=%08x\n", int_status);
337 iova = av1_iommu_read(iommu->bases[i], AV1_MMU_PAGE_FAULT_ADDR);
338
339 if (iommu->domain)
340 report_iommu_fault(iommu->domain, iommu->dev, iova, int_status);
341 else
342 dev_err(iommu->dev,
343 "Page fault while iommu not attached to domain?\n");
344 }
345 av1_iommu_write(iommu->bases[i], AV1_MMU_STATUS_BASE, 0);
346 ret = IRQ_HANDLED;
347 }
348
349 clk_bulk_disable(iommu->num_clocks, iommu->clocks);
350
351 out:
352 pm_runtime_put(iommu->dev);
353 return ret;
354 }
355
av1_iommu_is_attach_deferred(struct iommu_domain * domain,struct device * dev)356 static bool av1_iommu_is_attach_deferred(struct iommu_domain *domain,
357 struct device *dev)
358 {
359 struct av1_iommudata *data = dev_iommu_priv_get(dev);
360
361 return data->defer_attach;
362 }
363
av1_iommu_domain_alloc(unsigned type)364 static struct iommu_domain *av1_iommu_domain_alloc(unsigned type)
365 {
366 struct av1_iommu_domain *av1_domain;
367
368 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
369 return NULL;
370
371 if (!dma_dev)
372 return NULL;
373
374 av1_domain = kzalloc(sizeof(*av1_domain), GFP_KERNEL);
375 if (!av1_domain)
376 return NULL;
377
378 if (type == IOMMU_DOMAIN_DMA &&
379 iommu_get_dma_cookie(&av1_domain->domain))
380 goto err_free_domain;
381
382 /*
383 * av132xx iommus use a 2 level pagetable.
384 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
385 * Allocate one 4 KiB page for each table.
386 */
387 av1_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
388 if (!av1_domain->dt)
389 goto err_put_cookie;
390
391 av1_domain->dt_dma = dma_map_single(dma_dev, av1_domain->dt,
392 SPAGE_SIZE, DMA_TO_DEVICE);
393 if (dma_mapping_error(dma_dev, av1_domain->dt_dma)) {
394 dev_err(dma_dev, "DMA map error for DT\n");
395 goto err_free_dt;
396 }
397
398 av1_domain->pta = (u64 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
399 if (!av1_domain->pta)
400 goto err_unmap_dt;
401
402 av1_domain->pta_dma = dma_map_single(dma_dev, av1_domain->pta,
403 SPAGE_SIZE, DMA_TO_DEVICE);
404 if (dma_mapping_error(dma_dev, av1_domain->pta_dma)) {
405 dev_err(dma_dev, "DMA map error for PTA\n");
406 goto err_free_pta;
407 }
408 av1_domain->pta[0] = av1_mk_pta(av1_domain->dt_dma);
409
410 av1_table_flush(av1_domain, av1_domain->pta_dma, 1024);
411 av1_table_flush(av1_domain, av1_domain->dt_dma, NUM_DT_ENTRIES);
412
413 spin_lock_init(&av1_domain->iommus_lock);
414 spin_lock_init(&av1_domain->dt_lock);
415 INIT_LIST_HEAD(&av1_domain->iommus);
416
417 av1_domain->domain.geometry.aperture_start = 0;
418 av1_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
419 av1_domain->domain.geometry.force_aperture = true;
420
421 return &av1_domain->domain;
422 err_free_pta:
423 free_page((unsigned long)av1_domain->pta);
424 err_unmap_dt:
425 dma_unmap_single(dma_dev, av1_domain->dt_dma,
426 SPAGE_SIZE, DMA_TO_DEVICE);
427 err_free_dt:
428 free_page((unsigned long)av1_domain->dt);
429 err_put_cookie:
430 if (type == IOMMU_DOMAIN_DMA)
431 iommu_put_dma_cookie(&av1_domain->domain);
432 err_free_domain:
433 kfree(av1_domain);
434
435 return NULL;
436 }
437
av1_iommu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)438 static phys_addr_t av1_iommu_iova_to_phys(struct iommu_domain *domain,
439 dma_addr_t iova)
440 {
441 struct av1_iommu_domain *av1_domain = to_av1_domain(domain);
442 unsigned long flags;
443 phys_addr_t pt_phys, phys = 0;
444 u32 dte, pte;
445 u32 *page_table;
446
447 spin_lock_irqsave(&av1_domain->dt_lock, flags);
448
449 dte = av1_domain->dt[av1_iova_dte_index(iova)];
450 if (!av1_dte_is_pt_valid(dte))
451 goto out;
452
453 pt_phys = av1_dte_pt_address(dte);
454 page_table = (u32 *)phys_to_virt(pt_phys);
455 pte = page_table[av1_iova_pte_index(iova)];
456 if (!av1_pte_is_page_valid(pte))
457 goto out;
458
459 phys = av1_pte_page_address(pte) + av1_iova_page_offset(iova);
460 out:
461 spin_unlock_irqrestore(&av1_domain->dt_lock, flags);
462
463 return phys;
464 }
465
av1_dte_get_page_table(struct av1_iommu_domain * av1_domain,dma_addr_t iova)466 static u32 *av1_dte_get_page_table(struct av1_iommu_domain *av1_domain, dma_addr_t iova)
467 {
468 u32 *page_table, *dte_addr;
469 u32 dte_index, dte;
470 phys_addr_t pt_phys;
471 dma_addr_t pt_dma;
472
473 assert_spin_locked(&av1_domain->dt_lock);
474
475 dte_index = av1_iova_dte_index(iova);
476 dte_addr = &av1_domain->dt[dte_index];
477 dte = *dte_addr;
478 if (av1_dte_is_pt_valid(dte))
479 goto done;
480
481 page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
482 if (!page_table)
483 return ERR_PTR(-ENOMEM);
484
485 pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE);
486 if (dma_mapping_error(dma_dev, pt_dma)) {
487 dev_err(dma_dev, "DMA mapping error while allocating page table\n");
488 free_page((unsigned long)page_table);
489 return ERR_PTR(-ENOMEM);
490 }
491
492 dte = av1_mk_dte(pt_dma);
493 *dte_addr = dte;
494
495 av1_table_flush(av1_domain, pt_dma, NUM_PT_ENTRIES);
496 av1_table_flush(av1_domain,
497 av1_domain->dt_dma + dte_index * sizeof(u32), 1);
498 done:
499 pt_phys = av1_dte_pt_address(dte);
500 return (u32 *)phys_to_virt(pt_phys);
501 }
502
av1_iommu_unmap_iova(struct av1_iommu_domain * av1_domain,u32 * pte_addr,dma_addr_t pte_dma,size_t size)503 static size_t av1_iommu_unmap_iova(struct av1_iommu_domain *av1_domain,
504 u32 *pte_addr, dma_addr_t pte_dma,
505 size_t size)
506 {
507 unsigned int pte_count;
508 unsigned int pte_total = size / SPAGE_SIZE;
509
510 assert_spin_locked(&av1_domain->dt_lock);
511
512 for (pte_count = 0; pte_count < pte_total; pte_count++) {
513 u32 pte = pte_addr[pte_count];
514
515 if (!av1_pte_is_page_valid(pte))
516 break;
517
518 pte_addr[pte_count] = av1_mk_pte_invalid(pte);
519 }
520
521 av1_table_flush(av1_domain, pte_dma, pte_count);
522
523 return pte_count * SPAGE_SIZE;
524 }
525
av1_iommu_map_iova(struct av1_iommu_domain * av1_domain,u32 * pte_addr,dma_addr_t pte_dma,dma_addr_t iova,phys_addr_t paddr,size_t size,int prot)526 static int av1_iommu_map_iova(struct av1_iommu_domain *av1_domain, u32 *pte_addr,
527 dma_addr_t pte_dma, dma_addr_t iova,
528 phys_addr_t paddr, size_t size, int prot)
529 {
530 unsigned int pte_count;
531 unsigned int pte_total = size / SPAGE_SIZE;
532 phys_addr_t page_phys;
533
534 assert_spin_locked(&av1_domain->dt_lock);
535
536 for (pte_count = 0; pte_count < pte_total; pte_count++) {
537 u32 pte = pte_addr[pte_count];
538
539 if (av1_pte_is_page_valid(pte))
540 goto unwind;
541
542 pte_addr[pte_count] = av1_mk_pte(paddr, prot);
543
544 paddr += SPAGE_SIZE;
545 }
546
547 av1_table_flush(av1_domain, pte_dma, pte_total);
548
549 return 0;
550 unwind:
551 /* Unmap the range of iovas that we just mapped */
552 av1_iommu_unmap_iova(av1_domain, pte_addr, pte_dma,
553 pte_count * SPAGE_SIZE);
554
555 iova += pte_count * SPAGE_SIZE;
556 page_phys = av1_pte_page_address(pte_addr[pte_count]);
557 pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n",
558 &iova, &page_phys, &paddr, prot);
559
560 return -EADDRINUSE;
561 }
562
av1_iommu_unmap(struct iommu_domain * domain,unsigned long _iova,size_t size,struct iommu_iotlb_gather * gather)563 static size_t av1_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
564 size_t size, struct iommu_iotlb_gather *gather)
565 {
566 struct av1_iommu_domain *av1_domain = to_av1_domain(domain);
567 unsigned long flags;
568 dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
569 phys_addr_t pt_phys;
570 u32 dte;
571 u32 *pte_addr;
572 size_t unmap_size;
573
574 spin_lock_irqsave(&av1_domain->dt_lock, flags);
575
576 dte = av1_domain->dt[av1_iova_dte_index(iova)];
577 /* Just return 0 if iova is unmapped */
578 if (!av1_dte_is_pt_valid(dte)) {
579 spin_unlock_irqrestore(&av1_domain->dt_lock, flags);
580 return 0;
581 }
582
583 pt_phys = av1_dte_pt_address(dte);
584 pte_addr = (u32 *)phys_to_virt(pt_phys) + av1_iova_pte_index(iova);
585 pte_dma = pt_phys + av1_iova_pte_index(iova) * sizeof(u32);
586 unmap_size = av1_iommu_unmap_iova(av1_domain, pte_addr, pte_dma, size);
587
588 spin_unlock_irqrestore(&av1_domain->dt_lock, flags);
589
590 return unmap_size;
591 }
592
av1_iommu_map(struct iommu_domain * domain,unsigned long _iova,phys_addr_t paddr,size_t size,int prot,gfp_t gfp)593 static int av1_iommu_map(struct iommu_domain *domain, unsigned long _iova,
594 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
595 {
596 struct av1_iommu_domain *av1_domain = to_av1_domain(domain);
597 unsigned long flags;
598 dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
599 u32 *page_table, *pte_addr;
600 u32 dte, pte_index;
601 int ret;
602
603 spin_lock_irqsave(&av1_domain->dt_lock, flags);
604
605 page_table = av1_dte_get_page_table(av1_domain, iova);
606 if (IS_ERR(page_table)) {
607 spin_unlock_irqrestore(&av1_domain->dt_lock, flags);
608 return PTR_ERR(page_table);
609 }
610
611 dte = av1_domain->dt[av1_iova_dte_index(iova)];
612 pte_index = av1_iova_pte_index(iova);
613 pte_addr = &page_table[pte_index];
614 pte_dma = av1_dte_pt_address(dte) + pte_index * sizeof(u32);
615 ret = av1_iommu_map_iova(av1_domain, pte_addr, pte_dma, iova,
616 paddr, size, prot);
617
618 spin_unlock_irqrestore(&av1_domain->dt_lock, flags);
619
620 return ret;
621 }
622
av1_iommu_detach_device(struct iommu_domain * domain,struct device * dev)623 static void av1_iommu_detach_device(struct iommu_domain *domain,
624 struct device *dev)
625 {
626 struct av1_iommu *iommu;
627 struct av1_iommu_domain *av1_domain = to_av1_domain(domain);
628 unsigned long flags;
629 int ret;
630
631 /* Allow 'virtual devices' (eg drm) to detach from domain */
632 iommu = av1_iommu_from_dev(dev);
633 if (WARN_ON(!iommu))
634 return;
635
636 dev_dbg(dev, "Detaching from iommu domain\n");
637
638 if (!iommu->domain)
639 return;
640
641 spin_lock_irqsave(&av1_domain->iommus_lock, flags);
642 list_del_init(&iommu->node);
643 spin_unlock_irqrestore(&av1_domain->iommus_lock, flags);
644
645 ret = pm_runtime_get_if_in_use(iommu->dev);
646 WARN_ON_ONCE(ret < 0);
647 if (ret > 0) {
648 av1_iommu_disable(iommu);
649 pm_runtime_put(iommu->dev);
650 }
651 iommu->domain = NULL;
652 }
653
av1_iommu_attach_device(struct iommu_domain * domain,struct device * dev)654 static int av1_iommu_attach_device(struct iommu_domain *domain,
655 struct device *dev)
656 {
657 struct av1_iommu *iommu;
658 struct av1_iommu_domain *av1_domain = to_av1_domain(domain);
659 unsigned long flags;
660 int ret;
661
662 iommu = av1_iommu_from_dev(dev);
663 if (WARN_ON(!iommu))
664 return -ENODEV;
665
666 if (iommu->domain)
667 av1_iommu_detach_device(iommu->domain, dev);
668
669 iommu->domain = domain;
670
671 /* Attach NULL for disable iommu */
672 if (!domain)
673 return 0;
674
675 spin_lock_irqsave(&av1_domain->iommus_lock, flags);
676 list_add_tail(&iommu->node, &av1_domain->iommus);
677 spin_unlock_irqrestore(&av1_domain->iommus_lock, flags);
678
679 ret = pm_runtime_get_if_in_use(iommu->dev);
680 if (!ret || WARN_ON_ONCE(ret < 0))
681 return 0;
682
683 ret = av1_iommu_enable(iommu);
684 if (ret)
685 av1_iommu_detach_device(iommu->domain, dev);
686
687 pm_runtime_put(iommu->dev);
688
689 return ret;
690 }
691
av1_iommu_domain_free(struct iommu_domain * domain)692 static void av1_iommu_domain_free(struct iommu_domain *domain)
693 {
694 struct av1_iommu_domain *av1_domain = to_av1_domain(domain);
695 int i;
696
697 WARN_ON(!list_empty(&av1_domain->iommus));
698
699 for (i = 0; i < NUM_DT_ENTRIES; i++) {
700 u32 dte = av1_domain->dt[i];
701
702 if (av1_dte_is_pt_valid(dte)) {
703 phys_addr_t pt_phys = av1_dte_pt_address(dte);
704 u32 *page_table = phys_to_virt(pt_phys);
705
706 dma_unmap_single(dma_dev, pt_phys,
707 SPAGE_SIZE, DMA_TO_DEVICE);
708 free_page((unsigned long)page_table);
709 }
710 }
711
712 dma_unmap_single(dma_dev, av1_domain->dt_dma,
713 SPAGE_SIZE, DMA_TO_DEVICE);
714 free_page((unsigned long)av1_domain->dt);
715
716 dma_unmap_single(dma_dev, av1_domain->pta_dma,
717 SPAGE_SIZE, DMA_TO_DEVICE);
718 free_page((unsigned long)av1_domain->pta);
719
720 if (domain->type == IOMMU_DOMAIN_DMA)
721 iommu_put_dma_cookie(&av1_domain->domain);
722 kfree(av1_domain);
723 }
724
av1_iommu_probe_device(struct device * dev)725 static struct iommu_device *av1_iommu_probe_device(struct device *dev)
726 {
727 struct av1_iommudata *data;
728 struct av1_iommu *iommu;
729
730 data = dev_iommu_priv_get(dev);
731 if (!data)
732 return ERR_PTR(-ENODEV);
733
734 iommu = av1_iommu_from_dev(dev);
735
736 pr_info("%s,%d, consumer : %s, supplier : %s\n",
737 __func__, __LINE__, dev_name(dev), dev_name(iommu->dev));
738
739 /*
740 * link will free by platform_device_del(master) via
741 * BUS_NOTIFY_REMOVED_DEVICE
742 */
743 data->link = device_link_add(dev, iommu->dev,
744 DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
745
746 /* set max segment size for dev, needed for single chunk map */
747 if (!dev->dma_parms)
748 dev->dma_parms = kzalloc(sizeof(*dev->dma_parms), GFP_KERNEL);
749 if (!dev->dma_parms)
750 return ERR_PTR(-ENOMEM);
751
752 dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
753
754 return &iommu->iommu;
755 }
756
av1_iommu_release_device(struct device * dev)757 static void av1_iommu_release_device(struct device *dev)
758 {
759 const struct iommu_ops *ops = dev->bus->iommu_ops;
760
761 /* hack for rmmod */
762 __module_get(ops->owner);
763 }
764
av1_iommu_device_group(struct device * dev)765 static struct iommu_group *av1_iommu_device_group(struct device *dev)
766 {
767 struct av1_iommu *iommu;
768
769 iommu = av1_iommu_from_dev(dev);
770
771 return iommu_group_ref_get(iommu->group);
772 }
773
av1_iommu_of_xlate(struct device * dev,struct of_phandle_args * args)774 static int av1_iommu_of_xlate(struct device *dev,
775 struct of_phandle_args *args)
776 {
777 struct platform_device *iommu_dev;
778 struct av1_iommudata *data;
779
780 data = devm_kzalloc(dma_dev, sizeof(*data), GFP_KERNEL);
781 if (!data)
782 return -ENOMEM;
783
784 dev_info(dev, "%s,%d\n", __func__, __LINE__);
785 iommu_dev = of_find_device_by_node(args->np);
786
787 data->iommu = platform_get_drvdata(iommu_dev);
788
789 dev_iommu_priv_set(dev, data);
790
791 platform_device_put(iommu_dev);
792
793 return 0;
794 }
795
av1_iommu_probe_finalize(struct device * dev)796 static void av1_iommu_probe_finalize(struct device *dev)
797 {
798 const struct iommu_ops *ops = dev->bus->iommu_ops;
799
800 /* hack for rmmod */
801 module_put(ops->owner);
802 }
803
804 static struct iommu_ops av1_iommu_ops = {
805 .domain_alloc = av1_iommu_domain_alloc,
806 .domain_free = av1_iommu_domain_free,
807 .attach_dev = av1_iommu_attach_device,
808 .detach_dev = av1_iommu_detach_device,
809 .map = av1_iommu_map,
810 .unmap = av1_iommu_unmap,
811 .flush_iotlb_all = av1_iommu_flush_tlb_all,
812 .probe_device = av1_iommu_probe_device,
813 .release_device = av1_iommu_release_device,
814 .iova_to_phys = av1_iommu_iova_to_phys,
815 .is_attach_deferred = av1_iommu_is_attach_deferred,
816 .device_group = av1_iommu_device_group,
817 .pgsize_bitmap = AV1_IOMMU_PGSIZE_BITMAP,
818 .of_xlate = av1_iommu_of_xlate,
819 .probe_finalize = av1_iommu_probe_finalize,
820 };
821
822 static const struct of_device_id av1_iommu_dt_ids[] = {
823 {
824 .compatible = "rockchip,iommu-av1",
825 },
826 { /* sentinel */ }
827 };
828
av1_iommu_probe(struct platform_device * pdev)829 static int av1_iommu_probe(struct platform_device *pdev)
830 {
831 struct device *dev = &pdev->dev;
832 struct av1_iommu *iommu;
833 struct resource *res;
834 int num_res = pdev->num_resources;
835 int err, i;
836 const struct of_device_id *match;
837
838 iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
839 if (!iommu)
840 return -ENOMEM;
841
842 match = of_match_device(av1_iommu_dt_ids, dev);
843 if (!match)
844 return -EINVAL;
845
846 platform_set_drvdata(pdev, iommu);
847 iommu->dev = dev;
848 iommu->num_mmu = 0;
849
850 iommu->bases = devm_kcalloc(dev, num_res, sizeof(*iommu->bases),
851 GFP_KERNEL);
852 if (!iommu->bases)
853 return -ENOMEM;
854
855 for (i = 0; i < num_res; i++) {
856 res = platform_get_resource(pdev, IORESOURCE_MEM, i);
857 if (!res)
858 continue;
859 iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res);
860 if (IS_ERR(iommu->bases[i]))
861 continue;
862 iommu->num_mmu++;
863 }
864 if (iommu->num_mmu == 0)
865 return PTR_ERR(iommu->bases[0]);
866
867 iommu->num_irq = platform_irq_count(pdev);
868 if (iommu->num_irq < 0)
869 return iommu->num_irq;
870
871 err = devm_clk_bulk_get_all(dev, &iommu->clocks);
872 if (err >= 0)
873 iommu->num_clocks = err;
874 else if (err == -ENOENT)
875 iommu->num_clocks = 0;
876 else
877 return err;
878
879 err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);
880 if (err)
881 return err;
882
883 iommu->group = iommu_group_alloc();
884 if (IS_ERR(iommu->group)) {
885 err = PTR_ERR(iommu->group);
886 goto err_unprepare_clocks;
887 }
888
889 err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
890 if (err)
891 goto err_put_group;
892
893 iommu_device_set_ops(&iommu->iommu, &av1_iommu_ops);
894 iommu_device_set_fwnode(&iommu->iommu, &dev->of_node->fwnode);
895
896 err = iommu_device_register(&iommu->iommu);
897 if (err)
898 goto err_remove_sysfs;
899
900 if (!dma_dev)
901 dma_dev = &pdev->dev;
902
903 bus_set_iommu(&av1dec_bus, &av1_iommu_ops);
904
905 pm_runtime_enable(dev);
906
907 for (i = 0; i < iommu->num_irq; i++) {
908 int irq = platform_get_irq(pdev, i);
909
910 if (irq < 0) {
911 err = -ENODEV;
912 goto err_diable_runtime;
913 }
914
915 err = devm_request_irq(iommu->dev, irq, av1_iommu_irq,
916 IRQF_SHARED, dev_name(dev), iommu);
917 if (err)
918 goto err_diable_runtime;
919
920 }
921
922 return 0;
923 err_diable_runtime:
924 pm_runtime_disable(dev);
925 iommu_device_unregister(&iommu->iommu);
926 err_remove_sysfs:
927 iommu_device_sysfs_remove(&iommu->iommu);
928 err_put_group:
929 iommu_group_put(iommu->group);
930 err_unprepare_clocks:
931 clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);
932 return err;
933 }
934
av1_iommu_remove(struct platform_device * pdev)935 static int av1_iommu_remove(struct platform_device *pdev)
936 {
937 struct device *dev = &pdev->dev;
938 struct av1_iommu *iommu = platform_get_drvdata(pdev);
939
940 iommu_device_unregister(&iommu->iommu);
941 iommu_device_sysfs_remove(&iommu->iommu);
942 pm_runtime_disable(dev);
943 return 0;
944 }
945
av1_iommu_shutdown(struct platform_device * pdev)946 static void av1_iommu_shutdown(struct platform_device *pdev)
947 {
948 struct av1_iommu *iommu = platform_get_drvdata(pdev);
949 int i;
950
951 for (i = 0; i < iommu->num_irq; i++) {
952 int irq = platform_get_irq(pdev, i);
953
954 devm_free_irq(iommu->dev, irq, iommu);
955 }
956
957 pm_runtime_force_suspend(&pdev->dev);
958 }
959
av1_iommu_suspend(struct device * dev)960 static int __maybe_unused av1_iommu_suspend(struct device *dev)
961 {
962 struct av1_iommu *iommu = dev_get_drvdata(dev);
963
964 if (!iommu->domain)
965 return 0;
966
967 av1_iommu_disable(iommu);
968 return 0;
969 }
970
av1_iommu_resume(struct device * dev)971 static int __maybe_unused av1_iommu_resume(struct device *dev)
972 {
973 struct av1_iommu *iommu = dev_get_drvdata(dev);
974
975 if (!iommu->domain)
976 return 0;
977
978 return av1_iommu_enable(iommu);
979 }
980
981 static const struct dev_pm_ops av1_iommu_pm_ops = {
982 SET_RUNTIME_PM_OPS(av1_iommu_suspend, av1_iommu_resume, NULL)
983 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
984 pm_runtime_force_resume)
985 };
986
987 struct platform_driver rockchip_av1_iommu_driver = {
988 .probe = av1_iommu_probe,
989 .remove = av1_iommu_remove,
990 .shutdown = av1_iommu_shutdown,
991 .driver = {
992 .name = "av1_iommu",
993 .of_match_table = av1_iommu_dt_ids,
994 .pm = &av1_iommu_pm_ops,
995 .suppress_bind_attrs = true,
996 },
997 };
998