1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) Rockchip Electronics Co., Ltd.
4 *
5 * Author: Huang Lee <Putin.li@rock-chips.com>
6 */
7
8 #define pr_fmt(fmt) "rga_iommu: " fmt
9
10 #include "rga_iommu.h"
11 #include "rga_dma_buf.h"
12 #include "rga_mm.h"
13 #include "rga_job.h"
14 #include "rga_common.h"
15 #include "rga_hw_config.h"
16
rga_user_memory_check(struct page ** pages,u32 w,u32 h,u32 format,int flag)17 int rga_user_memory_check(struct page **pages, u32 w, u32 h, u32 format, int flag)
18 {
19 int bits;
20 void *vaddr = NULL;
21 int taipage_num;
22 int taidata_num;
23 int *tai_vaddr = NULL;
24
25 bits = rga_get_format_bits(format);
26 if (bits < 0)
27 return -1;
28
29 taipage_num = w * h * bits / 8 / (1024 * 4);
30 taidata_num = w * h * bits / 8 % (1024 * 4);
31 if (taidata_num == 0) {
32 vaddr = kmap(pages[taipage_num - 1]);
33 tai_vaddr = (int *)vaddr + 1023;
34 } else {
35 vaddr = kmap(pages[taipage_num]);
36 tai_vaddr = (int *)vaddr + taidata_num / 4 - 1;
37 }
38
39 if (flag == 1) {
40 pr_info("src user memory check\n");
41 pr_info("tai data is %d\n", *tai_vaddr);
42 } else {
43 pr_info("dst user memory check\n");
44 pr_info("tai data is %d\n", *tai_vaddr);
45 }
46
47 if (taidata_num == 0)
48 kunmap(pages[taipage_num - 1]);
49 else
50 kunmap(pages[taipage_num]);
51
52 return 0;
53 }
54
rga_set_mmu_base(struct rga_job * job,struct rga2_req * req)55 int rga_set_mmu_base(struct rga_job *job, struct rga2_req *req)
56 {
57 if (job->src_buffer.page_table) {
58 rga_dma_sync_flush_range(job->src_buffer.page_table,
59 (job->src_buffer.page_table +
60 job->src_buffer.page_count),
61 job->scheduler);
62 req->mmu_info.src0_base_addr = virt_to_phys(job->src_buffer.page_table);
63 }
64
65 if (job->src1_buffer.page_table) {
66 rga_dma_sync_flush_range(job->src1_buffer.page_table,
67 (job->src1_buffer.page_table +
68 job->src1_buffer.page_count),
69 job->scheduler);
70 req->mmu_info.src1_base_addr = virt_to_phys(job->src1_buffer.page_table);
71 }
72
73 if (job->dst_buffer.page_table) {
74 rga_dma_sync_flush_range(job->dst_buffer.page_table,
75 (job->dst_buffer.page_table +
76 job->dst_buffer.page_count),
77 job->scheduler);
78 req->mmu_info.dst_base_addr = virt_to_phys(job->dst_buffer.page_table);
79
80 if (((req->alpha_rop_flag & 1) == 1) && (req->bitblt_mode == 0)) {
81 req->mmu_info.src1_base_addr = req->mmu_info.dst_base_addr;
82 req->mmu_info.src1_mmu_flag = req->mmu_info.dst_mmu_flag;
83 }
84 }
85
86 if (job->els_buffer.page_table) {
87 rga_dma_sync_flush_range(job->els_buffer.page_table,
88 (job->els_buffer.page_table +
89 job->els_buffer.page_count),
90 job->scheduler);
91 req->mmu_info.els_base_addr = virt_to_phys(job->els_buffer.page_table);
92 }
93
94 return 0;
95 }
96
rga_mmu_buf_get_try(struct rga_mmu_base * t,uint32_t size)97 static int rga_mmu_buf_get_try(struct rga_mmu_base *t, uint32_t size)
98 {
99 int ret = 0;
100
101 if ((t->back - t->front) > t->size) {
102 if (t->front + size > t->back - t->size) {
103 pr_info("front %d, back %d dsize %d size %d",
104 t->front, t->back, t->size, size);
105 ret = -ENOMEM;
106 goto out;
107 }
108 } else {
109 if ((t->front + size) > t->back) {
110 pr_info("front %d, back %d dsize %d size %d",
111 t->front, t->back, t->size, size);
112 ret = -ENOMEM;
113 goto out;
114 }
115
116 if (t->front + size > t->size) {
117 if (size > (t->back - t->size)) {
118 pr_info("front %d, back %d dsize %d size %d",
119 t->front, t->back, t->size, size);
120 ret = -ENOMEM;
121 goto out;
122 }
123 t->front = 0;
124 }
125 }
126 out:
127 return ret;
128 }
129
rga_mmu_buf_get(struct rga_mmu_base * mmu_base,uint32_t size)130 unsigned int *rga_mmu_buf_get(struct rga_mmu_base *mmu_base, uint32_t size)
131 {
132 int ret;
133 unsigned int *buf = NULL;
134
135 WARN_ON(!mutex_is_locked(&rga_drvdata->lock));
136
137 size = ALIGN(size, 16);
138
139 ret = rga_mmu_buf_get_try(mmu_base, size);
140 if (ret < 0) {
141 pr_err("Get MMU mem failed\n");
142 return NULL;
143 }
144
145 buf = mmu_base->buf_virtual + mmu_base->front;
146
147 mmu_base->front += size;
148
149 if (mmu_base->back + size > 2 * mmu_base->size)
150 mmu_base->back = size + mmu_base->size;
151 else
152 mmu_base->back += size;
153
154 return buf;
155 }
156
rga_mmu_base_init(size_t size)157 struct rga_mmu_base *rga_mmu_base_init(size_t size)
158 {
159 int order = 0;
160 struct rga_mmu_base *mmu_base;
161
162 mmu_base = kzalloc(sizeof(*mmu_base), GFP_KERNEL);
163 if (mmu_base == NULL) {
164 pr_err("Cannot alloc mmu_base!\n");
165 return ERR_PTR(-ENOMEM);
166 }
167
168 /*
169 * malloc pre scale mid buf mmu table:
170 * size * channel_num * address_size
171 */
172 order = get_order(size * 3 * sizeof(*mmu_base->buf_virtual));
173 if (order >= MAX_ORDER) {
174 pr_err("Can not alloc pages with order[%d] for mmu_page_table, max_order = %d\n",
175 order, MAX_ORDER);
176 goto err_free_mmu_base;
177 }
178
179 mmu_base->buf_virtual = (uint32_t *) __get_free_pages(GFP_KERNEL | GFP_DMA32, order);
180 if (mmu_base->buf_virtual == NULL) {
181 pr_err("Can not alloc pages for mmu_page_table\n");
182 goto err_free_mmu_base;
183 }
184 mmu_base->buf_order = order;
185
186 order = get_order(size * sizeof(*mmu_base->pages));
187 if (order >= MAX_ORDER) {
188 pr_err("Can not alloc pages with order[%d] for mmu_base->pages, max_order = %d\n",
189 order, MAX_ORDER);
190 goto err_free_buf_virtual;
191 }
192
193 mmu_base->pages = (struct page **)__get_free_pages(GFP_KERNEL | GFP_DMA32, order);
194 if (mmu_base->pages == NULL) {
195 pr_err("Can not alloc pages for mmu_base->pages\n");
196 goto err_free_buf_virtual;
197 }
198 mmu_base->pages_order = order;
199
200 mmu_base->front = 0;
201 mmu_base->back = RGA2_PHY_PAGE_SIZE * 3;
202 mmu_base->size = RGA2_PHY_PAGE_SIZE * 3;
203
204 return mmu_base;
205
206 err_free_buf_virtual:
207 free_pages((unsigned long)mmu_base->buf_virtual, mmu_base->buf_order);
208 mmu_base->buf_order = 0;
209
210 err_free_mmu_base:
211 kfree(mmu_base);
212
213 return ERR_PTR(-ENOMEM);
214 }
215
rga_mmu_base_free(struct rga_mmu_base ** mmu_base)216 void rga_mmu_base_free(struct rga_mmu_base **mmu_base)
217 {
218 struct rga_mmu_base *base = *mmu_base;
219
220 if (base->buf_virtual != NULL) {
221 free_pages((unsigned long)base->buf_virtual, base->buf_order);
222 base->buf_virtual = NULL;
223 base->buf_order = 0;
224 }
225
226 if (base->pages != NULL) {
227 free_pages((unsigned long)base->pages, base->pages_order);
228 base->pages = NULL;
229 base->pages_order = 0;
230 }
231
232 kfree(base);
233 *mmu_base = NULL;
234 }
235
rga_iommu_intr_fault_handler(struct iommu_domain * iommu,struct device * iommu_dev,unsigned long iova,int status,void * arg)236 static int rga_iommu_intr_fault_handler(struct iommu_domain *iommu, struct device *iommu_dev,
237 unsigned long iova, int status, void *arg)
238 {
239 struct rga_scheduler_t *scheduler = (struct rga_scheduler_t *)arg;
240 struct rga_job *job = scheduler->running_job;
241
242 if (job == NULL)
243 return 0;
244
245 pr_err("IOMMU intr fault, IOVA[0x%lx], STATUS[0x%x]\n", iova, status);
246 if (scheduler->ops->irq)
247 scheduler->ops->irq(scheduler);
248
249 /* iommu interrupts on rga2 do not affect rga2 itself. */
250 if (!test_bit(RGA_JOB_STATE_INTR_ERR, &job->state)) {
251 set_bit(RGA_JOB_STATE_INTR_ERR, &job->state);
252 scheduler->ops->soft_reset(scheduler);
253 }
254
255 if (status & RGA_IOMMU_IRQ_PAGE_FAULT) {
256 pr_err("RGA IOMMU: page fault! Please check the memory size.\n");
257 job->ret = -EACCES;
258 } else if (status & RGA_IOMMU_IRQ_BUS_ERROR) {
259 pr_err("RGA IOMMU: bus error! Please check if the memory is invalid or has been freed.\n");
260 job->ret = -EACCES;
261 } else {
262 pr_err("RGA IOMMU: Wrong IOMMU interrupt signal!\n");
263 }
264
265 return 0;
266 }
267
rga_iommu_detach(struct rga_iommu_info * info)268 int rga_iommu_detach(struct rga_iommu_info *info)
269 {
270 if (!info)
271 return 0;
272
273 iommu_detach_group(info->domain, info->group);
274 return 0;
275 }
276
rga_iommu_attach(struct rga_iommu_info * info)277 int rga_iommu_attach(struct rga_iommu_info *info)
278 {
279 if (!info)
280 return 0;
281
282 return iommu_attach_group(info->domain, info->group);
283 }
284
rga_iommu_probe(struct device * dev)285 struct rga_iommu_info *rga_iommu_probe(struct device *dev)
286 {
287 int ret = 0;
288 struct rga_iommu_info *info = NULL;
289 struct iommu_domain *domain = NULL;
290 struct iommu_group *group = NULL;
291
292 group = iommu_group_get(dev);
293 if (!group)
294 return ERR_PTR(-EINVAL);
295
296 domain = iommu_get_domain_for_dev(dev);
297 if (!domain) {
298 ret = -EINVAL;
299 goto err_put_group;
300 }
301
302 info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
303 if (!info) {
304 ret = -ENOMEM;
305 goto err_put_group;
306 }
307
308 info->dev = dev;
309 info->default_dev = info->dev;
310 info->group = group;
311 info->domain = domain;
312
313 return info;
314
315 err_put_group:
316 if (group)
317 iommu_group_put(group);
318
319 return ERR_PTR(ret);
320 }
321
rga_iommu_remove(struct rga_iommu_info * info)322 int rga_iommu_remove(struct rga_iommu_info *info)
323 {
324 if (!info)
325 return 0;
326
327 iommu_group_put(info->group);
328
329 return 0;
330 }
331
rga_iommu_bind(void)332 int rga_iommu_bind(void)
333 {
334 int i;
335 int ret;
336 struct rga_scheduler_t *scheduler = NULL;
337 struct rga_iommu_info *main_iommu = NULL;
338 int main_iommu_index = -1;
339 int main_mmu_index = -1;
340 int another_index = -1;
341
342 for (i = 0; i < rga_drvdata->num_of_scheduler; i++) {
343 scheduler = rga_drvdata->scheduler[i];
344
345 switch (scheduler->data->mmu) {
346 case RGA_IOMMU:
347 if (scheduler->iommu_info == NULL)
348 continue;
349
350 if (main_iommu == NULL) {
351 main_iommu = scheduler->iommu_info;
352 main_iommu_index = i;
353 iommu_set_fault_handler(main_iommu->domain,
354 rga_iommu_intr_fault_handler,
355 (void *)scheduler);
356 } else {
357 scheduler->iommu_info->domain = main_iommu->domain;
358 scheduler->iommu_info->default_dev = main_iommu->default_dev;
359 rga_iommu_attach(scheduler->iommu_info);
360 }
361
362 break;
363
364 case RGA_MMU:
365 if (rga_drvdata->mmu_base != NULL)
366 continue;
367
368 rga_drvdata->mmu_base = rga_mmu_base_init(RGA2_PHY_PAGE_SIZE);
369 if (IS_ERR(rga_drvdata->mmu_base)) {
370 dev_err(scheduler->dev, "rga mmu base init failed!\n");
371 ret = PTR_ERR(rga_drvdata->mmu_base);
372 rga_drvdata->mmu_base = NULL;
373
374 return ret;
375 }
376
377 main_mmu_index = i;
378
379 break;
380 default:
381 if (another_index != RGA_NONE_CORE)
382 another_index = i;
383
384 break;
385 }
386 }
387
388 /*
389 * priority order: iommu > mmu > another
390 * The scheduler core with IOMMU will be used preferentially as the
391 * default memory-mapped core. This ensures that all cores can obtain
392 * the required memory data when they are equipped with different
393 * versions of cores.
394 */
395 if (main_iommu_index >= 0) {
396 rga_drvdata->map_scheduler_index = main_iommu_index;
397 } else if (main_mmu_index >= 0) {
398 rga_drvdata->map_scheduler_index = main_mmu_index;
399 } else if (another_index >= 0) {
400 rga_drvdata->map_scheduler_index = another_index;
401 } else {
402 rga_drvdata->map_scheduler_index = -1;
403 pr_err("%s, binding map scheduler failed!\n", __func__);
404 return -EFAULT;
405 }
406
407 pr_info("IOMMU binding successfully, default mapping core[0x%x]\n",
408 rga_drvdata->scheduler[rga_drvdata->map_scheduler_index]->core);
409
410 return 0;
411 }
412
rga_iommu_unbind(void)413 void rga_iommu_unbind(void)
414 {
415 int i;
416
417 for (i = 0; i < rga_drvdata->num_of_scheduler; i++)
418 if (rga_drvdata->scheduler[i]->iommu_info != NULL)
419 rga_iommu_detach(rga_drvdata->scheduler[i]->iommu_info);
420
421 if (rga_drvdata->mmu_base)
422 rga_mmu_base_free(&rga_drvdata->mmu_base);
423
424 rga_drvdata->map_scheduler_index = -1;
425 }
426