1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) Rockchip Electronics Co., Ltd.
4 *
5 * Author: Huang Lee <Putin.li@rock-chips.com>
6 */
7
8 #define pr_fmt(fmt) "rga_dma_buf: " fmt
9
10 #include "rga_dma_buf.h"
11 #include "rga.h"
12 #include "rga_common.h"
13 #include "rga_job.h"
14 #include "rga_debugger.h"
15
rga_dma_info_to_prot(enum dma_data_direction dir)16 static int rga_dma_info_to_prot(enum dma_data_direction dir)
17 {
18 switch (dir) {
19 case DMA_BIDIRECTIONAL:
20 return IOMMU_READ | IOMMU_WRITE;
21 case DMA_TO_DEVICE:
22 return IOMMU_READ;
23 case DMA_FROM_DEVICE:
24 return IOMMU_WRITE;
25 default:
26 return 0;
27 }
28 }
29
rga_buf_size_cal(unsigned long yrgb_addr,unsigned long uv_addr,unsigned long v_addr,int format,uint32_t w,uint32_t h,unsigned long * StartAddr,unsigned long * size)30 int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr,
31 unsigned long v_addr, int format, uint32_t w,
32 uint32_t h, unsigned long *StartAddr, unsigned long *size)
33 {
34 uint32_t size_yrgb = 0;
35 uint32_t size_uv = 0;
36 uint32_t size_v = 0;
37 uint32_t stride = 0;
38 unsigned long start, end;
39 uint32_t pageCount;
40
41 switch (format) {
42 case RGA_FORMAT_RGBA_8888:
43 case RGA_FORMAT_RGBX_8888:
44 case RGA_FORMAT_BGRA_8888:
45 case RGA_FORMAT_BGRX_8888:
46 case RGA_FORMAT_ARGB_8888:
47 case RGA_FORMAT_XRGB_8888:
48 case RGA_FORMAT_ABGR_8888:
49 case RGA_FORMAT_XBGR_8888:
50 stride = (w * 4 + 3) & (~3);
51 size_yrgb = stride * h;
52 start = yrgb_addr >> PAGE_SHIFT;
53 end = yrgb_addr + size_yrgb;
54 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
55 pageCount = end - start;
56 break;
57 case RGA_FORMAT_RGB_888:
58 case RGA_FORMAT_BGR_888:
59 stride = (w * 3 + 3) & (~3);
60 size_yrgb = stride * h;
61 start = yrgb_addr >> PAGE_SHIFT;
62 end = yrgb_addr + size_yrgb;
63 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
64 pageCount = end - start;
65 break;
66 case RGA_FORMAT_RGB_565:
67 case RGA_FORMAT_RGBA_5551:
68 case RGA_FORMAT_RGBA_4444:
69 case RGA_FORMAT_BGR_565:
70 case RGA_FORMAT_BGRA_5551:
71 case RGA_FORMAT_BGRA_4444:
72 case RGA_FORMAT_ARGB_5551:
73 case RGA_FORMAT_ARGB_4444:
74 case RGA_FORMAT_ABGR_5551:
75 case RGA_FORMAT_ABGR_4444:
76 stride = (w * 2 + 3) & (~3);
77 size_yrgb = stride * h;
78 start = yrgb_addr >> PAGE_SHIFT;
79 end = yrgb_addr + size_yrgb;
80 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
81 pageCount = end - start;
82 break;
83
84 /* YUV FORMAT */
85 case RGA_FORMAT_YCbCr_422_SP:
86 case RGA_FORMAT_YCrCb_422_SP:
87 stride = (w + 3) & (~3);
88 size_yrgb = stride * h;
89 size_uv = stride * h;
90 start = min(yrgb_addr, uv_addr);
91 start >>= PAGE_SHIFT;
92 end = max((yrgb_addr + size_yrgb), (uv_addr + size_uv));
93 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
94 pageCount = end - start;
95 break;
96 case RGA_FORMAT_YCbCr_422_P:
97 case RGA_FORMAT_YCrCb_422_P:
98 stride = (w + 3) & (~3);
99 size_yrgb = stride * h;
100 size_uv = ((stride >> 1) * h);
101 size_v = ((stride >> 1) * h);
102 start = min3(yrgb_addr, uv_addr, v_addr);
103 start = start >> PAGE_SHIFT;
104 end =
105 max3((yrgb_addr + size_yrgb), (uv_addr + size_uv),
106 (v_addr + size_v));
107 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
108 pageCount = end - start;
109 break;
110 case RGA_FORMAT_YCbCr_420_SP:
111 case RGA_FORMAT_YCrCb_420_SP:
112 stride = (w + 3) & (~3);
113 size_yrgb = stride * h;
114 size_uv = (stride * (h >> 1));
115 start = min(yrgb_addr, uv_addr);
116 start >>= PAGE_SHIFT;
117 end = max((yrgb_addr + size_yrgb), (uv_addr + size_uv));
118 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
119 pageCount = end - start;
120 break;
121 case RGA_FORMAT_YCbCr_420_P:
122 case RGA_FORMAT_YCrCb_420_P:
123 stride = (w + 3) & (~3);
124 size_yrgb = stride * h;
125 size_uv = ((stride >> 1) * (h >> 1));
126 size_v = ((stride >> 1) * (h >> 1));
127 start = min3(yrgb_addr, uv_addr, v_addr);
128 start >>= PAGE_SHIFT;
129 end =
130 max3((yrgb_addr + size_yrgb), (uv_addr + size_uv),
131 (v_addr + size_v));
132 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
133 pageCount = end - start;
134 break;
135 case RGA_FORMAT_YCbCr_400:
136 stride = (w + 3) & (~3);
137 size_yrgb = stride * h;
138 start = yrgb_addr >> PAGE_SHIFT;
139 end = yrgb_addr + size_yrgb;
140 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
141 pageCount = end - start;
142 break;
143 case RGA_FORMAT_Y4:
144 stride = ((w + 3) & (~3)) >> 1;
145 size_yrgb = stride * h;
146 start = yrgb_addr >> PAGE_SHIFT;
147 end = yrgb_addr + size_yrgb;
148 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
149 pageCount = end - start;
150 break;
151 case RGA_FORMAT_YVYU_422:
152 case RGA_FORMAT_VYUY_422:
153 case RGA_FORMAT_YUYV_422:
154 case RGA_FORMAT_UYVY_422:
155 stride = (w + 3) & (~3);
156 size_yrgb = stride * h;
157 size_uv = stride * h;
158 start = min(yrgb_addr, uv_addr);
159 start >>= PAGE_SHIFT;
160 end = max((yrgb_addr + size_yrgb), (uv_addr + size_uv));
161 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
162 pageCount = end - start;
163 break;
164 case RGA_FORMAT_YVYU_420:
165 case RGA_FORMAT_VYUY_420:
166 case RGA_FORMAT_YUYV_420:
167 case RGA_FORMAT_UYVY_420:
168 stride = (w + 3) & (~3);
169 size_yrgb = stride * h;
170 size_uv = (stride * (h >> 1));
171 start = min(yrgb_addr, uv_addr);
172 start >>= PAGE_SHIFT;
173 end = max((yrgb_addr + size_yrgb), (uv_addr + size_uv));
174 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
175 pageCount = end - start;
176 break;
177 case RGA_FORMAT_YCbCr_420_SP_10B:
178 case RGA_FORMAT_YCrCb_420_SP_10B:
179 stride = (w + 3) & (~3);
180 size_yrgb = stride * h;
181 size_uv = (stride * (h >> 1));
182 start = min(yrgb_addr, uv_addr);
183 start >>= PAGE_SHIFT;
184 end = max((yrgb_addr + size_yrgb), (uv_addr + size_uv));
185 end = (end + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
186 pageCount = end - start;
187 break;
188 default:
189 pageCount = 0;
190 start = 0;
191 break;
192 }
193
194 *StartAddr = start;
195
196 if (size != NULL)
197 *size = size_yrgb + size_uv + size_v;
198
199 return pageCount;
200 }
201
rga_iommu_dma_alloc_iova(struct iommu_domain * domain,size_t size,u64 dma_limit,struct device * dev)202 static dma_addr_t rga_iommu_dma_alloc_iova(struct iommu_domain *domain,
203 size_t size, u64 dma_limit,
204 struct device *dev)
205 {
206 struct rga_iommu_dma_cookie *cookie = (void *)domain->iova_cookie;
207 struct iova_domain *iovad = &cookie->iovad;
208 unsigned long shift, iova_len, iova = 0;
209
210 shift = iova_shift(iovad);
211 iova_len = size >> shift;
212
213 #if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0))
214 /*
215 * Freeing non-power-of-two-sized allocations back into the IOVA caches
216 * will come back to bite us badly, so we have to waste a bit of space
217 * rounding up anything cacheable to make sure that can't happen. The
218 * order of the unadjusted size will still match upon freeing.
219 */
220 if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
221 iova_len = roundup_pow_of_two(iova_len);
222 #endif
223
224 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
225 dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit);
226 #else
227 if (dev->bus_dma_mask)
228 dma_limit &= dev->bus_dma_mask;
229 #endif
230
231 if (domain->geometry.force_aperture)
232 dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);
233
234 #if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 19, 111) && \
235 LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
236 iova = alloc_iova_fast(iovad, iova_len,
237 min_t(dma_addr_t, dma_limit >> shift, iovad->end_pfn),
238 true);
239 #else
240 iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, true);
241 #endif
242
243 return (dma_addr_t)iova << shift;
244 }
245
rga_iommu_dma_free_iova(struct iommu_domain * domain,dma_addr_t iova,size_t size)246 static void rga_iommu_dma_free_iova(struct iommu_domain *domain,
247 dma_addr_t iova, size_t size)
248 {
249 struct rga_iommu_dma_cookie *cookie = (void *)domain->iova_cookie;
250 struct iova_domain *iovad = &cookie->iovad;
251
252 free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad));
253 }
254
rga_iommu_get_dma_domain(struct device * dev)255 static inline struct iommu_domain *rga_iommu_get_dma_domain(struct device *dev)
256 {
257 return iommu_get_domain_for_dev(dev);
258 }
259
rga_iommu_unmap(struct rga_dma_buffer * buffer)260 void rga_iommu_unmap(struct rga_dma_buffer *buffer)
261 {
262 if (buffer == NULL)
263 return;
264 if (buffer->iova == 0)
265 return;
266
267 iommu_unmap(buffer->domain, buffer->iova, buffer->size);
268 rga_iommu_dma_free_iova(buffer->domain, buffer->iova, buffer->size);
269 }
270
rga_iommu_map_sgt(struct sg_table * sgt,size_t size,struct rga_dma_buffer * buffer,struct device * rga_dev)271 int rga_iommu_map_sgt(struct sg_table *sgt, size_t size,
272 struct rga_dma_buffer *buffer,
273 struct device *rga_dev)
274 {
275 struct iommu_domain *domain = NULL;
276 struct rga_iommu_dma_cookie *cookie;
277 struct iova_domain *iovad;
278 dma_addr_t iova;
279 size_t map_size;
280 unsigned long align_size;
281
282 if (sgt == NULL) {
283 pr_err("can not map iommu, because sgt is null!\n");
284 return -EINVAL;
285 }
286
287 domain = rga_iommu_get_dma_domain(rga_dev);
288 cookie = (void *)domain->iova_cookie;
289 iovad = &cookie->iovad;
290 align_size = iova_align(iovad, size);
291
292 if (DEBUGGER_EN(MSG))
293 pr_info("iova_align size = %ld", align_size);
294
295 iova = rga_iommu_dma_alloc_iova(domain, align_size, rga_dev->coherent_dma_mask, rga_dev);
296 if (!iova) {
297 pr_err("rga_iommu_dma_alloc_iova failed");
298 return -ENOMEM;
299 }
300
301 map_size = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents,
302 rga_dma_info_to_prot(DMA_BIDIRECTIONAL));
303 if (map_size < align_size) {
304 pr_err("iommu can not map sgt to iova");
305 rga_iommu_dma_free_iova(domain, iova, align_size);
306 return -EINVAL;
307 }
308
309 buffer->domain = domain;
310 buffer->iova = iova;
311 buffer->size = align_size;
312
313 return 0;
314 }
315
rga_iommu_map(phys_addr_t paddr,size_t size,struct rga_dma_buffer * buffer,struct device * rga_dev)316 int rga_iommu_map(phys_addr_t paddr, size_t size,
317 struct rga_dma_buffer *buffer,
318 struct device *rga_dev)
319 {
320 int ret;
321 struct iommu_domain *domain = NULL;
322 struct rga_iommu_dma_cookie *cookie;
323 struct iova_domain *iovad;
324 dma_addr_t iova;
325 unsigned long align_size;
326
327 if (paddr == 0) {
328 pr_err("can not map iommu, because phys_addr is 0!\n");
329 return -EINVAL;
330 }
331
332 domain = rga_iommu_get_dma_domain(rga_dev);
333 cookie = (void *)domain->iova_cookie;
334 iovad = &cookie->iovad;
335 align_size = iova_align(iovad, size);
336
337 if (DEBUGGER_EN(MSG))
338 pr_info("iova_align size = %ld", align_size);
339
340 iova = rga_iommu_dma_alloc_iova(domain, align_size, rga_dev->coherent_dma_mask, rga_dev);
341 if (!iova) {
342 pr_err("rga_iommu_dma_alloc_iova failed");
343 return -ENOMEM;
344 }
345
346 ret = iommu_map(domain, iova, paddr, align_size,
347 rga_dma_info_to_prot(DMA_BIDIRECTIONAL));
348 if (ret) {
349 pr_err("iommu can not map phys_addr to iova");
350 rga_iommu_dma_free_iova(domain, iova, align_size);
351 return ret;
352 }
353
354 buffer->domain = domain;
355 buffer->iova = iova;
356 buffer->size = align_size;
357
358 return 0;
359 }
360
rga_virtual_memory_check(void * vaddr,u32 w,u32 h,u32 format,int fd)361 int rga_virtual_memory_check(void *vaddr, u32 w, u32 h, u32 format, int fd)
362 {
363 int bits = 32;
364 int temp_data = 0;
365 void *one_line = NULL;
366
367 bits = rga_get_format_bits(format);
368 if (bits < 0)
369 return -1;
370
371 one_line = kzalloc(w * 4, GFP_KERNEL);
372 if (!one_line) {
373 pr_err("kzalloc fail %s[%d]\n", __func__, __LINE__);
374 return 0;
375 }
376
377 temp_data = w * (h - 1) * bits >> 3;
378 if (fd > 0) {
379 pr_info("vaddr is%p, bits is %d, fd check\n", vaddr, bits);
380 memcpy(one_line, (char *)vaddr + temp_data, w * bits >> 3);
381 pr_info("fd check ok\n");
382 } else {
383 pr_info("vir addr memory check.\n");
384 memcpy((void *)((char *)vaddr + temp_data), one_line,
385 w * bits >> 3);
386 pr_info("vir addr check ok.\n");
387 }
388
389 kfree(one_line);
390 return 0;
391 }
392
rga_dma_memory_check(struct rga_dma_buffer * rga_dma_buffer,struct rga_img_info_t * img)393 int rga_dma_memory_check(struct rga_dma_buffer *rga_dma_buffer, struct rga_img_info_t *img)
394 {
395 int ret = 0;
396 void *vaddr;
397 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
398 struct iosys_map map;
399 #endif
400 struct dma_buf *dma_buf;
401
402 dma_buf = rga_dma_buffer->dma_buf;
403
404 if (!IS_ERR_OR_NULL(dma_buf)) {
405 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
406 ret = dma_buf_vmap(dma_buf, &map);
407 vaddr = ret ? NULL : map.vaddr;
408 #else
409 vaddr = dma_buf_vmap(dma_buf);
410 #endif
411 if (vaddr) {
412 ret = rga_virtual_memory_check(vaddr, img->vir_w,
413 img->vir_h, img->format, img->yrgb_addr);
414 } else {
415 pr_err("can't vmap the dma buffer!\n");
416 return -EINVAL;
417 }
418 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
419 dma_buf_vunmap(dma_buf, &map);
420 #else
421 dma_buf_vunmap(dma_buf, vaddr);
422 #endif
423 }
424
425 return ret;
426 }
427
rga_dma_map_buf(struct dma_buf * dma_buf,struct rga_dma_buffer * rga_dma_buffer,enum dma_data_direction dir,struct device * rga_dev)428 int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buffer,
429 enum dma_data_direction dir, struct device *rga_dev)
430 {
431 struct dma_buf_attachment *attach = NULL;
432 struct sg_table *sgt = NULL;
433 struct scatterlist *sg = NULL;
434 int i, ret = 0;
435
436 if (dma_buf != NULL) {
437 get_dma_buf(dma_buf);
438 } else {
439 pr_err("dma_buf is invalid[%p]\n", dma_buf);
440 return -EINVAL;
441 }
442
443 attach = dma_buf_attach(dma_buf, rga_dev);
444 if (IS_ERR(attach)) {
445 ret = PTR_ERR(attach);
446 pr_err("Failed to attach dma_buf, ret[%d]\n", ret);
447 goto err_get_attach;
448 }
449
450 sgt = dma_buf_map_attachment(attach, dir);
451 if (IS_ERR(sgt)) {
452 ret = PTR_ERR(sgt);
453 pr_err("Failed to map attachment, ret[%d]\n", ret);
454 goto err_get_sgt;
455 }
456
457 rga_dma_buffer->dma_buf = dma_buf;
458 rga_dma_buffer->attach = attach;
459 rga_dma_buffer->sgt = sgt;
460 rga_dma_buffer->iova = sg_dma_address(sgt->sgl);
461 rga_dma_buffer->dir = dir;
462 rga_dma_buffer->size = 0;
463 for_each_sgtable_sg(sgt, sg, i)
464 rga_dma_buffer->size += sg_dma_len(sg);
465
466 return ret;
467
468 err_get_sgt:
469 if (attach)
470 dma_buf_detach(dma_buf, attach);
471 err_get_attach:
472 if (dma_buf)
473 dma_buf_put(dma_buf);
474
475 return ret;
476 }
477
rga_dma_map_fd(int fd,struct rga_dma_buffer * rga_dma_buffer,enum dma_data_direction dir,struct device * rga_dev)478 int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer,
479 enum dma_data_direction dir, struct device *rga_dev)
480 {
481 struct dma_buf *dma_buf = NULL;
482 struct dma_buf_attachment *attach = NULL;
483 struct sg_table *sgt = NULL;
484 struct scatterlist *sg = NULL;
485 int i, ret = 0;
486
487 dma_buf = dma_buf_get(fd);
488 if (IS_ERR(dma_buf)) {
489 ret = PTR_ERR(dma_buf);
490 pr_err("Fail to get dma_buf from fd[%d], ret[%d]\n", fd, ret);
491 return ret;
492 }
493
494 attach = dma_buf_attach(dma_buf, rga_dev);
495 if (IS_ERR(attach)) {
496 ret = PTR_ERR(attach);
497 pr_err("Failed to attach dma_buf, ret[%d]\n", ret);
498 goto err_get_attach;
499 }
500
501 sgt = dma_buf_map_attachment(attach, dir);
502 if (IS_ERR(sgt)) {
503 ret = PTR_ERR(sgt);
504 pr_err("Failed to map attachment, ret[%d]\n", ret);
505 goto err_get_sgt;
506 }
507
508 rga_dma_buffer->dma_buf = dma_buf;
509 rga_dma_buffer->attach = attach;
510 rga_dma_buffer->sgt = sgt;
511 rga_dma_buffer->iova = sg_dma_address(sgt->sgl);
512 rga_dma_buffer->dir = dir;
513 rga_dma_buffer->size = 0;
514 for_each_sgtable_sg(sgt, sg, i)
515 rga_dma_buffer->size += sg_dma_len(sg);
516
517 return ret;
518
519 err_get_sgt:
520 if (attach)
521 dma_buf_detach(dma_buf, attach);
522 err_get_attach:
523 if (dma_buf)
524 dma_buf_put(dma_buf);
525
526 return ret;
527 }
528
rga_dma_unmap_buf(struct rga_dma_buffer * rga_dma_buffer)529 void rga_dma_unmap_buf(struct rga_dma_buffer *rga_dma_buffer)
530 {
531 if (rga_dma_buffer->attach && rga_dma_buffer->sgt)
532 dma_buf_unmap_attachment(rga_dma_buffer->attach,
533 rga_dma_buffer->sgt,
534 rga_dma_buffer->dir);
535
536 if (rga_dma_buffer->attach) {
537 dma_buf_detach(rga_dma_buffer->dma_buf, rga_dma_buffer->attach);
538 dma_buf_put(rga_dma_buffer->dma_buf);
539 }
540 }
541
rga_dma_sync_flush_range(void * pstart,void * pend,struct rga_scheduler_t * scheduler)542 void rga_dma_sync_flush_range(void *pstart, void *pend, struct rga_scheduler_t *scheduler)
543 {
544 dma_sync_single_for_device(scheduler->dev, virt_to_phys(pstart),
545 pend - pstart, DMA_TO_DEVICE);
546 }
547