1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) Rockchip Electronics Co.Ltd
4 * Author: Felix Zeng <felix.zeng@rock-chips.com>
5 */
6
7 #include <drm/drm_device.h>
8 #include <drm/drm_vma_manager.h>
9 #include <drm/drm_prime.h>
10 #include <drm/drm_file.h>
11 #include <drm/drm_drv.h>
12
13 #include <linux/shmem_fs.h>
14 #include <linux/dma-buf.h>
15 #include <linux/iommu.h>
16 #include <linux/pfn_t.h>
17 #include <linux/version.h>
18 #include <asm/cacheflush.h>
19
20 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
21 #include <linux/dma-map-ops.h>
22 #endif
23
24 #include "rknpu_drv.h"
25 #include "rknpu_ioctl.h"
26 #include "rknpu_gem.h"
27 #include "rknpu_iommu.h"
28
29 #define RKNPU_GEM_ALLOC_FROM_PAGES 1
30
31 #if RKNPU_GEM_ALLOC_FROM_PAGES
rknpu_gem_get_pages(struct rknpu_gem_object * rknpu_obj)32 static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj)
33 {
34 struct drm_device *drm = rknpu_obj->base.dev;
35 struct scatterlist *s = NULL;
36 dma_addr_t dma_addr = 0;
37 dma_addr_t phys = 0;
38 int ret = -EINVAL, i = 0;
39
40 rknpu_obj->pages = drm_gem_get_pages(&rknpu_obj->base);
41 if (IS_ERR(rknpu_obj->pages)) {
42 ret = PTR_ERR(rknpu_obj->pages);
43 LOG_ERROR("failed to get pages: %d\n", ret);
44 return ret;
45 }
46
47 rknpu_obj->num_pages = rknpu_obj->size >> PAGE_SHIFT;
48
49 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
50 rknpu_obj->sgt = drm_prime_pages_to_sg(drm, rknpu_obj->pages,
51 rknpu_obj->num_pages);
52 #else
53 rknpu_obj->sgt =
54 drm_prime_pages_to_sg(rknpu_obj->pages, rknpu_obj->num_pages);
55 #endif
56 if (IS_ERR(rknpu_obj->sgt)) {
57 ret = PTR_ERR(rknpu_obj->sgt);
58 LOG_ERROR("failed to allocate sgt: %d\n", ret);
59 goto put_pages;
60 }
61
62 ret = dma_map_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
63 DMA_BIDIRECTIONAL);
64 if (ret == 0) {
65 ret = -EFAULT;
66 LOG_DEV_ERROR(drm->dev, "%s: dma map %zu fail\n", __func__,
67 rknpu_obj->size);
68 goto free_sgt;
69 }
70 iommu_flush_iotlb_all(iommu_get_domain_for_dev(drm->dev));
71
72 if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) {
73 rknpu_obj->cookie = vmap(rknpu_obj->pages, rknpu_obj->num_pages,
74 VM_MAP, PAGE_KERNEL);
75 if (!rknpu_obj->cookie) {
76 ret = -ENOMEM;
77 LOG_ERROR("failed to vmap: %d\n", ret);
78 goto unmap_sg;
79 }
80 rknpu_obj->kv_addr = rknpu_obj->cookie;
81 }
82
83 dma_addr = sg_dma_address(rknpu_obj->sgt->sgl);
84 rknpu_obj->dma_addr = dma_addr;
85
86 for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) {
87 dma_addr += s->length;
88 phys = sg_phys(s);
89 LOG_DEBUG(
90 "gem pages alloc sgt[%d], dma_address: %pad, length: %#x, phys: %pad, virt: %p\n",
91 i, &dma_addr, s->length, &phys, sg_virt(s));
92 }
93
94 return 0;
95
96 unmap_sg:
97 dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
98 DMA_BIDIRECTIONAL);
99
100 free_sgt:
101 sg_free_table(rknpu_obj->sgt);
102 kfree(rknpu_obj->sgt);
103
104 put_pages:
105 drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false);
106
107 return ret;
108 }
109
rknpu_gem_put_pages(struct rknpu_gem_object * rknpu_obj)110 static void rknpu_gem_put_pages(struct rknpu_gem_object *rknpu_obj)
111 {
112 struct drm_device *drm = rknpu_obj->base.dev;
113
114 if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) {
115 vunmap(rknpu_obj->kv_addr);
116 rknpu_obj->kv_addr = NULL;
117 }
118
119 if (rknpu_obj->sgt != NULL) {
120 dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl,
121 rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL);
122 sg_free_table(rknpu_obj->sgt);
123 kfree(rknpu_obj->sgt);
124 }
125
126 drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true, true);
127 }
128 #endif
129
rknpu_gem_alloc_buf(struct rknpu_gem_object * rknpu_obj)130 static int rknpu_gem_alloc_buf(struct rknpu_gem_object *rknpu_obj)
131 {
132 struct drm_device *drm = rknpu_obj->base.dev;
133 struct rknpu_device *rknpu_dev = drm->dev_private;
134 unsigned int nr_pages = 0;
135 struct sg_table *sgt = NULL;
136 struct scatterlist *s = NULL;
137 gfp_t gfp_mask = GFP_KERNEL;
138 int ret = -EINVAL, i = 0;
139
140 if (rknpu_obj->dma_addr) {
141 LOG_DEBUG("buffer already allocated.\n");
142 return 0;
143 }
144
145 rknpu_obj->dma_attrs = 0;
146
147 /*
148 * if RKNPU_MEM_CONTIGUOUS, fully physically contiguous memory
149 * region will be allocated else physically contiguous
150 * as possible.
151 */
152 if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS))
153 rknpu_obj->dma_attrs |= DMA_ATTR_FORCE_CONTIGUOUS;
154
155 // cacheable mapping or writecombine mapping
156 if (rknpu_obj->flags & RKNPU_MEM_CACHEABLE) {
157 #ifdef DMA_ATTR_NON_CONSISTENT
158 rknpu_obj->dma_attrs |= DMA_ATTR_NON_CONSISTENT;
159 #endif
160 #ifdef DMA_ATTR_SYS_CACHE_ONLY
161 rknpu_obj->dma_attrs |= DMA_ATTR_SYS_CACHE_ONLY;
162 #endif
163 } else if (rknpu_obj->flags & RKNPU_MEM_WRITE_COMBINE) {
164 rknpu_obj->dma_attrs |= DMA_ATTR_WRITE_COMBINE;
165 }
166
167 if (!(rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING))
168 rknpu_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;
169
170 #ifdef DMA_ATTR_SKIP_ZEROING
171 if (!(rknpu_obj->flags & RKNPU_MEM_ZEROING))
172 rknpu_obj->dma_attrs |= DMA_ATTR_SKIP_ZEROING;
173 #endif
174
175 #if RKNPU_GEM_ALLOC_FROM_PAGES
176 if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
177 rknpu_dev->iommu_en) {
178 return rknpu_gem_get_pages(rknpu_obj);
179 }
180 #endif
181
182 if (rknpu_obj->flags & RKNPU_MEM_ZEROING)
183 gfp_mask |= __GFP_ZERO;
184
185 if (!rknpu_dev->iommu_en ||
186 rknpu_dev->config->dma_mask <= DMA_BIT_MASK(32) ||
187 (rknpu_obj->flags & RKNPU_MEM_DMA32)) {
188 gfp_mask &= ~__GFP_HIGHMEM;
189 gfp_mask |= __GFP_DMA32;
190 }
191
192 nr_pages = rknpu_obj->size >> PAGE_SHIFT;
193
194 rknpu_obj->pages = rknpu_gem_alloc_page(nr_pages);
195 if (!rknpu_obj->pages) {
196 LOG_ERROR("failed to allocate pages.\n");
197 return -ENOMEM;
198 }
199
200 rknpu_obj->cookie =
201 dma_alloc_attrs(drm->dev, rknpu_obj->size, &rknpu_obj->dma_addr,
202 gfp_mask, rknpu_obj->dma_attrs);
203 if (!rknpu_obj->cookie) {
204 /*
205 * when RKNPU_MEM_CONTIGUOUS and IOMMU is available
206 * try to fallback to allocate non-contiguous buffer
207 */
208 if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
209 rknpu_dev->iommu_en) {
210 LOG_DEV_WARN(
211 drm->dev,
212 "try to fallback to allocate non-contiguous %lu buffer.\n",
213 rknpu_obj->size);
214 rknpu_obj->dma_attrs &= ~DMA_ATTR_FORCE_CONTIGUOUS;
215 rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS;
216 rknpu_obj->cookie =
217 dma_alloc_attrs(drm->dev, rknpu_obj->size,
218 &rknpu_obj->dma_addr, gfp_mask,
219 rknpu_obj->dma_attrs);
220 if (!rknpu_obj->cookie) {
221 LOG_DEV_ERROR(
222 drm->dev,
223 "failed to allocate non-contiguous %lu buffer.\n",
224 rknpu_obj->size);
225 goto err_free;
226 }
227 } else {
228 LOG_DEV_ERROR(drm->dev,
229 "failed to allocate %lu buffer.\n",
230 rknpu_obj->size);
231 goto err_free;
232 }
233 }
234
235 if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING)
236 rknpu_obj->kv_addr = rknpu_obj->cookie;
237
238 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
239 if (!sgt) {
240 ret = -ENOMEM;
241 goto err_free_dma;
242 }
243
244 ret = dma_get_sgtable_attrs(drm->dev, sgt, rknpu_obj->cookie,
245 rknpu_obj->dma_addr, rknpu_obj->size,
246 rknpu_obj->dma_attrs);
247 if (ret < 0) {
248 LOG_DEV_ERROR(drm->dev, "failed to get sgtable.\n");
249 goto err_free_sgt;
250 }
251
252 for_each_sg(sgt->sgl, s, sgt->nents, i) {
253 sg_dma_address(s) = sg_phys(s);
254 LOG_DEBUG("dma alloc sgt[%d], phys_address: %pad, length: %u\n",
255 i, &s->dma_address, s->length);
256 }
257
258 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
259 ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
260 nr_pages);
261 #else
262 ret = drm_prime_sg_to_page_array(sgt, rknpu_obj->pages, nr_pages);
263 #endif
264
265 if (ret < 0) {
266 LOG_DEV_ERROR(drm->dev, "invalid sgtable, ret: %d\n", ret);
267 goto err_free_sg_table;
268 }
269
270 rknpu_obj->sgt = sgt;
271
272 return ret;
273
274 err_free_sg_table:
275 sg_free_table(sgt);
276 err_free_sgt:
277 kfree(sgt);
278 err_free_dma:
279 dma_free_attrs(drm->dev, rknpu_obj->size, rknpu_obj->cookie,
280 rknpu_obj->dma_addr, rknpu_obj->dma_attrs);
281 err_free:
282 rknpu_gem_free_page(rknpu_obj->pages);
283
284 return ret;
285 }
286
rknpu_gem_free_buf(struct rknpu_gem_object * rknpu_obj)287 static void rknpu_gem_free_buf(struct rknpu_gem_object *rknpu_obj)
288 {
289 struct drm_device *drm = rknpu_obj->base.dev;
290 #if RKNPU_GEM_ALLOC_FROM_PAGES
291 struct rknpu_device *rknpu_dev = drm->dev_private;
292 #endif
293
294 if (!rknpu_obj->dma_addr) {
295 LOG_DEBUG("dma handle is invalid.\n");
296 return;
297 }
298
299 #if RKNPU_GEM_ALLOC_FROM_PAGES
300 if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
301 rknpu_dev->iommu_en) {
302 rknpu_gem_put_pages(rknpu_obj);
303 return;
304 }
305 #endif
306
307 sg_free_table(rknpu_obj->sgt);
308 kfree(rknpu_obj->sgt);
309
310 dma_free_attrs(drm->dev, rknpu_obj->size, rknpu_obj->cookie,
311 rknpu_obj->dma_addr, rknpu_obj->dma_attrs);
312
313 rknpu_gem_free_page(rknpu_obj->pages);
314
315 rknpu_obj->dma_addr = 0;
316 }
317
rknpu_gem_handle_create(struct drm_gem_object * obj,struct drm_file * file_priv,unsigned int * handle)318 static int rknpu_gem_handle_create(struct drm_gem_object *obj,
319 struct drm_file *file_priv,
320 unsigned int *handle)
321 {
322 int ret = -EINVAL;
323 /*
324 * allocate a id of idr table where the obj is registered
325 * and handle has the id what user can see.
326 */
327 ret = drm_gem_handle_create(file_priv, obj, handle);
328 if (ret)
329 return ret;
330
331 LOG_DEBUG("gem handle: %#x\n", *handle);
332
333 /* drop reference from allocate - handle holds it now. */
334 rknpu_gem_object_put(obj);
335
336 return 0;
337 }
338
rknpu_gem_handle_destroy(struct drm_file * file_priv,unsigned int handle)339 static int rknpu_gem_handle_destroy(struct drm_file *file_priv,
340 unsigned int handle)
341 {
342 return drm_gem_handle_delete(file_priv, handle);
343 }
344
345 #if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
346 static const struct vm_operations_struct vm_ops = {
347 .fault = rknpu_gem_fault,
348 .open = drm_gem_vm_open,
349 .close = drm_gem_vm_close,
350 };
351
352 static const struct drm_gem_object_funcs rknpu_gem_object_funcs = {
353 .free = rknpu_gem_free_object,
354 .export = drm_gem_prime_export,
355 .get_sg_table = rknpu_gem_prime_get_sg_table,
356 .vmap = rknpu_gem_prime_vmap,
357 .vunmap = rknpu_gem_prime_vunmap,
358 .mmap = rknpu_gem_mmap_obj,
359 .vm_ops = &vm_ops,
360 };
361 #endif
362
rknpu_gem_init(struct drm_device * drm,unsigned long size)363 static struct rknpu_gem_object *rknpu_gem_init(struct drm_device *drm,
364 unsigned long size)
365 {
366 struct rknpu_device *rknpu_dev = drm->dev_private;
367 struct rknpu_gem_object *rknpu_obj = NULL;
368 struct drm_gem_object *obj = NULL;
369 gfp_t gfp_mask;
370 int ret = -EINVAL;
371
372 rknpu_obj = kzalloc(sizeof(*rknpu_obj), GFP_KERNEL);
373 if (!rknpu_obj)
374 return ERR_PTR(-ENOMEM);
375
376 obj = &rknpu_obj->base;
377 #if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
378 obj->funcs = &rknpu_gem_object_funcs;
379 #endif
380
381 ret = drm_gem_object_init(drm, obj, size);
382 if (ret < 0) {
383 LOG_DEV_ERROR(drm->dev, "failed to initialize gem object\n");
384 kfree(rknpu_obj);
385 return ERR_PTR(ret);
386 }
387
388 rknpu_obj->size = rknpu_obj->base.size;
389
390 gfp_mask = mapping_gfp_mask(obj->filp->f_mapping);
391
392 if (rknpu_obj->flags & RKNPU_MEM_ZEROING)
393 gfp_mask |= __GFP_ZERO;
394
395 if (!rknpu_dev->iommu_en ||
396 rknpu_dev->config->dma_mask <= DMA_BIT_MASK(32) ||
397 (rknpu_obj->flags & RKNPU_MEM_DMA32)) {
398 gfp_mask &= ~__GFP_HIGHMEM;
399 gfp_mask |= __GFP_DMA32;
400 }
401
402 mapping_set_gfp_mask(obj->filp->f_mapping, gfp_mask);
403
404 return rknpu_obj;
405 }
406
rknpu_gem_release(struct rknpu_gem_object * rknpu_obj)407 static void rknpu_gem_release(struct rknpu_gem_object *rknpu_obj)
408 {
409 /* release file pointer to gem object. */
410 drm_gem_object_release(&rknpu_obj->base);
411 kfree(rknpu_obj);
412 }
413
rknpu_gem_alloc_buf_with_cache(struct rknpu_gem_object * rknpu_obj,enum rknpu_cache_type cache_type)414 static int rknpu_gem_alloc_buf_with_cache(struct rknpu_gem_object *rknpu_obj,
415 enum rknpu_cache_type cache_type)
416 {
417 struct drm_device *drm = rknpu_obj->base.dev;
418 struct rknpu_device *rknpu_dev = drm->dev_private;
419 struct iommu_domain *domain = NULL;
420 struct rknpu_iommu_dma_cookie *cookie = NULL;
421 struct iova_domain *iovad = NULL;
422 struct scatterlist *s = NULL;
423 unsigned long length = 0;
424 unsigned long size = 0;
425 unsigned long offset = 0;
426 int i = 0;
427 int ret = -EINVAL;
428 phys_addr_t cache_start = 0;
429 unsigned long cache_offset = 0;
430 unsigned long cache_size = 0;
431
432 switch (cache_type) {
433 case RKNPU_CACHE_SRAM:
434 cache_start = rknpu_dev->sram_start;
435 cache_offset = rknpu_obj->sram_obj->range_start *
436 rknpu_dev->sram_mm->chunk_size;
437 cache_size = rknpu_obj->sram_size;
438 break;
439 case RKNPU_CACHE_NBUF:
440 cache_start = rknpu_dev->nbuf_start;
441 cache_offset = 0;
442 cache_size = rknpu_obj->nbuf_size;
443 break;
444 default:
445 LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
446 return -EINVAL;
447 }
448
449 /* iova map to cache */
450 domain = iommu_get_domain_for_dev(rknpu_dev->dev);
451 if (!domain) {
452 LOG_ERROR("failed to get iommu domain!");
453 return -EINVAL;
454 }
455
456 cookie = (void *)domain->iova_cookie;
457 iovad = &cookie->iovad;
458 rknpu_obj->iova_size = iova_align(iovad, cache_size + rknpu_obj->size);
459 rknpu_obj->iova_start = rknpu_iommu_dma_alloc_iova(
460 domain, rknpu_obj->iova_size, dma_get_mask(drm->dev), drm->dev);
461 if (!rknpu_obj->iova_start) {
462 LOG_ERROR("iommu_dma_alloc_iova failed\n");
463 return -ENOMEM;
464 }
465
466 LOG_INFO("allocate iova start: %pad, size: %lu\n",
467 &rknpu_obj->iova_start, rknpu_obj->iova_size);
468
469 /*
470 * Overview cache + DDR map to IOVA
471 * --------
472 * cache_size:
473 * - allocate from CACHE, this size value has been page-aligned
474 * size: rknpu_obj->size
475 * - allocate from DDR pages, this size value has been page-aligned
476 * iova_size: rknpu_obj->iova_size
477 * - from iova_align(cache_size + size)
478 * - it may be larger than the (cache_size + size), and the larger part is not mapped
479 * --------
480 *
481 * |<- cache_size ->| |<- - - - size - - - ->|
482 * +---------------+ +----------------------+
483 * | CACHE | | DDR |
484 * +---------------+ +----------------------+
485 * | |
486 * | V | V |
487 * +---------------------------------------+
488 * | IOVA range |
489 * +---------------------------------------+
490 * |<- - - - - - - iova_size - - - - - - ->|
491 *
492 */
493 ret = iommu_map(domain, rknpu_obj->iova_start,
494 cache_start + cache_offset, cache_size,
495 IOMMU_READ | IOMMU_WRITE);
496 if (ret) {
497 LOG_ERROR("cache iommu_map error: %d\n", ret);
498 goto free_iova;
499 }
500
501 rknpu_obj->dma_addr = rknpu_obj->iova_start;
502
503 if (rknpu_obj->size == 0) {
504 LOG_INFO("allocate cache size: %lu\n", cache_size);
505 return 0;
506 }
507
508 rknpu_obj->pages = drm_gem_get_pages(&rknpu_obj->base);
509 if (IS_ERR(rknpu_obj->pages)) {
510 ret = PTR_ERR(rknpu_obj->pages);
511 LOG_ERROR("failed to get pages: %d\n", ret);
512 goto cache_unmap;
513 }
514
515 rknpu_obj->num_pages = rknpu_obj->size >> PAGE_SHIFT;
516
517 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
518 rknpu_obj->sgt = drm_prime_pages_to_sg(drm, rknpu_obj->pages,
519 rknpu_obj->num_pages);
520 #else
521 rknpu_obj->sgt =
522 drm_prime_pages_to_sg(rknpu_obj->pages, rknpu_obj->num_pages);
523 #endif
524 if (IS_ERR(rknpu_obj->sgt)) {
525 ret = PTR_ERR(rknpu_obj->sgt);
526 LOG_ERROR("failed to allocate sgt: %d\n", ret);
527 goto put_pages;
528 }
529
530 length = rknpu_obj->size;
531 offset = rknpu_obj->iova_start + cache_size;
532
533 for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) {
534 size = (length < s->length) ? length : s->length;
535
536 ret = iommu_map(domain, offset, sg_phys(s), size,
537 IOMMU_READ | IOMMU_WRITE);
538 if (ret) {
539 LOG_ERROR("ddr iommu_map error: %d\n", ret);
540 goto sgl_unmap;
541 }
542
543 length -= size;
544 offset += size;
545
546 if (length == 0)
547 break;
548 }
549
550 LOG_INFO("allocate size: %lu with cache size: %lu\n", rknpu_obj->size,
551 cache_size);
552
553 return 0;
554
555 sgl_unmap:
556 iommu_unmap(domain, rknpu_obj->iova_start + cache_size,
557 rknpu_obj->size - length);
558 sg_free_table(rknpu_obj->sgt);
559 kfree(rknpu_obj->sgt);
560
561 put_pages:
562 drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false);
563
564 cache_unmap:
565 iommu_unmap(domain, rknpu_obj->iova_start, cache_size);
566
567 free_iova:
568 rknpu_iommu_dma_free_iova((void *)domain->iova_cookie,
569 rknpu_obj->iova_start, rknpu_obj->iova_size);
570
571 return ret;
572 }
573
rknpu_gem_free_buf_with_cache(struct rknpu_gem_object * rknpu_obj,enum rknpu_cache_type cache_type)574 static void rknpu_gem_free_buf_with_cache(struct rknpu_gem_object *rknpu_obj,
575 enum rknpu_cache_type cache_type)
576 {
577 struct drm_device *drm = rknpu_obj->base.dev;
578 struct rknpu_device *rknpu_dev = drm->dev_private;
579 struct iommu_domain *domain = NULL;
580 unsigned long cache_size = 0;
581
582 switch (cache_type) {
583 case RKNPU_CACHE_SRAM:
584 cache_size = rknpu_obj->sram_size;
585 break;
586 case RKNPU_CACHE_NBUF:
587 cache_size = rknpu_obj->nbuf_size;
588 break;
589 default:
590 LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
591 return;
592 }
593
594 domain = iommu_get_domain_for_dev(rknpu_dev->dev);
595 if (domain) {
596 iommu_unmap(domain, rknpu_obj->iova_start, cache_size);
597 if (rknpu_obj->size > 0)
598 iommu_unmap(domain, rknpu_obj->iova_start + cache_size,
599 rknpu_obj->size);
600 rknpu_iommu_dma_free_iova((void *)domain->iova_cookie,
601 rknpu_obj->iova_start,
602 rknpu_obj->iova_size);
603 }
604
605 if (rknpu_obj->pages)
606 drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true,
607 true);
608
609 if (rknpu_obj->sgt != NULL) {
610 sg_free_table(rknpu_obj->sgt);
611 kfree(rknpu_obj->sgt);
612 }
613 }
614
rknpu_gem_object_create(struct drm_device * drm,unsigned int flags,unsigned long size,unsigned long sram_size)615 struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
616 unsigned int flags,
617 unsigned long size,
618 unsigned long sram_size)
619 {
620 struct rknpu_device *rknpu_dev = drm->dev_private;
621 struct rknpu_gem_object *rknpu_obj = NULL;
622 size_t remain_ddr_size = 0;
623 int ret = -EINVAL;
624
625 if (!size) {
626 LOG_DEV_ERROR(drm->dev, "invalid buffer size: %lu\n", size);
627 return ERR_PTR(-EINVAL);
628 }
629
630 remain_ddr_size = round_up(size, PAGE_SIZE);
631
632 if (!rknpu_dev->iommu_en && (flags & RKNPU_MEM_NON_CONTIGUOUS)) {
633 /*
634 * when no IOMMU is available, all allocated buffers are
635 * contiguous anyway, so drop RKNPU_MEM_NON_CONTIGUOUS flag
636 */
637 flags &= ~RKNPU_MEM_NON_CONTIGUOUS;
638 LOG_WARN(
639 "non-contiguous allocation is not supported without IOMMU, falling back to contiguous buffer\n");
640 }
641
642 if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
643 (flags & RKNPU_MEM_TRY_ALLOC_SRAM) && rknpu_dev->sram_size > 0) {
644 size_t sram_free_size = 0;
645 size_t real_sram_size = 0;
646
647 if (sram_size != 0)
648 sram_size = round_up(sram_size, PAGE_SIZE);
649
650 rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
651 if (IS_ERR(rknpu_obj))
652 return rknpu_obj;
653
654 /* set memory type and cache attribute from user side. */
655 rknpu_obj->flags = flags;
656
657 sram_free_size = rknpu_dev->sram_mm->free_chunks *
658 rknpu_dev->sram_mm->chunk_size;
659 if (sram_free_size > 0) {
660 real_sram_size = remain_ddr_size;
661 if (sram_size != 0 && remain_ddr_size > sram_size)
662 real_sram_size = sram_size;
663 if (real_sram_size > sram_free_size)
664 real_sram_size = sram_free_size;
665 ret = rknpu_mm_alloc(rknpu_dev->sram_mm, real_sram_size,
666 &rknpu_obj->sram_obj);
667 if (ret != 0) {
668 sram_free_size =
669 rknpu_dev->sram_mm->free_chunks *
670 rknpu_dev->sram_mm->chunk_size;
671 LOG_WARN(
672 "mm allocate %zu failed, ret: %d, free size: %zu\n",
673 real_sram_size, ret, sram_free_size);
674 real_sram_size = 0;
675 }
676 }
677
678 if (real_sram_size > 0) {
679 rknpu_obj->sram_size = real_sram_size;
680
681 ret = rknpu_gem_alloc_buf_with_cache(rknpu_obj,
682 RKNPU_CACHE_SRAM);
683 if (ret < 0)
684 goto mm_free;
685 remain_ddr_size = 0;
686 }
687 } else if (IS_ENABLED(CONFIG_NO_GKI) &&
688 (flags & RKNPU_MEM_TRY_ALLOC_NBUF) &&
689 rknpu_dev->nbuf_size > 0) {
690 size_t nbuf_size = 0;
691
692 rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
693 if (IS_ERR(rknpu_obj))
694 return rknpu_obj;
695
696 nbuf_size = remain_ddr_size <= rknpu_dev->nbuf_size ?
697 remain_ddr_size :
698 rknpu_dev->nbuf_size;
699
700 /* set memory type and cache attribute from user side. */
701 rknpu_obj->flags = flags;
702
703 if (nbuf_size > 0) {
704 rknpu_obj->nbuf_size = nbuf_size;
705
706 ret = rknpu_gem_alloc_buf_with_cache(rknpu_obj,
707 RKNPU_CACHE_NBUF);
708 if (ret < 0)
709 goto gem_release;
710 remain_ddr_size = 0;
711 }
712 }
713
714 if (remain_ddr_size > 0) {
715 rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
716 if (IS_ERR(rknpu_obj))
717 return rknpu_obj;
718
719 /* set memory type and cache attribute from user side. */
720 rknpu_obj->flags = flags;
721
722 ret = rknpu_gem_alloc_buf(rknpu_obj);
723 if (ret < 0)
724 goto gem_release;
725 }
726
727 if (rknpu_obj)
728 LOG_DEBUG(
729 "created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, nbuf size: %lu, attrs: %#lx, flags: %#x\n",
730 &rknpu_obj->dma_addr, rknpu_obj->cookie,
731 rknpu_obj->size, rknpu_obj->sram_size,
732 rknpu_obj->nbuf_size, rknpu_obj->dma_attrs,
733 rknpu_obj->flags);
734
735 return rknpu_obj;
736
737 mm_free:
738 if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
739 rknpu_obj->sram_obj != NULL)
740 rknpu_mm_free(rknpu_dev->sram_mm, rknpu_obj->sram_obj);
741
742 gem_release:
743 rknpu_gem_release(rknpu_obj);
744
745 return ERR_PTR(ret);
746 }
747
rknpu_gem_object_destroy(struct rknpu_gem_object * rknpu_obj)748 void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj)
749 {
750 struct drm_gem_object *obj = &rknpu_obj->base;
751
752 LOG_DEBUG(
753 "destroy dma addr: %pad, cookie: %p, size: %lu, attrs: %#lx, flags: %#x, handle count: %d\n",
754 &rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size,
755 rknpu_obj->dma_attrs, rknpu_obj->flags, obj->handle_count);
756
757 /*
758 * do not release memory region from exporter.
759 *
760 * the region will be released by exporter
761 * once dmabuf's refcount becomes 0.
762 */
763 if (obj->import_attach) {
764 drm_prime_gem_destroy(obj, rknpu_obj->sgt);
765 rknpu_gem_free_page(rknpu_obj->pages);
766 } else {
767 if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
768 rknpu_obj->sram_size > 0) {
769 struct rknpu_device *rknpu_dev = obj->dev->dev_private;
770
771 if (rknpu_obj->sram_obj != NULL)
772 rknpu_mm_free(rknpu_dev->sram_mm,
773 rknpu_obj->sram_obj);
774 rknpu_gem_free_buf_with_cache(rknpu_obj,
775 RKNPU_CACHE_SRAM);
776 } else if (IS_ENABLED(CONFIG_NO_GKI) &&
777 rknpu_obj->nbuf_size > 0) {
778 rknpu_gem_free_buf_with_cache(rknpu_obj,
779 RKNPU_CACHE_NBUF);
780 } else {
781 rknpu_gem_free_buf(rknpu_obj);
782 }
783 }
784
785 rknpu_gem_release(rknpu_obj);
786 }
787
rknpu_gem_create_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)788 int rknpu_gem_create_ioctl(struct drm_device *dev, void *data,
789 struct drm_file *file_priv)
790 {
791 struct rknpu_mem_create *args = data;
792 struct rknpu_gem_object *rknpu_obj = NULL;
793 int ret = -EINVAL;
794
795 rknpu_obj = rknpu_gem_object_find(file_priv, args->handle);
796 if (!rknpu_obj) {
797 rknpu_obj = rknpu_gem_object_create(
798 dev, args->flags, args->size, args->sram_size);
799 if (IS_ERR(rknpu_obj))
800 return PTR_ERR(rknpu_obj);
801
802 ret = rknpu_gem_handle_create(&rknpu_obj->base, file_priv,
803 &args->handle);
804 if (ret) {
805 rknpu_gem_object_destroy(rknpu_obj);
806 return ret;
807 }
808 }
809
810 // rknpu_gem_object_get(&rknpu_obj->base);
811
812 args->size = rknpu_obj->size;
813 args->sram_size = rknpu_obj->sram_size;
814 args->obj_addr = (__u64)(uintptr_t)rknpu_obj;
815 args->dma_addr = rknpu_obj->dma_addr;
816
817 return 0;
818 }
819
rknpu_gem_map_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)820 int rknpu_gem_map_ioctl(struct drm_device *dev, void *data,
821 struct drm_file *file_priv)
822 {
823 struct rknpu_mem_map *args = data;
824
825 #if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
826 return rknpu_gem_dumb_map_offset(file_priv, dev, args->handle,
827 &args->offset);
828 #else
829 return drm_gem_dumb_map_offset(file_priv, dev, args->handle,
830 &args->offset);
831 #endif
832 }
833
rknpu_gem_destroy_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)834 int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data,
835 struct drm_file *file_priv)
836 {
837 struct rknpu_gem_object *rknpu_obj = NULL;
838 struct rknpu_mem_destroy *args = data;
839
840 rknpu_obj = rknpu_gem_object_find(file_priv, args->handle);
841 if (!rknpu_obj)
842 return -EINVAL;
843
844 // rknpu_gem_object_put(&rknpu_obj->base);
845
846 return rknpu_gem_handle_destroy(file_priv, args->handle);
847 }
848
849 #if RKNPU_GEM_ALLOC_FROM_PAGES
850 /*
851 * __vm_map_pages - maps range of kernel pages into user vma
852 * @vma: user vma to map to
853 * @pages: pointer to array of source kernel pages
854 * @num: number of pages in page array
855 * @offset: user's requested vm_pgoff
856 *
857 * This allows drivers to map range of kernel pages into a user vma.
858 *
859 * Return: 0 on success and error code otherwise.
860 */
__vm_map_pages(struct vm_area_struct * vma,struct page ** pages,unsigned long num,unsigned long offset)861 static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
862 unsigned long num, unsigned long offset)
863 {
864 unsigned long count = vma_pages(vma);
865 unsigned long uaddr = vma->vm_start;
866 int ret = -EINVAL, i = 0;
867
868 /* Fail if the user requested offset is beyond the end of the object */
869 if (offset >= num)
870 return -ENXIO;
871
872 /* Fail if the user requested size exceeds available object size */
873 if (count > num - offset)
874 return -ENXIO;
875
876 for (i = 0; i < count; i++) {
877 ret = vm_insert_page(vma, uaddr, pages[offset + i]);
878 if (ret < 0)
879 return ret;
880 uaddr += PAGE_SIZE;
881 }
882
883 return 0;
884 }
885
rknpu_gem_mmap_pages(struct rknpu_gem_object * rknpu_obj,struct vm_area_struct * vma)886 static int rknpu_gem_mmap_pages(struct rknpu_gem_object *rknpu_obj,
887 struct vm_area_struct *vma)
888 {
889 struct drm_device *drm = rknpu_obj->base.dev;
890 int ret = -EINVAL;
891
892 vma->vm_flags |= VM_MIXEDMAP;
893
894 ret = __vm_map_pages(vma, rknpu_obj->pages, rknpu_obj->num_pages,
895 vma->vm_pgoff);
896 if (ret < 0)
897 LOG_DEV_ERROR(drm->dev, "failed to map pages into vma: %d\n",
898 ret);
899
900 return ret;
901 }
902 #endif
903
rknpu_gem_mmap_cache(struct rknpu_gem_object * rknpu_obj,struct vm_area_struct * vma,enum rknpu_cache_type cache_type)904 static int rknpu_gem_mmap_cache(struct rknpu_gem_object *rknpu_obj,
905 struct vm_area_struct *vma,
906 enum rknpu_cache_type cache_type)
907 {
908 struct drm_device *drm = rknpu_obj->base.dev;
909 #if RKNPU_GEM_ALLOC_FROM_PAGES
910 struct rknpu_device *rknpu_dev = drm->dev_private;
911 #endif
912 unsigned long vm_size = 0;
913 int ret = -EINVAL;
914 unsigned long offset = 0;
915 unsigned long num_pages = 0;
916 int i = 0;
917 phys_addr_t cache_start = 0;
918 unsigned long cache_offset = 0;
919 unsigned long cache_size = 0;
920
921 switch (cache_type) {
922 case RKNPU_CACHE_SRAM:
923 cache_start = rknpu_dev->sram_start;
924 cache_offset = rknpu_obj->sram_obj->range_start *
925 rknpu_dev->sram_mm->chunk_size;
926 cache_size = rknpu_obj->sram_size;
927 break;
928 case RKNPU_CACHE_NBUF:
929 cache_start = rknpu_dev->nbuf_start;
930 cache_offset = 0;
931 cache_size = rknpu_obj->nbuf_size;
932 break;
933 default:
934 LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
935 return -EINVAL;
936 }
937
938 vma->vm_flags |= VM_MIXEDMAP;
939
940 vm_size = vma->vm_end - vma->vm_start;
941
942 /*
943 * Convert a physical address in a cache area to a page frame number (PFN),
944 * and store the resulting PFN in the vm_pgoff field of the given VMA.
945 *
946 * NOTE: This conversion carries a risk because the resulting PFN is not a true
947 * page frame number and may not be valid or usable in all contexts.
948 */
949 vma->vm_pgoff = __phys_to_pfn(cache_start + cache_offset);
950
951 ret = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, cache_size,
952 vma->vm_page_prot);
953 if (ret)
954 return -EAGAIN;
955
956 if (rknpu_obj->size == 0)
957 return 0;
958
959 offset = cache_size;
960
961 num_pages = (vm_size - cache_size) / PAGE_SIZE;
962 for (i = 0; i < num_pages; ++i) {
963 ret = vm_insert_page(vma, vma->vm_start + offset,
964 rknpu_obj->pages[i]);
965 if (ret < 0)
966 return ret;
967 offset += PAGE_SIZE;
968 }
969
970 return 0;
971 }
972
rknpu_gem_mmap_buffer(struct rknpu_gem_object * rknpu_obj,struct vm_area_struct * vma)973 static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj,
974 struct vm_area_struct *vma)
975 {
976 struct drm_device *drm = rknpu_obj->base.dev;
977 #if RKNPU_GEM_ALLOC_FROM_PAGES
978 struct rknpu_device *rknpu_dev = drm->dev_private;
979 #endif
980 unsigned long vm_size = 0;
981 int ret = -EINVAL;
982
983 /*
984 * clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
985 * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
986 * the whole buffer.
987 */
988 vma->vm_flags &= ~VM_PFNMAP;
989 vma->vm_pgoff = 0;
990
991 vm_size = vma->vm_end - vma->vm_start;
992
993 /* check if user-requested size is valid. */
994 if (vm_size > rknpu_obj->size)
995 return -EINVAL;
996
997 if (rknpu_obj->sram_size > 0)
998 return rknpu_gem_mmap_cache(rknpu_obj, vma, RKNPU_CACHE_SRAM);
999 else if (rknpu_obj->nbuf_size > 0)
1000 return rknpu_gem_mmap_cache(rknpu_obj, vma, RKNPU_CACHE_NBUF);
1001
1002 #if RKNPU_GEM_ALLOC_FROM_PAGES
1003 if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
1004 rknpu_dev->iommu_en) {
1005 return rknpu_gem_mmap_pages(rknpu_obj, vma);
1006 }
1007 #endif
1008
1009 ret = dma_mmap_attrs(drm->dev, vma, rknpu_obj->cookie,
1010 rknpu_obj->dma_addr, rknpu_obj->size,
1011 rknpu_obj->dma_attrs);
1012 if (ret < 0) {
1013 LOG_DEV_ERROR(drm->dev, "failed to mmap, ret: %d\n", ret);
1014 return ret;
1015 }
1016
1017 return 0;
1018 }
1019
rknpu_gem_free_object(struct drm_gem_object * obj)1020 void rknpu_gem_free_object(struct drm_gem_object *obj)
1021 {
1022 rknpu_gem_object_destroy(to_rknpu_obj(obj));
1023 }
1024
rknpu_gem_dumb_create(struct drm_file * file_priv,struct drm_device * drm,struct drm_mode_create_dumb * args)1025 int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm,
1026 struct drm_mode_create_dumb *args)
1027 {
1028 struct rknpu_device *rknpu_dev = drm->dev_private;
1029 struct rknpu_gem_object *rknpu_obj = NULL;
1030 unsigned int flags = 0;
1031 int ret = -EINVAL;
1032
1033 /*
1034 * allocate memory to be used for framebuffer.
1035 * - this callback would be called by user application
1036 * with DRM_IOCTL_MODE_CREATE_DUMB command.
1037 */
1038 args->pitch = args->width * ((args->bpp + 7) / 8);
1039 args->size = args->pitch * args->height;
1040
1041 if (rknpu_dev->iommu_en)
1042 flags = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE;
1043 else
1044 flags = RKNPU_MEM_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE;
1045
1046 rknpu_obj = rknpu_gem_object_create(drm, flags, args->size, 0);
1047 if (IS_ERR(rknpu_obj)) {
1048 LOG_DEV_ERROR(drm->dev, "gem object allocate failed.\n");
1049 return PTR_ERR(rknpu_obj);
1050 }
1051
1052 ret = rknpu_gem_handle_create(&rknpu_obj->base, file_priv,
1053 &args->handle);
1054 if (ret) {
1055 rknpu_gem_object_destroy(rknpu_obj);
1056 return ret;
1057 }
1058
1059 return 0;
1060 }
1061
1062 #if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
rknpu_gem_dumb_map_offset(struct drm_file * file_priv,struct drm_device * drm,uint32_t handle,uint64_t * offset)1063 int rknpu_gem_dumb_map_offset(struct drm_file *file_priv,
1064 struct drm_device *drm, uint32_t handle,
1065 uint64_t *offset)
1066 {
1067 struct rknpu_gem_object *rknpu_obj = NULL;
1068 struct drm_gem_object *obj = NULL;
1069 int ret = -EINVAL;
1070
1071 rknpu_obj = rknpu_gem_object_find(file_priv, handle);
1072 if (!rknpu_obj)
1073 return 0;
1074
1075 /* Don't allow imported objects to be mapped */
1076 obj = &rknpu_obj->base;
1077 if (obj->import_attach)
1078 return -EINVAL;
1079
1080 ret = drm_gem_create_mmap_offset(obj);
1081 if (ret)
1082 return ret;
1083
1084 *offset = drm_vma_node_offset_addr(&obj->vma_node);
1085
1086 return 0;
1087 }
1088 #endif
1089
1090 #if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
rknpu_gem_fault(struct vm_fault * vmf)1091 vm_fault_t rknpu_gem_fault(struct vm_fault *vmf)
1092 {
1093 struct vm_area_struct *vma = vmf->vma;
1094 struct drm_gem_object *obj = vma->vm_private_data;
1095 struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1096 struct drm_device *drm = rknpu_obj->base.dev;
1097 unsigned long pfn = 0;
1098 pgoff_t page_offset = 0;
1099
1100 page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
1101
1102 if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
1103 LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
1104 return VM_FAULT_SIGBUS;
1105 }
1106
1107 pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
1108 return vmf_insert_mixed(vma, vmf->address,
1109 __pfn_to_pfn_t(pfn, PFN_DEV));
1110 }
1111 #elif KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE
rknpu_gem_fault(struct vm_fault * vmf)1112 int rknpu_gem_fault(struct vm_fault *vmf)
1113 {
1114 struct vm_area_struct *vma = vmf->vma;
1115 struct drm_gem_object *obj = vma->vm_private_data;
1116 struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1117 struct drm_device *drm = rknpu_obj->base.dev;
1118 unsigned long pfn = 0;
1119 pgoff_t page_offset = 0;
1120 int ret = -EINVAL;
1121
1122 page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
1123
1124 if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
1125 LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
1126 ret = -EINVAL;
1127 goto out;
1128 }
1129
1130 pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
1131 ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
1132
1133 out:
1134 switch (ret) {
1135 case 0:
1136 case -ERESTARTSYS:
1137 case -EINTR:
1138 return VM_FAULT_NOPAGE;
1139 case -ENOMEM:
1140 return VM_FAULT_OOM;
1141 default:
1142 return VM_FAULT_SIGBUS;
1143 }
1144 }
1145 #else
rknpu_gem_fault(struct vm_area_struct * vma,struct vm_fault * vmf)1146 int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1147 {
1148 struct drm_gem_object *obj = vma->vm_private_data;
1149 struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1150 struct drm_device *drm = rknpu_obj->base.dev;
1151 unsigned long pfn = 0;
1152 pgoff_t page_offset = 0;
1153 int ret = -EINVAL;
1154
1155 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1156 PAGE_SHIFT;
1157
1158 if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
1159 LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
1160 ret = -EINVAL;
1161 goto out;
1162 }
1163
1164 pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
1165 ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
1166 __pfn_to_pfn_t(pfn, PFN_DEV));
1167
1168 out:
1169 switch (ret) {
1170 case 0:
1171 case -ERESTARTSYS:
1172 case -EINTR:
1173 return VM_FAULT_NOPAGE;
1174 case -ENOMEM:
1175 return VM_FAULT_OOM;
1176 default:
1177 return VM_FAULT_SIGBUS;
1178 }
1179 }
1180 #endif
1181
rknpu_gem_mmap_obj(struct drm_gem_object * obj,struct vm_area_struct * vma)1182 int rknpu_gem_mmap_obj(struct drm_gem_object *obj, struct vm_area_struct *vma)
1183 {
1184 struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1185 int ret = -EINVAL;
1186
1187 LOG_DEBUG("flags: %#x\n", rknpu_obj->flags);
1188
1189 /* non-cacheable as default. */
1190 if (rknpu_obj->flags & RKNPU_MEM_CACHEABLE) {
1191 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
1192 } else if (rknpu_obj->flags & RKNPU_MEM_WRITE_COMBINE) {
1193 vma->vm_page_prot =
1194 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1195 } else {
1196 vma->vm_page_prot =
1197 pgprot_noncached(vm_get_page_prot(vma->vm_flags));
1198 }
1199
1200 ret = rknpu_gem_mmap_buffer(rknpu_obj, vma);
1201 if (ret)
1202 goto err_close_vm;
1203
1204 return 0;
1205
1206 err_close_vm:
1207 drm_gem_vm_close(vma);
1208
1209 return ret;
1210 }
1211
rknpu_gem_mmap(struct file * filp,struct vm_area_struct * vma)1212 int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma)
1213 {
1214 struct drm_gem_object *obj = NULL;
1215 int ret = -EINVAL;
1216
1217 /* set vm_area_struct. */
1218 ret = drm_gem_mmap(filp, vma);
1219 if (ret < 0) {
1220 LOG_ERROR("failed to mmap, ret: %d\n", ret);
1221 return ret;
1222 }
1223
1224 obj = vma->vm_private_data;
1225
1226 if (obj->import_attach)
1227 return dma_buf_mmap(obj->dma_buf, vma, 0);
1228
1229 return rknpu_gem_mmap_obj(obj, vma);
1230 }
1231
1232 /* low-level interface prime helpers */
1233 #if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
rknpu_gem_prime_import(struct drm_device * dev,struct dma_buf * dma_buf)1234 struct drm_gem_object *rknpu_gem_prime_import(struct drm_device *dev,
1235 struct dma_buf *dma_buf)
1236 {
1237 return drm_gem_prime_import_dev(dev, dma_buf, dev->dev);
1238 }
1239 #endif
1240
rknpu_gem_prime_get_sg_table(struct drm_gem_object * obj)1241 struct sg_table *rknpu_gem_prime_get_sg_table(struct drm_gem_object *obj)
1242 {
1243 struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1244 int npages = 0;
1245
1246 npages = rknpu_obj->size >> PAGE_SHIFT;
1247
1248 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1249 return drm_prime_pages_to_sg(obj->dev, rknpu_obj->pages, npages);
1250 #else
1251 return drm_prime_pages_to_sg(rknpu_obj->pages, npages);
1252 #endif
1253 }
1254
1255 struct drm_gem_object *
rknpu_gem_prime_import_sg_table(struct drm_device * dev,struct dma_buf_attachment * attach,struct sg_table * sgt)1256 rknpu_gem_prime_import_sg_table(struct drm_device *dev,
1257 struct dma_buf_attachment *attach,
1258 struct sg_table *sgt)
1259 {
1260 struct rknpu_gem_object *rknpu_obj = NULL;
1261 int npages = 0;
1262 int ret = -EINVAL;
1263
1264 rknpu_obj = rknpu_gem_init(dev, attach->dmabuf->size);
1265 if (IS_ERR(rknpu_obj)) {
1266 ret = PTR_ERR(rknpu_obj);
1267 return ERR_PTR(ret);
1268 }
1269
1270 rknpu_obj->dma_addr = sg_dma_address(sgt->sgl);
1271
1272 npages = rknpu_obj->size >> PAGE_SHIFT;
1273 rknpu_obj->pages = rknpu_gem_alloc_page(npages);
1274 if (!rknpu_obj->pages) {
1275 ret = -ENOMEM;
1276 goto err;
1277 }
1278
1279 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
1280 ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
1281 npages);
1282 #else
1283 ret = drm_prime_sg_to_page_array(sgt, rknpu_obj->pages, npages);
1284 #endif
1285 if (ret < 0)
1286 goto err_free_large;
1287
1288 rknpu_obj->sgt = sgt;
1289
1290 if (sgt->nents == 1) {
1291 /* always physically continuous memory if sgt->nents is 1. */
1292 rknpu_obj->flags |= RKNPU_MEM_CONTIGUOUS;
1293 } else {
1294 /*
1295 * this case could be CONTIG or NONCONTIG type but for now
1296 * sets NONCONTIG.
1297 * TODO. we have to find a way that exporter can notify
1298 * the type of its own buffer to importer.
1299 */
1300 rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS;
1301 }
1302
1303 return &rknpu_obj->base;
1304
1305 err_free_large:
1306 rknpu_gem_free_page(rknpu_obj->pages);
1307 err:
1308 rknpu_gem_release(rknpu_obj);
1309 return ERR_PTR(ret);
1310 }
1311
1312 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
rknpu_gem_prime_vmap(struct drm_gem_object * obj)1313 void *rknpu_gem_prime_vmap(struct drm_gem_object *obj)
1314 {
1315 struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1316
1317 if (!rknpu_obj->pages)
1318 return NULL;
1319
1320 return vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
1321 PAGE_KERNEL);
1322 }
1323
rknpu_gem_prime_vunmap(struct drm_gem_object * obj,void * vaddr)1324 void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
1325 {
1326 vunmap(vaddr);
1327 }
1328 #else
rknpu_gem_prime_vmap(struct drm_gem_object * obj,struct iosys_map * map)1329 int rknpu_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
1330 {
1331 struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1332
1333 if (!rknpu_obj->pages)
1334 return -EINVAL;
1335
1336 map->vaddr = vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
1337 PAGE_KERNEL);
1338
1339 return 0;
1340 }
1341
rknpu_gem_prime_vunmap(struct drm_gem_object * obj,struct iosys_map * map)1342 void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
1343 {
1344 vunmap(map->vaddr);
1345 }
1346 #endif
1347
rknpu_gem_prime_mmap(struct drm_gem_object * obj,struct vm_area_struct * vma)1348 int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
1349 {
1350 int ret = -EINVAL;
1351
1352 ret = drm_gem_mmap_obj(obj, obj->size, vma);
1353 if (ret < 0)
1354 return ret;
1355
1356 return rknpu_gem_mmap_obj(obj, vma);
1357 }
1358
rknpu_cache_sync(struct rknpu_gem_object * rknpu_obj,unsigned long * length,unsigned long * offset,enum rknpu_cache_type cache_type)1359 static int rknpu_cache_sync(struct rknpu_gem_object *rknpu_obj,
1360 unsigned long *length, unsigned long *offset,
1361 enum rknpu_cache_type cache_type)
1362 {
1363 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
1364 struct drm_gem_object *obj = &rknpu_obj->base;
1365 struct rknpu_device *rknpu_dev = obj->dev->dev_private;
1366 void __iomem *cache_base_io = NULL;
1367 unsigned long cache_offset = 0;
1368 unsigned long cache_size = 0;
1369
1370 switch (cache_type) {
1371 case RKNPU_CACHE_SRAM:
1372 cache_base_io = rknpu_dev->sram_base_io;
1373 cache_offset = rknpu_obj->sram_obj->range_start *
1374 rknpu_dev->sram_mm->chunk_size;
1375 cache_size = rknpu_obj->sram_size;
1376 break;
1377 case RKNPU_CACHE_NBUF:
1378 cache_base_io = rknpu_dev->nbuf_base_io;
1379 cache_offset = 0;
1380 cache_size = rknpu_obj->nbuf_size;
1381 break;
1382 default:
1383 LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
1384 return -EINVAL;
1385 }
1386
1387 if ((*offset + *length) <= cache_size) {
1388 __dma_map_area(cache_base_io + *offset + cache_offset, *length,
1389 DMA_TO_DEVICE);
1390 __dma_unmap_area(cache_base_io + *offset + cache_offset,
1391 *length, DMA_FROM_DEVICE);
1392 *length = 0;
1393 *offset = 0;
1394 } else if (*offset >= cache_size) {
1395 *offset -= cache_size;
1396 } else {
1397 unsigned long cache_length = cache_size - *offset;
1398
1399 __dma_map_area(cache_base_io + *offset + cache_offset,
1400 cache_length, DMA_TO_DEVICE);
1401 __dma_unmap_area(cache_base_io + *offset + cache_offset,
1402 cache_length, DMA_FROM_DEVICE);
1403 *length -= cache_length;
1404 *offset = 0;
1405 }
1406 #endif
1407
1408 return 0;
1409 }
1410
rknpu_gem_sync_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1411 int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
1412 struct drm_file *file_priv)
1413 {
1414 struct rknpu_gem_object *rknpu_obj = NULL;
1415 struct rknpu_mem_sync *args = data;
1416 struct scatterlist *sg;
1417 unsigned long length, offset = 0;
1418 unsigned long sg_left, size = 0;
1419 unsigned long len = 0;
1420 int i;
1421
1422 rknpu_obj = (struct rknpu_gem_object *)(uintptr_t)args->obj_addr;
1423 if (!rknpu_obj)
1424 return -EINVAL;
1425
1426 if (!(rknpu_obj->flags & RKNPU_MEM_CACHEABLE))
1427 return -EINVAL;
1428
1429 if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS)) {
1430 if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) {
1431 dma_sync_single_range_for_device(
1432 dev->dev, rknpu_obj->dma_addr, args->offset,
1433 args->size, DMA_TO_DEVICE);
1434 }
1435 if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
1436 dma_sync_single_range_for_cpu(dev->dev,
1437 rknpu_obj->dma_addr,
1438 args->offset, args->size,
1439 DMA_FROM_DEVICE);
1440 }
1441 } else {
1442 length = args->size;
1443 offset = args->offset;
1444
1445 if (IS_ENABLED(CONFIG_NO_GKI) &&
1446 IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
1447 rknpu_obj->sram_size > 0) {
1448 rknpu_cache_sync(rknpu_obj, &length, &offset,
1449 RKNPU_CACHE_SRAM);
1450 } else if (IS_ENABLED(CONFIG_NO_GKI) &&
1451 rknpu_obj->nbuf_size > 0) {
1452 rknpu_cache_sync(rknpu_obj, &length, &offset,
1453 RKNPU_CACHE_NBUF);
1454 }
1455
1456 for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents,
1457 i) {
1458 if (length == 0)
1459 break;
1460
1461 len += sg->length;
1462 if (len <= offset)
1463 continue;
1464
1465 sg_left = len - offset;
1466 size = (length < sg_left) ? length : sg_left;
1467
1468 if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) {
1469 dma_sync_sg_for_device(dev->dev, sg, 1,
1470 DMA_TO_DEVICE);
1471 }
1472
1473 if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
1474 dma_sync_sg_for_cpu(dev->dev, sg, 1,
1475 DMA_FROM_DEVICE);
1476 }
1477
1478 offset += size;
1479 length -= size;
1480 }
1481 }
1482
1483 return 0;
1484 }
1485