1 /*
2 * Copyright (C) 2012-2017 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10
11 #include <linux/fs.h> /* file system operations */
12 #include <linux/uaccess.h> /* user space access */
13 #include <linux/dma-buf.h>
14 #include <linux/scatterlist.h>
15 #include <linux/rbtree.h>
16 #include <linux/platform_device.h>
17 #include <linux/wait.h>
18 #include <linux/sched.h>
19 #include <linux/mutex.h>
20
21 #include "mali_ukk.h"
22 #include "mali_osk.h"
23 #include "mali_kernel_common.h"
24 #include "mali_session.h"
25 #include "mali_kernel_linux.h"
26
27 #include "mali_memory.h"
28 #include "mali_memory_dma_buf.h"
29 #include "mali_memory_virtual.h"
30 #include "mali_pp_job.h"
31
32 /*
33 * Map DMA buf attachment \a mem into \a session at virtual address \a virt.
34 */
mali_dma_buf_map(mali_mem_backend * mem_backend)35 static int mali_dma_buf_map(mali_mem_backend *mem_backend)
36 {
37 mali_mem_allocation *alloc;
38 struct mali_dma_buf_attachment *mem;
39 struct mali_session_data *session;
40 struct mali_page_directory *pagedir;
41 _mali_osk_errcode_t err;
42 struct scatterlist *sg;
43 u32 virt, flags;
44 int i;
45
46 MALI_DEBUG_ASSERT_POINTER(mem_backend);
47
48 alloc = mem_backend->mali_allocation;
49 MALI_DEBUG_ASSERT_POINTER(alloc);
50
51 mem = mem_backend->dma_buf.attachment;
52 MALI_DEBUG_ASSERT_POINTER(mem);
53
54 session = alloc->session;
55 MALI_DEBUG_ASSERT_POINTER(session);
56 MALI_DEBUG_ASSERT(mem->session == session);
57
58 virt = alloc->mali_vma_node.vm_node.start;
59 flags = alloc->flags;
60
61 mali_session_memory_lock(session);
62 mem->map_ref++;
63
64 MALI_DEBUG_PRINT(5, ("Mali DMA-buf: map attachment %p, new map_ref = %d\n", mem, mem->map_ref));
65
66 if (1 == mem->map_ref) {
67
68 /* First reference taken, so we need to map the dma buf */
69 MALI_DEBUG_ASSERT(!mem->is_mapped);
70
71 mem->sgt = dma_buf_map_attachment(mem->attachment, DMA_BIDIRECTIONAL);
72 if (IS_ERR_OR_NULL(mem->sgt)) {
73 MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf attachment\n"));
74 mem->map_ref--;
75 mali_session_memory_unlock(session);
76 return -EFAULT;
77 }
78
79 err = mali_mem_mali_map_prepare(alloc);
80 if (_MALI_OSK_ERR_OK != err) {
81 MALI_DEBUG_PRINT(1, ("Mapping of DMA memory failed\n"));
82 mem->map_ref--;
83 mali_session_memory_unlock(session);
84 return -ENOMEM;
85 }
86
87 pagedir = mali_session_get_page_directory(session);
88 MALI_DEBUG_ASSERT_POINTER(pagedir);
89
90 for_each_sg(mem->sgt->sgl, sg, mem->sgt->nents, i) {
91 u32 size = sg_dma_len(sg);
92 dma_addr_t phys = sg_dma_address(sg);
93
94 /* sg must be page aligned. */
95 MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE);
96 MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF));
97
98 mali_mmu_pagedir_update(pagedir, virt, phys, size, MALI_MMU_FLAGS_DEFAULT);
99
100 virt += size;
101 }
102
103 if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) {
104 u32 guard_phys;
105 MALI_DEBUG_PRINT(7, ("Mapping in extra guard page\n"));
106
107 guard_phys = sg_dma_address(mem->sgt->sgl);
108 mali_mmu_pagedir_update(pagedir, virt, guard_phys, MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT);
109 }
110
111 mem->is_mapped = MALI_TRUE;
112 mali_session_memory_unlock(session);
113 /* Wake up any thread waiting for buffer to become mapped */
114 wake_up_all(&mem->wait_queue);
115 } else {
116 MALI_DEBUG_ASSERT(mem->is_mapped);
117 mali_session_memory_unlock(session);
118 }
119
120 return 0;
121 }
122
mali_dma_buf_unmap(mali_mem_allocation * alloc,struct mali_dma_buf_attachment * mem)123 static void mali_dma_buf_unmap(mali_mem_allocation *alloc, struct mali_dma_buf_attachment *mem)
124 {
125 MALI_DEBUG_ASSERT_POINTER(alloc);
126 MALI_DEBUG_ASSERT_POINTER(mem);
127 MALI_DEBUG_ASSERT_POINTER(mem->attachment);
128 MALI_DEBUG_ASSERT_POINTER(mem->buf);
129 MALI_DEBUG_ASSERT_POINTER(alloc->session);
130
131 mali_session_memory_lock(alloc->session);
132 mem->map_ref--;
133
134 MALI_DEBUG_PRINT(5, ("Mali DMA-buf: unmap attachment %p, new map_ref = %d\n", mem, mem->map_ref));
135
136 if (0 == mem->map_ref) {
137 dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL);
138 if (MALI_TRUE == mem->is_mapped) {
139 mali_mem_mali_map_free(alloc->session, alloc->psize, alloc->mali_vma_node.vm_node.start,
140 alloc->flags);
141 }
142 mem->is_mapped = MALI_FALSE;
143 }
144 mali_session_memory_unlock(alloc->session);
145 /* Wake up any thread waiting for buffer to become unmapped */
146 wake_up_all(&mem->wait_queue);
147 }
148
149 #if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
mali_dma_buf_map_job(struct mali_pp_job * job)150 int mali_dma_buf_map_job(struct mali_pp_job *job)
151 {
152 struct mali_dma_buf_attachment *mem;
153 _mali_osk_errcode_t err;
154 int i;
155 int ret = 0;
156 u32 num_memory_cookies;
157 struct mali_session_data *session;
158 struct mali_vma_node *mali_vma_node = NULL;
159 mali_mem_allocation *mali_alloc = NULL;
160 mali_mem_backend *mem_bkend = NULL;
161
162 MALI_DEBUG_ASSERT_POINTER(job);
163
164 num_memory_cookies = mali_pp_job_num_memory_cookies(job);
165
166 session = mali_pp_job_get_session(job);
167
168 MALI_DEBUG_ASSERT_POINTER(session);
169
170 for (i = 0; i < num_memory_cookies; i++) {
171 u32 mali_addr = mali_pp_job_get_memory_cookie(job, i);
172 mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
173 MALI_DEBUG_ASSERT(NULL != mali_vma_node);
174 mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
175 MALI_DEBUG_ASSERT(NULL != mali_alloc);
176 if (MALI_MEM_DMA_BUF != mali_alloc->type) {
177 continue;
178 }
179
180 /* Get backend memory & Map on CPU */
181 mutex_lock(&mali_idr_mutex);
182 mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
183 mutex_unlock(&mali_idr_mutex);
184 MALI_DEBUG_ASSERT(NULL != mem_bkend);
185
186 mem = mem_bkend->dma_buf.attachment;
187
188 MALI_DEBUG_ASSERT_POINTER(mem);
189 MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job));
190
191 err = mali_dma_buf_map(mem_bkend);
192 if (0 != err) {
193 MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to map dma-buf for mali address %x\n", mali_addr));
194 ret = -EFAULT;
195 continue;
196 }
197 }
198 return ret;
199 }
200
mali_dma_buf_unmap_job(struct mali_pp_job * job)201 void mali_dma_buf_unmap_job(struct mali_pp_job *job)
202 {
203 struct mali_dma_buf_attachment *mem;
204 int i;
205 u32 num_memory_cookies;
206 struct mali_session_data *session;
207 struct mali_vma_node *mali_vma_node = NULL;
208 mali_mem_allocation *mali_alloc = NULL;
209 mali_mem_backend *mem_bkend = NULL;
210
211 MALI_DEBUG_ASSERT_POINTER(job);
212
213 num_memory_cookies = mali_pp_job_num_memory_cookies(job);
214
215 session = mali_pp_job_get_session(job);
216
217 MALI_DEBUG_ASSERT_POINTER(session);
218
219 for (i = 0; i < num_memory_cookies; i++) {
220 u32 mali_addr = mali_pp_job_get_memory_cookie(job, i);
221 mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
222 MALI_DEBUG_ASSERT(NULL != mali_vma_node);
223 mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
224 MALI_DEBUG_ASSERT(NULL != mali_alloc);
225 if (MALI_MEM_DMA_BUF != mali_alloc->type) {
226 continue;
227 }
228
229 /* Get backend memory & Map on CPU */
230 mutex_lock(&mali_idr_mutex);
231 mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
232 mutex_unlock(&mali_idr_mutex);
233 MALI_DEBUG_ASSERT(NULL != mem_bkend);
234
235 mem = mem_bkend->dma_buf.attachment;
236
237 MALI_DEBUG_ASSERT_POINTER(mem);
238 MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job));
239 mali_dma_buf_unmap(mem_bkend->mali_allocation, mem);
240 }
241 }
242 #endif /* !CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH */
243
mali_dma_buf_get_size(struct mali_session_data * session,_mali_uk_dma_buf_get_size_s __user * user_arg)244 int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg)
245 {
246 _mali_uk_dma_buf_get_size_s args;
247 int fd;
248 struct dma_buf *buf;
249
250 /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */
251 if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s))) {
252 return -EFAULT;
253 }
254
255 /* Do DMA-BUF stuff */
256 fd = args.mem_fd;
257
258 buf = dma_buf_get(fd);
259 if (IS_ERR_OR_NULL(buf)) {
260 MALI_DEBUG_PRINT_ERROR(("Failed to get dma-buf from fd: %d\n", fd));
261 return PTR_ERR_OR_ZERO(buf);
262 }
263
264 if (0 != put_user(buf->size, &user_arg->size)) {
265 dma_buf_put(buf);
266 return -EFAULT;
267 }
268
269 dma_buf_put(buf);
270
271 return 0;
272 }
273
mali_mem_bind_dma_buf(mali_mem_allocation * alloc,mali_mem_backend * mem_backend,int fd,u32 flags)274 _mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc,
275 mali_mem_backend *mem_backend,
276 int fd, u32 flags)
277 {
278 struct dma_buf *buf;
279 struct mali_dma_buf_attachment *dma_mem;
280 struct mali_session_data *session = alloc->session;
281
282 MALI_DEBUG_ASSERT_POINTER(session);
283 MALI_DEBUG_ASSERT_POINTER(mem_backend);
284 MALI_DEBUG_ASSERT_POINTER(alloc);
285
286 /* get dma buffer */
287 buf = dma_buf_get(fd);
288 if (IS_ERR_OR_NULL(buf)) {
289 return _MALI_OSK_ERR_FAULT;
290 }
291
292 /* Currently, mapping of the full buffer are supported. */
293 if (alloc->psize != buf->size) {
294 goto failed_alloc_mem;
295 }
296
297 dma_mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment));
298 if (NULL == dma_mem) {
299 goto failed_alloc_mem;
300 }
301
302 dma_mem->buf = buf;
303 dma_mem->session = session;
304 dma_mem->map_ref = 0;
305 init_waitqueue_head(&dma_mem->wait_queue);
306
307 dma_mem->attachment = dma_buf_attach(dma_mem->buf, &mali_platform_device->dev);
308 if (NULL == dma_mem->attachment) {
309 goto failed_dma_attach;
310 }
311
312 mem_backend->dma_buf.attachment = dma_mem;
313
314 alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP;
315 if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) {
316 alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE;
317 }
318
319
320 #if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
321 /* Map memory into session's Mali virtual address space. */
322 if (0 != mali_dma_buf_map(mem_backend)) {
323 goto Failed_dma_map;
324 }
325 #endif
326
327 return _MALI_OSK_ERR_OK;
328
329 #if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
330 Failed_dma_map:
331 mali_dma_buf_unmap(alloc, dma_mem);
332 #endif
333 /* Wait for buffer to become unmapped */
334 wait_event(dma_mem->wait_queue, !dma_mem->is_mapped);
335 MALI_DEBUG_ASSERT(!dma_mem->is_mapped);
336 dma_buf_detach(dma_mem->buf, dma_mem->attachment);
337 failed_dma_attach:
338 _mali_osk_free(dma_mem);
339 failed_alloc_mem:
340 dma_buf_put(buf);
341 return _MALI_OSK_ERR_FAULT;
342 }
343
mali_mem_unbind_dma_buf(mali_mem_backend * mem_backend)344 void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend)
345 {
346 struct mali_dma_buf_attachment *mem;
347 MALI_DEBUG_ASSERT_POINTER(mem_backend);
348 MALI_DEBUG_ASSERT(MALI_MEM_DMA_BUF == mem_backend->type);
349
350 mem = mem_backend->dma_buf.attachment;
351 MALI_DEBUG_ASSERT_POINTER(mem);
352 MALI_DEBUG_ASSERT_POINTER(mem->attachment);
353 MALI_DEBUG_ASSERT_POINTER(mem->buf);
354 MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release attachment %p\n", mem));
355
356 #if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
357 MALI_DEBUG_ASSERT_POINTER(mem_backend->mali_allocation);
358 /* We mapped implicitly on attach, so we need to unmap on release */
359 mali_dma_buf_unmap(mem_backend->mali_allocation, mem);
360 #endif
361 /* Wait for buffer to become unmapped */
362 wait_event(mem->wait_queue, !mem->is_mapped);
363 MALI_DEBUG_ASSERT(!mem->is_mapped);
364
365 dma_buf_detach(mem->buf, mem->attachment);
366 dma_buf_put(mem->buf);
367
368 _mali_osk_free(mem);
369 }
370