1 /*
2 *
3 * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18
19
20 /**
21 * @file mali_kbase_mem_linux.c
22 * Base kernel memory APIs, Linux implementation.
23 */
24
25 #include <linux/compat.h>
26 #include <linux/kernel.h>
27 #include <linux/bug.h>
28 #include <linux/mm.h>
29 #include <linux/mman.h>
30 #include <linux/fs.h>
31 #include <linux/version.h>
32 #include <linux/dma-mapping.h>
33 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \
34 (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
35 #include <linux/dma-attrs.h>
36 #endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */
37 #ifdef CONFIG_DMA_SHARED_BUFFER
38 #include <linux/dma-buf.h>
39 #endif /* defined(CONFIG_DMA_SHARED_BUFFER) */
40 #include <linux/shrinker.h>
41 #include <linux/cache.h>
42
43 #include <mali_kbase.h>
44 #include <mali_kbase_mem_linux.h>
45 #include <mali_kbase_config_defaults.h>
46 #include <mali_kbase_hwaccess_time.h>
47 #include <mali_kbase_tlstream.h>
48
49 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma);
50
51 /**
52 * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation
53 * @kctx: Context the region belongs to
54 * @reg: The GPU region
55 * @new_pages: The number of pages after the shrink
56 * @old_pages: The number of pages before the shrink
57 *
58 * Shrink (or completely remove) all CPU mappings which reference the shrunk
59 * part of the allocation.
60 *
61 * Note: Caller must be holding the processes mmap_lock lock.
62 */
63 static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
64 struct kbase_va_region *reg,
65 u64 new_pages, u64 old_pages);
66
67 /**
68 * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation
69 * @kctx: Context the region belongs to
70 * @reg: The GPU region or NULL if there isn't one
71 * @new_pages: The number of pages after the shrink
72 * @old_pages: The number of pages before the shrink
73 *
74 * Return: 0 on success, negative -errno on error
75 *
76 * Unmap the shrunk pages from the GPU mapping. Note that the size of the region
77 * itself is unmodified as we still need to reserve the VA, only the page tables
78 * will be modified by this function.
79 */
80 static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
81 struct kbase_va_region *reg,
82 u64 new_pages, u64 old_pages);
83
kbase_mem_alloc(struct kbase_context * kctx,u64 va_pages,u64 commit_pages,u64 extent,u64 * flags,u64 * gpu_va)84 struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx,
85 u64 va_pages, u64 commit_pages, u64 extent, u64 *flags,
86 u64 *gpu_va)
87 {
88 int zone;
89 int gpu_pc_bits;
90 int cpu_va_bits;
91 struct kbase_va_region *reg;
92 struct device *dev;
93
94 KBASE_DEBUG_ASSERT(kctx);
95 KBASE_DEBUG_ASSERT(flags);
96 KBASE_DEBUG_ASSERT(gpu_va);
97
98 dev = kctx->kbdev->dev;
99 *gpu_va = 0; /* return 0 on failure */
100
101 gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size;
102 cpu_va_bits = BITS_PER_LONG;
103
104 if (0 == va_pages) {
105 dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!");
106 goto bad_size;
107 }
108
109 if (va_pages > (U64_MAX / PAGE_SIZE))
110 /* 64-bit address range is the max */
111 goto bad_size;
112
113 #if defined(CONFIG_64BIT)
114 if (kbase_ctx_flag(kctx, KCTX_COMPAT))
115 cpu_va_bits = 32;
116 #endif
117
118 if (!kbase_check_alloc_flags(*flags)) {
119 dev_warn(dev,
120 "kbase_mem_alloc called with bad flags (%llx)",
121 (unsigned long long)*flags);
122 goto bad_flags;
123 }
124
125 if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
126 !kbase_device_is_cpu_coherent(kctx->kbdev)) {
127 dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable");
128 goto bad_flags;
129 }
130 if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
131 !kbase_device_is_cpu_coherent(kctx->kbdev)) {
132 /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */
133 *flags &= ~BASE_MEM_COHERENT_SYSTEM;
134 }
135
136 /* Limit GPU executable allocs to GPU PC size */
137 if ((*flags & BASE_MEM_PROT_GPU_EX) &&
138 (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT)))
139 goto bad_ex_size;
140
141 /* find out which VA zone to use */
142 if (*flags & BASE_MEM_SAME_VA)
143 zone = KBASE_REG_ZONE_SAME_VA;
144 else if (*flags & BASE_MEM_PROT_GPU_EX)
145 zone = KBASE_REG_ZONE_EXEC;
146 else
147 zone = KBASE_REG_ZONE_CUSTOM_VA;
148
149 reg = kbase_alloc_free_region(kctx, 0, va_pages, zone);
150 if (!reg) {
151 dev_err(dev, "Failed to allocate free region");
152 goto no_region;
153 }
154
155 if (kbase_update_region_flags(kctx, reg, *flags) != 0)
156 goto invalid_flags;
157
158 if (kbase_reg_prepare_native(reg, kctx) != 0) {
159 dev_err(dev, "Failed to prepare region");
160 goto prepare_failed;
161 }
162
163 if (*flags & BASE_MEM_GROW_ON_GPF)
164 reg->extent = extent;
165 else
166 reg->extent = 0;
167
168 if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) {
169 dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)",
170 (unsigned long long)commit_pages,
171 (unsigned long long)va_pages);
172 goto no_mem;
173 }
174
175 kbase_gpu_vm_lock(kctx);
176
177 /* mmap needed to setup VA? */
178 if (*flags & BASE_MEM_SAME_VA) {
179 unsigned long prot = PROT_NONE;
180 unsigned long va_size = va_pages << PAGE_SHIFT;
181 unsigned long va_map = va_size;
182 unsigned long cookie, cookie_nr;
183 unsigned long cpu_addr;
184
185 /* Bind to a cookie */
186 if (!kctx->cookies) {
187 dev_err(dev, "No cookies available for allocation!");
188 kbase_gpu_vm_unlock(kctx);
189 goto no_cookie;
190 }
191 /* return a cookie */
192 cookie_nr = __ffs(kctx->cookies);
193 kctx->cookies &= ~(1UL << cookie_nr);
194 BUG_ON(kctx->pending_regions[cookie_nr]);
195 kctx->pending_regions[cookie_nr] = reg;
196
197 kbase_gpu_vm_unlock(kctx);
198
199 /* relocate to correct base */
200 cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE);
201 cookie <<= PAGE_SHIFT;
202
203 /*
204 * 10.1-10.4 UKU userland relies on the kernel to call mmap.
205 * For all other versions we can just return the cookie
206 */
207 if (kctx->api_version < KBASE_API_VERSION(10, 1) ||
208 kctx->api_version > KBASE_API_VERSION(10, 4)) {
209 *gpu_va = (u64) cookie;
210 return reg;
211 }
212 if (*flags & BASE_MEM_PROT_CPU_RD)
213 prot |= PROT_READ;
214 if (*flags & BASE_MEM_PROT_CPU_WR)
215 prot |= PROT_WRITE;
216
217 cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot,
218 MAP_SHARED, cookie);
219
220 if (IS_ERR_VALUE(cpu_addr)) {
221 kbase_gpu_vm_lock(kctx);
222 kctx->pending_regions[cookie_nr] = NULL;
223 kctx->cookies |= (1UL << cookie_nr);
224 kbase_gpu_vm_unlock(kctx);
225 goto no_mmap;
226 }
227
228 *gpu_va = (u64) cpu_addr;
229 } else /* we control the VA */ {
230 if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) {
231 dev_warn(dev, "Failed to map memory on GPU");
232 kbase_gpu_vm_unlock(kctx);
233 goto no_mmap;
234 }
235 /* return real GPU VA */
236 *gpu_va = reg->start_pfn << PAGE_SHIFT;
237
238 kbase_gpu_vm_unlock(kctx);
239 }
240
241 return reg;
242
243 no_mmap:
244 no_cookie:
245 no_mem:
246 kbase_mem_phy_alloc_put(reg->cpu_alloc);
247 kbase_mem_phy_alloc_put(reg->gpu_alloc);
248 invalid_flags:
249 prepare_failed:
250 kfree(reg);
251 no_region:
252 bad_ex_size:
253 bad_flags:
254 bad_size:
255 return NULL;
256 }
257 KBASE_EXPORT_TEST_API(kbase_mem_alloc);
258
kbase_mem_query(struct kbase_context * kctx,u64 gpu_addr,int query,u64 * const out)259 int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out)
260 {
261 struct kbase_va_region *reg;
262 int ret = -EINVAL;
263
264 KBASE_DEBUG_ASSERT(kctx);
265 KBASE_DEBUG_ASSERT(out);
266
267 if (gpu_addr & ~PAGE_MASK) {
268 dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid");
269 return -EINVAL;
270 }
271
272 kbase_gpu_vm_lock(kctx);
273
274 /* Validate the region */
275 reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
276 if (!reg || (reg->flags & KBASE_REG_FREE))
277 goto out_unlock;
278
279 switch (query) {
280 case KBASE_MEM_QUERY_COMMIT_SIZE:
281 if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) {
282 *out = kbase_reg_current_backed_size(reg);
283 } else {
284 size_t i;
285 struct kbase_aliased *aliased;
286 *out = 0;
287 aliased = reg->cpu_alloc->imported.alias.aliased;
288 for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++)
289 *out += aliased[i].length;
290 }
291 break;
292 case KBASE_MEM_QUERY_VA_SIZE:
293 *out = reg->nr_pages;
294 break;
295 case KBASE_MEM_QUERY_FLAGS:
296 {
297 *out = 0;
298 if (KBASE_REG_CPU_WR & reg->flags)
299 *out |= BASE_MEM_PROT_CPU_WR;
300 if (KBASE_REG_CPU_RD & reg->flags)
301 *out |= BASE_MEM_PROT_CPU_RD;
302 if (KBASE_REG_CPU_CACHED & reg->flags)
303 *out |= BASE_MEM_CACHED_CPU;
304 if (KBASE_REG_GPU_WR & reg->flags)
305 *out |= BASE_MEM_PROT_GPU_WR;
306 if (KBASE_REG_GPU_RD & reg->flags)
307 *out |= BASE_MEM_PROT_GPU_RD;
308 if (!(KBASE_REG_GPU_NX & reg->flags))
309 *out |= BASE_MEM_PROT_GPU_EX;
310 if (KBASE_REG_SHARE_BOTH & reg->flags)
311 *out |= BASE_MEM_COHERENT_SYSTEM;
312 if (KBASE_REG_SHARE_IN & reg->flags)
313 *out |= BASE_MEM_COHERENT_LOCAL;
314 break;
315 }
316 default:
317 *out = 0;
318 goto out_unlock;
319 }
320
321 ret = 0;
322
323 out_unlock:
324 kbase_gpu_vm_unlock(kctx);
325 return ret;
326 }
327
328 /**
329 * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the
330 * Ephemeral memory eviction list.
331 * @s: Shrinker
332 * @sc: Shrinker control
333 *
334 * Return: Number of pages which can be freed.
335 */
336 static
kbase_mem_evictable_reclaim_count_objects(struct shrinker * s,struct shrink_control * sc)337 unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s,
338 struct shrink_control *sc)
339 {
340 struct kbase_context *kctx;
341 struct kbase_mem_phy_alloc *alloc;
342 unsigned long pages = 0;
343
344 kctx = container_of(s, struct kbase_context, reclaim);
345
346 mutex_lock(&kctx->jit_evict_lock);
347
348 list_for_each_entry(alloc, &kctx->evict_list, evict_node)
349 pages += alloc->nents;
350
351 mutex_unlock(&kctx->jit_evict_lock);
352 return pages;
353 }
354
355 /**
356 * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction
357 * list for pages and try to reclaim them.
358 * @s: Shrinker
359 * @sc: Shrinker control
360 *
361 * Return: Number of pages freed (can be less then requested) or -1 if the
362 * shrinker failed to free pages in its pool.
363 *
364 * Note:
365 * This function accesses region structures without taking the region lock,
366 * this is required as the OOM killer can call the shrinker after the region
367 * lock has already been held.
368 * This is safe as we can guarantee that a region on the eviction list will
369 * not be freed (kbase_mem_free_region removes the allocation from the list
370 * before destroying it), or modified by other parts of the driver.
371 * The eviction list itself is guarded by the eviction lock and the MMU updates
372 * are protected by their own lock.
373 */
374 static
kbase_mem_evictable_reclaim_scan_objects(struct shrinker * s,struct shrink_control * sc)375 unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s,
376 struct shrink_control *sc)
377 {
378 struct kbase_context *kctx;
379 struct kbase_mem_phy_alloc *alloc;
380 struct kbase_mem_phy_alloc *tmp;
381 unsigned long freed = 0;
382
383 kctx = container_of(s, struct kbase_context, reclaim);
384 mutex_lock(&kctx->jit_evict_lock);
385
386 list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) {
387 int err;
388
389 err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg,
390 0, alloc->nents);
391 if (err != 0) {
392 /*
393 * Failed to remove GPU mapping, tell the shrinker
394 * to stop trying to shrink our slab even though we
395 * have pages in it.
396 */
397 freed = -1;
398 goto out_unlock;
399 }
400
401 /*
402 * Update alloc->evicted before freeing the backing so the
403 * helper can determine that it needs to bypass the accounting
404 * and memory pool.
405 */
406 alloc->evicted = alloc->nents;
407
408 kbase_free_phy_pages_helper(alloc, alloc->evicted);
409 freed += alloc->evicted;
410 list_del_init(&alloc->evict_node);
411
412 /*
413 * Inform the JIT allocator this region has lost backing
414 * as it might need to free the allocation.
415 */
416 kbase_jit_backing_lost(alloc->reg);
417
418 /* Enough pages have been freed so stop now */
419 if (freed > sc->nr_to_scan)
420 break;
421 }
422 out_unlock:
423 mutex_unlock(&kctx->jit_evict_lock);
424
425 return freed;
426 }
427
428 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
kbase_mem_evictable_reclaim_shrink(struct shrinker * s,struct shrink_control * sc)429 static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s,
430 struct shrink_control *sc)
431 {
432 if (sc->nr_to_scan == 0)
433 return kbase_mem_evictable_reclaim_count_objects(s, sc);
434
435 return kbase_mem_evictable_reclaim_scan_objects(s, sc);
436 }
437 #endif
438
kbase_mem_evictable_init(struct kbase_context * kctx)439 int kbase_mem_evictable_init(struct kbase_context *kctx)
440 {
441 INIT_LIST_HEAD(&kctx->evict_list);
442 mutex_init(&kctx->jit_evict_lock);
443
444 /* Register shrinker */
445 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
446 kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink;
447 #else
448 kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects;
449 kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects;
450 #endif
451 kctx->reclaim.seeks = DEFAULT_SEEKS;
452 /* Kernel versions prior to 3.1 :
453 * struct shrinker does not define batch */
454 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0)
455 kctx->reclaim.batch = 0;
456 #endif
457 register_shrinker(&kctx->reclaim);
458 return 0;
459 }
460
kbase_mem_evictable_deinit(struct kbase_context * kctx)461 void kbase_mem_evictable_deinit(struct kbase_context *kctx)
462 {
463 unregister_shrinker(&kctx->reclaim);
464 }
465
466 /**
467 * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable.
468 * @alloc: The physical allocation
469 */
kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc * alloc)470 static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc)
471 {
472 struct kbase_context *kctx = alloc->imported.kctx;
473 int __maybe_unused new_page_count;
474
475 kbase_process_page_usage_dec(kctx, alloc->nents);
476 new_page_count = kbase_atomic_sub_pages(alloc->nents,
477 &kctx->used_pages);
478 kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
479
480 KBASE_TLSTREAM_AUX_PAGESALLOC(
481 (u32)kctx->id,
482 (u64)new_page_count);
483 }
484
485 /**
486 * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable.
487 * @alloc: The physical allocation
488 */
489 static
kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc * alloc)490 void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc)
491 {
492 struct kbase_context *kctx = alloc->imported.kctx;
493 int __maybe_unused new_page_count;
494
495 new_page_count = kbase_atomic_add_pages(alloc->nents,
496 &kctx->used_pages);
497 kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
498
499 /* Increase mm counters so that the allocation is accounted for
500 * against the process and thus is visible to the OOM killer.
501 */
502 kbase_process_page_usage_inc(kctx, alloc->nents);
503
504 KBASE_TLSTREAM_AUX_PAGESALLOC(
505 (u32)kctx->id,
506 (u64)new_page_count);
507 }
508
kbase_mem_evictable_make(struct kbase_mem_phy_alloc * gpu_alloc)509 int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc)
510 {
511 struct kbase_context *kctx = gpu_alloc->imported.kctx;
512
513 lockdep_assert_held(&kctx->reg_lock);
514
515 /* This alloction can't already be on a list. */
516 WARN_ON(!list_empty(&gpu_alloc->evict_node));
517
518 kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg,
519 0, gpu_alloc->nents);
520
521 /*
522 * Add the allocation to the eviction list, after this point the shrink
523 * can reclaim it.
524 */
525 mutex_lock(&kctx->jit_evict_lock);
526 list_add(&gpu_alloc->evict_node, &kctx->evict_list);
527 mutex_unlock(&kctx->jit_evict_lock);
528 kbase_mem_evictable_mark_reclaim(gpu_alloc);
529
530 gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED;
531 return 0;
532 }
533
kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc * gpu_alloc)534 bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc)
535 {
536 struct kbase_context *kctx = gpu_alloc->imported.kctx;
537 int err = 0;
538
539 lockdep_assert_held(&kctx->reg_lock);
540
541 /*
542 * First remove the allocation from the eviction list as it's no
543 * longer eligible for eviction.
544 */
545 list_del_init(&gpu_alloc->evict_node);
546
547 if (gpu_alloc->evicted == 0) {
548 /*
549 * The backing is still present, update the VM stats as it's
550 * in use again.
551 */
552 kbase_mem_evictable_unmark_reclaim(gpu_alloc);
553 } else {
554 /* If the region is still alive ... */
555 if (gpu_alloc->reg) {
556 /* ... allocate replacement backing ... */
557 err = kbase_alloc_phy_pages_helper(gpu_alloc,
558 gpu_alloc->evicted);
559
560 /*
561 * ... and grow the mapping back to its
562 * pre-eviction size.
563 */
564 if (!err)
565 err = kbase_mem_grow_gpu_mapping(kctx,
566 gpu_alloc->reg,
567 gpu_alloc->evicted, 0);
568
569 gpu_alloc->evicted = 0;
570 }
571 }
572
573 /* If the region is still alive remove the DONT_NEED attribute. */
574 if (gpu_alloc->reg)
575 gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED;
576
577 return (err == 0);
578 }
579
kbase_mem_flags_change(struct kbase_context * kctx,u64 gpu_addr,unsigned int flags,unsigned int mask)580 int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask)
581 {
582 struct kbase_va_region *reg;
583 int ret = -EINVAL;
584 unsigned int real_flags = 0;
585 unsigned int prev_flags = 0;
586 bool prev_needed, new_needed;
587
588 KBASE_DEBUG_ASSERT(kctx);
589
590 if (!gpu_addr)
591 return -EINVAL;
592
593 if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE))
594 return -EINVAL;
595
596 /* nuke other bits */
597 flags &= mask;
598
599 /* check for only supported flags */
600 if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE))
601 goto out;
602
603 /* mask covers bits we don't support? */
604 if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE))
605 goto out;
606
607 /* convert flags */
608 if (BASE_MEM_COHERENT_SYSTEM & flags)
609 real_flags |= KBASE_REG_SHARE_BOTH;
610 else if (BASE_MEM_COHERENT_LOCAL & flags)
611 real_flags |= KBASE_REG_SHARE_IN;
612
613 /* now we can lock down the context, and find the region */
614 down_write(¤t->mm->mmap_lock);
615 kbase_gpu_vm_lock(kctx);
616
617 /* Validate the region */
618 reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
619 if (!reg || (reg->flags & KBASE_REG_FREE))
620 goto out_unlock;
621
622 /* Is the region being transitioning between not needed and needed? */
623 prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED;
624 new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED;
625 if (prev_needed != new_needed) {
626 /* Aliased allocations can't be made ephemeral */
627 if (atomic_read(®->cpu_alloc->gpu_mappings) > 1)
628 goto out_unlock;
629
630 if (new_needed) {
631 /* Only native allocations can be marked not needed */
632 if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
633 ret = -EINVAL;
634 goto out_unlock;
635 }
636 ret = kbase_mem_evictable_make(reg->gpu_alloc);
637 if (ret)
638 goto out_unlock;
639 } else {
640 kbase_mem_evictable_unmake(reg->gpu_alloc);
641 }
642 }
643
644 /* limit to imported memory */
645 if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) &&
646 (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM))
647 goto out_unlock;
648
649 /* no change? */
650 if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) {
651 ret = 0;
652 goto out_unlock;
653 }
654
655 /* save for roll back */
656 prev_flags = reg->flags;
657 reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH);
658 reg->flags |= real_flags;
659
660 /* Currently supporting only imported memory */
661 switch (reg->gpu_alloc->type) {
662 #ifdef CONFIG_UMP
663 case KBASE_MEM_TYPE_IMPORTED_UMP:
664 ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_cpu_phy_pages(reg), reg->gpu_alloc->nents, reg->flags);
665 break;
666 #endif
667 #ifdef CONFIG_DMA_SHARED_BUFFER
668 case KBASE_MEM_TYPE_IMPORTED_UMM:
669 /* Future use will use the new flags, existing mapping will NOT be updated
670 * as memory should not be in use by the GPU when updating the flags.
671 */
672 ret = 0;
673 WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count);
674 break;
675 #endif
676 default:
677 break;
678 }
679
680 /* roll back on error, i.e. not UMP */
681 if (ret)
682 reg->flags = prev_flags;
683
684 out_unlock:
685 kbase_gpu_vm_unlock(kctx);
686 up_write(¤t->mm->mmap_lock);
687 out:
688 return ret;
689 }
690
691 #define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS)
692
693 #ifdef CONFIG_UMP
kbase_mem_from_ump(struct kbase_context * kctx,ump_secure_id id,u64 * va_pages,u64 * flags)694 static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags)
695 {
696 struct kbase_va_region *reg;
697 ump_dd_handle umph;
698 u64 block_count;
699 const ump_dd_physical_block_64 *block_array;
700 u64 i, j;
701 int page = 0;
702 ump_alloc_flags ump_flags;
703 ump_alloc_flags cpu_flags;
704 ump_alloc_flags gpu_flags;
705
706 if (*flags & BASE_MEM_SECURE)
707 goto bad_flags;
708
709 umph = ump_dd_from_secure_id(id);
710 if (UMP_DD_INVALID_MEMORY_HANDLE == umph)
711 goto bad_id;
712
713 ump_flags = ump_dd_allocation_flags_get(umph);
714 cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK;
715 gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) &
716 UMP_DEVICE_MASK;
717
718 *va_pages = ump_dd_size_get_64(umph);
719 *va_pages >>= PAGE_SHIFT;
720
721 if (!*va_pages)
722 goto bad_size;
723
724 if (*va_pages > (U64_MAX / PAGE_SIZE))
725 /* 64-bit address range is the max */
726 goto bad_size;
727
728 if (*flags & BASE_MEM_SAME_VA)
729 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
730 else
731 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
732
733 if (!reg)
734 goto no_region;
735
736 /* we've got pages to map now, and support SAME_VA */
737 *flags |= KBASE_MEM_IMPORT_HAVE_PAGES;
738
739 reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP);
740 if (IS_ERR_OR_NULL(reg->gpu_alloc))
741 goto no_alloc_obj;
742
743 reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
744
745 reg->gpu_alloc->imported.ump_handle = umph;
746
747 reg->flags &= ~KBASE_REG_FREE;
748 reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */
749 reg->flags &= ~KBASE_REG_GROWABLE; /* UMP cannot be grown */
750
751 /* Override import flags based on UMP flags */
752 *flags &= ~(BASE_MEM_CACHED_CPU);
753 *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR);
754 *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR);
755
756 if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ==
757 (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) {
758 reg->flags |= KBASE_REG_CPU_CACHED;
759 *flags |= BASE_MEM_CACHED_CPU;
760 }
761
762 if (cpu_flags & UMP_PROT_CPU_WR) {
763 reg->flags |= KBASE_REG_CPU_WR;
764 *flags |= BASE_MEM_PROT_CPU_WR;
765 }
766
767 if (cpu_flags & UMP_PROT_CPU_RD) {
768 reg->flags |= KBASE_REG_CPU_RD;
769 *flags |= BASE_MEM_PROT_CPU_RD;
770 }
771
772 if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ==
773 (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR))
774 reg->flags |= KBASE_REG_GPU_CACHED;
775
776 if (gpu_flags & UMP_PROT_DEVICE_WR) {
777 reg->flags |= KBASE_REG_GPU_WR;
778 *flags |= BASE_MEM_PROT_GPU_WR;
779 }
780
781 if (gpu_flags & UMP_PROT_DEVICE_RD) {
782 reg->flags |= KBASE_REG_GPU_RD;
783 *flags |= BASE_MEM_PROT_GPU_RD;
784 }
785
786 /* ump phys block query */
787 ump_dd_phys_blocks_get_64(umph, &block_count, &block_array);
788
789 for (i = 0; i < block_count; i++) {
790 for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) {
791 reg->gpu_alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT);
792 page++;
793 }
794 }
795 reg->gpu_alloc->nents = *va_pages;
796 reg->extent = 0;
797
798 return reg;
799
800 no_alloc_obj:
801 kfree(reg);
802 no_region:
803 bad_size:
804 ump_dd_release(umph);
805 bad_id:
806 bad_flags:
807 return NULL;
808 }
809 #endif /* CONFIG_UMP */
810
811 #ifdef CONFIG_DMA_SHARED_BUFFER
kbase_mem_from_umm(struct kbase_context * kctx,int fd,u64 * va_pages,u64 * flags,u32 padding)812 static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx,
813 int fd, u64 *va_pages, u64 *flags, u32 padding)
814 {
815 struct kbase_va_region *reg;
816 struct dma_buf *dma_buf;
817 struct dma_buf_attachment *dma_attachment;
818 bool shared_zone = false;
819
820 dma_buf = dma_buf_get(fd);
821 if (IS_ERR_OR_NULL(dma_buf))
822 goto no_buf;
823
824 dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev);
825 if (!dma_attachment)
826 goto no_attachment;
827
828 *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding;
829 if (!*va_pages)
830 goto bad_size;
831
832 if (*va_pages > (U64_MAX / PAGE_SIZE))
833 /* 64-bit address range is the max */
834 goto bad_size;
835
836 /* ignore SAME_VA */
837 *flags &= ~BASE_MEM_SAME_VA;
838
839 if (*flags & BASE_MEM_IMPORT_SHARED)
840 shared_zone = true;
841
842 #ifdef CONFIG_64BIT
843 if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
844 /*
845 * 64-bit tasks require us to reserve VA on the CPU that we use
846 * on the GPU.
847 */
848 shared_zone = true;
849 }
850 #endif
851
852 if (shared_zone) {
853 *flags |= BASE_MEM_NEED_MMAP;
854 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
855 } else {
856 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
857 }
858
859 if (!reg)
860 goto no_region;
861
862 reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM);
863 if (IS_ERR_OR_NULL(reg->gpu_alloc))
864 goto no_alloc_obj;
865
866 reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
867
868 /* No pages to map yet */
869 reg->gpu_alloc->nents = 0;
870
871 if (kbase_update_region_flags(kctx, reg, *flags) != 0)
872 goto invalid_flags;
873
874 reg->flags &= ~KBASE_REG_FREE;
875 reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */
876 reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */
877 reg->flags |= KBASE_REG_GPU_CACHED;
878
879 if (*flags & BASE_MEM_SECURE)
880 reg->flags |= KBASE_REG_SECURE;
881
882 if (padding)
883 reg->flags |= KBASE_REG_IMPORT_PAD;
884
885 reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM;
886 reg->gpu_alloc->imported.umm.sgt = NULL;
887 reg->gpu_alloc->imported.umm.dma_buf = dma_buf;
888 reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment;
889 reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0;
890 reg->extent = 0;
891
892 return reg;
893
894 invalid_flags:
895 kbase_mem_phy_alloc_put(reg->gpu_alloc);
896 no_alloc_obj:
897 kfree(reg);
898 no_region:
899 bad_size:
900 dma_buf_detach(dma_buf, dma_attachment);
901 no_attachment:
902 dma_buf_put(dma_buf);
903 no_buf:
904 return NULL;
905 }
906 #endif /* CONFIG_DMA_SHARED_BUFFER */
907
kbase_get_cache_line_alignment(struct kbase_context * kctx)908 static u32 kbase_get_cache_line_alignment(struct kbase_context *kctx)
909 {
910 u32 cpu_cache_line_size = cache_line_size();
911 u32 gpu_cache_line_size =
912 (1UL << kctx->kbdev->gpu_props.props.l2_props.log2_line_size);
913
914 return ((cpu_cache_line_size > gpu_cache_line_size) ?
915 cpu_cache_line_size :
916 gpu_cache_line_size);
917 }
918
kbase_mem_from_user_buffer(struct kbase_context * kctx,unsigned long address,unsigned long size,u64 * va_pages,u64 * flags)919 static struct kbase_va_region *kbase_mem_from_user_buffer(
920 struct kbase_context *kctx, unsigned long address,
921 unsigned long size, u64 *va_pages, u64 *flags)
922 {
923 long i;
924 struct kbase_va_region *reg;
925 long faulted_pages;
926 int zone = KBASE_REG_ZONE_CUSTOM_VA;
927 bool shared_zone = false;
928 u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx);
929 struct kbase_alloc_import_user_buf *user_buf;
930 struct page **pages = NULL;
931
932 if ((address & (cache_line_alignment - 1)) != 0 ||
933 (size & (cache_line_alignment - 1)) != 0) {
934 /* Coherency must be enabled to handle partial cache lines */
935 if (*flags & (BASE_MEM_COHERENT_SYSTEM |
936 BASE_MEM_COHERENT_SYSTEM_REQUIRED)) {
937 /* Force coherent system required flag, import will
938 * then fail if coherency isn't available
939 */
940 *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED;
941 } else {
942 dev_warn(kctx->kbdev->dev,
943 "User buffer is not cache line aligned and no coherency enabled\n");
944 goto bad_size;
945 }
946 }
947
948 *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) -
949 PFN_DOWN(address);
950 if (!*va_pages)
951 goto bad_size;
952
953 if (*va_pages > (UINT64_MAX / PAGE_SIZE))
954 /* 64-bit address range is the max */
955 goto bad_size;
956
957 /* SAME_VA generally not supported with imported memory (no known use cases) */
958 *flags &= ~BASE_MEM_SAME_VA;
959
960 if (*flags & BASE_MEM_IMPORT_SHARED)
961 shared_zone = true;
962
963 #ifdef CONFIG_64BIT
964 if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
965 /*
966 * 64-bit tasks require us to reserve VA on the CPU that we use
967 * on the GPU.
968 */
969 shared_zone = true;
970 }
971 #endif
972
973 if (shared_zone) {
974 *flags |= BASE_MEM_NEED_MMAP;
975 zone = KBASE_REG_ZONE_SAME_VA;
976 }
977
978 reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone);
979
980 if (!reg)
981 goto no_region;
982
983 reg->gpu_alloc = kbase_alloc_create(*va_pages,
984 KBASE_MEM_TYPE_IMPORTED_USER_BUF);
985 if (IS_ERR_OR_NULL(reg->gpu_alloc))
986 goto no_alloc_obj;
987
988 reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
989
990 if (kbase_update_region_flags(kctx, reg, *flags) != 0)
991 goto invalid_flags;
992
993 reg->flags &= ~KBASE_REG_FREE;
994 reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */
995 reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */
996 reg->flags &= ~KBASE_REG_CPU_CACHED;
997
998 user_buf = ®->gpu_alloc->imported.user_buf;
999
1000 user_buf->size = size;
1001 user_buf->address = address;
1002 user_buf->nr_pages = *va_pages;
1003 user_buf->mm = current->mm;
1004 user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *),
1005 GFP_KERNEL);
1006
1007 if (!user_buf->pages)
1008 goto no_page_array;
1009
1010 /* If the region is coherent with the CPU then the memory is imported
1011 * and mapped onto the GPU immediately.
1012 * Otherwise get_user_pages is called as a sanity check, but with
1013 * NULL as the pages argument which will fault the pages, but not
1014 * pin them. The memory will then be pinned only around the jobs that
1015 * specify the region as an external resource.
1016 */
1017 if (reg->flags & KBASE_REG_SHARE_BOTH) {
1018 pages = user_buf->pages;
1019 *flags |= KBASE_MEM_IMPORT_HAVE_PAGES;
1020 }
1021
1022 down_read(¤t->mm->mmap_lock);
1023
1024 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1025 faulted_pages = get_user_pages(current, current->mm, address, *va_pages,
1026 reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL);
1027 #elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0)
1028 faulted_pages = get_user_pages(address, *va_pages,
1029 reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL);
1030 #else
1031 faulted_pages = get_user_pages(address, *va_pages,
1032 reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0,
1033 pages, NULL);
1034 #endif
1035
1036 up_read(¤t->mm->mmap_lock);
1037
1038 if (faulted_pages != *va_pages)
1039 goto fault_mismatch;
1040
1041 atomic_inc(¤t->mm->mm_count);
1042
1043 reg->gpu_alloc->nents = 0;
1044 reg->extent = 0;
1045
1046 if (pages) {
1047 struct device *dev = kctx->kbdev->dev;
1048 unsigned long local_size = user_buf->size;
1049 unsigned long offset = user_buf->address & ~PAGE_MASK;
1050 phys_addr_t *pa = kbase_get_gpu_phy_pages(reg);
1051
1052 /* Top bit signifies that this was pinned on import */
1053 user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT;
1054
1055 for (i = 0; i < faulted_pages; i++) {
1056 dma_addr_t dma_addr;
1057 unsigned long min;
1058
1059 min = MIN(PAGE_SIZE - offset, local_size);
1060 dma_addr = dma_map_page(dev, pages[i],
1061 offset, min,
1062 DMA_BIDIRECTIONAL);
1063 if (dma_mapping_error(dev, dma_addr))
1064 goto unwind_dma_map;
1065
1066 user_buf->dma_addrs[i] = dma_addr;
1067 pa[i] = page_to_phys(pages[i]);
1068
1069 local_size -= min;
1070 offset = 0;
1071 }
1072
1073 reg->gpu_alloc->nents = faulted_pages;
1074 }
1075
1076 return reg;
1077
1078 unwind_dma_map:
1079 while (i--) {
1080 dma_unmap_page(kctx->kbdev->dev,
1081 user_buf->dma_addrs[i],
1082 PAGE_SIZE, DMA_BIDIRECTIONAL);
1083 }
1084 fault_mismatch:
1085 if (pages) {
1086 for (i = 0; i < faulted_pages; i++)
1087 put_page(pages[i]);
1088 }
1089 kfree(user_buf->pages);
1090 no_page_array:
1091 invalid_flags:
1092 kbase_mem_phy_alloc_put(reg->cpu_alloc);
1093 kbase_mem_phy_alloc_put(reg->gpu_alloc);
1094 no_alloc_obj:
1095 kfree(reg);
1096 no_region:
1097 bad_size:
1098 return NULL;
1099
1100 }
1101
1102
kbase_mem_alias(struct kbase_context * kctx,u64 * flags,u64 stride,u64 nents,struct base_mem_aliasing_info * ai,u64 * num_pages)1103 u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride,
1104 u64 nents, struct base_mem_aliasing_info *ai,
1105 u64 *num_pages)
1106 {
1107 struct kbase_va_region *reg;
1108 u64 gpu_va;
1109 size_t i;
1110 bool coherent;
1111
1112 KBASE_DEBUG_ASSERT(kctx);
1113 KBASE_DEBUG_ASSERT(flags);
1114 KBASE_DEBUG_ASSERT(ai);
1115 KBASE_DEBUG_ASSERT(num_pages);
1116
1117 /* mask to only allowed flags */
1118 *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR |
1119 BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL |
1120 BASE_MEM_COHERENT_SYSTEM_REQUIRED);
1121
1122 if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) {
1123 dev_warn(kctx->kbdev->dev,
1124 "kbase_mem_alias called with bad flags (%llx)",
1125 (unsigned long long)*flags);
1126 goto bad_flags;
1127 }
1128 coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 ||
1129 (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0;
1130
1131 if (!stride)
1132 goto bad_stride;
1133
1134 if (!nents)
1135 goto bad_nents;
1136
1137 if ((nents * stride) > (U64_MAX / PAGE_SIZE))
1138 /* 64-bit address range is the max */
1139 goto bad_size;
1140
1141 /* calculate the number of pages this alias will cover */
1142 *num_pages = nents * stride;
1143
1144 #ifdef CONFIG_64BIT
1145 if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1146 /* 64-bit tasks must MMAP anyway, but not expose this address to
1147 * clients */
1148 *flags |= BASE_MEM_NEED_MMAP;
1149 reg = kbase_alloc_free_region(kctx, 0, *num_pages,
1150 KBASE_REG_ZONE_SAME_VA);
1151 } else {
1152 #else
1153 if (1) {
1154 #endif
1155 reg = kbase_alloc_free_region(kctx, 0, *num_pages,
1156 KBASE_REG_ZONE_CUSTOM_VA);
1157 }
1158
1159 if (!reg)
1160 goto no_reg;
1161
1162 /* zero-sized page array, as we don't need one/can support one */
1163 reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS);
1164 if (IS_ERR_OR_NULL(reg->gpu_alloc))
1165 goto no_alloc_obj;
1166
1167 reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1168
1169 if (kbase_update_region_flags(kctx, reg, *flags) != 0)
1170 goto invalid_flags;
1171
1172 reg->gpu_alloc->imported.alias.nents = nents;
1173 reg->gpu_alloc->imported.alias.stride = stride;
1174 reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents);
1175 if (!reg->gpu_alloc->imported.alias.aliased)
1176 goto no_aliased_array;
1177
1178 kbase_gpu_vm_lock(kctx);
1179
1180 /* validate and add src handles */
1181 for (i = 0; i < nents; i++) {
1182 if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) {
1183 if (ai[i].handle.basep.handle !=
1184 BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE)
1185 goto bad_handle; /* unsupported magic handle */
1186 if (!ai[i].length)
1187 goto bad_handle; /* must be > 0 */
1188 if (ai[i].length > stride)
1189 goto bad_handle; /* can't be larger than the
1190 stride */
1191 reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
1192 } else {
1193 struct kbase_va_region *aliasing_reg;
1194 struct kbase_mem_phy_alloc *alloc;
1195
1196 aliasing_reg = kbase_region_tracker_find_region_base_address(
1197 kctx,
1198 (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT);
1199
1200 /* validate found region */
1201 if (!aliasing_reg)
1202 goto bad_handle; /* Not found */
1203 if (aliasing_reg->flags & KBASE_REG_FREE)
1204 goto bad_handle; /* Free region */
1205 if (aliasing_reg->flags & KBASE_REG_DONT_NEED)
1206 goto bad_handle; /* Ephemeral region */
1207 if (!aliasing_reg->gpu_alloc)
1208 goto bad_handle; /* No alloc */
1209 if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
1210 goto bad_handle; /* Not a native alloc */
1211 if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0))
1212 goto bad_handle;
1213 /* Non-coherent memory cannot alias
1214 coherent memory, and vice versa.*/
1215
1216 /* check size against stride */
1217 if (!ai[i].length)
1218 goto bad_handle; /* must be > 0 */
1219 if (ai[i].length > stride)
1220 goto bad_handle; /* can't be larger than the
1221 stride */
1222
1223 alloc = aliasing_reg->gpu_alloc;
1224
1225 /* check against the alloc's size */
1226 if (ai[i].offset > alloc->nents)
1227 goto bad_handle; /* beyond end */
1228 if (ai[i].offset + ai[i].length > alloc->nents)
1229 goto bad_handle; /* beyond end */
1230
1231 reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc);
1232 reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
1233 reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset;
1234 }
1235 }
1236
1237 #ifdef CONFIG_64BIT
1238 if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1239 /* Bind to a cookie */
1240 if (!kctx->cookies) {
1241 dev_err(kctx->kbdev->dev, "No cookies available for allocation!");
1242 goto no_cookie;
1243 }
1244 /* return a cookie */
1245 gpu_va = __ffs(kctx->cookies);
1246 kctx->cookies &= ~(1UL << gpu_va);
1247 BUG_ON(kctx->pending_regions[gpu_va]);
1248 kctx->pending_regions[gpu_va] = reg;
1249
1250 /* relocate to correct base */
1251 gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
1252 gpu_va <<= PAGE_SHIFT;
1253 } else /* we control the VA */ {
1254 #else
1255 if (1) {
1256 #endif
1257 if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) {
1258 dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU");
1259 goto no_mmap;
1260 }
1261 /* return real GPU VA */
1262 gpu_va = reg->start_pfn << PAGE_SHIFT;
1263 }
1264
1265 reg->flags &= ~KBASE_REG_FREE;
1266 reg->flags &= ~KBASE_REG_GROWABLE;
1267
1268 kbase_gpu_vm_unlock(kctx);
1269
1270 return gpu_va;
1271
1272 #ifdef CONFIG_64BIT
1273 no_cookie:
1274 #endif
1275 no_mmap:
1276 bad_handle:
1277 kbase_gpu_vm_unlock(kctx);
1278 no_aliased_array:
1279 invalid_flags:
1280 kbase_mem_phy_alloc_put(reg->cpu_alloc);
1281 kbase_mem_phy_alloc_put(reg->gpu_alloc);
1282 no_alloc_obj:
1283 kfree(reg);
1284 no_reg:
1285 bad_size:
1286 bad_nents:
1287 bad_stride:
1288 bad_flags:
1289 return 0;
1290 }
1291
1292 int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
1293 void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages,
1294 u64 *flags)
1295 {
1296 struct kbase_va_region *reg;
1297
1298 KBASE_DEBUG_ASSERT(kctx);
1299 KBASE_DEBUG_ASSERT(gpu_va);
1300 KBASE_DEBUG_ASSERT(va_pages);
1301 KBASE_DEBUG_ASSERT(flags);
1302
1303 #ifdef CONFIG_64BIT
1304 if (!kbase_ctx_flag(kctx, KCTX_COMPAT))
1305 *flags |= BASE_MEM_SAME_VA;
1306 #endif
1307
1308 if (!kbase_check_import_flags(*flags)) {
1309 dev_warn(kctx->kbdev->dev,
1310 "kbase_mem_import called with bad flags (%llx)",
1311 (unsigned long long)*flags);
1312 goto bad_flags;
1313 }
1314
1315 if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
1316 !kbase_device_is_cpu_coherent(kctx->kbdev)) {
1317 dev_warn(kctx->kbdev->dev,
1318 "kbase_mem_import call required coherent mem when unavailable");
1319 goto bad_flags;
1320 }
1321 if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
1322 !kbase_device_is_cpu_coherent(kctx->kbdev)) {
1323 /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */
1324 *flags &= ~BASE_MEM_COHERENT_SYSTEM;
1325 }
1326
1327 if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) {
1328 dev_warn(kctx->kbdev->dev,
1329 "padding is only supported for UMM");
1330 goto bad_flags;
1331 }
1332
1333 switch (type) {
1334 #ifdef CONFIG_UMP
1335 case BASE_MEM_IMPORT_TYPE_UMP: {
1336 ump_secure_id id;
1337
1338 if (get_user(id, (ump_secure_id __user *)phandle))
1339 reg = NULL;
1340 else
1341 reg = kbase_mem_from_ump(kctx, id, va_pages, flags);
1342 }
1343 break;
1344 #endif /* CONFIG_UMP */
1345 #ifdef CONFIG_DMA_SHARED_BUFFER
1346 case BASE_MEM_IMPORT_TYPE_UMM: {
1347 int fd;
1348
1349 if (get_user(fd, (int __user *)phandle))
1350 reg = NULL;
1351 else
1352 reg = kbase_mem_from_umm(kctx, fd, va_pages, flags,
1353 padding);
1354 }
1355 break;
1356 #endif /* CONFIG_DMA_SHARED_BUFFER */
1357 case BASE_MEM_IMPORT_TYPE_USER_BUFFER: {
1358 struct base_mem_import_user_buffer user_buffer;
1359 void __user *uptr;
1360
1361 if (copy_from_user(&user_buffer, phandle,
1362 sizeof(user_buffer))) {
1363 reg = NULL;
1364 } else {
1365 #ifdef CONFIG_COMPAT
1366 if (kbase_ctx_flag(kctx, KCTX_COMPAT))
1367 uptr = compat_ptr(user_buffer.ptr.compat_value);
1368 else
1369 #endif
1370 uptr = user_buffer.ptr.value;
1371
1372 reg = kbase_mem_from_user_buffer(kctx,
1373 (unsigned long)uptr, user_buffer.length,
1374 va_pages, flags);
1375 }
1376 break;
1377 }
1378 default: {
1379 reg = NULL;
1380 break;
1381 }
1382 }
1383
1384 if (!reg)
1385 goto no_reg;
1386
1387 kbase_gpu_vm_lock(kctx);
1388
1389 /* mmap needed to setup VA? */
1390 if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) {
1391 /* Bind to a cookie */
1392 if (!kctx->cookies)
1393 goto no_cookie;
1394 /* return a cookie */
1395 *gpu_va = __ffs(kctx->cookies);
1396 kctx->cookies &= ~(1UL << *gpu_va);
1397 BUG_ON(kctx->pending_regions[*gpu_va]);
1398 kctx->pending_regions[*gpu_va] = reg;
1399
1400 /* relocate to correct base */
1401 *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
1402 *gpu_va <<= PAGE_SHIFT;
1403
1404 } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) {
1405 /* we control the VA, mmap now to the GPU */
1406 if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0)
1407 goto no_gpu_va;
1408 /* return real GPU VA */
1409 *gpu_va = reg->start_pfn << PAGE_SHIFT;
1410 } else {
1411 /* we control the VA, but nothing to mmap yet */
1412 if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0)
1413 goto no_gpu_va;
1414 /* return real GPU VA */
1415 *gpu_va = reg->start_pfn << PAGE_SHIFT;
1416 }
1417
1418 /* clear out private flags */
1419 *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1);
1420
1421 kbase_gpu_vm_unlock(kctx);
1422
1423 return 0;
1424
1425 no_gpu_va:
1426 no_cookie:
1427 kbase_gpu_vm_unlock(kctx);
1428 kbase_mem_phy_alloc_put(reg->cpu_alloc);
1429 kbase_mem_phy_alloc_put(reg->gpu_alloc);
1430 kfree(reg);
1431 no_reg:
1432 bad_flags:
1433 *gpu_va = 0;
1434 *va_pages = 0;
1435 *flags = 0;
1436 return -ENOMEM;
1437 }
1438
1439 int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx,
1440 struct kbase_va_region *reg,
1441 u64 new_pages, u64 old_pages)
1442 {
1443 phys_addr_t *phy_pages;
1444 u64 delta = new_pages - old_pages;
1445 int ret = 0;
1446
1447 lockdep_assert_held(&kctx->reg_lock);
1448
1449 /* Map the new pages into the GPU */
1450 phy_pages = kbase_get_gpu_phy_pages(reg);
1451 ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages,
1452 phy_pages + old_pages, delta, reg->flags);
1453
1454 return ret;
1455 }
1456
1457 static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
1458 struct kbase_va_region *reg,
1459 u64 new_pages, u64 old_pages)
1460 {
1461 u64 gpu_va_start = reg->start_pfn;
1462
1463 if (new_pages == old_pages)
1464 /* Nothing to do */
1465 return;
1466
1467 unmap_mapping_range(kctx->filp->f_inode->i_mapping,
1468 (gpu_va_start + new_pages)<<PAGE_SHIFT,
1469 (old_pages - new_pages)<<PAGE_SHIFT, 1);
1470 }
1471
1472 static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
1473 struct kbase_va_region *reg,
1474 u64 new_pages, u64 old_pages)
1475 {
1476 u64 delta = old_pages - new_pages;
1477 int ret = 0;
1478
1479 ret = kbase_mmu_teardown_pages(kctx,
1480 reg->start_pfn + new_pages, delta);
1481
1482 return ret;
1483 }
1484
1485 int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages)
1486 {
1487 u64 old_pages;
1488 u64 delta;
1489 int res = -EINVAL;
1490 struct kbase_va_region *reg;
1491 bool read_locked = false;
1492
1493 KBASE_DEBUG_ASSERT(kctx);
1494 KBASE_DEBUG_ASSERT(gpu_addr != 0);
1495
1496 if (gpu_addr & ~PAGE_MASK) {
1497 dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid");
1498 return -EINVAL;
1499 }
1500
1501 down_write(¤t->mm->mmap_lock);
1502 kbase_gpu_vm_lock(kctx);
1503
1504 /* Validate the region */
1505 reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
1506 if (!reg || (reg->flags & KBASE_REG_FREE))
1507 goto out_unlock;
1508
1509 KBASE_DEBUG_ASSERT(reg->cpu_alloc);
1510 KBASE_DEBUG_ASSERT(reg->gpu_alloc);
1511
1512 if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
1513 goto out_unlock;
1514
1515 if (0 == (reg->flags & KBASE_REG_GROWABLE))
1516 goto out_unlock;
1517
1518 /* Would overflow the VA region */
1519 if (new_pages > reg->nr_pages)
1520 goto out_unlock;
1521
1522 /* can't be mapped more than once on the GPU */
1523 if (atomic_read(®->gpu_alloc->gpu_mappings) > 1)
1524 goto out_unlock;
1525 /* can't grow regions which are ephemeral */
1526 if (reg->flags & KBASE_REG_DONT_NEED)
1527 goto out_unlock;
1528
1529 if (new_pages == reg->gpu_alloc->nents) {
1530 /* no change */
1531 res = 0;
1532 goto out_unlock;
1533 }
1534
1535 old_pages = kbase_reg_current_backed_size(reg);
1536 if (new_pages > old_pages) {
1537 delta = new_pages - old_pages;
1538
1539 /*
1540 * No update to the mm so downgrade the writer lock to a read
1541 * lock so other readers aren't blocked after this point.
1542 */
1543 downgrade_write(¤t->mm->mmap_lock);
1544 read_locked = true;
1545
1546 /* Allocate some more pages */
1547 if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) {
1548 res = -ENOMEM;
1549 goto out_unlock;
1550 }
1551 if (reg->cpu_alloc != reg->gpu_alloc) {
1552 if (kbase_alloc_phy_pages_helper(
1553 reg->gpu_alloc, delta) != 0) {
1554 res = -ENOMEM;
1555 kbase_free_phy_pages_helper(reg->cpu_alloc,
1556 delta);
1557 goto out_unlock;
1558 }
1559 }
1560
1561 /* No update required for CPU mappings, that's done on fault. */
1562
1563 /* Update GPU mapping. */
1564 res = kbase_mem_grow_gpu_mapping(kctx, reg,
1565 new_pages, old_pages);
1566
1567 /* On error free the new pages */
1568 if (res) {
1569 kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
1570 if (reg->cpu_alloc != reg->gpu_alloc)
1571 kbase_free_phy_pages_helper(reg->gpu_alloc,
1572 delta);
1573 res = -ENOMEM;
1574 goto out_unlock;
1575 }
1576 } else {
1577 delta = old_pages - new_pages;
1578
1579 /* Update all CPU mapping(s) */
1580 kbase_mem_shrink_cpu_mapping(kctx, reg,
1581 new_pages, old_pages);
1582
1583 /* Update the GPU mapping */
1584 res = kbase_mem_shrink_gpu_mapping(kctx, reg,
1585 new_pages, old_pages);
1586 if (res) {
1587 res = -ENOMEM;
1588 goto out_unlock;
1589 }
1590
1591 kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
1592 if (reg->cpu_alloc != reg->gpu_alloc)
1593 kbase_free_phy_pages_helper(reg->gpu_alloc, delta);
1594 }
1595
1596 out_unlock:
1597 kbase_gpu_vm_unlock(kctx);
1598 if (read_locked)
1599 up_read(¤t->mm->mmap_lock);
1600 else
1601 up_write(¤t->mm->mmap_lock);
1602
1603 return res;
1604 }
1605
1606 static void kbase_cpu_vm_open(struct vm_area_struct *vma)
1607 {
1608 struct kbase_cpu_mapping *map = vma->vm_private_data;
1609
1610 KBASE_DEBUG_ASSERT(map);
1611 KBASE_DEBUG_ASSERT(map->count > 0);
1612 /* non-atomic as we're under Linux' mm lock */
1613 map->count++;
1614 }
1615
1616 static void kbase_cpu_vm_close(struct vm_area_struct *vma)
1617 {
1618 struct kbase_cpu_mapping *map = vma->vm_private_data;
1619
1620 KBASE_DEBUG_ASSERT(map);
1621 KBASE_DEBUG_ASSERT(map->count > 0);
1622
1623 /* non-atomic as we're under Linux' mm lock */
1624 if (--map->count)
1625 return;
1626
1627 KBASE_DEBUG_ASSERT(map->kctx);
1628 KBASE_DEBUG_ASSERT(map->alloc);
1629
1630 kbase_gpu_vm_lock(map->kctx);
1631
1632 if (map->free_on_close) {
1633 KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) ==
1634 KBASE_REG_ZONE_SAME_VA);
1635 /* Avoid freeing memory on the process death which results in
1636 * GPU Page Fault. Memory will be freed in kbase_destroy_context
1637 */
1638 if (!(current->flags & PF_EXITING))
1639 kbase_mem_free_region(map->kctx, map->region);
1640 }
1641
1642 list_del(&map->mappings_list);
1643
1644 kbase_gpu_vm_unlock(map->kctx);
1645
1646 kbase_mem_phy_alloc_put(map->alloc);
1647 kfree(map);
1648 }
1649
1650 KBASE_EXPORT_TEST_API(kbase_cpu_vm_close);
1651
1652
1653 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
1654 static vm_fault_t kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1655 {
1656 #else
1657 static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf)
1658 {
1659 struct vm_area_struct *vma = vmf->vma;
1660 #endif
1661 struct kbase_cpu_mapping *map = vma->vm_private_data;
1662 pgoff_t rel_pgoff;
1663 size_t i;
1664 pgoff_t addr;
1665 vm_fault_t ret = VM_FAULT_SIGBUS;
1666
1667 KBASE_DEBUG_ASSERT(map);
1668 KBASE_DEBUG_ASSERT(map->count > 0);
1669 KBASE_DEBUG_ASSERT(map->kctx);
1670 KBASE_DEBUG_ASSERT(map->alloc);
1671
1672 rel_pgoff = vmf->pgoff - map->region->start_pfn;
1673
1674 kbase_gpu_vm_lock(map->kctx);
1675 if (rel_pgoff >= map->alloc->nents)
1676 goto locked_bad_fault;
1677
1678 /* Fault on access to DONT_NEED regions */
1679 if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED))
1680 goto locked_bad_fault;
1681
1682 /* insert all valid pages from the fault location */
1683 i = rel_pgoff;
1684 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
1685 addr = (pgoff_t)((uintptr_t)vmf->virtual_address >> PAGE_SHIFT);
1686 #else
1687 addr = (pgoff_t)(vmf->address >> PAGE_SHIFT);
1688 #endif
1689 while (i < map->alloc->nents && (addr < vma->vm_end >> PAGE_SHIFT)) {
1690 ret = vmf_insert_pfn(vma, addr << PAGE_SHIFT,
1691 PFN_DOWN(map->alloc->pages[i]));
1692 if (ret != VM_FAULT_NOPAGE)
1693 goto locked_bad_fault;
1694
1695 i++; addr++;
1696 }
1697
1698 kbase_gpu_vm_unlock(map->kctx);
1699 /* we resolved it, nothing for VM to do */
1700 return VM_FAULT_NOPAGE;
1701
1702 locked_bad_fault:
1703 kbase_gpu_vm_unlock(map->kctx);
1704 return ret;
1705 }
1706
1707 const struct vm_operations_struct kbase_vm_ops = {
1708 .open = kbase_cpu_vm_open,
1709 .close = kbase_cpu_vm_close,
1710 .fault = kbase_cpu_vm_fault
1711 };
1712
1713 static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close)
1714 {
1715 struct kbase_cpu_mapping *map;
1716 phys_addr_t *page_array;
1717 int err = 0;
1718 int i;
1719
1720 map = kzalloc(sizeof(*map), GFP_KERNEL);
1721
1722 if (!map) {
1723 WARN_ON(1);
1724 err = -ENOMEM;
1725 goto out;
1726 }
1727
1728 /*
1729 * VM_DONTCOPY - don't make this mapping available in fork'ed processes
1730 * VM_DONTEXPAND - disable mremap on this region
1731 * VM_IO - disables paging
1732 * VM_DONTDUMP - Don't include in core dumps (3.7 only)
1733 * VM_MIXEDMAP - Support mixing struct page*s and raw pfns.
1734 * This is needed to support using the dedicated and
1735 * the OS based memory backends together.
1736 */
1737 /*
1738 * This will need updating to propagate coherency flags
1739 * See MIDBASE-1057
1740 */
1741
1742 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
1743 vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO;
1744 #else
1745 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO;
1746 #endif
1747 vma->vm_ops = &kbase_vm_ops;
1748 vma->vm_private_data = map;
1749
1750 page_array = kbase_get_cpu_phy_pages(reg);
1751
1752 if (!(reg->flags & KBASE_REG_CPU_CACHED) &&
1753 (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) {
1754 /* We can't map vmalloc'd memory uncached.
1755 * Other memory will have been returned from
1756 * kbase_mem_pool which would be
1757 * suitable for mapping uncached.
1758 */
1759 BUG_ON(kaddr);
1760 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1761 }
1762
1763 if (!kaddr) {
1764 unsigned long addr = vma->vm_start + aligned_offset;
1765 u64 start_off = vma->vm_pgoff - reg->start_pfn +
1766 (aligned_offset>>PAGE_SHIFT);
1767
1768 vma->vm_flags |= VM_PFNMAP;
1769 for (i = 0; i < nr_pages; i++) {
1770 unsigned long pfn = PFN_DOWN(page_array[i + start_off]);
1771 vm_fault_t ret;
1772
1773 ret = vmf_insert_pfn(vma, addr, pfn);
1774 if (WARN_ON(ret != VM_FAULT_NOPAGE)) {
1775 if (ret == VM_FAULT_OOM)
1776 err = -ENOMEM;
1777 else
1778 err = -EFAULT;
1779 break;
1780 }
1781
1782 addr += PAGE_SIZE;
1783 }
1784 } else {
1785 WARN_ON(aligned_offset);
1786 /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */
1787 vma->vm_flags |= VM_MIXEDMAP;
1788 /* vmalloc remaping is easy... */
1789 err = remap_vmalloc_range(vma, kaddr, 0);
1790 WARN_ON(err);
1791 }
1792
1793 if (err) {
1794 kfree(map);
1795 goto out;
1796 }
1797
1798 map->region = reg;
1799 map->free_on_close = free_on_close;
1800 map->kctx = reg->kctx;
1801 map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
1802 map->count = 1; /* start with one ref */
1803
1804 if (reg->flags & KBASE_REG_CPU_CACHED)
1805 map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED;
1806
1807 list_add(&map->mappings_list, &map->alloc->mappings);
1808
1809 out:
1810 return err;
1811 }
1812
1813 static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr)
1814 {
1815 struct kbase_va_region *new_reg;
1816 u32 nr_pages;
1817 size_t size;
1818 int err = 0;
1819 u32 *tb;
1820 int owns_tb = 1;
1821
1822 dev_dbg(kctx->kbdev->dev, "in %s\n", __func__);
1823 size = (vma->vm_end - vma->vm_start);
1824 nr_pages = size >> PAGE_SHIFT;
1825
1826 if (!kctx->jctx.tb) {
1827 KBASE_DEBUG_ASSERT(0 != size);
1828 tb = vmalloc_user(size);
1829
1830 if (NULL == tb) {
1831 err = -ENOMEM;
1832 goto out;
1833 }
1834
1835 err = kbase_device_trace_buffer_install(kctx, tb, size);
1836 if (err) {
1837 vfree(tb);
1838 goto out;
1839 }
1840 } else {
1841 err = -EINVAL;
1842 goto out;
1843 }
1844
1845 *kaddr = kctx->jctx.tb;
1846
1847 new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA);
1848 if (!new_reg) {
1849 err = -ENOMEM;
1850 WARN_ON(1);
1851 goto out_no_region;
1852 }
1853
1854 new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB);
1855 if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
1856 err = -ENOMEM;
1857 new_reg->cpu_alloc = NULL;
1858 WARN_ON(1);
1859 goto out_no_alloc;
1860 }
1861
1862 new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
1863
1864 new_reg->cpu_alloc->imported.kctx = kctx;
1865 new_reg->flags &= ~KBASE_REG_FREE;
1866 new_reg->flags |= KBASE_REG_CPU_CACHED;
1867
1868 /* alloc now owns the tb */
1869 owns_tb = 0;
1870
1871 if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
1872 err = -ENOMEM;
1873 WARN_ON(1);
1874 goto out_no_va_region;
1875 }
1876
1877 *reg = new_reg;
1878
1879 /* map read only, noexec */
1880 vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
1881 /* the rest of the flags is added by the cpu_mmap handler */
1882
1883 dev_dbg(kctx->kbdev->dev, "%s done\n", __func__);
1884 return 0;
1885
1886 out_no_va_region:
1887 out_no_alloc:
1888 kbase_free_alloced_region(new_reg);
1889 out_no_region:
1890 if (owns_tb) {
1891 kbase_device_trace_buffer_uninstall(kctx);
1892 vfree(tb);
1893 }
1894 out:
1895 return err;
1896 }
1897
1898 static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr)
1899 {
1900 struct kbase_va_region *new_reg;
1901 void *kaddr;
1902 u32 nr_pages;
1903 size_t size;
1904 int err = 0;
1905
1906 dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n");
1907 size = (vma->vm_end - vma->vm_start);
1908 nr_pages = size >> PAGE_SHIFT;
1909
1910 kaddr = kbase_mmu_dump(kctx, nr_pages);
1911
1912 if (!kaddr) {
1913 err = -ENOMEM;
1914 goto out;
1915 }
1916
1917 new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA);
1918 if (!new_reg) {
1919 err = -ENOMEM;
1920 WARN_ON(1);
1921 goto out;
1922 }
1923
1924 new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW);
1925 if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
1926 err = -ENOMEM;
1927 new_reg->cpu_alloc = NULL;
1928 WARN_ON(1);
1929 goto out_no_alloc;
1930 }
1931
1932 new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
1933
1934 new_reg->flags &= ~KBASE_REG_FREE;
1935 new_reg->flags |= KBASE_REG_CPU_CACHED;
1936 if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
1937 err = -ENOMEM;
1938 WARN_ON(1);
1939 goto out_va_region;
1940 }
1941
1942 *kmap_addr = kaddr;
1943 *reg = new_reg;
1944
1945 dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n");
1946 return 0;
1947
1948 out_no_alloc:
1949 out_va_region:
1950 kbase_free_alloced_region(new_reg);
1951 out:
1952 return err;
1953 }
1954
1955
1956 void kbase_os_mem_map_lock(struct kbase_context *kctx)
1957 {
1958 struct mm_struct *mm = current->mm;
1959 (void)kctx;
1960 down_read(&mm->mmap_lock);
1961 }
1962
1963 void kbase_os_mem_map_unlock(struct kbase_context *kctx)
1964 {
1965 struct mm_struct *mm = current->mm;
1966 (void)kctx;
1967 up_read(&mm->mmap_lock);
1968 }
1969
1970 static int kbasep_reg_mmap(struct kbase_context *kctx,
1971 struct vm_area_struct *vma,
1972 struct kbase_va_region **regm,
1973 size_t *nr_pages, size_t *aligned_offset)
1974
1975 {
1976 int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE);
1977 struct kbase_va_region *reg;
1978 int err = 0;
1979
1980 *aligned_offset = 0;
1981
1982 dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n");
1983
1984 /* SAME_VA stuff, fetch the right region */
1985 reg = kctx->pending_regions[cookie];
1986 if (!reg) {
1987 err = -ENOMEM;
1988 goto out;
1989 }
1990
1991 if ((reg->flags & KBASE_REG_GPU_NX) && (reg->nr_pages != *nr_pages)) {
1992 /* incorrect mmap size */
1993 /* leave the cookie for a potential later
1994 * mapping, or to be reclaimed later when the
1995 * context is freed */
1996 err = -ENOMEM;
1997 goto out;
1998 }
1999
2000 if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) ||
2001 (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) {
2002 /* VM flags inconsistent with region flags */
2003 err = -EPERM;
2004 dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n",
2005 __FILE__, __LINE__);
2006 goto out;
2007 }
2008
2009 /* adjust down nr_pages to what we have physically */
2010 *nr_pages = kbase_reg_current_backed_size(reg);
2011
2012 if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset,
2013 reg->nr_pages, 1) != 0) {
2014 dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__);
2015 /* Unable to map in GPU space. */
2016 WARN_ON(1);
2017 err = -ENOMEM;
2018 goto out;
2019 }
2020 /* no need for the cookie anymore */
2021 kctx->pending_regions[cookie] = NULL;
2022 kctx->cookies |= (1UL << cookie);
2023
2024 /*
2025 * Overwrite the offset with the region start_pfn, so we effectively
2026 * map from offset 0 in the region. However subtract the aligned
2027 * offset so that when user space trims the mapping the beginning of
2028 * the trimmed VMA has the correct vm_pgoff;
2029 */
2030 vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT);
2031 out:
2032 *regm = reg;
2033 dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n");
2034
2035 return err;
2036 }
2037
2038 int kbase_mmap(struct file *file, struct vm_area_struct *vma)
2039 {
2040 struct kbase_context *kctx = file->private_data;
2041 struct kbase_va_region *reg = NULL;
2042 void *kaddr = NULL;
2043 size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2044 int err = 0;
2045 int free_on_close = 0;
2046 struct device *dev = kctx->kbdev->dev;
2047 size_t aligned_offset = 0;
2048
2049 dev_dbg(dev, "kbase_mmap\n");
2050
2051 /* strip away corresponding VM_MAY% flags to the VM_% flags requested */
2052 vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4);
2053
2054 if (0 == nr_pages) {
2055 err = -EINVAL;
2056 goto out;
2057 }
2058
2059 if (!(vma->vm_flags & VM_SHARED)) {
2060 err = -EINVAL;
2061 goto out;
2062 }
2063
2064 kbase_gpu_vm_lock(kctx);
2065
2066 if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) {
2067 /* The non-mapped tracking helper page */
2068 err = kbase_tracking_page_setup(kctx, vma);
2069 goto out_unlock;
2070 }
2071
2072 /* if not the MTP, verify that the MTP has been mapped */
2073 rcu_read_lock();
2074 /* catches both when the special page isn't present or
2075 * when we've forked */
2076 if (rcu_dereference(kctx->process_mm) != current->mm) {
2077 err = -EINVAL;
2078 rcu_read_unlock();
2079 goto out_unlock;
2080 }
2081 rcu_read_unlock();
2082
2083 switch (vma->vm_pgoff) {
2084 case PFN_DOWN(BASEP_MEM_INVALID_HANDLE):
2085 case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE):
2086 /* Illegal handle for direct map */
2087 err = -EINVAL;
2088 goto out_unlock;
2089 case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE):
2090 err = kbase_trace_buffer_mmap(kctx, vma, ®, &kaddr);
2091 if (0 != err)
2092 goto out_unlock;
2093 dev_dbg(dev, "kbase_trace_buffer_mmap ok\n");
2094 /* free the region on munmap */
2095 free_on_close = 1;
2096 break;
2097 case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE):
2098 /* MMU dump */
2099 err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr);
2100 if (0 != err)
2101 goto out_unlock;
2102 /* free the region on munmap */
2103 free_on_close = 1;
2104 break;
2105 case PFN_DOWN(BASE_MEM_COOKIE_BASE) ...
2106 PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: {
2107 err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages,
2108 &aligned_offset);
2109 if (0 != err)
2110 goto out_unlock;
2111 /* free the region on munmap */
2112 free_on_close = 1;
2113 break;
2114 }
2115 default: {
2116 reg = kbase_region_tracker_find_region_enclosing_address(kctx,
2117 (u64)vma->vm_pgoff << PAGE_SHIFT);
2118
2119 if (reg && !(reg->flags & KBASE_REG_FREE)) {
2120 /* will this mapping overflow the size of the region? */
2121 if (nr_pages > (reg->nr_pages -
2122 (vma->vm_pgoff - reg->start_pfn))) {
2123 err = -ENOMEM;
2124 goto out_unlock;
2125 }
2126
2127 if ((vma->vm_flags & VM_READ &&
2128 !(reg->flags & KBASE_REG_CPU_RD)) ||
2129 (vma->vm_flags & VM_WRITE &&
2130 !(reg->flags & KBASE_REG_CPU_WR))) {
2131 /* VM flags inconsistent with region flags */
2132 err = -EPERM;
2133 dev_err(dev, "%s:%d inconsistent VM flags\n",
2134 __FILE__, __LINE__);
2135 goto out_unlock;
2136 }
2137
2138 #ifdef CONFIG_DMA_SHARED_BUFFER
2139 if (KBASE_MEM_TYPE_IMPORTED_UMM ==
2140 reg->cpu_alloc->type) {
2141 err = dma_buf_mmap(
2142 reg->cpu_alloc->imported.umm.dma_buf,
2143 vma, vma->vm_pgoff - reg->start_pfn);
2144 goto out_unlock;
2145 }
2146 #endif /* CONFIG_DMA_SHARED_BUFFER */
2147
2148 /* limit what we map to the amount currently backed */
2149 if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) {
2150 if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents)
2151 nr_pages = 0;
2152 else
2153 nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn);
2154 }
2155 } else {
2156 err = -ENOMEM;
2157 goto out_unlock;
2158 }
2159 } /* default */
2160 } /* switch */
2161
2162 err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close);
2163
2164 if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) {
2165 /* MMU dump - userspace should now have a reference on
2166 * the pages, so we can now free the kernel mapping */
2167 vfree(kaddr);
2168 }
2169
2170 out_unlock:
2171 kbase_gpu_vm_unlock(kctx);
2172 out:
2173 if (err)
2174 dev_err(dev, "mmap failed %d\n", err);
2175
2176 return err;
2177 }
2178
2179 KBASE_EXPORT_TEST_API(kbase_mmap);
2180
2181 void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size,
2182 unsigned long prot_request, struct kbase_vmap_struct *map)
2183 {
2184 struct kbase_va_region *reg;
2185 unsigned long page_index;
2186 unsigned int offset = gpu_addr & ~PAGE_MASK;
2187 size_t page_count = PFN_UP(offset + size);
2188 phys_addr_t *page_array;
2189 struct page **pages;
2190 void *cpu_addr = NULL;
2191 pgprot_t prot;
2192 size_t i;
2193 bool sync_needed;
2194
2195 if (!size || !map)
2196 return NULL;
2197
2198 /* check if page_count calculation will wrap */
2199 if (size > ((size_t)-1 / PAGE_SIZE))
2200 return NULL;
2201
2202 kbase_gpu_vm_lock(kctx);
2203
2204 reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr);
2205 if (!reg || (reg->flags & KBASE_REG_FREE))
2206 goto out_unlock;
2207
2208 page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn;
2209
2210 /* check if page_index + page_count will wrap */
2211 if (-1UL - page_count < page_index)
2212 goto out_unlock;
2213
2214 if (page_index + page_count > kbase_reg_current_backed_size(reg))
2215 goto out_unlock;
2216
2217 if (reg->flags & KBASE_REG_DONT_NEED)
2218 goto out_unlock;
2219
2220 /* check access permissions can be satisfied
2221 * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */
2222 if ((reg->flags & prot_request) != prot_request)
2223 goto out_unlock;
2224
2225 page_array = kbase_get_cpu_phy_pages(reg);
2226 if (!page_array)
2227 goto out_unlock;
2228
2229 pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL);
2230 if (!pages)
2231 goto out_unlock;
2232
2233 for (i = 0; i < page_count; i++)
2234 pages[i] = pfn_to_page(PFN_DOWN(page_array[page_index + i]));
2235
2236 prot = PAGE_KERNEL;
2237 if (!(reg->flags & KBASE_REG_CPU_CACHED)) {
2238 /* Map uncached */
2239 prot = pgprot_writecombine(prot);
2240 }
2241 /* Note: enforcing a RO prot_request onto prot is not done, since:
2242 * - CPU-arch-specific integration required
2243 * - kbase_vmap() requires no access checks to be made/enforced */
2244
2245 cpu_addr = vmap(pages, page_count, VM_MAP, prot);
2246
2247 kfree(pages);
2248
2249 if (!cpu_addr)
2250 goto out_unlock;
2251
2252 map->gpu_addr = gpu_addr;
2253 map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2254 map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index];
2255 map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
2256 map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index];
2257 map->addr = (void *)((uintptr_t)cpu_addr + offset);
2258 map->size = size;
2259 map->is_cached = (reg->flags & KBASE_REG_CPU_CACHED) != 0;
2260 sync_needed = map->is_cached;
2261
2262 #ifdef CONFIG_MALI_COH_KERN
2263 /* kernel can use coherent memory if supported */
2264 if (kctx->kbdev->system_coherency == COHERENCY_ACE)
2265 sync_needed = false;
2266 #endif
2267
2268 if (sync_needed) {
2269 /* Sync first page */
2270 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
2271 phys_addr_t cpu_pa = map->cpu_pages[0];
2272 phys_addr_t gpu_pa = map->gpu_pages[0];
2273
2274 kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
2275 KBASE_SYNC_TO_CPU);
2276
2277 /* Sync middle pages (if any) */
2278 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
2279 cpu_pa = map->cpu_pages[i];
2280 gpu_pa = map->gpu_pages[i];
2281 kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
2282 KBASE_SYNC_TO_CPU);
2283 }
2284
2285 /* Sync last page (if any) */
2286 if (page_count > 1) {
2287 cpu_pa = map->cpu_pages[page_count - 1];
2288 gpu_pa = map->gpu_pages[page_count - 1];
2289 sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
2290 kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
2291 KBASE_SYNC_TO_CPU);
2292 }
2293 }
2294 kbase_gpu_vm_unlock(kctx);
2295
2296 return map->addr;
2297
2298 out_unlock:
2299 kbase_gpu_vm_unlock(kctx);
2300 return NULL;
2301 }
2302
2303 void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size,
2304 struct kbase_vmap_struct *map)
2305 {
2306 /* 0 is specified for prot_request to indicate no access checks should
2307 * be made.
2308 *
2309 * As mentioned in kbase_vmap_prot() this means that a kernel-side
2310 * CPU-RO mapping is not enforced to allow this to work */
2311 return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map);
2312 }
2313 KBASE_EXPORT_TEST_API(kbase_vmap);
2314
2315 void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map)
2316 {
2317 void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK);
2318 bool sync_needed = map->is_cached;
2319 vunmap(addr);
2320 #ifdef CONFIG_MALI_COH_KERN
2321 /* kernel can use coherent memory if supported */
2322 if (kctx->kbdev->system_coherency == COHERENCY_ACE)
2323 sync_needed = false;
2324 #endif
2325 if (sync_needed) {
2326 off_t offset = (uintptr_t)map->addr & ~PAGE_MASK;
2327 size_t size = map->size;
2328 size_t page_count = PFN_UP(offset + size);
2329 size_t i;
2330
2331 /* Sync first page */
2332 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
2333 phys_addr_t cpu_pa = map->cpu_pages[0];
2334 phys_addr_t gpu_pa = map->gpu_pages[0];
2335
2336 kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
2337 KBASE_SYNC_TO_DEVICE);
2338
2339 /* Sync middle pages (if any) */
2340 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
2341 cpu_pa = map->cpu_pages[i];
2342 gpu_pa = map->gpu_pages[i];
2343 kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
2344 KBASE_SYNC_TO_DEVICE);
2345 }
2346
2347 /* Sync last page (if any) */
2348 if (page_count > 1) {
2349 cpu_pa = map->cpu_pages[page_count - 1];
2350 gpu_pa = map->gpu_pages[page_count - 1];
2351 sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
2352 kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
2353 KBASE_SYNC_TO_DEVICE);
2354 }
2355 }
2356 map->gpu_addr = 0;
2357 map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc);
2358 map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc);
2359 map->cpu_pages = NULL;
2360 map->gpu_pages = NULL;
2361 map->addr = NULL;
2362 map->size = 0;
2363 map->is_cached = false;
2364 }
2365 KBASE_EXPORT_TEST_API(kbase_vunmap);
2366
2367 void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages)
2368 {
2369 struct mm_struct *mm;
2370
2371 rcu_read_lock();
2372 mm = rcu_dereference(kctx->process_mm);
2373 if (mm) {
2374 atomic_add(pages, &kctx->nonmapped_pages);
2375 #ifdef SPLIT_RSS_COUNTING
2376 add_mm_counter(mm, MM_FILEPAGES, pages);
2377 #else
2378 spin_lock(&mm->page_table_lock);
2379 add_mm_counter(mm, MM_FILEPAGES, pages);
2380 spin_unlock(&mm->page_table_lock);
2381 #endif
2382 }
2383 rcu_read_unlock();
2384 }
2385
2386 static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx)
2387 {
2388 int pages;
2389 struct mm_struct *mm;
2390
2391 spin_lock(&kctx->mm_update_lock);
2392 mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock));
2393 if (!mm) {
2394 spin_unlock(&kctx->mm_update_lock);
2395 return;
2396 }
2397
2398 rcu_assign_pointer(kctx->process_mm, NULL);
2399 spin_unlock(&kctx->mm_update_lock);
2400 synchronize_rcu();
2401
2402 pages = atomic_xchg(&kctx->nonmapped_pages, 0);
2403 #ifdef SPLIT_RSS_COUNTING
2404 add_mm_counter(mm, MM_FILEPAGES, -pages);
2405 #else
2406 spin_lock(&mm->page_table_lock);
2407 add_mm_counter(mm, MM_FILEPAGES, -pages);
2408 spin_unlock(&mm->page_table_lock);
2409 #endif
2410 }
2411
2412 static void kbase_special_vm_close(struct vm_area_struct *vma)
2413 {
2414 struct kbase_context *kctx;
2415
2416 kctx = vma->vm_private_data;
2417 kbasep_os_process_page_usage_drain(kctx);
2418 }
2419
2420 static const struct vm_operations_struct kbase_vm_special_ops = {
2421 .close = kbase_special_vm_close,
2422 };
2423
2424 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma)
2425 {
2426 /* check that this is the only tracking page */
2427 spin_lock(&kctx->mm_update_lock);
2428 if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) {
2429 spin_unlock(&kctx->mm_update_lock);
2430 return -EFAULT;
2431 }
2432
2433 rcu_assign_pointer(kctx->process_mm, current->mm);
2434
2435 spin_unlock(&kctx->mm_update_lock);
2436
2437 /* no real access */
2438 vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
2439 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
2440 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
2441 #else
2442 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO;
2443 #endif
2444 vma->vm_ops = &kbase_vm_special_ops;
2445 vma->vm_private_data = kctx;
2446
2447 return 0;
2448 }
2449 void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle)
2450 {
2451 int i;
2452 int res;
2453 void *va;
2454 dma_addr_t dma_pa;
2455 struct kbase_va_region *reg;
2456 phys_addr_t *page_array;
2457 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
2458 unsigned long attrs = DMA_ATTR_WRITE_COMBINE;
2459 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2460 DEFINE_DMA_ATTRS(attrs);
2461 #endif
2462
2463 u32 pages = ((size - 1) >> PAGE_SHIFT) + 1;
2464 u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR |
2465 BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR;
2466
2467 KBASE_DEBUG_ASSERT(kctx != NULL);
2468 KBASE_DEBUG_ASSERT(0 != size);
2469 KBASE_DEBUG_ASSERT(0 != pages);
2470
2471 if (size == 0)
2472 goto err;
2473
2474 /* All the alloc calls return zeroed memory */
2475 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
2476 va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL,
2477 attrs);
2478 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2479 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
2480 va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL,
2481 &attrs);
2482 #else
2483 va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL);
2484 #endif
2485 if (!va)
2486 goto err;
2487
2488 /* Store the state so we can free it later. */
2489 handle->cpu_va = va;
2490 handle->dma_pa = dma_pa;
2491 handle->size = size;
2492
2493
2494 reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA);
2495 if (!reg)
2496 goto no_reg;
2497
2498 reg->flags &= ~KBASE_REG_FREE;
2499 if (kbase_update_region_flags(kctx, reg, flags) != 0)
2500 goto invalid_flags;
2501
2502 reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW);
2503 if (IS_ERR_OR_NULL(reg->cpu_alloc))
2504 goto no_alloc;
2505
2506 reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2507
2508 page_array = kbase_get_cpu_phy_pages(reg);
2509
2510 for (i = 0; i < pages; i++)
2511 page_array[i] = dma_pa + (i << PAGE_SHIFT);
2512
2513 reg->cpu_alloc->nents = pages;
2514
2515 kbase_gpu_vm_lock(kctx);
2516 res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1);
2517 kbase_gpu_vm_unlock(kctx);
2518 if (res)
2519 goto no_mmap;
2520
2521 return va;
2522
2523 no_mmap:
2524 kbase_mem_phy_alloc_put(reg->cpu_alloc);
2525 kbase_mem_phy_alloc_put(reg->gpu_alloc);
2526 no_alloc:
2527 invalid_flags:
2528 kfree(reg);
2529 no_reg:
2530 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
2531 dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, attrs);
2532 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2533 dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs);
2534 #else
2535 dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa);
2536 #endif
2537 err:
2538 return NULL;
2539 }
2540 KBASE_EXPORT_SYMBOL(kbase_va_alloc);
2541
2542 void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle)
2543 {
2544 struct kbase_va_region *reg;
2545 int err;
2546 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \
2547 (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
2548 DEFINE_DMA_ATTRS(attrs);
2549 #endif
2550
2551 KBASE_DEBUG_ASSERT(kctx != NULL);
2552 KBASE_DEBUG_ASSERT(handle->cpu_va != NULL);
2553
2554 kbase_gpu_vm_lock(kctx);
2555 reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va);
2556 KBASE_DEBUG_ASSERT(reg);
2557 err = kbase_gpu_munmap(kctx, reg);
2558 kbase_gpu_vm_unlock(kctx);
2559 KBASE_DEBUG_ASSERT(!err);
2560
2561 kbase_mem_phy_alloc_put(reg->cpu_alloc);
2562 kbase_mem_phy_alloc_put(reg->gpu_alloc);
2563 kfree(reg);
2564
2565 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
2566 dma_free_attrs(kctx->kbdev->dev, handle->size,
2567 handle->cpu_va, handle->dma_pa, DMA_ATTR_WRITE_COMBINE);
2568 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2569 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
2570 dma_free_attrs(kctx->kbdev->dev, handle->size,
2571 handle->cpu_va, handle->dma_pa, &attrs);
2572 #else
2573 dma_free_writecombine(kctx->kbdev->dev, handle->size,
2574 handle->cpu_va, handle->dma_pa);
2575 #endif
2576 }
2577 KBASE_EXPORT_SYMBOL(kbase_va_free);
2578
2579