1 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2 /*
3 *
4 * (C) COPYRIGHT 2010, 2012-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /**
23 * DOC: Base kernel memory APIs, Linux implementation.
24 */
25
26 #ifndef _KBASE_MEM_LINUX_H_
27 #define _KBASE_MEM_LINUX_H_
28
29 /* A HWC dump mapping */
30 struct kbase_hwc_dma_mapping {
31 void *cpu_va;
32 dma_addr_t dma_pa;
33 size_t size;
34 };
35
36 /**
37 * kbase_mem_alloc - Create a new allocation for GPU
38 *
39 * @kctx: The kernel context
40 * @va_pages: The number of pages of virtual address space to reserve
41 * @commit_pages: The number of physical pages to allocate upfront
42 * @extension: The number of extra pages to allocate on each GPU fault which
43 * grows the region.
44 * @flags: bitmask of BASE_MEM_* flags to convey special requirements &
45 * properties for the new allocation.
46 * @gpu_va: Start address of the memory region which was allocated from GPU
47 * virtual address space. If the BASE_MEM_FLAG_MAP_FIXED is set
48 * then this parameter shall be provided by the caller.
49 * @mmu_sync_info: Indicates whether this call is synchronous wrt MMU ops.
50 *
51 * Return: 0 on success or error code
52 */
53 struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages,
54 u64 extension, u64 *flags, u64 *gpu_va,
55 enum kbase_caller_mmu_sync_info mmu_sync_info);
56
57 /**
58 * kbase_mem_query - Query properties of a GPU memory region
59 *
60 * @kctx: The kernel context
61 * @gpu_addr: A GPU address contained within the memory region
62 * @query: The type of query, from KBASE_MEM_QUERY_* flags, which could be
63 * regarding the amount of backing physical memory allocated so far
64 * for the region or the size of the region or the flags associated
65 * with the region.
66 * @out: Pointer to the location to store the result of query.
67 *
68 * Return: 0 on success or error code
69 */
70 int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query,
71 u64 *const out);
72
73 /**
74 * kbase_mem_import - Import the external memory for use by the GPU
75 *
76 * @kctx: The kernel context
77 * @type: Type of external memory
78 * @phandle: Handle to the external memory interpreted as per the type.
79 * @padding: Amount of extra VA pages to append to the imported buffer
80 * @gpu_va: GPU address assigned to the imported external memory
81 * @va_pages: Size of the memory region reserved from the GPU address space
82 * @flags: bitmask of BASE_MEM_* flags to convey special requirements &
83 * properties for the new allocation representing the external
84 * memory.
85 * Return: 0 on success or error code
86 */
87 int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
88 void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages,
89 u64 *flags);
90
91 /**
92 * kbase_mem_alias - Create a new allocation for GPU, aliasing one or more
93 * memory regions
94 *
95 * @kctx: The kernel context
96 * @flags: bitmask of BASE_MEM_* flags.
97 * @stride: Bytes between start of each memory region
98 * @nents: The number of regions to pack together into the alias
99 * @ai: Pointer to the struct containing the memory aliasing info
100 * @num_pages: Number of pages the alias will cover
101 *
102 * Return: 0 on failure or otherwise the GPU VA for the alias
103 */
104 u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages);
105
106 /**
107 * kbase_mem_flags_change - Change the flags for a memory region
108 *
109 * @kctx: The kernel context
110 * @gpu_addr: A GPU address contained within the memory region to modify.
111 * @flags: The new flags to set
112 * @mask: Mask of the flags, from BASE_MEM_*, to modify.
113 *
114 * Return: 0 on success or error code
115 */
116 int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask);
117
118 /**
119 * kbase_mem_commit - Change the physical backing size of a region
120 *
121 * @kctx: The kernel context
122 * @gpu_addr: Handle to the memory region
123 * @new_pages: Number of physical pages to back the region with
124 *
125 * Return: 0 on success or error code
126 */
127 int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages);
128
129 /**
130 * kbase_mem_shrink - Shrink the physical backing size of a region
131 *
132 * @kctx: The kernel context
133 * @reg: The GPU region
134 * @new_pages: Number of physical pages to back the region with
135 *
136 * Return: 0 on success or error code
137 */
138 int kbase_mem_shrink(struct kbase_context *kctx,
139 struct kbase_va_region *reg, u64 new_pages);
140
141 /**
142 * kbase_context_mmap - Memory map method, gets invoked when mmap system call is
143 * issued on device file /dev/malixx.
144 * @kctx: The kernel context
145 * @vma: Pointer to the struct containing the info where the GPU allocation
146 * will be mapped in virtual address space of CPU.
147 *
148 * Return: 0 on success or error code
149 */
150 int kbase_context_mmap(struct kbase_context *kctx, struct vm_area_struct *vma);
151
152 /**
153 * kbase_mem_evictable_init - Initialize the Ephemeral memory eviction
154 * mechanism.
155 * @kctx: The kbase context to initialize.
156 *
157 * Return: Zero on success or -errno on failure.
158 */
159 int kbase_mem_evictable_init(struct kbase_context *kctx);
160
161 /**
162 * kbase_mem_evictable_deinit - De-initialize the Ephemeral memory eviction
163 * mechanism.
164 * @kctx: The kbase context to de-initialize.
165 */
166 void kbase_mem_evictable_deinit(struct kbase_context *kctx);
167
168 /**
169 * kbase_mem_grow_gpu_mapping - Grow the GPU mapping of an allocation
170 * @kctx: Context the region belongs to
171 * @reg: The GPU region
172 * @new_pages: The number of pages after the grow
173 * @old_pages: The number of pages before the grow
174 * @mmu_sync_info: Indicates whether this call is synchronous wrt MMU ops.
175 *
176 * Return: 0 on success, -errno on error.
177 *
178 * Expand the GPU mapping to encompass the new psychical pages which have
179 * been added to the allocation.
180 *
181 * Note: Caller must be holding the region lock.
182 */
183 int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx,
184 struct kbase_va_region *reg, u64 new_pages,
185 u64 old_pages,
186 enum kbase_caller_mmu_sync_info mmu_sync_info);
187
188 /**
189 * kbase_mem_evictable_make - Make a physical allocation eligible for eviction
190 * @gpu_alloc: The physical allocation to make evictable
191 *
192 * Return: 0 on success, -errno on error.
193 *
194 * Take the provided region and make all the physical pages within it
195 * reclaimable by the kernel, updating the per-process VM stats as well.
196 * Remove any CPU mappings (as these can't be removed in the shrinker callback
197 * as mmap_sem/mmap_lock might already be taken) but leave the GPU mapping
198 * intact as and until the shrinker reclaims the allocation.
199 *
200 * Note: Must be called with the region lock of the containing context.
201 */
202 int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc);
203
204 /**
205 * kbase_mem_evictable_unmake - Remove a physical allocations eligibility for
206 * eviction.
207 * @alloc: The physical allocation to remove eviction eligibility from.
208 *
209 * Return: True if the allocation had its backing restored and false if
210 * it hasn't.
211 *
212 * Make the physical pages in the region no longer reclaimable and update the
213 * per-process stats, if the shrinker has already evicted the memory then
214 * re-allocate it if the region is still alive.
215 *
216 * Note: Must be called with the region lock of the containing context.
217 */
218 bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *alloc);
219
220 typedef unsigned int kbase_vmap_flag;
221
222 /* Sync operations are needed on beginning and ending of access to kernel-mapped GPU memory.
223 *
224 * This is internal to the struct kbase_vmap_struct and should not be passed in by callers of
225 * kbase_vmap-related functions.
226 */
227 #define KBASE_VMAP_FLAG_SYNC_NEEDED (((kbase_vmap_flag)1) << 0)
228
229 /* Permanently mapped memory accounting (including enforcing limits) should be done on the
230 * kernel-mapped GPU memory.
231 *
232 * This should be used if the kernel mapping is going to live for a potentially long time, for
233 * example if it will persist after the caller has returned.
234 */
235 #define KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING (((kbase_vmap_flag)1) << 1)
236
237 /* Set of flags that can be passed into kbase_vmap-related functions */
238 #define KBASE_VMAP_INPUT_FLAGS (KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING)
239
240 struct kbase_vmap_struct {
241 off_t offset_in_page;
242 struct kbase_mem_phy_alloc *cpu_alloc;
243 struct kbase_mem_phy_alloc *gpu_alloc;
244 struct tagged_addr *cpu_pages;
245 struct tagged_addr *gpu_pages;
246 void *addr;
247 size_t size;
248 kbase_vmap_flag flags;
249 };
250
251 /**
252 * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation
253 * @kctx: Context the region belongs to
254 * @reg: The GPU region or NULL if there isn't one
255 * @new_pages: The number of pages after the shrink
256 * @old_pages: The number of pages before the shrink
257 *
258 * Return: 0 on success, negative -errno on error
259 *
260 * Unmap the shrunk pages from the GPU mapping. Note that the size of the region
261 * itself is unmodified as we still need to reserve the VA, only the page tables
262 * will be modified by this function.
263 */
264 int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, struct kbase_va_region *reg,
265 u64 new_pages, u64 old_pages);
266
267 /**
268 * kbase_vmap_reg - Map part of an existing region into the kernel safely, only if the requested
269 * access permissions are supported
270 * @kctx: Context @reg belongs to
271 * @reg: The GPU region to map part of
272 * @gpu_addr: Start address of VA range to map, which must be within @reg
273 * @size: Size of VA range, which when added to @gpu_addr must be within @reg
274 * @prot_request: Flags indicating how the caller will then access the memory
275 * @map: Structure to be given to kbase_vunmap() on freeing
276 * @vmap_flags: Flags of type kbase_vmap_flag
277 *
278 * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error
279 *
280 * Variant of kbase_vmap_prot() that can be used given an existing region.
281 *
282 * The caller must satisfy one of the following for @reg:
283 * * It must have been obtained by finding it on the region tracker, and the region lock must not
284 * have been released in the mean time.
285 * * Or, it must have been refcounted with a call to kbase_va_region_alloc_get(), and the region
286 * lock is now held again.
287 * * Or, @reg has had NO_USER_FREE set at creation time or under the region lock, and the
288 * region lock is now held again.
289 *
290 * The acceptable @vmap_flags are those in %KBASE_VMAP_INPUT_FLAGS.
291 *
292 * Refer to kbase_vmap_prot() for more information on the operation of this function.
293 */
294 void *kbase_vmap_reg(struct kbase_context *kctx, struct kbase_va_region *reg, u64 gpu_addr,
295 size_t size, unsigned long prot_request, struct kbase_vmap_struct *map,
296 kbase_vmap_flag vmap_flags);
297
298 /**
299 * kbase_vmap_prot - Map a GPU VA range into the kernel safely, only if the
300 * requested access permissions are supported
301 * @kctx: Context the VA range belongs to
302 * @gpu_addr: Start address of VA range
303 * @size: Size of VA range
304 * @prot_request: Flags indicating how the caller will then access the memory
305 * @map: Structure to be given to kbase_vunmap() on freeing
306 *
307 * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error
308 *
309 * Map a GPU VA Range into the kernel. The VA range must be contained within a
310 * GPU memory region. Appropriate CPU cache-flushing operations are made as
311 * required, dependent on the CPU mapping for the memory region.
312 *
313 * This is safer than using kmap() on the pages directly,
314 * because the pages here are refcounted to prevent freeing (and hence reuse
315 * elsewhere in the system) until an kbase_vunmap()
316 *
317 * The flags in @prot_request should use KBASE_REG_{CPU,GPU}_{RD,WR}, to check
318 * whether the region should allow the intended access, and return an error if
319 * disallowed. This is essential for security of imported memory, particularly
320 * a user buf from SHM mapped into the process as RO. In that case, write
321 * access must be checked if the intention is for kernel to write to the
322 * memory.
323 *
324 * The checks are also there to help catch access errors on memory where
325 * security is not a concern: imported memory that is always RW, and memory
326 * that was allocated and owned by the process attached to @kctx. In this case,
327 * it helps to identify memory that was mapped with the wrong access type.
328 *
329 * Note: KBASE_REG_GPU_{RD,WR} flags are currently supported for legacy cases
330 * where either the security of memory is solely dependent on those flags, or
331 * when userspace code was expecting only the GPU to access the memory (e.g. HW
332 * workarounds).
333 *
334 * All cache maintenance operations shall be ignored if the
335 * memory region has been imported.
336 *
337 */
338 void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size,
339 unsigned long prot_request, struct kbase_vmap_struct *map);
340
341 /**
342 * kbase_vmap - Map a GPU VA range into the kernel safely
343 * @kctx: Context the VA range belongs to
344 * @gpu_addr: Start address of VA range
345 * @size: Size of VA range
346 * @map: Structure to be given to kbase_vunmap() on freeing
347 *
348 * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error
349 *
350 * Map a GPU VA Range into the kernel. The VA range must be contained within a
351 * GPU memory region. Appropriate CPU cache-flushing operations are made as
352 * required, dependent on the CPU mapping for the memory region.
353 *
354 * This is safer than using kmap() on the pages directly,
355 * because the pages here are refcounted to prevent freeing (and hence reuse
356 * elsewhere in the system) until an kbase_vunmap()
357 *
358 * kbase_vmap_prot() should be used in preference, since kbase_vmap() makes no
359 * checks to ensure the security of e.g. imported user bufs from RO SHM.
360 *
361 * Note: All cache maintenance operations shall be ignored if the memory region
362 * has been imported.
363 */
364 void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size,
365 struct kbase_vmap_struct *map);
366
367 /**
368 * kbase_vunmap - Unmap a GPU VA range from the kernel
369 * @kctx: Context the VA range belongs to
370 * @map: Structure describing the mapping from the corresponding kbase_vmap()
371 * call
372 *
373 * Unmaps a GPU VA range from the kernel, given its @map structure obtained
374 * from kbase_vmap(). Appropriate CPU cache-flushing operations are made as
375 * required, dependent on the CPU mapping for the memory region.
376 *
377 * The reference taken on pages during kbase_vmap() is released.
378 *
379 * Note: All cache maintenance operations shall be ignored if the memory region
380 * has been imported.
381 */
382 void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map);
383
384 extern const struct vm_operations_struct kbase_vm_ops;
385
386 /**
387 * kbase_sync_mem_regions - Perform the cache maintenance for the kernel mode
388 * CPU mapping.
389 * @kctx: Context the CPU mapping belongs to.
390 * @map: Structure describing the CPU mapping, setup previously by the
391 * kbase_vmap() call.
392 * @dest: Indicates the type of maintenance required (i.e. flush or invalidate)
393 *
394 * Note: The caller shall ensure that CPU mapping is not revoked & remains
395 * active whilst the maintenance is in progress.
396 */
397 void kbase_sync_mem_regions(struct kbase_context *kctx,
398 struct kbase_vmap_struct *map, enum kbase_sync_type dest);
399
400 /**
401 * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation
402 * @kctx: Context the region belongs to
403 * @reg: The GPU region
404 * @new_pages: The number of pages after the shrink
405 * @old_pages: The number of pages before the shrink
406 *
407 * Shrink (or completely remove) all CPU mappings which reference the shrunk
408 * part of the allocation.
409 */
410 void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
411 struct kbase_va_region *reg,
412 u64 new_pages, u64 old_pages);
413
414 /**
415 * kbase_phy_alloc_mapping_term - Terminate the kernel side mapping of a
416 * physical allocation
417 * @kctx: The kernel base context associated with the mapping
418 * @alloc: Pointer to the allocation to terminate
419 *
420 * This function will unmap the kernel mapping, and free any structures used to
421 * track it.
422 */
423 void kbase_phy_alloc_mapping_term(struct kbase_context *kctx,
424 struct kbase_mem_phy_alloc *alloc);
425
426 /**
427 * kbase_phy_alloc_mapping_get - Get a kernel-side CPU pointer to the permanent
428 * mapping of a physical allocation
429 * @kctx: The kernel base context @gpu_addr will be looked up in
430 * @gpu_addr: The gpu address to lookup for the kernel-side CPU mapping
431 * @out_kern_mapping: Pointer to storage for a struct kbase_vmap_struct pointer
432 * which will be used for a call to
433 * kbase_phy_alloc_mapping_put()
434 *
435 * Return: Pointer to a kernel-side accessible location that directly
436 * corresponds to @gpu_addr, or NULL on failure
437 *
438 * Looks up @gpu_addr to retrieve the CPU pointer that can be used to access
439 * that location kernel-side. Only certain kinds of memory have a permanent
440 * kernel mapping, refer to the internal functions
441 * kbase_reg_needs_kernel_mapping() and kbase_phy_alloc_mapping_init() for more
442 * information.
443 *
444 * If this function succeeds, a CPU access to the returned pointer will access
445 * the actual location represented by @gpu_addr. That is, the return value does
446 * not require any offset added to it to access the location specified in
447 * @gpu_addr
448 *
449 * The client must take care to either apply any necessary sync operations when
450 * accessing the data, or ensure that the enclosing region was coherent with
451 * the GPU, or uncached in the CPU.
452 *
453 * The refcount on the physical allocations backing the region are taken, so
454 * that they do not disappear whilst the client is accessing it. Once the
455 * client has finished accessing the memory, it must be released with a call to
456 * kbase_phy_alloc_mapping_put()
457 *
458 * Whilst this is expected to execute quickly (the mapping was already setup
459 * when the physical allocation was created), the call is not IRQ-safe due to
460 * the region lookup involved.
461 *
462 * An error code may indicate that:
463 * - a userside process has freed the allocation, and so @gpu_addr is no longer
464 * valid
465 * - the region containing @gpu_addr does not support a permanent kernel mapping
466 */
467 void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx, u64 gpu_addr,
468 struct kbase_vmap_struct **out_kern_mapping);
469
470 /**
471 * kbase_phy_alloc_mapping_put - Put a reference to the kernel-side mapping of a
472 * physical allocation
473 * @kctx: The kernel base context associated with the mapping
474 * @kern_mapping: Pointer to a struct kbase_phy_alloc_mapping pointer obtained
475 * from a call to kbase_phy_alloc_mapping_get()
476 *
477 * Releases the reference to the allocations backing @kern_mapping that was
478 * obtained through a call to kbase_phy_alloc_mapping_get(). This must be used
479 * when the client no longer needs to access the kernel-side CPU pointer.
480 *
481 * If this was the last reference on the underlying physical allocations, they
482 * will go through the normal allocation free steps, which also includes an
483 * unmap of the permanent kernel mapping for those allocations.
484 *
485 * Due to these operations, the function is not IRQ-safe. However it is
486 * expected to execute quickly in the normal case, i.e. when the region holding
487 * the physical allocation is still present.
488 */
489 void kbase_phy_alloc_mapping_put(struct kbase_context *kctx,
490 struct kbase_vmap_struct *kern_mapping);
491
492 /**
493 * kbase_get_cache_line_alignment - Return cache line alignment
494 *
495 * @kbdev: Device pointer.
496 *
497 * Helper function to return the maximum cache line alignment considering
498 * both CPU and GPU cache sizes.
499 *
500 * Return: CPU and GPU cache line alignment, in bytes.
501 */
502 u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev);
503
504 #if (KERNEL_VERSION(4, 20, 0) > LINUX_VERSION_CODE)
vmf_insert_pfn_prot(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,pgprot_t pgprot)505 static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma,
506 unsigned long addr, unsigned long pfn, pgprot_t pgprot)
507 {
508 int err = vm_insert_pfn_prot(vma, addr, pfn, pgprot);
509
510 if (unlikely(err == -ENOMEM))
511 return VM_FAULT_OOM;
512 if (unlikely(err < 0 && err != -EBUSY))
513 return VM_FAULT_SIGBUS;
514
515 return VM_FAULT_NOPAGE;
516 }
517 #endif
518
519 /**
520 * kbase_mem_get_process_mmap_lock - Return the mmap lock for the current process
521 *
522 * Return: the mmap lock for the current process
523 */
kbase_mem_get_process_mmap_lock(void)524 static inline struct rw_semaphore *kbase_mem_get_process_mmap_lock(void)
525 {
526 #if KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE
527 return ¤t->mm->mmap_sem;
528 #else /* KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE */
529 return ¤t->mm->mmap_lock;
530 #endif /* KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE */
531 }
532
533 #endif /* _KBASE_MEM_LINUX_H_ */
534