1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /*
23 * Base kernel context APIs
24 */
25 #include <linux/version.h>
26 #if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE
27 #include <linux/sched/task.h>
28 #else
29 #include <linux/sched.h>
30 #endif
31
32 #include <mali_kbase.h>
33 #include <gpu/mali_kbase_gpu_regmap.h>
34 #include <mali_kbase_mem_linux.h>
35 #include <mali_kbase_ctx_sched.h>
36 #include <mali_kbase_mem_pool_group.h>
37 #include <tl/mali_kbase_timeline.h>
38 #include <mmu/mali_kbase_mmu.h>
39 #include <context/mali_kbase_context_internal.h>
40
41 /**
42 * find_process_node - Used to traverse the process rb_tree to find if
43 * process exists already in process rb_tree.
44 *
45 * @node: Pointer to root node to start search.
46 * @tgid: Thread group PID to search for.
47 *
48 * Return: Pointer to kbase_process if exists otherwise NULL.
49 */
find_process_node(struct rb_node * node,pid_t tgid)50 static struct kbase_process *find_process_node(struct rb_node *node, pid_t tgid)
51 {
52 struct kbase_process *kprcs = NULL;
53
54 /* Check if the kctx creation request is from a existing process.*/
55 while (node) {
56 struct kbase_process *prcs_node =
57 rb_entry(node, struct kbase_process, kprcs_node);
58 if (prcs_node->tgid == tgid) {
59 kprcs = prcs_node;
60 break;
61 }
62
63 if (tgid < prcs_node->tgid)
64 node = node->rb_left;
65 else
66 node = node->rb_right;
67 }
68
69 return kprcs;
70 }
71
72 /**
73 * kbase_insert_kctx_to_process - Initialise kbase process context.
74 *
75 * @kctx: Pointer to kbase context.
76 *
77 * Here we initialise per process rb_tree managed by kbase_device.
78 * We maintain a rb_tree of each unique process that gets created.
79 * and Each process maintains a list of kbase context.
80 * This setup is currently used by kernel trace functionality
81 * to trace and visualise gpu memory consumption.
82 *
83 * Return: 0 on success and error number on failure.
84 */
kbase_insert_kctx_to_process(struct kbase_context * kctx)85 static int kbase_insert_kctx_to_process(struct kbase_context *kctx)
86 {
87 struct rb_root *const prcs_root = &kctx->kbdev->process_root;
88 const pid_t tgid = kctx->tgid;
89 struct kbase_process *kprcs = NULL;
90
91 lockdep_assert_held(&kctx->kbdev->kctx_list_lock);
92
93 kprcs = find_process_node(prcs_root->rb_node, tgid);
94
95 /* if the kctx is from new process then create a new kbase_process
96 * and add it to the &kbase_device->rb_tree
97 */
98 if (!kprcs) {
99 struct rb_node **new = &prcs_root->rb_node, *parent = NULL;
100
101 kprcs = kzalloc(sizeof(*kprcs), GFP_KERNEL);
102 if (kprcs == NULL)
103 return -ENOMEM;
104 kprcs->tgid = tgid;
105 INIT_LIST_HEAD(&kprcs->kctx_list);
106 kprcs->dma_buf_root = RB_ROOT;
107 kprcs->total_gpu_pages = 0;
108
109 while (*new) {
110 struct kbase_process *prcs_node;
111
112 parent = *new;
113 prcs_node = rb_entry(parent, struct kbase_process,
114 kprcs_node);
115 if (tgid < prcs_node->tgid)
116 new = &(*new)->rb_left;
117 else
118 new = &(*new)->rb_right;
119 }
120 rb_link_node(&kprcs->kprcs_node, parent, new);
121 rb_insert_color(&kprcs->kprcs_node, prcs_root);
122 }
123
124 kctx->kprcs = kprcs;
125 list_add(&kctx->kprcs_link, &kprcs->kctx_list);
126
127 return 0;
128 }
129
kbase_context_common_init(struct kbase_context * kctx)130 int kbase_context_common_init(struct kbase_context *kctx)
131 {
132 const unsigned long cookies_mask = KBASE_COOKIE_MASK;
133 int err = 0;
134
135 /* creating a context is considered a disjoint event */
136 kbase_disjoint_event(kctx->kbdev);
137
138 kctx->process_mm = NULL;
139 kctx->task = NULL;
140 atomic_set(&kctx->nonmapped_pages, 0);
141 atomic_set(&kctx->permanent_mapped_pages, 0);
142 kctx->tgid = task_tgid_vnr(current);
143 kctx->pid = task_pid_vnr(current);
144
145 /* Check if this is a Userspace created context */
146 if (likely(kctx->filp)) {
147 struct pid *pid_struct;
148
149 rcu_read_lock();
150 pid_struct = find_get_pid(kctx->tgid);
151 if (likely(pid_struct)) {
152 struct task_struct *task = pid_task(pid_struct, PIDTYPE_PID);
153
154 if (likely(task)) {
155 /* Take a reference on the task to avoid slow lookup
156 * later on from the page allocation loop.
157 */
158 get_task_struct(task);
159 kctx->task = task;
160 } else {
161 dev_err(kctx->kbdev->dev,
162 "Failed to get task pointer for %s/%d",
163 current->comm, kctx->pid);
164 err = -ESRCH;
165 }
166
167 put_pid(pid_struct);
168 } else {
169 dev_err(kctx->kbdev->dev,
170 "Failed to get pid pointer for %s/%d",
171 current->comm, kctx->pid);
172 err = -ESRCH;
173 }
174 rcu_read_unlock();
175
176 if (unlikely(err))
177 return err;
178
179 kbase_mem_mmgrab();
180 kctx->process_mm = current->mm;
181 }
182
183 atomic_set(&kctx->used_pages, 0);
184
185 mutex_init(&kctx->reg_lock);
186
187 spin_lock_init(&kctx->mem_partials_lock);
188 INIT_LIST_HEAD(&kctx->mem_partials);
189
190 spin_lock_init(&kctx->waiting_soft_jobs_lock);
191 INIT_LIST_HEAD(&kctx->waiting_soft_jobs);
192
193 init_waitqueue_head(&kctx->event_queue);
194 atomic_set(&kctx->event_count, 0);
195
196 #if !MALI_USE_CSF
197 atomic_set(&kctx->event_closed, false);
198 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
199 atomic_set(&kctx->jctx.work_id, 0);
200 #endif
201 #endif
202
203 #if MALI_USE_CSF
204 atomic64_set(&kctx->num_fixable_allocs, 0);
205 atomic64_set(&kctx->num_fixed_allocs, 0);
206 #endif
207
208 kbase_gpu_vm_lock(kctx);
209 bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG);
210 kbase_gpu_vm_unlock(kctx);
211
212 kctx->id = atomic_add_return(1, &(kctx->kbdev->ctx_num)) - 1;
213
214 mutex_lock(&kctx->kbdev->kctx_list_lock);
215 err = kbase_insert_kctx_to_process(kctx);
216 mutex_unlock(&kctx->kbdev->kctx_list_lock);
217 if (err) {
218 dev_err(kctx->kbdev->dev,
219 "(err:%d) failed to insert kctx to kbase_process", err);
220 if (likely(kctx->filp)) {
221 mmdrop(kctx->process_mm);
222 put_task_struct(kctx->task);
223 }
224 }
225
226 return err;
227 }
228
kbase_context_add_to_dev_list(struct kbase_context * kctx)229 int kbase_context_add_to_dev_list(struct kbase_context *kctx)
230 {
231 if (WARN_ON(!kctx))
232 return -EINVAL;
233
234 if (WARN_ON(!kctx->kbdev))
235 return -EINVAL;
236
237 mutex_lock(&kctx->kbdev->kctx_list_lock);
238 list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list);
239 mutex_unlock(&kctx->kbdev->kctx_list_lock);
240
241 kbase_timeline_post_kbase_context_create(kctx);
242
243 return 0;
244 }
245
kbase_context_remove_from_dev_list(struct kbase_context * kctx)246 void kbase_context_remove_from_dev_list(struct kbase_context *kctx)
247 {
248 if (WARN_ON(!kctx))
249 return;
250
251 if (WARN_ON(!kctx->kbdev))
252 return;
253
254 kbase_timeline_pre_kbase_context_destroy(kctx);
255
256 mutex_lock(&kctx->kbdev->kctx_list_lock);
257 list_del_init(&kctx->kctx_list_link);
258 mutex_unlock(&kctx->kbdev->kctx_list_lock);
259 }
260
261 /**
262 * kbase_remove_kctx_from_process - remove a terminating context from
263 * the process list.
264 *
265 * @kctx: Pointer to kbase context.
266 *
267 * Remove the tracking of context from the list of contexts maintained under
268 * kbase process and if the list if empty then there no outstanding contexts
269 * we can remove the process node as well.
270 */
271
kbase_remove_kctx_from_process(struct kbase_context * kctx)272 static void kbase_remove_kctx_from_process(struct kbase_context *kctx)
273 {
274 struct kbase_process *kprcs = kctx->kprcs;
275
276 lockdep_assert_held(&kctx->kbdev->kctx_list_lock);
277 list_del(&kctx->kprcs_link);
278
279 /* if there are no outstanding contexts in current process node,
280 * we can remove it from the process rb_tree.
281 */
282 if (list_empty(&kprcs->kctx_list)) {
283 rb_erase(&kprcs->kprcs_node, &kctx->kbdev->process_root);
284 /* Add checks, so that the terminating process Should not
285 * hold any gpu_memory.
286 */
287 spin_lock(&kctx->kbdev->gpu_mem_usage_lock);
288 WARN_ON(kprcs->total_gpu_pages);
289 spin_unlock(&kctx->kbdev->gpu_mem_usage_lock);
290 WARN_ON(!RB_EMPTY_ROOT(&kprcs->dma_buf_root));
291 kfree(kprcs);
292 }
293 }
294
kbase_context_common_term(struct kbase_context * kctx)295 void kbase_context_common_term(struct kbase_context *kctx)
296 {
297 int pages;
298
299 pages = atomic_read(&kctx->used_pages);
300 if (pages != 0)
301 dev_warn(kctx->kbdev->dev,
302 "%s: %d pages in use!\n", __func__, pages);
303
304 WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
305
306 mutex_lock(&kctx->kbdev->kctx_list_lock);
307 kbase_remove_kctx_from_process(kctx);
308 mutex_unlock(&kctx->kbdev->kctx_list_lock);
309
310 if (likely(kctx->filp)) {
311 mmdrop(kctx->process_mm);
312 put_task_struct(kctx->task);
313 }
314
315 KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u);
316 }
317
kbase_context_mem_pool_group_init(struct kbase_context * kctx)318 int kbase_context_mem_pool_group_init(struct kbase_context *kctx)
319 {
320 return kbase_mem_pool_group_init(&kctx->mem_pools, kctx->kbdev,
321 &kctx->kbdev->mem_pool_defaults, &kctx->kbdev->mem_pools);
322 }
323
kbase_context_mem_pool_group_term(struct kbase_context * kctx)324 void kbase_context_mem_pool_group_term(struct kbase_context *kctx)
325 {
326 kbase_mem_pool_group_term(&kctx->mem_pools);
327 }
328
kbase_context_mmu_init(struct kbase_context * kctx)329 int kbase_context_mmu_init(struct kbase_context *kctx)
330 {
331 return kbase_mmu_init(
332 kctx->kbdev, &kctx->mmu, kctx,
333 kbase_context_mmu_group_id_get(kctx->create_flags));
334 }
335
kbase_context_mmu_term(struct kbase_context * kctx)336 void kbase_context_mmu_term(struct kbase_context *kctx)
337 {
338 kbase_mmu_term(kctx->kbdev, &kctx->mmu);
339 }
340
kbase_context_mem_alloc_page(struct kbase_context * kctx)341 int kbase_context_mem_alloc_page(struct kbase_context *kctx)
342 {
343 struct page *p;
344
345 p = kbase_mem_alloc_page(&kctx->mem_pools.small[KBASE_MEM_GROUP_SINK]);
346 if (!p)
347 return -ENOMEM;
348
349 kctx->aliasing_sink_page = as_tagged(page_to_phys(p));
350
351 return 0;
352 }
353
kbase_context_mem_pool_free(struct kbase_context * kctx)354 void kbase_context_mem_pool_free(struct kbase_context *kctx)
355 {
356 /* drop the aliasing sink page now that it can't be mapped anymore */
357 kbase_mem_pool_free(
358 &kctx->mem_pools.small[KBASE_MEM_GROUP_SINK],
359 as_page(kctx->aliasing_sink_page),
360 false);
361 }
362
kbase_context_sticky_resource_term(struct kbase_context * kctx)363 void kbase_context_sticky_resource_term(struct kbase_context *kctx)
364 {
365 unsigned long pending_regions_to_clean;
366
367 kbase_gpu_vm_lock(kctx);
368 kbase_sticky_resource_term(kctx);
369
370 /* free pending region setups */
371 pending_regions_to_clean = KBASE_COOKIE_MASK;
372 bitmap_andnot(&pending_regions_to_clean, &pending_regions_to_clean,
373 kctx->cookies, BITS_PER_LONG);
374 while (pending_regions_to_clean) {
375 unsigned int cookie = find_first_bit(&pending_regions_to_clean,
376 BITS_PER_LONG);
377
378 if (!WARN_ON(!kctx->pending_regions[cookie])) {
379 dev_dbg(kctx->kbdev->dev, "Freeing pending unmapped region\n");
380 kbase_mem_phy_alloc_put(
381 kctx->pending_regions[cookie]->cpu_alloc);
382 kbase_mem_phy_alloc_put(
383 kctx->pending_regions[cookie]->gpu_alloc);
384 kfree(kctx->pending_regions[cookie]);
385
386 kctx->pending_regions[cookie] = NULL;
387 }
388
389 bitmap_clear(&pending_regions_to_clean, cookie, 1);
390 }
391 kbase_gpu_vm_unlock(kctx);
392 }
393