1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 #include <linux/dma-buf.h>
23 #if IS_ENABLED(CONFIG_COMPAT)
24 #include <linux/compat.h>
25 #endif
26 #include <mali_kbase.h>
27 #include <linux/random.h>
28 #include <linux/version.h>
29 #include <linux/ratelimit.h>
30 #include <linux/priority_control_manager.h>
31 #if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE
32 #include <linux/sched/signal.h>
33 #else
34 #include <linux/signal.h>
35 #endif
36
37 #include <mali_kbase_jm.h>
38 #include <mali_kbase_kinstr_jm.h>
39 #include <mali_kbase_hwaccess_jm.h>
40 #include <tl/mali_kbase_tracepoints.h>
41 #include <mali_linux_trace.h>
42
43 #include <mali_kbase_cs_experimental.h>
44
45 #include <mali_kbase_caps.h>
46
47 /* Return whether katom will run on the GPU or not. Currently only soft jobs and
48 * dependency-only atoms do not run on the GPU
49 */
50 #define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \
51 ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \
52 BASE_JD_REQ_DEP)))
53
54 /*
55 * This is the kernel side of the API. Only entry points are:
56 * - kbase_jd_submit(): Called from userspace to submit a single bag
57 * - kbase_jd_done(): Called from interrupt context to track the
58 * completion of a job.
59 * Callouts:
60 * - to the job manager (enqueue a job)
61 * - to the event subsystem (signals the completion/failure of bag/job-chains).
62 */
63
64 static void __user *
get_compat_pointer(struct kbase_context * kctx,const u64 p)65 get_compat_pointer(struct kbase_context *kctx, const u64 p)
66 {
67 #if IS_ENABLED(CONFIG_COMPAT)
68 if (kbase_ctx_flag(kctx, KCTX_COMPAT))
69 return compat_ptr(p);
70 #endif
71 return u64_to_user_ptr(p);
72 }
73
74 /* Mark an atom as complete, and trace it in kinstr_jm */
jd_mark_atom_complete(struct kbase_jd_atom * katom)75 static void jd_mark_atom_complete(struct kbase_jd_atom *katom)
76 {
77 katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
78 kbase_kinstr_jm_atom_complete(katom);
79 dev_dbg(katom->kctx->kbdev->dev, "Atom %pK status to completed\n",
80 (void *)katom);
81 KBASE_TLSTREAM_TL_JD_ATOM_COMPLETE(katom->kctx->kbdev, katom);
82 }
83
84 /* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs
85 *
86 * Returns whether the JS needs a reschedule.
87 *
88 * Note that the caller must also check the atom status and
89 * if it is KBASE_JD_ATOM_STATE_COMPLETED must call kbase_jd_done_nolock
90 */
jd_run_atom(struct kbase_jd_atom * katom)91 static bool jd_run_atom(struct kbase_jd_atom *katom)
92 {
93 struct kbase_context *kctx = katom->kctx;
94
95 dev_dbg(kctx->kbdev->dev, "JD run atom %pK in kctx %pK\n",
96 (void *)katom, (void *)kctx);
97
98 KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED);
99
100 if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) {
101 /* Dependency only atom */
102 trace_sysgraph(SGR_SUBMIT, kctx->id,
103 kbase_jd_atom_id(katom->kctx, katom));
104 jd_mark_atom_complete(katom);
105 return false;
106 } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
107 /* Soft-job */
108 if (katom->will_fail_event_code) {
109 kbase_finish_soft_job(katom);
110 jd_mark_atom_complete(katom);
111 return false;
112 }
113 if (kbase_process_soft_job(katom) == 0) {
114 kbase_finish_soft_job(katom);
115 jd_mark_atom_complete(katom);
116 }
117 return false;
118 }
119
120 katom->status = KBASE_JD_ATOM_STATE_IN_JS;
121 dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", (void *)katom);
122 /* Queue an action about whether we should try scheduling a context */
123 return kbasep_js_add_job(kctx, katom);
124 }
125
kbase_jd_dep_clear_locked(struct kbase_jd_atom * katom)126 void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom)
127 {
128 struct kbase_device *kbdev;
129
130 KBASE_DEBUG_ASSERT(katom);
131 kbdev = katom->kctx->kbdev;
132 KBASE_DEBUG_ASSERT(kbdev);
133
134 /* Check whether the atom's other dependencies were already met. If
135 * katom is a GPU atom then the job scheduler may be able to represent
136 * the dependencies, hence we may attempt to submit it before they are
137 * met. Other atoms must have had both dependencies resolved.
138 */
139 if (IS_GPU_ATOM(katom) ||
140 (!kbase_jd_katom_dep_atom(&katom->dep[0]) &&
141 !kbase_jd_katom_dep_atom(&katom->dep[1]))) {
142 /* katom dep complete, attempt to run it */
143 bool resched = false;
144
145 KBASE_TLSTREAM_TL_RUN_ATOM_START(
146 katom->kctx->kbdev, katom,
147 kbase_jd_atom_id(katom->kctx, katom));
148 resched = jd_run_atom(katom);
149 KBASE_TLSTREAM_TL_RUN_ATOM_END(katom->kctx->kbdev, katom,
150 kbase_jd_atom_id(katom->kctx,
151 katom));
152
153 if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
154 /* The atom has already finished */
155 resched |= kbase_jd_done_nolock(katom, true);
156 }
157
158 if (resched)
159 kbase_js_sched_all(kbdev);
160 }
161 }
162
kbase_jd_free_external_resources(struct kbase_jd_atom * katom)163 void kbase_jd_free_external_resources(struct kbase_jd_atom *katom)
164 {
165 }
166
kbase_jd_post_external_resources(struct kbase_jd_atom * katom)167 static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom)
168 {
169 KBASE_DEBUG_ASSERT(katom);
170 KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);
171
172 kbase_gpu_vm_lock(katom->kctx);
173 /* only roll back if extres is non-NULL */
174 if (katom->extres) {
175 u32 res_no;
176
177 res_no = katom->nr_extres;
178 while (res_no-- > 0) {
179 kbase_unmap_external_resource(katom->kctx, katom->extres[res_no]);
180 }
181 kfree(katom->extres);
182 katom->extres = NULL;
183 }
184 kbase_gpu_vm_unlock(katom->kctx);
185 }
186
187 /*
188 * Set up external resources needed by this job.
189 *
190 * jctx.lock must be held when this is called.
191 */
192
kbase_jd_pre_external_resources(struct kbase_jd_atom * katom,const struct base_jd_atom * user_atom)193 static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom *user_atom)
194 {
195 int err = -EINVAL;
196 u32 res_no;
197 struct base_external_resource *input_extres;
198
199 KBASE_DEBUG_ASSERT(katom);
200 KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);
201
202 /* no resources encoded, early out */
203 if (!katom->nr_extres)
204 return -EINVAL;
205
206 katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL);
207 if (!katom->extres)
208 return -ENOMEM;
209
210 input_extres = kmalloc_array(katom->nr_extres, sizeof(*input_extres), GFP_KERNEL);
211 if (!input_extres) {
212 err = -ENOMEM;
213 goto failed_input_alloc;
214 }
215
216 if (copy_from_user(input_extres,
217 get_compat_pointer(katom->kctx, user_atom->extres_list),
218 sizeof(*input_extres) * katom->nr_extres) != 0) {
219 err = -EINVAL;
220 goto failed_input_copy;
221 }
222
223 /* Take the processes mmap lock */
224 down_read(kbase_mem_get_process_mmap_lock());
225
226 /* need to keep the GPU VM locked while we set up UMM buffers */
227 kbase_gpu_vm_lock(katom->kctx);
228 for (res_no = 0; res_no < katom->nr_extres; res_no++) {
229 struct base_external_resource *user_res = &input_extres[res_no];
230 struct kbase_va_region *reg;
231
232 reg = kbase_region_tracker_find_region_enclosing_address(
233 katom->kctx, user_res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE);
234 /* did we find a matching region object? */
235 if (unlikely(kbase_is_region_invalid_or_free(reg))) {
236 /* roll back */
237 goto failed_loop;
238 }
239
240 if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) &&
241 (reg->flags & KBASE_REG_PROTECTED)) {
242 katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED;
243 }
244
245 err = kbase_map_external_resource(katom->kctx, reg, current->mm);
246 if (err)
247 goto failed_loop;
248
249 katom->extres[res_no] = reg;
250 }
251 /* successfully parsed the extres array */
252 /* drop the vm lock now */
253 kbase_gpu_vm_unlock(katom->kctx);
254
255 /* Release the processes mmap lock */
256 up_read(kbase_mem_get_process_mmap_lock());
257
258 /* Free the buffer holding data from userspace */
259 kfree(input_extres);
260
261 /* all done OK */
262 return 0;
263
264 /* error handling section */
265 failed_loop:
266 /* undo the loop work. We are guaranteed to have access to the VA region
267 * as we hold a reference to it until it's unmapped
268 */
269 while (res_no-- > 0) {
270 struct kbase_va_region *reg = katom->extres[res_no];
271
272 kbase_unmap_external_resource(katom->kctx, reg);
273 }
274 kbase_gpu_vm_unlock(katom->kctx);
275
276 /* Release the processes mmap lock */
277 up_read(kbase_mem_get_process_mmap_lock());
278
279 failed_input_copy:
280 kfree(input_extres);
281 failed_input_alloc:
282 kfree(katom->extres);
283 katom->extres = NULL;
284 return err;
285 }
286
jd_resolve_dep(struct list_head * out_list,struct kbase_jd_atom * katom,u8 d,bool ctx_is_dying)287 static inline void jd_resolve_dep(struct list_head *out_list,
288 struct kbase_jd_atom *katom,
289 u8 d, bool ctx_is_dying)
290 {
291 u8 other_d = !d;
292
293 while (!list_empty(&katom->dep_head[d])) {
294 struct kbase_jd_atom *dep_atom;
295 struct kbase_jd_atom *other_dep_atom;
296 u8 dep_type;
297
298 dep_atom = list_entry(katom->dep_head[d].next,
299 struct kbase_jd_atom, dep_item[d]);
300 list_del(katom->dep_head[d].next);
301
302 dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]);
303 kbase_jd_katom_dep_clear(&dep_atom->dep[d]);
304
305 if (katom->event_code != BASE_JD_EVENT_DONE &&
306 (dep_type != BASE_JD_DEP_TYPE_ORDER)) {
307 dep_atom->event_code = katom->event_code;
308 KBASE_DEBUG_ASSERT(dep_atom->status !=
309 KBASE_JD_ATOM_STATE_UNUSED);
310
311 dep_atom->will_fail_event_code = dep_atom->event_code;
312 }
313 other_dep_atom = (struct kbase_jd_atom *)
314 kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]);
315
316 if (!dep_atom->in_jd_list && (!other_dep_atom ||
317 (IS_GPU_ATOM(dep_atom) && !ctx_is_dying &&
318 !dep_atom->will_fail_event_code &&
319 !other_dep_atom->will_fail_event_code))) {
320 dep_atom->in_jd_list = true;
321 list_add_tail(&dep_atom->jd_item, out_list);
322 }
323 }
324 }
325
326 /**
327 * is_dep_valid - Validate that a dependency is valid for early dependency
328 * submission
329 * @katom: Dependency atom to validate
330 *
331 * A dependency is valid if any of the following are true :
332 * - It does not exist (a non-existent dependency does not block submission)
333 * - It is in the job scheduler
334 * - It has completed, does not have a failure event code, and has not been
335 * marked to fail in the future
336 *
337 * Return: true if valid, false otherwise
338 */
is_dep_valid(struct kbase_jd_atom * katom)339 static bool is_dep_valid(struct kbase_jd_atom *katom)
340 {
341 /* If there's no dependency then this is 'valid' from the perspective of
342 * early dependency submission
343 */
344 if (!katom)
345 return true;
346
347 /* Dependency must have reached the job scheduler */
348 if (katom->status < KBASE_JD_ATOM_STATE_IN_JS)
349 return false;
350
351 /* If dependency has completed and has failed or will fail then it is
352 * not valid
353 */
354 if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED &&
355 (katom->event_code != BASE_JD_EVENT_DONE ||
356 katom->will_fail_event_code))
357 return false;
358
359 return true;
360 }
361
jd_try_submitting_deps(struct list_head * out_list,struct kbase_jd_atom * node)362 static void jd_try_submitting_deps(struct list_head *out_list,
363 struct kbase_jd_atom *node)
364 {
365 int i;
366
367 for (i = 0; i < 2; i++) {
368 struct list_head *pos;
369
370 list_for_each(pos, &node->dep_head[i]) {
371 struct kbase_jd_atom *dep_atom = list_entry(pos,
372 struct kbase_jd_atom, dep_item[i]);
373
374 if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) {
375 /*Check if atom deps look sane*/
376 bool dep0_valid = is_dep_valid(
377 dep_atom->dep[0].atom);
378 bool dep1_valid = is_dep_valid(
379 dep_atom->dep[1].atom);
380
381 if (dep0_valid && dep1_valid) {
382 dep_atom->in_jd_list = true;
383 list_add(&dep_atom->jd_item, out_list);
384 }
385 }
386 }
387 }
388 }
389
390 #if MALI_JIT_PRESSURE_LIMIT_BASE
391 /**
392 * jd_update_jit_usage - Update just-in-time physical memory usage for an atom.
393 *
394 * @katom: An atom that has just finished.
395 *
396 * Read back actual just-in-time memory region usage from atoms that provide
397 * this information, and update the current physical page pressure.
398 *
399 * The caller must hold the kbase_jd_context.lock.
400 */
jd_update_jit_usage(struct kbase_jd_atom * katom)401 static void jd_update_jit_usage(struct kbase_jd_atom *katom)
402 {
403 struct kbase_context *kctx = katom->kctx;
404 struct kbase_va_region *reg;
405 struct kbase_vmap_struct mapping;
406 u64 *ptr;
407 u64 used_pages;
408 unsigned int idx;
409
410 lockdep_assert_held(&kctx->jctx.lock);
411
412 /* If this atom wrote to JIT memory, find out how much it has written
413 * and update the usage information in the region.
414 */
415 for (idx = 0;
416 idx < ARRAY_SIZE(katom->jit_ids) && katom->jit_ids[idx];
417 idx++) {
418 enum heap_pointer { LOW = 0, HIGH, COUNT };
419 size_t size_to_read;
420 u64 read_val;
421
422 reg = kctx->jit_alloc[katom->jit_ids[idx]];
423
424 if (!reg) {
425 dev_warn(kctx->kbdev->dev,
426 "%s: JIT id[%u]=%u has no region\n",
427 __func__, idx, katom->jit_ids[idx]);
428 continue;
429 }
430
431 if (reg == KBASE_RESERVED_REG_JIT_ALLOC) {
432 dev_warn(kctx->kbdev->dev,
433 "%s: JIT id[%u]=%u has failed to allocate a region\n",
434 __func__, idx, katom->jit_ids[idx]);
435 continue;
436 }
437
438 if (!reg->heap_info_gpu_addr)
439 continue;
440
441 size_to_read = sizeof(*ptr);
442 if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE)
443 size_to_read = sizeof(u32);
444 else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP)
445 size_to_read = sizeof(u64[COUNT]);
446
447 ptr = kbase_vmap_prot(kctx, reg->heap_info_gpu_addr, size_to_read,
448 KBASE_REG_CPU_RD, &mapping);
449
450 if (!ptr) {
451 dev_warn(kctx->kbdev->dev,
452 "%s: JIT id[%u]=%u start=0x%llx unable to map end marker %llx\n",
453 __func__, idx, katom->jit_ids[idx],
454 reg->start_pfn << PAGE_SHIFT,
455 reg->heap_info_gpu_addr);
456 continue;
457 }
458
459 if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) {
460 read_val = READ_ONCE(*(u32 *)ptr);
461 used_pages = PFN_UP(read_val);
462 } else {
463 u64 addr_end;
464
465 if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) {
466 const unsigned long extension_bytes =
467 reg->extension << PAGE_SHIFT;
468 const u64 low_ptr = ptr[LOW];
469 const u64 high_ptr = ptr[HIGH];
470
471 /* As either the low or high pointer could
472 * consume their partition and move onto the
473 * next chunk, we need to account for both.
474 * In the case where nothing has been allocated
475 * from the high pointer the whole chunk could
476 * be backed unnecessarily - but the granularity
477 * is the chunk size anyway and any non-zero
478 * offset of low pointer from the start of the
479 * chunk would result in the whole chunk being
480 * backed.
481 */
482 read_val = max(high_ptr, low_ptr);
483
484 /* kbase_check_alloc_sizes() already satisfies
485 * this, but here to avoid future maintenance
486 * hazards
487 */
488 WARN_ON(!is_power_of_2(extension_bytes));
489 addr_end = ALIGN(read_val, extension_bytes);
490 } else {
491 addr_end = read_val = READ_ONCE(*ptr);
492 }
493
494 if (addr_end >= (reg->start_pfn << PAGE_SHIFT))
495 used_pages = PFN_UP(addr_end) - reg->start_pfn;
496 else
497 used_pages = reg->used_pages;
498 }
499
500 trace_mali_jit_report(katom, reg, idx, read_val, used_pages);
501 kbase_trace_jit_report_gpu_mem(kctx, reg, 0u);
502
503 /* We can never have used more pages than the VA size of the
504 * region
505 */
506 if (used_pages > reg->nr_pages) {
507 dev_warn(kctx->kbdev->dev,
508 "%s: JIT id[%u]=%u start=0x%llx used_pages %llx > %zx (read 0x%llx as %s%s)\n",
509 __func__, idx, katom->jit_ids[idx],
510 reg->start_pfn << PAGE_SHIFT,
511 used_pages, reg->nr_pages, read_val,
512 (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) ?
513 "size" : "addr",
514 (reg->flags & KBASE_REG_TILER_ALIGN_TOP) ?
515 " with align" : "");
516 used_pages = reg->nr_pages;
517 }
518 /* Note: one real use case has an atom correctly reporting 0
519 * pages in use. This happens in normal use-cases but may only
520 * happen for a few of the application's frames.
521 */
522
523 kbase_vunmap(kctx, &mapping);
524
525 kbase_jit_report_update_pressure(kctx, reg, used_pages, 0u);
526 }
527
528 kbase_jit_retry_pending_alloc(kctx);
529 }
530 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
531
kbase_jd_done_nolock(struct kbase_jd_atom * katom,bool post_immediately)532 bool kbase_jd_done_nolock(struct kbase_jd_atom *katom, bool post_immediately)
533 {
534 struct kbase_context *kctx = katom->kctx;
535 struct list_head completed_jobs;
536 struct list_head runnable_jobs;
537 bool need_to_try_schedule_context = false;
538 int i;
539
540 lockdep_assert_held(&kctx->jctx.lock);
541
542 KBASE_TLSTREAM_TL_JD_DONE_NO_LOCK_START(kctx->kbdev, katom);
543
544 INIT_LIST_HEAD(&completed_jobs);
545 INIT_LIST_HEAD(&runnable_jobs);
546
547 KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED);
548
549 #if MALI_JIT_PRESSURE_LIMIT_BASE
550 if (kbase_ctx_flag(kctx, KCTX_JPL_ENABLED))
551 jd_update_jit_usage(katom);
552 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
553
554 /* This is needed in case an atom is failed due to being invalid, this
555 * can happen *before* the jobs that the atom depends on have completed
556 */
557 for (i = 0; i < 2; i++) {
558 if (kbase_jd_katom_dep_atom(&katom->dep[i])) {
559 list_del(&katom->dep_item[i]);
560 kbase_jd_katom_dep_clear(&katom->dep[i]);
561 }
562 }
563
564 jd_mark_atom_complete(katom);
565
566 list_add_tail(&katom->jd_item, &completed_jobs);
567
568 while (!list_empty(&completed_jobs)) {
569 katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item);
570 list_del(completed_jobs.prev);
571 KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED);
572
573 for (i = 0; i < 2; i++)
574 jd_resolve_dep(&runnable_jobs, katom, i,
575 kbase_ctx_flag(kctx, KCTX_DYING));
576
577 if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES)
578 kbase_jd_post_external_resources(katom);
579
580 while (!list_empty(&runnable_jobs)) {
581 struct kbase_jd_atom *node;
582
583 node = list_entry(runnable_jobs.next,
584 struct kbase_jd_atom, jd_item);
585 list_del(runnable_jobs.next);
586 node->in_jd_list = false;
587
588 dev_dbg(kctx->kbdev->dev, "List node %pK has status %d\n",
589 node, node->status);
590
591 KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED);
592 if (node->status == KBASE_JD_ATOM_STATE_IN_JS)
593 continue;
594
595 if (node->status != KBASE_JD_ATOM_STATE_COMPLETED &&
596 !kbase_ctx_flag(kctx, KCTX_DYING)) {
597 KBASE_TLSTREAM_TL_RUN_ATOM_START(
598 kctx->kbdev, node,
599 kbase_jd_atom_id(kctx, node));
600 need_to_try_schedule_context |= jd_run_atom(node);
601 KBASE_TLSTREAM_TL_RUN_ATOM_END(
602 kctx->kbdev, node,
603 kbase_jd_atom_id(kctx, node));
604 } else {
605 node->event_code = katom->event_code;
606
607 if (node->core_req &
608 BASE_JD_REQ_SOFT_JOB) {
609 WARN_ON(!list_empty(&node->queue));
610 kbase_finish_soft_job(node);
611 }
612 node->status = KBASE_JD_ATOM_STATE_COMPLETED;
613 }
614
615 if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) {
616 list_add_tail(&node->jd_item, &completed_jobs);
617 } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS &&
618 !node->will_fail_event_code) {
619 /* Node successfully submitted, try submitting
620 * dependencies as they may now be representable
621 * in JS
622 */
623 jd_try_submitting_deps(&runnable_jobs, node);
624 }
625 }
626
627 /* Register a completed job as a disjoint event when the GPU
628 * is in a disjoint state (ie. being reset).
629 */
630 kbase_disjoint_event_potential(kctx->kbdev);
631 if (post_immediately && list_empty(&kctx->completed_jobs))
632 kbase_event_post(kctx, katom);
633 else
634 list_add_tail(&katom->jd_item, &kctx->completed_jobs);
635
636 /* Decrement and check the TOTAL number of jobs. This includes
637 * those not tracked by the scheduler: 'not ready to run' and
638 * 'dependency-only' jobs.
639 */
640 if (--kctx->jctx.job_nr == 0)
641 /* All events are safely queued now, and we can signal
642 * any waiter that we've got no more jobs (so we can be
643 * safely terminated)
644 */
645 wake_up(&kctx->jctx.zero_jobs_wait);
646 }
647 KBASE_TLSTREAM_TL_JD_DONE_NO_LOCK_END(kctx->kbdev, katom);
648 return need_to_try_schedule_context;
649 }
650
651 KBASE_EXPORT_TEST_API(kbase_jd_done_nolock);
652
653 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
654 enum {
655 CORE_REQ_DEP_ONLY,
656 CORE_REQ_SOFT,
657 CORE_REQ_COMPUTE,
658 CORE_REQ_FRAGMENT,
659 CORE_REQ_VERTEX,
660 CORE_REQ_TILER,
661 CORE_REQ_FRAGMENT_VERTEX,
662 CORE_REQ_FRAGMENT_VERTEX_TILER,
663 CORE_REQ_FRAGMENT_TILER,
664 CORE_REQ_VERTEX_TILER,
665 CORE_REQ_UNKNOWN
666 };
667 static const char * const core_req_strings[] = {
668 "Dependency Only Job",
669 "Soft Job",
670 "Compute Shader Job",
671 "Fragment Shader Job",
672 "Vertex/Geometry Shader Job",
673 "Tiler Job",
674 "Fragment Shader + Vertex/Geometry Shader Job",
675 "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job",
676 "Fragment Shader + Tiler Job",
677 "Vertex/Geometry Shader Job + Tiler Job",
678 "Unknown Job"
679 };
kbasep_map_core_reqs_to_string(base_jd_core_req core_req)680 static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req)
681 {
682 if (core_req & BASE_JD_REQ_SOFT_JOB)
683 return core_req_strings[CORE_REQ_SOFT];
684 if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
685 return core_req_strings[CORE_REQ_COMPUTE];
686 switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) {
687 case BASE_JD_REQ_DEP:
688 return core_req_strings[CORE_REQ_DEP_ONLY];
689 case BASE_JD_REQ_FS:
690 return core_req_strings[CORE_REQ_FRAGMENT];
691 case BASE_JD_REQ_CS:
692 return core_req_strings[CORE_REQ_VERTEX];
693 case BASE_JD_REQ_T:
694 return core_req_strings[CORE_REQ_TILER];
695 case (BASE_JD_REQ_FS | BASE_JD_REQ_CS):
696 return core_req_strings[CORE_REQ_FRAGMENT_VERTEX];
697 case (BASE_JD_REQ_FS | BASE_JD_REQ_T):
698 return core_req_strings[CORE_REQ_FRAGMENT_TILER];
699 case (BASE_JD_REQ_CS | BASE_JD_REQ_T):
700 return core_req_strings[CORE_REQ_VERTEX_TILER];
701 case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T):
702 return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER];
703 }
704 return core_req_strings[CORE_REQ_UNKNOWN];
705 }
706 #endif
707
708 /* Trace an atom submission. */
jd_trace_atom_submit(struct kbase_context * const kctx,struct kbase_jd_atom * const katom,int * priority)709 static void jd_trace_atom_submit(struct kbase_context *const kctx,
710 struct kbase_jd_atom *const katom,
711 int *priority)
712 {
713 struct kbase_device *const kbdev = kctx->kbdev;
714
715 KBASE_TLSTREAM_TL_NEW_ATOM(kbdev, katom, kbase_jd_atom_id(kctx, katom));
716 KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx);
717 if (priority)
718 KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, *priority);
719 KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_IDLE);
720 kbase_kinstr_jm_atom_queue(katom);
721 }
722
jd_submit_atom(struct kbase_context * const kctx,const struct base_jd_atom * const user_atom,const struct base_jd_fragment * const user_jc_incr,struct kbase_jd_atom * const katom)723 static bool jd_submit_atom(struct kbase_context *const kctx,
724 const struct base_jd_atom *const user_atom,
725 const struct base_jd_fragment *const user_jc_incr,
726 struct kbase_jd_atom *const katom)
727 {
728 struct kbase_device *kbdev = kctx->kbdev;
729 struct kbase_jd_context *jctx = &kctx->jctx;
730 int queued = 0;
731 int i;
732 int sched_prio;
733 bool will_fail = false;
734 unsigned long flags;
735 enum kbase_jd_atom_state status;
736
737 dev_dbg(kbdev->dev, "User did JD submit atom %pK\n", (void *)katom);
738
739 /* Update the TOTAL number of jobs. This includes those not tracked by
740 * the scheduler: 'not ready to run' and 'dependency-only' jobs.
741 */
742 jctx->job_nr++;
743
744 #if KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE
745 katom->start_timestamp.tv64 = 0;
746 #else
747 katom->start_timestamp = 0;
748 #endif
749 katom->udata = user_atom->udata;
750 katom->kctx = kctx;
751 katom->nr_extres = user_atom->nr_extres;
752 katom->extres = NULL;
753 katom->device_nr = user_atom->device_nr;
754 katom->jc = user_atom->jc;
755 katom->core_req = user_atom->core_req;
756 katom->jobslot = user_atom->jobslot;
757 katom->seq_nr = user_atom->seq_nr;
758 katom->atom_flags = 0;
759 katom->need_cache_flush_cores_retained = 0;
760 katom->pre_dep = NULL;
761 katom->post_dep = NULL;
762 katom->x_pre_dep = NULL;
763 katom->x_post_dep = NULL;
764 katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED;
765 katom->softjob_data = NULL;
766
767 trace_sysgraph(SGR_ARRIVE, kctx->id, user_atom->atom_number);
768
769 #if MALI_JIT_PRESSURE_LIMIT_BASE
770 /* Older API version atoms might have random values where jit_id now
771 * lives, but we must maintain backwards compatibility - handle the
772 * issue.
773 */
774 if (!mali_kbase_supports_jit_pressure_limit(kctx->api_version)) {
775 katom->jit_ids[0] = 0;
776 katom->jit_ids[1] = 0;
777 } else {
778 katom->jit_ids[0] = user_atom->jit_id[0];
779 katom->jit_ids[1] = user_atom->jit_id[1];
780 }
781 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
782
783 katom->renderpass_id = user_atom->renderpass_id;
784
785 /* Implicitly sets katom->protected_state.enter as well. */
786 katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
787
788 katom->age = kctx->age_count++;
789
790 INIT_LIST_HEAD(&katom->queue);
791 INIT_LIST_HEAD(&katom->jd_item);
792
793 /* Don't do anything if there is a mess up with dependencies.
794 * This is done in a separate cycle to check both the dependencies at ones, otherwise
795 * it will be extra complexity to deal with 1st dependency ( just added to the list )
796 * if only the 2nd one has invalid config.
797 */
798 for (i = 0; i < 2; i++) {
799 int dep_atom_number = user_atom->pre_dep[i].atom_id;
800 base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type;
801
802 if (dep_atom_number) {
803 if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER &&
804 dep_atom_type != BASE_JD_DEP_TYPE_DATA) {
805 katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT;
806 katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
807 dev_dbg(kbdev->dev,
808 "Atom %pK status to completed\n",
809 (void *)katom);
810
811 /* Wrong dependency setup. Atom will be sent
812 * back to user space. Do not record any
813 * dependencies.
814 */
815 jd_trace_atom_submit(kctx, katom, NULL);
816 return kbase_jd_done_nolock(katom, true);
817 }
818 }
819 }
820
821 /* Add dependencies */
822 for (i = 0; i < 2; i++) {
823 int dep_atom_number = user_atom->pre_dep[i].atom_id;
824 base_jd_dep_type dep_atom_type;
825 struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number];
826
827 dep_atom_type = user_atom->pre_dep[i].dependency_type;
828 kbase_jd_katom_dep_clear(&katom->dep[i]);
829
830 if (!dep_atom_number)
831 continue;
832
833 if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED ||
834 dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
835
836 if (dep_atom->event_code == BASE_JD_EVENT_DONE)
837 continue;
838 /* don't stop this atom if it has an order dependency
839 * only to the failed one, try to submit it through
840 * the normal path
841 */
842 if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER &&
843 dep_atom->event_code > BASE_JD_EVENT_ACTIVE) {
844 continue;
845 }
846
847 /* Atom has completed, propagate the error code if any */
848 katom->event_code = dep_atom->event_code;
849 katom->status = KBASE_JD_ATOM_STATE_QUEUED;
850 dev_dbg(kbdev->dev, "Atom %pK status to queued\n",
851 (void *)katom);
852
853 /* This atom will be sent back to user space.
854 * Do not record any dependencies.
855 */
856 jd_trace_atom_submit(kctx, katom, NULL);
857
858 will_fail = true;
859
860 } else {
861 /* Atom is in progress, add this atom to the list */
862 list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]);
863 kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type);
864 queued = 1;
865 }
866 }
867
868 if (will_fail) {
869 if (!queued) {
870 if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
871 /* This softjob has failed due to a previous
872 * dependency, however we should still run the
873 * prepare & finish functions
874 */
875 int err = kbase_prepare_soft_job(katom);
876
877 if (err >= 0)
878 kbase_finish_soft_job(katom);
879 }
880 return kbase_jd_done_nolock(katom, true);
881 }
882
883 katom->will_fail_event_code = katom->event_code;
884 }
885
886 /* These must occur after the above loop to ensure that an atom
887 * that depends on a previous atom with the same number behaves
888 * as expected
889 */
890 katom->event_code = BASE_JD_EVENT_DONE;
891 katom->status = KBASE_JD_ATOM_STATE_QUEUED;
892 dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)katom);
893
894 /* For invalid priority, be most lenient and choose the default */
895 sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio);
896 if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID)
897 sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT;
898
899 /* Cap the priority to jctx.max_priority */
900 katom->sched_priority = (sched_prio < kctx->jctx.max_priority) ?
901 kctx->jctx.max_priority : sched_prio;
902
903 /* Create a new atom. */
904 jd_trace_atom_submit(kctx, katom, &katom->sched_priority);
905
906 #if !MALI_INCREMENTAL_RENDERING_JM
907 /* Reject atoms for incremental rendering if not supported */
908 if (katom->core_req &
909 (BASE_JD_REQ_START_RENDERPASS|BASE_JD_REQ_END_RENDERPASS)) {
910 dev_err(kctx->kbdev->dev,
911 "Rejecting atom with unsupported core_req 0x%x\n",
912 katom->core_req);
913 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
914 return kbase_jd_done_nolock(katom, true);
915 }
916 #endif /* !MALI_INCREMENTAL_RENDERING_JM */
917
918 if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) {
919 WARN_ON(katom->jc != 0);
920 katom->jc_fragment = *user_jc_incr;
921 } else if (!katom->jc &&
922 (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
923 /* Reject atoms with job chain = NULL, as these cause issues
924 * with soft-stop
925 */
926 dev_err(kctx->kbdev->dev, "Rejecting atom with jc = NULL\n");
927 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
928 return kbase_jd_done_nolock(katom, true);
929 }
930
931 /* Reject atoms with an invalid device_nr */
932 if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) &&
933 (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) {
934 dev_err(kctx->kbdev->dev,
935 "Rejecting atom with invalid device_nr %d\n",
936 katom->device_nr);
937 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
938 return kbase_jd_done_nolock(katom, true);
939 }
940
941 /* Reject atoms with invalid core requirements */
942 if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) &&
943 (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) {
944 dev_err(kctx->kbdev->dev,
945 "Rejecting atom with invalid core requirements\n");
946 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
947 katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE;
948 return kbase_jd_done_nolock(katom, true);
949 }
950
951 /* Reject soft-job atom of certain types from accessing external resources */
952 if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) &&
953 (((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_FENCE_WAIT) ||
954 ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_ALLOC) ||
955 ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_FREE))) {
956 dev_err(kctx->kbdev->dev,
957 "Rejecting soft-job atom accessing external resources\n");
958 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
959 return kbase_jd_done_nolock(katom, true);
960 }
961
962 if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
963 /* handle what we need to do to access the external resources */
964 if (kbase_jd_pre_external_resources(katom, user_atom) != 0) {
965 /* setup failed (no access, bad resource, unknown resource types, etc.) */
966 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
967 return kbase_jd_done_nolock(katom, true);
968 }
969 }
970
971 #if !MALI_JIT_PRESSURE_LIMIT_BASE
972 if (mali_kbase_supports_jit_pressure_limit(kctx->api_version) &&
973 (user_atom->jit_id[0] || user_atom->jit_id[1])) {
974 /* JIT pressure limit is disabled, but we are receiving non-0
975 * JIT IDs - atom is invalid.
976 */
977 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
978 return kbase_jd_done_nolock(katom, true);
979 }
980 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
981
982 /* Validate the atom. Function will return error if the atom is
983 * malformed.
984 *
985 * Soft-jobs never enter the job scheduler but have their own initialize method.
986 *
987 * If either fail then we immediately complete the atom with an error.
988 */
989 if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) {
990 if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) {
991 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
992 return kbase_jd_done_nolock(katom, true);
993 }
994 } else {
995 /* Soft-job */
996 if (kbase_prepare_soft_job(katom) != 0) {
997 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
998 return kbase_jd_done_nolock(katom, true);
999 }
1000 }
1001
1002 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
1003 katom->work_id = atomic_inc_return(&jctx->work_id);
1004 trace_gpu_job_enqueue(kctx->id, katom->work_id,
1005 kbasep_map_core_reqs_to_string(katom->core_req));
1006 #endif
1007
1008 if (queued && !IS_GPU_ATOM(katom))
1009 return false;
1010
1011 if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
1012 if (kbase_process_soft_job(katom) == 0) {
1013 kbase_finish_soft_job(katom);
1014 return kbase_jd_done_nolock(katom, true);
1015 }
1016 return false;
1017 }
1018
1019 if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
1020 bool need_to_try_schedule_context;
1021
1022 katom->status = KBASE_JD_ATOM_STATE_IN_JS;
1023 dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n",
1024 (void *)katom);
1025
1026 need_to_try_schedule_context = kbasep_js_add_job(kctx, katom);
1027 /* If job was cancelled then resolve immediately */
1028 if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
1029 return need_to_try_schedule_context;
1030
1031 /* Synchronize with backend reset */
1032 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1033 status = katom->status;
1034 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1035 if (status == KBASE_JD_ATOM_STATE_HW_COMPLETED) {
1036 dev_dbg(kctx->kbdev->dev,
1037 "Atom %d cancelled on HW\n",
1038 kbase_jd_atom_id(katom->kctx, katom));
1039 return need_to_try_schedule_context;
1040 }
1041 }
1042
1043 /* This is a pure dependency. Resolve it immediately */
1044 return kbase_jd_done_nolock(katom, true);
1045 }
1046
kbase_jd_submit(struct kbase_context * kctx,void __user * user_addr,u32 nr_atoms,u32 stride,bool uk6_atom)1047 int kbase_jd_submit(struct kbase_context *kctx,
1048 void __user *user_addr, u32 nr_atoms, u32 stride,
1049 bool uk6_atom)
1050 {
1051 struct kbase_jd_context *jctx = &kctx->jctx;
1052 int err = 0;
1053 int i;
1054 bool need_to_try_schedule_context = false;
1055 struct kbase_device *kbdev;
1056 u32 latest_flush;
1057
1058 bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2) ||
1059 stride == offsetof(struct base_jd_atom_v2, renderpass_id));
1060
1061 /*
1062 * kbase_jd_submit isn't expected to fail and so all errors with the
1063 * jobs are reported by immediately failing them (through event system)
1064 */
1065 kbdev = kctx->kbdev;
1066
1067 if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
1068 dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it\n");
1069 return -EINVAL;
1070 }
1071
1072 if (stride != offsetof(struct base_jd_atom_v2, renderpass_id) &&
1073 stride != sizeof(struct base_jd_atom_v2) &&
1074 stride != offsetof(struct base_jd_atom, renderpass_id) &&
1075 stride != sizeof(struct base_jd_atom)) {
1076 dev_err(kbdev->dev,
1077 "Stride %u passed to job_submit isn't supported by the kernel\n",
1078 stride);
1079 return -EINVAL;
1080 }
1081
1082 if (nr_atoms > BASE_JD_ATOM_COUNT) {
1083 dev_dbg(kbdev->dev, "Invalid attempt to submit %u atoms at once for kctx %d_%d",
1084 nr_atoms, kctx->tgid, kctx->id);
1085 return -EINVAL;
1086 }
1087
1088 /* All atoms submitted in this call have the same flush ID */
1089 latest_flush = kbase_backend_get_current_flush_id(kbdev);
1090
1091 for (i = 0; i < nr_atoms; i++) {
1092 struct base_jd_atom user_atom = {
1093 .seq_nr = 0,
1094 };
1095 struct base_jd_fragment user_jc_incr;
1096 struct kbase_jd_atom *katom;
1097
1098 if (unlikely(jd_atom_is_v2)) {
1099 if (copy_from_user(&user_atom.jc, user_addr, sizeof(struct base_jd_atom_v2)) != 0) {
1100 dev_dbg(kbdev->dev,
1101 "Invalid atom address %pK passed to job_submit\n",
1102 user_addr);
1103 err = -EFAULT;
1104 break;
1105 }
1106
1107 /* no seq_nr in v2 */
1108 user_atom.seq_nr = 0;
1109 } else {
1110 if (copy_from_user(&user_atom, user_addr, stride) != 0) {
1111 dev_dbg(kbdev->dev,
1112 "Invalid atom address %pK passed to job_submit\n",
1113 user_addr);
1114 err = -EFAULT;
1115 break;
1116 }
1117 }
1118
1119 if (stride == offsetof(struct base_jd_atom_v2, renderpass_id)) {
1120 dev_dbg(kbdev->dev, "No renderpass ID: use 0\n");
1121 user_atom.renderpass_id = 0;
1122 } else {
1123 /* Ensure all padding bytes are 0 for potential future
1124 * extension
1125 */
1126 size_t j;
1127
1128 dev_dbg(kbdev->dev, "Renderpass ID is %d\n",
1129 user_atom.renderpass_id);
1130 for (j = 0; j < sizeof(user_atom.padding); j++) {
1131 if (user_atom.padding[j]) {
1132 dev_err(kbdev->dev,
1133 "Bad padding byte %zu: %d\n",
1134 j, user_atom.padding[j]);
1135 err = -EINVAL;
1136 break;
1137 }
1138 }
1139 if (err)
1140 break;
1141 }
1142
1143 /* In this case 'jc' is the CPU address of a struct
1144 * instead of a GPU address of a job chain.
1145 */
1146 if (user_atom.core_req & BASE_JD_REQ_END_RENDERPASS) {
1147 if (copy_from_user(&user_jc_incr,
1148 u64_to_user_ptr(user_atom.jc),
1149 sizeof(user_jc_incr))) {
1150 dev_err(kbdev->dev,
1151 "Invalid jc address 0x%llx passed to job_submit\n",
1152 user_atom.jc);
1153 err = -EFAULT;
1154 break;
1155 }
1156 dev_dbg(kbdev->dev, "Copied IR jobchain addresses\n");
1157 user_atom.jc = 0;
1158 }
1159
1160 user_addr = (void __user *)((uintptr_t) user_addr + stride);
1161
1162 mutex_lock(&jctx->lock);
1163 #ifndef compiletime_assert
1164 #define compiletime_assert_defined
1165 #define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \
1166 while (false)
1167 #endif
1168 compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) ==
1169 BASE_JD_ATOM_COUNT,
1170 "BASE_JD_ATOM_COUNT and base_atom_id type out of sync");
1171 compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) ==
1172 sizeof(user_atom.atom_number),
1173 "BASE_JD_ATOM_COUNT and base_atom_id type out of sync");
1174 #ifdef compiletime_assert_defined
1175 #undef compiletime_assert
1176 #undef compiletime_assert_defined
1177 #endif
1178 katom = &jctx->atoms[user_atom.atom_number];
1179
1180 /* Record the flush ID for the cache flush optimisation */
1181 katom->flush_id = latest_flush;
1182
1183 while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) {
1184 /* Atom number is already in use, wait for the atom to
1185 * complete
1186 */
1187 mutex_unlock(&jctx->lock);
1188
1189 /* This thread will wait for the atom to complete. Due
1190 * to thread scheduling we are not sure that the other
1191 * thread that owns the atom will also schedule the
1192 * context, so we force the scheduler to be active and
1193 * hence eventually schedule this context at some point
1194 * later.
1195 */
1196 kbase_js_sched_all(kbdev);
1197
1198 if (wait_event_killable(katom->completed,
1199 katom->status ==
1200 KBASE_JD_ATOM_STATE_UNUSED) != 0) {
1201 /* We're being killed so the result code
1202 * doesn't really matter
1203 */
1204 return 0;
1205 }
1206 mutex_lock(&jctx->lock);
1207 }
1208 KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_START(kbdev, katom);
1209 need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom,
1210 &user_jc_incr, katom);
1211 KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_END(kbdev, katom);
1212 /* Register a completed job as a disjoint event when the GPU is in a disjoint state
1213 * (ie. being reset).
1214 */
1215 kbase_disjoint_event_potential(kbdev);
1216
1217 mutex_unlock(&jctx->lock);
1218 if (fatal_signal_pending(current)) {
1219 dev_dbg(kbdev->dev, "Fatal signal pending for kctx %d_%d",
1220 kctx->tgid, kctx->id);
1221 /* We're being killed so the result code doesn't really matter */
1222 return 0;
1223 }
1224 }
1225
1226 if (need_to_try_schedule_context)
1227 kbase_js_sched_all(kbdev);
1228
1229 return err;
1230 }
1231
1232 KBASE_EXPORT_TEST_API(kbase_jd_submit);
1233
kbase_jd_done_worker(struct work_struct * data)1234 void kbase_jd_done_worker(struct work_struct *data)
1235 {
1236 struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work);
1237 struct kbase_jd_context *jctx;
1238 struct kbase_context *kctx;
1239 struct kbasep_js_kctx_info *js_kctx_info;
1240 struct kbase_device *kbdev;
1241 struct kbasep_js_device_data *js_devdata;
1242 u64 cache_jc = katom->jc;
1243 struct kbasep_js_atom_retained_state katom_retained_state;
1244 bool context_idle;
1245 base_jd_core_req core_req = katom->core_req;
1246
1247 /* Soft jobs should never reach this function */
1248 KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);
1249
1250 kctx = katom->kctx;
1251 jctx = &kctx->jctx;
1252 kbdev = kctx->kbdev;
1253 js_kctx_info = &kctx->jctx.sched_info;
1254 js_devdata = &kbdev->js_data;
1255
1256 dev_dbg(kbdev->dev, "Enter atom %pK done worker for kctx %pK\n",
1257 (void *)katom, (void *)kctx);
1258
1259 KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0);
1260
1261 kbase_backend_complete_wq(kbdev, katom);
1262
1263 /*
1264 * Begin transaction on JD context and JS context
1265 */
1266 mutex_lock(&jctx->lock);
1267 KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_DONE);
1268 mutex_lock(&js_devdata->queue_mutex);
1269 mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
1270
1271 /* This worker only gets called on contexts that are scheduled *in*. This is
1272 * because it only happens in response to an IRQ from a job that was
1273 * running.
1274 */
1275 KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
1276
1277 if (katom->event_code == BASE_JD_EVENT_STOPPED) {
1278 unsigned long flags;
1279
1280 dev_dbg(kbdev->dev, "Atom %pK has been promoted to stopped\n",
1281 (void *)katom);
1282 mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
1283 mutex_unlock(&js_devdata->queue_mutex);
1284
1285 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1286
1287 katom->status = KBASE_JD_ATOM_STATE_IN_JS;
1288 dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n",
1289 (void *)katom);
1290 kbase_js_unpull(kctx, katom);
1291
1292 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1293 mutex_unlock(&jctx->lock);
1294
1295 return;
1296 }
1297
1298 if ((katom->event_code != BASE_JD_EVENT_DONE) &&
1299 (!kbase_ctx_flag(katom->kctx, KCTX_DYING))) {
1300 if (!kbase_is_quick_reset_enabled(kbdev))
1301 dev_err(kbdev->dev,
1302 "t6xx: GPU fault 0x%02lx from job slot %d\n",
1303 (unsigned long)katom->event_code,
1304 katom->slot_nr);
1305 }
1306
1307 /* Retain state before the katom disappears */
1308 kbasep_js_atom_retained_state_copy(&katom_retained_state, katom);
1309
1310 context_idle = kbase_js_complete_atom_wq(kctx, katom);
1311
1312 KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state));
1313
1314 kbasep_js_remove_job(kbdev, kctx, katom);
1315 mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
1316 mutex_unlock(&js_devdata->queue_mutex);
1317 /* kbase_jd_done_nolock() requires the jsctx_mutex lock to be dropped */
1318 kbase_jd_done_nolock(katom, false);
1319
1320 /* katom may have been freed now, do not use! */
1321
1322 if (context_idle) {
1323 unsigned long flags;
1324
1325 context_idle = false;
1326 mutex_lock(&js_devdata->queue_mutex);
1327 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1328
1329 /* If kbase_sched() has scheduled this context back in then
1330 * KCTX_ACTIVE will have been set after we marked it as
1331 * inactive, and another pm reference will have been taken, so
1332 * drop our reference. But do not call kbase_jm_idle_ctx(), as
1333 * the context is active and fast-starting is allowed.
1334 *
1335 * If an atom has been fast-started then
1336 * kbase_jsctx_atoms_pulled(kctx) will return non-zero but
1337 * KCTX_ACTIVE will still be false (as the previous pm
1338 * reference has been inherited). Do NOT drop our reference, as
1339 * it has been re-used, and leave the context as active.
1340 *
1341 * If no new atoms have been started then KCTX_ACTIVE will
1342 * still be false and kbase_jsctx_atoms_pulled(kctx) will
1343 * return zero, so drop the reference and call
1344 * kbase_jm_idle_ctx().
1345 *
1346 * As the checks are done under both the queue_mutex and
1347 * hwaccess_lock is should be impossible for this to race
1348 * with the scheduler code.
1349 */
1350 if (kbase_ctx_flag(kctx, KCTX_ACTIVE) ||
1351 !kbase_jsctx_atoms_pulled(kctx)) {
1352 /* Calling kbase_jm_idle_ctx() here will ensure that
1353 * atoms are not fast-started when we drop the
1354 * hwaccess_lock. This is not performed if
1355 * KCTX_ACTIVE is set as in that case another pm
1356 * reference has been taken and a fast-start would be
1357 * valid.
1358 */
1359 if (!kbase_ctx_flag(kctx, KCTX_ACTIVE))
1360 kbase_jm_idle_ctx(kbdev, kctx);
1361 context_idle = true;
1362 } else {
1363 kbase_ctx_flag_set(kctx, KCTX_ACTIVE);
1364 }
1365 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1366 mutex_unlock(&js_devdata->queue_mutex);
1367 }
1368
1369 /*
1370 * Transaction complete
1371 */
1372 mutex_unlock(&jctx->lock);
1373
1374 /* Job is now no longer running, so can now safely release the context
1375 * reference, and handle any actions that were logged against the
1376 * atom's retained state
1377 */
1378
1379 kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state);
1380
1381 kbase_js_sched_all(kbdev);
1382
1383 if (!atomic_dec_return(&kctx->work_count)) {
1384 /* If worker now idle then post all events that kbase_jd_done_nolock()
1385 * has queued
1386 */
1387 mutex_lock(&jctx->lock);
1388 while (!list_empty(&kctx->completed_jobs)) {
1389 struct kbase_jd_atom *atom = list_entry(
1390 kctx->completed_jobs.next,
1391 struct kbase_jd_atom, jd_item);
1392 list_del(kctx->completed_jobs.next);
1393
1394 kbase_event_post(kctx, atom);
1395 }
1396 mutex_unlock(&jctx->lock);
1397 }
1398
1399 kbase_backend_complete_wq_post_sched(kbdev, core_req);
1400
1401 if (context_idle)
1402 kbase_pm_context_idle(kbdev);
1403
1404 KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0);
1405
1406 dev_dbg(kbdev->dev, "Leave atom %pK done worker for kctx %pK\n",
1407 (void *)katom, (void *)kctx);
1408 }
1409
1410 /**
1411 * jd_cancel_worker - Work queue job cancel function.
1412 * @data: a &struct work_struct
1413 *
1414 * Only called as part of 'Zapping' a context (which occurs on termination).
1415 * Operates serially with the kbase_jd_done_worker() on the work queue.
1416 *
1417 * This can only be called on contexts that aren't scheduled.
1418 *
1419 * We don't need to release most of the resources that would occur on
1420 * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be
1421 * running (by virtue of only being called on contexts that aren't
1422 * scheduled).
1423 */
jd_cancel_worker(struct work_struct * data)1424 static void jd_cancel_worker(struct work_struct *data)
1425 {
1426 struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work);
1427 struct kbase_jd_context *jctx;
1428 struct kbase_context *kctx;
1429 struct kbasep_js_kctx_info *js_kctx_info;
1430 bool need_to_try_schedule_context;
1431 bool attr_state_changed;
1432 struct kbase_device *kbdev;
1433 CSTD_UNUSED(need_to_try_schedule_context);
1434
1435 /* Soft jobs should never reach this function */
1436 KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);
1437
1438 kctx = katom->kctx;
1439 kbdev = kctx->kbdev;
1440 jctx = &kctx->jctx;
1441 js_kctx_info = &kctx->jctx.sched_info;
1442
1443 KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0);
1444
1445 /* This only gets called on contexts that are scheduled out. Hence, we must
1446 * make sure we don't de-ref the number of running jobs (there aren't
1447 * any), nor must we try to schedule out the context (it's already
1448 * scheduled out).
1449 */
1450 KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED));
1451
1452 /* Scheduler: Remove the job from the system */
1453 mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
1454 attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom);
1455 mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
1456
1457 mutex_lock(&jctx->lock);
1458
1459 need_to_try_schedule_context = kbase_jd_done_nolock(katom, true);
1460 /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to
1461 * schedule the context. There's also no need for the jsctx_mutex to have been taken
1462 * around this too.
1463 */
1464 KBASE_DEBUG_ASSERT(!need_to_try_schedule_context);
1465
1466 /* katom may have been freed now, do not use! */
1467 mutex_unlock(&jctx->lock);
1468
1469 if (attr_state_changed)
1470 kbase_js_sched_all(kbdev);
1471 }
1472
1473 /**
1474 * kbase_jd_done - Complete a job that has been removed from the Hardware
1475 * @katom: atom which has been completed
1476 * @slot_nr: slot the atom was on
1477 * @end_timestamp: completion time
1478 * @done_code: completion code
1479 *
1480 * This must be used whenever a job has been removed from the Hardware, e.g.:
1481 * An IRQ indicates that the job finished (for both error and 'done' codes), or
1482 * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop.
1483 *
1484 * Some work is carried out immediately, and the rest is deferred onto a
1485 * workqueue
1486 *
1487 * Context:
1488 * This can be called safely from atomic context.
1489 * The caller must hold kbdev->hwaccess_lock
1490 */
kbase_jd_done(struct kbase_jd_atom * katom,int slot_nr,ktime_t * end_timestamp,kbasep_js_atom_done_code done_code)1491 void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr,
1492 ktime_t *end_timestamp, kbasep_js_atom_done_code done_code)
1493 {
1494 struct kbase_context *kctx;
1495 struct kbase_device *kbdev;
1496
1497 KBASE_DEBUG_ASSERT(katom);
1498 kctx = katom->kctx;
1499 KBASE_DEBUG_ASSERT(kctx);
1500 kbdev = kctx->kbdev;
1501 KBASE_DEBUG_ASSERT(kbdev);
1502
1503 lockdep_assert_held(&kbdev->hwaccess_lock);
1504
1505 if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT)
1506 katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1507
1508 KBASE_KTRACE_ADD_JM(kbdev, JD_DONE, kctx, katom, katom->jc, 0);
1509
1510 kbase_job_check_leave_disjoint(kbdev, katom);
1511
1512 katom->slot_nr = slot_nr;
1513
1514 atomic_inc(&kctx->work_count);
1515
1516 #if IS_ENABLED(CONFIG_DEBUG_FS)
1517 /* a failed job happened and is waiting for dumping*/
1518 if (!katom->will_fail_event_code &&
1519 kbase_debug_job_fault_process(katom, katom->event_code))
1520 return;
1521 #endif
1522
1523 WARN_ON(work_pending(&katom->work));
1524 INIT_WORK(&katom->work, kbase_jd_done_worker);
1525 queue_work(kctx->jctx.job_done_wq, &katom->work);
1526 }
1527
1528 KBASE_EXPORT_TEST_API(kbase_jd_done);
1529
kbase_jd_cancel(struct kbase_device * kbdev,struct kbase_jd_atom * katom)1530 void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
1531 {
1532 struct kbase_context *kctx;
1533
1534 KBASE_DEBUG_ASSERT(kbdev != NULL);
1535 KBASE_DEBUG_ASSERT(katom != NULL);
1536 kctx = katom->kctx;
1537 KBASE_DEBUG_ASSERT(kctx != NULL);
1538
1539 dev_dbg(kbdev->dev, "JD: cancelling atom %pK\n", (void *)katom);
1540 KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);
1541
1542 /* This should only be done from a context that is not scheduled */
1543 KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED));
1544
1545 WARN_ON(work_pending(&katom->work));
1546
1547 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1548
1549 INIT_WORK(&katom->work, jd_cancel_worker);
1550 queue_work(kctx->jctx.job_done_wq, &katom->work);
1551 }
1552
1553
kbase_jd_zap_context(struct kbase_context * kctx)1554 void kbase_jd_zap_context(struct kbase_context *kctx)
1555 {
1556 struct kbase_jd_atom *katom;
1557 struct list_head *entry, *tmp;
1558 struct kbase_device *kbdev;
1559
1560 KBASE_DEBUG_ASSERT(kctx);
1561
1562 kbdev = kctx->kbdev;
1563
1564 KBASE_KTRACE_ADD_JM(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u);
1565
1566 kbase_js_zap_context(kctx);
1567
1568 mutex_lock(&kctx->jctx.lock);
1569
1570 /*
1571 * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are
1572 * queued outside the job scheduler.
1573 */
1574
1575 del_timer_sync(&kctx->soft_job_timeout);
1576 list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) {
1577 katom = list_entry(entry, struct kbase_jd_atom, queue);
1578 kbase_cancel_soft_job(katom);
1579 }
1580
1581 mutex_unlock(&kctx->jctx.lock);
1582
1583 #if IS_ENABLED(CONFIG_DEBUG_FS)
1584 kbase_debug_job_fault_kctx_unblock(kctx);
1585 #endif
1586
1587 kbase_jm_wait_for_zero_jobs(kctx);
1588 }
1589
1590 KBASE_EXPORT_TEST_API(kbase_jd_zap_context);
1591
kbase_jd_init(struct kbase_context * kctx)1592 int kbase_jd_init(struct kbase_context *kctx)
1593 {
1594 int i;
1595 int mali_err = 0;
1596 struct priority_control_manager_device *pcm_device = NULL;
1597
1598 KBASE_DEBUG_ASSERT(kctx);
1599 pcm_device = kctx->kbdev->pcm_dev;
1600 kctx->jctx.max_priority = KBASE_JS_ATOM_SCHED_PRIO_REALTIME;
1601
1602 kctx->jctx.job_done_wq = alloc_workqueue("mali_jd",
1603 WQ_HIGHPRI | WQ_UNBOUND, 1);
1604 if (kctx->jctx.job_done_wq == NULL) {
1605 mali_err = -ENOMEM;
1606 goto out1;
1607 }
1608
1609 for (i = 0; i < BASE_JD_ATOM_COUNT; i++) {
1610 init_waitqueue_head(&kctx->jctx.atoms[i].completed);
1611
1612 INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]);
1613 INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]);
1614
1615 /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */
1616 kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID;
1617 kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED;
1618
1619 #if IS_ENABLED(CONFIG_SYNC_FILE)
1620 kctx->jctx.atoms[i].dma_fence.context =
1621 dma_fence_context_alloc(1);
1622 atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0);
1623 #endif
1624 }
1625
1626 for (i = 0; i < BASE_JD_RP_COUNT; i++)
1627 kctx->jctx.renderpasses[i].state = KBASE_JD_RP_COMPLETE;
1628
1629 mutex_init(&kctx->jctx.lock);
1630
1631 init_waitqueue_head(&kctx->jctx.zero_jobs_wait);
1632
1633 spin_lock_init(&kctx->jctx.tb_lock);
1634
1635 kctx->jctx.job_nr = 0;
1636 INIT_LIST_HEAD(&kctx->completed_jobs);
1637 atomic_set(&kctx->work_count, 0);
1638
1639 /* Check if there are platform rules for maximum priority */
1640 if (pcm_device)
1641 kctx->jctx.max_priority = pcm_device->ops.pcm_scheduler_priority_check(
1642 pcm_device, current, KBASE_JS_ATOM_SCHED_PRIO_REALTIME);
1643
1644 return 0;
1645
1646 out1:
1647 return mali_err;
1648 }
1649
1650 KBASE_EXPORT_TEST_API(kbase_jd_init);
1651
kbase_jd_exit(struct kbase_context * kctx)1652 void kbase_jd_exit(struct kbase_context *kctx)
1653 {
1654 KBASE_DEBUG_ASSERT(kctx);
1655
1656 /* Work queue is emptied by this */
1657 destroy_workqueue(kctx->jctx.job_done_wq);
1658 }
1659
1660 KBASE_EXPORT_TEST_API(kbase_jd_exit);
1661