1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /*
23 * Register-based HW access backend specific APIs
24 */
25
26 #include <mali_kbase.h>
27 #include <gpu/mali_kbase_gpu_fault.h>
28 #include <mali_kbase_hwaccess_jm.h>
29 #include <mali_kbase_jm.h>
30 #include <mali_kbase_js.h>
31 #include <tl/mali_kbase_tracepoints.h>
32 #include <hwcnt/mali_kbase_hwcnt_context.h>
33 #include <mali_kbase_reset_gpu.h>
34 #include <mali_kbase_kinstr_jm.h>
35 #include <backend/gpu/mali_kbase_cache_policy_backend.h>
36 #include <device/mali_kbase_device.h>
37 #include <backend/gpu/mali_kbase_jm_internal.h>
38 #include <backend/gpu/mali_kbase_pm_internal.h>
39
40 /**
41 * SLOT_RB_EMPTY - Return whether the specified ringbuffer is empty.
42 *
43 * @rb: ring buffer
44 *
45 * Note: HW access lock must be held
46 */
47 #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx)
48
49 /**
50 * SLOT_RB_ENTRIES - Return number of atoms currently in the specified ringbuffer.
51 *
52 * @rb: ring buffer
53 *
54 * Note: HW access lock must be held
55 */
56 #define SLOT_RB_ENTRIES(rb) ((int)(s8)(rb->write_idx - rb->read_idx))
57
58 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
59 struct kbase_jd_atom *katom,
60 ktime_t *end_timestamp);
61
62 /**
63 * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer
64 * @kbdev: Device pointer
65 * @katom: Atom to enqueue
66 *
67 * Context: Caller must hold the HW access lock
68 */
kbase_gpu_enqueue_atom(struct kbase_device * kbdev,struct kbase_jd_atom * katom)69 static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev,
70 struct kbase_jd_atom *katom)
71 {
72 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr];
73
74 WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE);
75
76 lockdep_assert_held(&kbdev->hwaccess_lock);
77
78 rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom;
79 rb->write_idx++;
80
81 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
82 }
83
84 /**
85 * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once
86 * it has been completed
87 * @kbdev: Device pointer
88 * @js: Job slot to remove atom from
89 * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in
90 * which case current time will be used.
91 *
92 * Context: Caller must hold the HW access lock
93 *
94 * Return: Atom removed from ringbuffer
95 */
kbase_gpu_dequeue_atom(struct kbase_device * kbdev,unsigned int js,ktime_t * end_timestamp)96 static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, unsigned int js,
97 ktime_t *end_timestamp)
98 {
99 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
100 struct kbase_jd_atom *katom;
101
102 if (SLOT_RB_EMPTY(rb)) {
103 WARN(1, "GPU ringbuffer unexpectedly empty\n");
104 return NULL;
105 }
106
107 lockdep_assert_held(&kbdev->hwaccess_lock);
108
109 katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom;
110
111 kbase_gpu_release_atom(kbdev, katom, end_timestamp);
112
113 rb->read_idx++;
114
115 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB;
116
117 return katom;
118 }
119
kbase_gpu_inspect(struct kbase_device * kbdev,unsigned int js,int idx)120 struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, unsigned int js, int idx)
121 {
122 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
123
124 lockdep_assert_held(&kbdev->hwaccess_lock);
125
126 if ((SLOT_RB_ENTRIES(rb) - 1) < idx)
127 return NULL; /* idx out of range */
128
129 return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom;
130 }
131
kbase_backend_inspect_tail(struct kbase_device * kbdev,unsigned int js)132 struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, unsigned int js)
133 {
134 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
135
136 if (SLOT_RB_EMPTY(rb))
137 return NULL;
138
139 return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom;
140 }
141
kbase_gpu_atoms_submitted_any(struct kbase_device * kbdev)142 bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev)
143 {
144 unsigned int js;
145
146 lockdep_assert_held(&kbdev->hwaccess_lock);
147
148 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
149 int i;
150
151 for (i = 0; i < SLOT_RB_SIZE; i++) {
152 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
153
154 if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)
155 return true;
156 }
157 }
158 return false;
159 }
160
kbase_backend_nr_atoms_submitted(struct kbase_device * kbdev,unsigned int js)161 int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js)
162 {
163 int nr = 0;
164 int i;
165
166 lockdep_assert_held(&kbdev->hwaccess_lock);
167
168 for (i = 0; i < SLOT_RB_SIZE; i++) {
169 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
170
171 if (katom && (katom->gpu_rb_state ==
172 KBASE_ATOM_GPU_RB_SUBMITTED))
173 nr++;
174 }
175
176 return nr;
177 }
178
kbase_backend_nr_atoms_on_slot(struct kbase_device * kbdev,unsigned int js)179 int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, unsigned int js)
180 {
181 int nr = 0;
182 int i;
183
184 lockdep_assert_held(&kbdev->hwaccess_lock);
185
186 for (i = 0; i < SLOT_RB_SIZE; i++) {
187 if (kbase_gpu_inspect(kbdev, js, i))
188 nr++;
189 }
190
191 return nr;
192 }
193
kbase_gpu_nr_atoms_on_slot_min(struct kbase_device * kbdev,unsigned int js,enum kbase_atom_gpu_rb_state min_rb_state)194 static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, unsigned int js,
195 enum kbase_atom_gpu_rb_state min_rb_state)
196 {
197 int nr = 0;
198 int i;
199
200 lockdep_assert_held(&kbdev->hwaccess_lock);
201
202 for (i = 0; i < SLOT_RB_SIZE; i++) {
203 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
204
205 if (katom && (katom->gpu_rb_state >= min_rb_state))
206 nr++;
207 }
208
209 return nr;
210 }
211
212 /**
213 * check_secure_atom - Check if the given atom is in the given secure state and
214 * has a ringbuffer state of at least
215 * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
216 * @katom: Atom pointer
217 * @secure: Desired secure state
218 *
219 * Return: true if atom is in the given state, false otherwise
220 */
check_secure_atom(struct kbase_jd_atom * katom,bool secure)221 static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure)
222 {
223 if (katom->gpu_rb_state >=
224 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION &&
225 ((kbase_jd_katom_is_protected(katom) && secure) ||
226 (!kbase_jd_katom_is_protected(katom) && !secure)))
227 return true;
228
229 return false;
230 }
231
232 /**
233 * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given
234 * secure state in the ringbuffers of at least
235 * state
236 * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE
237 * @kbdev: Device pointer
238 * @secure: Desired secure state
239 *
240 * Return: true if any atoms are in the given state, false otherwise
241 */
kbase_gpu_check_secure_atoms(struct kbase_device * kbdev,bool secure)242 static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev,
243 bool secure)
244 {
245 unsigned int js;
246
247 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
248 int i;
249
250 for (i = 0; i < SLOT_RB_SIZE; i++) {
251 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
252 js, i);
253
254 if (katom) {
255 if (check_secure_atom(katom, secure))
256 return true;
257 }
258 }
259 }
260
261 return false;
262 }
263
kbase_backend_slot_free(struct kbase_device * kbdev,unsigned int js)264 int kbase_backend_slot_free(struct kbase_device *kbdev, unsigned int js)
265 {
266 lockdep_assert_held(&kbdev->hwaccess_lock);
267
268 if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) !=
269 KBASE_RESET_GPU_NOT_PENDING) {
270 /* The GPU is being reset - so prevent submission */
271 return 0;
272 }
273
274 return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js);
275 }
276
277
kbase_gpu_release_atom(struct kbase_device * kbdev,struct kbase_jd_atom * katom,ktime_t * end_timestamp)278 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
279 struct kbase_jd_atom *katom,
280 ktime_t *end_timestamp)
281 {
282 struct kbase_context *kctx = katom->kctx;
283
284 lockdep_assert_held(&kbdev->hwaccess_lock);
285
286 switch (katom->gpu_rb_state) {
287 case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
288 /* Should be impossible */
289 WARN(1, "Attempting to release atom not in ringbuffer\n");
290 break;
291
292 case KBASE_ATOM_GPU_RB_SUBMITTED:
293 kbase_kinstr_jm_atom_hw_release(katom);
294 /* Inform power management at start/finish of atom so it can
295 * update its GPU utilisation metrics. Mark atom as not
296 * submitted beforehand.
297 */
298 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
299 kbase_pm_metrics_update(kbdev, end_timestamp);
300
301 /* Inform platform at start/finish of atom */
302 kbasep_platform_event_atom_complete(katom);
303
304 if (katom->core_req & BASE_JD_REQ_PERMON)
305 kbase_pm_release_gpu_cycle_counter_nolock(kbdev);
306
307 KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, katom,
308 &kbdev->gpu_props.props.raw_props.js_features
309 [katom->slot_nr]);
310 KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]);
311 KBASE_TLSTREAM_TL_NRET_CTX_LPU(kbdev, kctx,
312 &kbdev->gpu_props.props.raw_props.js_features
313 [katom->slot_nr]);
314
315 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
316 fallthrough;
317 case KBASE_ATOM_GPU_RB_READY:
318 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
319 fallthrough;
320 case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
321 break;
322
323 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION:
324 if (kbase_jd_katom_is_protected(katom) &&
325 (katom->protected_state.enter !=
326 KBASE_ATOM_ENTER_PROTECTED_CHECK) &&
327 (katom->protected_state.enter !=
328 KBASE_ATOM_ENTER_PROTECTED_HWCNT)) {
329 kbase_pm_protected_override_disable(kbdev);
330 kbase_pm_update_cores_state_nolock(kbdev);
331 }
332 if (kbase_jd_katom_is_protected(katom) &&
333 (katom->protected_state.enter ==
334 KBASE_ATOM_ENTER_PROTECTED_IDLE_L2))
335 kbase_pm_protected_entry_override_disable(kbdev);
336 if (!kbase_jd_katom_is_protected(katom) &&
337 (katom->protected_state.exit !=
338 KBASE_ATOM_EXIT_PROTECTED_CHECK) &&
339 (katom->protected_state.exit !=
340 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT)) {
341 kbase_pm_protected_override_disable(kbdev);
342 kbase_pm_update_cores_state_nolock(kbdev);
343 }
344
345 if (katom->protected_state.enter !=
346 KBASE_ATOM_ENTER_PROTECTED_CHECK ||
347 katom->protected_state.exit !=
348 KBASE_ATOM_EXIT_PROTECTED_CHECK)
349 kbdev->protected_mode_transition = false;
350
351 /* If the atom is at KBASE_ATOM_ENTER_PROTECTED_HWCNT state, it means
352 * one of two events prevented it from progressing to the next state and
353 * ultimately reach protected mode:
354 * - hwcnts were enabled, and the atom had to schedule a worker to
355 * disable them.
356 * - the hwcnts were already disabled, but some other error occurred.
357 * In the first case, if the worker has not yet completed
358 * (kbdev->protected_mode_hwcnt_disabled == false), we need to re-enable
359 * them and signal to the worker they have already been enabled
360 */
361 if (kbase_jd_katom_is_protected(katom) &&
362 (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_HWCNT)) {
363 kbdev->protected_mode_hwcnt_desired = true;
364 if (kbdev->protected_mode_hwcnt_disabled) {
365 kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
366 kbdev->protected_mode_hwcnt_disabled = false;
367 }
368 }
369
370 /* If the atom has suspended hwcnt but has not yet entered
371 * protected mode, then resume hwcnt now. If the GPU is now in
372 * protected mode then hwcnt will be resumed by GPU reset so
373 * don't resume it here.
374 */
375 if (kbase_jd_katom_is_protected(katom) &&
376 ((katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_IDLE_L2) ||
377 (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY) ||
378 (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_FINISHED))) {
379 WARN_ON(!kbdev->protected_mode_hwcnt_disabled);
380 kbdev->protected_mode_hwcnt_desired = true;
381 if (kbdev->protected_mode_hwcnt_disabled) {
382 kbase_hwcnt_context_enable(
383 kbdev->hwcnt_gpu_ctx);
384 kbdev->protected_mode_hwcnt_disabled = false;
385 }
386 }
387
388 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) {
389 if (katom->atom_flags &
390 KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) {
391 kbase_pm_protected_l2_override(kbdev, false);
392 katom->atom_flags &=
393 ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT;
394 }
395 }
396
397 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
398 fallthrough;
399 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV:
400 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
401 fallthrough;
402 case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
403 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
404 fallthrough;
405 case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
406 break;
407 }
408
409 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
410 katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
411 }
412
kbase_gpu_mark_atom_for_return(struct kbase_device * kbdev,struct kbase_jd_atom * katom)413 static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev,
414 struct kbase_jd_atom *katom)
415 {
416 lockdep_assert_held(&kbdev->hwaccess_lock);
417
418 KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_MARK_FOR_RETURN_TO_JS,
419 katom->kctx, katom, katom->jc,
420 katom->slot_nr, katom->event_code);
421 kbase_gpu_release_atom(kbdev, katom, NULL);
422 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS;
423 }
424
425 /**
426 * other_slots_busy - Determine if any job slots other than @js are currently
427 * running atoms
428 * @kbdev: Device pointer
429 * @js: Job slot
430 *
431 * Return: true if any slots other than @js are busy, false otherwise
432 */
other_slots_busy(struct kbase_device * kbdev,unsigned int js)433 static inline bool other_slots_busy(struct kbase_device *kbdev, unsigned int js)
434 {
435 unsigned int slot;
436
437 for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) {
438 if (slot == js)
439 continue;
440
441 if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot,
442 KBASE_ATOM_GPU_RB_SUBMITTED))
443 return true;
444 }
445
446 return false;
447 }
448
kbase_gpu_in_protected_mode(struct kbase_device * kbdev)449 static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev)
450 {
451 return kbdev->protected_mode;
452 }
453
kbase_gpu_disable_coherent(struct kbase_device * kbdev)454 static void kbase_gpu_disable_coherent(struct kbase_device *kbdev)
455 {
456 lockdep_assert_held(&kbdev->hwaccess_lock);
457
458 /*
459 * When entering into protected mode, we must ensure that the
460 * GPU is not operating in coherent mode as well. This is to
461 * ensure that no protected memory can be leaked.
462 */
463 if (kbdev->system_coherency == COHERENCY_ACE)
464 kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE);
465 }
466
kbase_gpu_protected_mode_enter(struct kbase_device * kbdev)467 static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev)
468 {
469 int err = -EINVAL;
470
471 lockdep_assert_held(&kbdev->hwaccess_lock);
472
473 WARN_ONCE(!kbdev->protected_ops,
474 "Cannot enter protected mode: protected callbacks not specified.\n");
475
476 if (kbdev->protected_ops) {
477 /* Switch GPU to protected mode */
478 err = kbdev->protected_ops->protected_mode_enable(
479 kbdev->protected_dev);
480
481 if (err) {
482 dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n",
483 err);
484 } else {
485 kbdev->protected_mode = true;
486 kbase_ipa_protection_mode_switch_event(kbdev);
487 }
488 }
489
490 return err;
491 }
492
kbase_gpu_protected_mode_reset(struct kbase_device * kbdev)493 static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev)
494 {
495 lockdep_assert_held(&kbdev->hwaccess_lock);
496
497 WARN_ONCE(!kbdev->protected_ops,
498 "Cannot exit protected mode: protected callbacks not specified.\n");
499
500 if (!kbdev->protected_ops)
501 return -EINVAL;
502
503 /* The protected mode disable callback will be called as part of reset
504 */
505 return kbase_reset_gpu_silent(kbdev);
506 }
507
kbase_jm_protected_entry(struct kbase_device * kbdev,struct kbase_jd_atom ** katom,int idx,int js)508 static int kbase_jm_protected_entry(struct kbase_device *kbdev,
509 struct kbase_jd_atom **katom, int idx, int js)
510 {
511 int err = 0;
512
513 lockdep_assert_held(&kbdev->hwaccess_lock);
514
515 err = kbase_gpu_protected_mode_enter(kbdev);
516
517 /*
518 * Regardless of result before this call, we are no longer
519 * transitioning the GPU.
520 */
521
522 kbdev->protected_mode_transition = false;
523 kbase_pm_protected_override_disable(kbdev);
524 kbase_pm_update_cores_state_nolock(kbdev);
525
526 KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev, kbdev);
527 if (err) {
528 /*
529 * Failed to switch into protected mode.
530 *
531 * At this point we expect:
532 * katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION &&
533 * katom->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED
534 * ==>
535 * kbdev->protected_mode_hwcnt_disabled = false
536 */
537 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
538 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
539 /*
540 * Only return if head atom or previous atom
541 * already removed - as atoms must be returned
542 * in order.
543 */
544 if (idx == 0 || katom[0]->gpu_rb_state ==
545 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
546 kbase_gpu_dequeue_atom(kbdev, js, NULL);
547 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
548 }
549
550 return -EINVAL;
551 }
552
553 /*
554 * Protected mode sanity checks.
555 */
556 WARN(kbase_jd_katom_is_protected(katom[idx]) != kbase_gpu_in_protected_mode(kbdev),
557 "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
558 kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev));
559 katom[idx]->gpu_rb_state =
560 KBASE_ATOM_GPU_RB_READY;
561
562 return err;
563 }
564
kbase_jm_enter_protected_mode(struct kbase_device * kbdev,struct kbase_jd_atom ** katom,int idx,int js)565 static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev,
566 struct kbase_jd_atom **katom, int idx, int js)
567 {
568 int err = 0;
569
570 lockdep_assert_held(&kbdev->hwaccess_lock);
571
572 switch (katom[idx]->protected_state.enter) {
573 case KBASE_ATOM_ENTER_PROTECTED_CHECK:
574 KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev);
575 /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV
576 * should ensure that we are not already transitiong, and that
577 * there are no atoms currently on the GPU.
578 */
579 WARN_ON(kbdev->protected_mode_transition);
580 WARN_ON(kbase_gpu_atoms_submitted_any(kbdev));
581 /* If hwcnt is disabled, it means we didn't clean up correctly
582 * during last exit from protected mode.
583 */
584 WARN_ON(kbdev->protected_mode_hwcnt_disabled);
585
586 katom[idx]->protected_state.enter =
587 KBASE_ATOM_ENTER_PROTECTED_HWCNT;
588
589 kbdev->protected_mode_transition = true;
590
591 /* ***TRANSITION TO HIGHER STATE*** */
592 fallthrough;
593 case KBASE_ATOM_ENTER_PROTECTED_HWCNT:
594 /* See if we can get away with disabling hwcnt atomically */
595 kbdev->protected_mode_hwcnt_desired = false;
596 if (!kbdev->protected_mode_hwcnt_disabled) {
597 if (kbase_hwcnt_context_disable_atomic(
598 kbdev->hwcnt_gpu_ctx))
599 kbdev->protected_mode_hwcnt_disabled = true;
600 }
601
602 /* We couldn't disable atomically, so kick off a worker */
603 if (!kbdev->protected_mode_hwcnt_disabled) {
604 kbase_hwcnt_context_queue_work(
605 kbdev->hwcnt_gpu_ctx,
606 &kbdev->protected_mode_hwcnt_disable_work);
607 return -EAGAIN;
608 }
609
610 /* Once reaching this point GPU must be switched to protected
611 * mode or hwcnt re-enabled.
612 */
613
614 if (kbase_pm_protected_entry_override_enable(kbdev))
615 return -EAGAIN;
616
617 /*
618 * Not in correct mode, begin protected mode switch.
619 * Entering protected mode requires us to power down the L2,
620 * and drop out of fully coherent mode.
621 */
622 katom[idx]->protected_state.enter =
623 KBASE_ATOM_ENTER_PROTECTED_IDLE_L2;
624
625 kbase_pm_protected_override_enable(kbdev);
626 /*
627 * Only if the GPU reset hasn't been initiated, there is a need
628 * to invoke the state machine to explicitly power down the
629 * shader cores and L2.
630 */
631 if (!kbdev->pm.backend.protected_entry_transition_override)
632 kbase_pm_update_cores_state_nolock(kbdev);
633
634 /* ***TRANSITION TO HIGHER STATE*** */
635 fallthrough;
636 case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2:
637 /* Avoid unnecessary waiting on non-ACE platforms. */
638 if (kbdev->system_coherency == COHERENCY_ACE) {
639 if (kbdev->pm.backend.l2_always_on) {
640 /*
641 * If the GPU reset hasn't completed, then L2
642 * could still be powered up.
643 */
644 if (kbase_reset_gpu_is_active(kbdev))
645 return -EAGAIN;
646 }
647
648 if (kbase_pm_get_ready_cores(kbdev,
649 KBASE_PM_CORE_L2) ||
650 kbase_pm_get_trans_cores(kbdev,
651 KBASE_PM_CORE_L2) ||
652 kbase_is_gpu_removed(kbdev)) {
653 /*
654 * The L2 is still powered, wait for all
655 * the users to finish with it before doing
656 * the actual reset.
657 */
658 return -EAGAIN;
659 }
660 }
661
662 katom[idx]->protected_state.enter =
663 KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY;
664
665 /* ***TRANSITION TO HIGHER STATE*** */
666 fallthrough;
667 case KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY:
668 /*
669 * When entering into protected mode, we must ensure that the
670 * GPU is not operating in coherent mode as well. This is to
671 * ensure that no protected memory can be leaked.
672 */
673 kbase_gpu_disable_coherent(kbdev);
674
675 kbase_pm_protected_entry_override_disable(kbdev);
676
677 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) {
678 /*
679 * Power on L2 caches; this will also result in the
680 * correct value written to coherency enable register.
681 */
682 kbase_pm_protected_l2_override(kbdev, true);
683
684 /*
685 * Set the flag on the atom that additional
686 * L2 references are taken.
687 */
688 katom[idx]->atom_flags |=
689 KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT;
690 }
691
692 katom[idx]->protected_state.enter =
693 KBASE_ATOM_ENTER_PROTECTED_FINISHED;
694
695 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234))
696 return -EAGAIN;
697
698 /* ***TRANSITION TO HIGHER STATE*** */
699 fallthrough;
700 case KBASE_ATOM_ENTER_PROTECTED_FINISHED:
701 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) {
702 /*
703 * Check that L2 caches are powered and, if so,
704 * enter protected mode.
705 */
706 if (kbdev->pm.backend.l2_state == KBASE_L2_ON) {
707 /*
708 * Remove additional L2 reference and reset
709 * the atom flag which denotes it.
710 */
711 if (katom[idx]->atom_flags &
712 KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) {
713 kbase_pm_protected_l2_override(kbdev,
714 false);
715 katom[idx]->atom_flags &=
716 ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT;
717 }
718
719 err = kbase_jm_protected_entry(kbdev, katom, idx, js);
720
721 if (err)
722 return err;
723 } else {
724 /*
725 * still waiting for L2 caches to power up
726 */
727 return -EAGAIN;
728 }
729 } else {
730 err = kbase_jm_protected_entry(kbdev, katom, idx, js);
731
732 if (err)
733 return err;
734 }
735 }
736
737 return 0;
738 }
739
kbase_jm_exit_protected_mode(struct kbase_device * kbdev,struct kbase_jd_atom ** katom,int idx,int js)740 static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev,
741 struct kbase_jd_atom **katom, int idx, int js)
742 {
743 int err = 0;
744
745 lockdep_assert_held(&kbdev->hwaccess_lock);
746
747 switch (katom[idx]->protected_state.exit) {
748 case KBASE_ATOM_EXIT_PROTECTED_CHECK:
749 KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev, kbdev);
750 /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV
751 * should ensure that we are not already transitiong, and that
752 * there are no atoms currently on the GPU.
753 */
754 WARN_ON(kbdev->protected_mode_transition);
755 WARN_ON(kbase_gpu_atoms_submitted_any(kbdev));
756
757 /*
758 * Exiting protected mode requires a reset, but first the L2
759 * needs to be powered down to ensure it's not active when the
760 * reset is issued.
761 */
762 katom[idx]->protected_state.exit =
763 KBASE_ATOM_EXIT_PROTECTED_IDLE_L2;
764
765 kbdev->protected_mode_transition = true;
766 kbase_pm_protected_override_enable(kbdev);
767 kbase_pm_update_cores_state_nolock(kbdev);
768
769 /* ***TRANSITION TO HIGHER STATE*** */
770 fallthrough;
771 case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2:
772 if (kbdev->pm.backend.l2_state != KBASE_L2_OFF) {
773 /*
774 * The L2 is still powered, wait for all the users to
775 * finish with it before doing the actual reset.
776 */
777 return -EAGAIN;
778 }
779 katom[idx]->protected_state.exit =
780 KBASE_ATOM_EXIT_PROTECTED_RESET;
781
782 /* ***TRANSITION TO HIGHER STATE*** */
783 fallthrough;
784 case KBASE_ATOM_EXIT_PROTECTED_RESET:
785 /* L2 cache has been turned off (which is needed prior to the reset of GPU
786 * to exit the protected mode), so the override flag can be safely cleared.
787 * Even if L2 cache is powered up again before the actual reset, it should
788 * not be an issue (there are no jobs running on the GPU).
789 */
790 kbase_pm_protected_override_disable(kbdev);
791
792 /* Issue the reset to the GPU */
793 err = kbase_gpu_protected_mode_reset(kbdev);
794
795 if (err == -EAGAIN)
796 return -EAGAIN;
797
798 if (err) {
799 kbdev->protected_mode_transition = false;
800
801 /* Failed to exit protected mode, fail atom */
802 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
803 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
804 /* Only return if head atom or previous atom
805 * already removed - as atoms must be returned in order
806 */
807 if (idx == 0 || katom[0]->gpu_rb_state ==
808 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
809 kbase_gpu_dequeue_atom(kbdev, js, NULL);
810 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
811 }
812
813 /* If we're exiting from protected mode, hwcnt must have
814 * been disabled during entry.
815 */
816 WARN_ON(!kbdev->protected_mode_hwcnt_disabled);
817 kbdev->protected_mode_hwcnt_desired = true;
818 if (kbdev->protected_mode_hwcnt_disabled) {
819 kbase_hwcnt_context_enable(
820 kbdev->hwcnt_gpu_ctx);
821 kbdev->protected_mode_hwcnt_disabled = false;
822 }
823
824 return -EINVAL;
825 }
826
827 katom[idx]->protected_state.exit =
828 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT;
829
830 /* ***TRANSITION TO HIGHER STATE*** */
831 fallthrough;
832 case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT:
833 /* A GPU reset is issued when exiting protected mode. Once the
834 * reset is done all atoms' state will also be reset. For this
835 * reason, if the atom is still in this state we can safely
836 * say that the reset has not completed i.e., we have not
837 * finished exiting protected mode yet.
838 */
839 return -EAGAIN;
840 }
841
842 return 0;
843 }
844
kbase_backend_slot_update(struct kbase_device * kbdev)845 void kbase_backend_slot_update(struct kbase_device *kbdev)
846 {
847 unsigned int js;
848
849 lockdep_assert_held(&kbdev->hwaccess_lock);
850
851 #ifdef CONFIG_MALI_ARBITER_SUPPORT
852 if (kbase_reset_gpu_is_active(kbdev) ||
853 kbase_is_gpu_removed(kbdev))
854 #else
855 if (kbase_reset_gpu_is_active(kbdev))
856 #endif
857 return;
858
859 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
860 struct kbase_jd_atom *katom[2];
861 int idx;
862
863 katom[0] = kbase_gpu_inspect(kbdev, js, 0);
864 katom[1] = kbase_gpu_inspect(kbdev, js, 1);
865 WARN_ON(katom[1] && !katom[0]);
866
867 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
868 bool cores_ready;
869 int ret;
870
871 if (!katom[idx])
872 continue;
873
874 switch (katom[idx]->gpu_rb_state) {
875 case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
876 /* Should be impossible */
877 WARN(1, "Attempting to update atom not in ringbuffer\n");
878 break;
879
880 case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
881 if (kbase_js_atom_blocked_on_x_dep(katom[idx]))
882 break;
883
884 katom[idx]->gpu_rb_state =
885 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV;
886
887 /* ***TRANSITION TO HIGHER STATE*** */
888 fallthrough;
889 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV:
890 if (kbase_gpu_check_secure_atoms(kbdev,
891 !kbase_jd_katom_is_protected(
892 katom[idx])))
893 break;
894
895 if ((idx == 1) && (kbase_jd_katom_is_protected(
896 katom[0]) !=
897 kbase_jd_katom_is_protected(
898 katom[1])))
899 break;
900
901 if (kbdev->protected_mode_transition)
902 break;
903
904 katom[idx]->gpu_rb_state =
905 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION;
906
907 /* ***TRANSITION TO HIGHER STATE*** */
908 fallthrough;
909 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION:
910
911 /*
912 * Exiting protected mode must be done before
913 * the references on the cores are taken as
914 * a power down the L2 is required which
915 * can't happen after the references for this
916 * atom are taken.
917 */
918
919 if (!kbase_gpu_in_protected_mode(kbdev) &&
920 kbase_jd_katom_is_protected(katom[idx])) {
921 /* Atom needs to transition into protected mode. */
922 ret = kbase_jm_enter_protected_mode(kbdev,
923 katom, idx, js);
924 if (ret)
925 break;
926 } else if (kbase_gpu_in_protected_mode(kbdev) &&
927 !kbase_jd_katom_is_protected(katom[idx])) {
928 /* Atom needs to transition out of protected mode. */
929 ret = kbase_jm_exit_protected_mode(kbdev,
930 katom, idx, js);
931 if (ret)
932 break;
933 }
934 katom[idx]->protected_state.exit =
935 KBASE_ATOM_EXIT_PROTECTED_CHECK;
936
937 /* Atom needs no protected mode transition. */
938
939 katom[idx]->gpu_rb_state =
940 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE;
941
942 /* ***TRANSITION TO HIGHER STATE*** */
943 fallthrough;
944 case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
945 if (katom[idx]->will_fail_event_code) {
946 kbase_gpu_mark_atom_for_return(kbdev,
947 katom[idx]);
948 /* Set EVENT_DONE so this atom will be
949 * completed, not unpulled.
950 */
951 katom[idx]->event_code =
952 BASE_JD_EVENT_DONE;
953 /* Only return if head atom or previous
954 * atom already removed - as atoms must
955 * be returned in order.
956 */
957 if (idx == 0 || katom[0]->gpu_rb_state ==
958 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
959 kbase_gpu_dequeue_atom(kbdev, js, NULL);
960 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
961 }
962 break;
963 }
964
965 cores_ready = kbase_pm_cores_requested(kbdev,
966 true);
967
968 if (!cores_ready)
969 break;
970
971 katom[idx]->gpu_rb_state =
972 KBASE_ATOM_GPU_RB_READY;
973
974 /* ***TRANSITION TO HIGHER STATE*** */
975 fallthrough;
976 case KBASE_ATOM_GPU_RB_READY:
977
978 if (idx == 1) {
979 /* Only submit if head atom or previous
980 * atom already submitted
981 */
982 if ((katom[0]->gpu_rb_state !=
983 KBASE_ATOM_GPU_RB_SUBMITTED &&
984 katom[0]->gpu_rb_state !=
985 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
986 break;
987
988 /* If intra-slot serialization in use
989 * then don't submit atom to NEXT slot
990 */
991 if (kbdev->serialize_jobs &
992 KBASE_SERIALIZE_INTRA_SLOT)
993 break;
994 }
995
996 /* If inter-slot serialization in use then don't
997 * submit atom if any other slots are in use
998 */
999 if ((kbdev->serialize_jobs &
1000 KBASE_SERIALIZE_INTER_SLOT) &&
1001 other_slots_busy(kbdev, js))
1002 break;
1003
1004 /* Check if this job needs the cycle counter
1005 * enabled before submission
1006 */
1007 if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
1008 kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev);
1009
1010 if (!kbase_job_hw_submit(kbdev, katom[idx], js)) {
1011 katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_SUBMITTED;
1012
1013 /* Inform power management at start/finish of
1014 * atom so it can update its GPU utilisation
1015 * metrics.
1016 */
1017 kbase_pm_metrics_update(kbdev,
1018 &katom[idx]->start_timestamp);
1019
1020 /* Inform platform at start/finish of atom */
1021 kbasep_platform_event_atom_submit(katom[idx]);
1022 } else {
1023 if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
1024 kbase_pm_release_gpu_cycle_counter_nolock(kbdev);
1025
1026 break;
1027 }
1028
1029 /* ***TRANSITION TO HIGHER STATE*** */
1030 fallthrough;
1031 case KBASE_ATOM_GPU_RB_SUBMITTED:
1032 break;
1033
1034 case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
1035 /* Only return if head atom or previous atom
1036 * already removed - as atoms must be returned
1037 * in order
1038 */
1039 if (idx == 0 || katom[0]->gpu_rb_state ==
1040 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
1041 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1042 kbase_jm_return_atom_to_js(kbdev,
1043 katom[idx]);
1044 }
1045 break;
1046 }
1047 }
1048 }
1049 }
1050
1051
kbase_backend_run_atom(struct kbase_device * kbdev,struct kbase_jd_atom * katom)1052 void kbase_backend_run_atom(struct kbase_device *kbdev,
1053 struct kbase_jd_atom *katom)
1054 {
1055 lockdep_assert_held(&kbdev->hwaccess_lock);
1056 dev_dbg(kbdev->dev, "Backend running atom %pK\n", (void *)katom);
1057
1058 kbase_gpu_enqueue_atom(kbdev, katom);
1059 kbase_backend_slot_update(kbdev);
1060 }
1061
1062 /**
1063 * kbase_rb_atom_might_depend - determine if one atom in the slot ringbuffer
1064 * might depend on another from the same kctx
1065 * @katom_a: dependee atom
1066 * @katom_b: atom to query
1067 *
1068 * This can be used on atoms that belong to different slot ringbuffers
1069 *
1070 * Return: true if @katom_b might depend on @katom_a, false if it cannot depend.
1071 */
1072 static inline bool
kbase_rb_atom_might_depend(const struct kbase_jd_atom * katom_a,const struct kbase_jd_atom * katom_b)1073 kbase_rb_atom_might_depend(const struct kbase_jd_atom *katom_a,
1074 const struct kbase_jd_atom *katom_b)
1075 {
1076 if (katom_a->kctx != katom_b->kctx)
1077 return false;
1078 return (katom_b->pre_dep ||
1079 (katom_b->atom_flags & (KBASE_KATOM_FLAG_X_DEP_BLOCKED |
1080 KBASE_KATOM_FLAG_FAIL_BLOCKER)));
1081 }
1082
1083 /**
1084 * kbase_gpu_irq_evict - evict a slot's JSn_HEAD_NEXT atom from the HW if it is
1085 * related to a failed JSn_HEAD atom
1086 * @kbdev: kbase device
1087 * @js: job slot to check
1088 * @completion_code: completion code of the failed atom
1089 *
1090 * Note: 'STOPPED' atoms are considered 'failed', as they are in the HW, but
1091 * unlike other failure codes we _can_ re-run them.
1092 *
1093 * This forms step 1 in a 2-step process of removing any related atoms from a
1094 * slot's JSn_HEAD_NEXT (ringbuffer index 1), should there have
1095 * been a 'failure' on an atom in JSn_HEAD (ringbuffer index 0).
1096 *
1097 * This step only removes the atoms from the HW, and marks them as
1098 * (potentially) ready to run again.
1099 *
1100 * Step 2 is on marking the JSn_HEAD atom as complete
1101 * (kbase_gpu_complete_hw()), to dequeue said atoms and return them to the JS
1102 * as appropriate, or re-submit them.
1103 *
1104 * Hence, this function must evict at a minimum the atoms related to the atom
1105 * in JSn_HEAD that kbase_gpu_complete_hw() will also dequeue. It is acceptable
1106 * if this function evicts more atoms than kbase_gpu_complete_hw() dequeues, as
1107 * the next kbase_backend_slot_update() will resubmit any remaining.
1108 *
1109 * Return: true if an atom was evicted, false otherwise.
1110 */
kbase_gpu_irq_evict(struct kbase_device * kbdev,unsigned int js,u32 completion_code)1111 bool kbase_gpu_irq_evict(struct kbase_device *kbdev, unsigned int js, u32 completion_code)
1112 {
1113 struct kbase_jd_atom *katom;
1114 struct kbase_jd_atom *next_katom;
1115
1116 lockdep_assert_held(&kbdev->hwaccess_lock);
1117
1118 katom = kbase_gpu_inspect(kbdev, js, 0);
1119 if (!katom) {
1120 dev_err(kbdev->dev, "Can't get a katom from js(%u)\n", js);
1121 return false;
1122 }
1123 next_katom = kbase_gpu_inspect(kbdev, js, 1);
1124
1125 if (next_katom &&
1126 next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
1127 (kbase_rb_atom_might_depend(katom, next_katom) ||
1128 kbase_js_atom_runs_before(kbdev, katom, next_katom, 0u)) &&
1129 (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO)) != 0 ||
1130 kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI)) != 0)) {
1131 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
1132 JS_COMMAND_NOP);
1133 next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
1134
1135 if (completion_code == BASE_JD_EVENT_STOPPED) {
1136 KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, next_katom,
1137 &kbdev->gpu_props.props.raw_props.js_features
1138 [next_katom->slot_nr]);
1139 KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, next_katom, &kbdev->as
1140 [next_katom->kctx->as_nr]);
1141 KBASE_TLSTREAM_TL_NRET_CTX_LPU(kbdev, next_katom->kctx,
1142 &kbdev->gpu_props.props.raw_props.js_features
1143 [next_katom->slot_nr]);
1144 }
1145
1146 if (next_katom->core_req & BASE_JD_REQ_PERMON)
1147 kbase_pm_release_gpu_cycle_counter_nolock(kbdev);
1148
1149 /* On evicting the next_katom, the last submission kctx on the
1150 * given job slot then reverts back to the one that owns katom.
1151 * The aim is to enable the next submission that can determine
1152 * if the read only shader core L1 cache should be invalidated.
1153 */
1154 kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged =
1155 SLOT_RB_TAG_KCTX(katom->kctx);
1156
1157 return true;
1158 }
1159
1160 return false;
1161 }
1162
1163 /**
1164 * kbase_gpu_complete_hw - complete the atom in a slot's JSn_HEAD
1165 * @kbdev: kbase device
1166 * @js: job slot to check
1167 * @completion_code: completion code of the completed atom
1168 * @job_tail: value read from JSn_TAIL, for STOPPED atoms
1169 * @end_timestamp: pointer to approximate ktime value when the katom completed
1170 *
1171 * Among other operations, this also executes step 2 of a 2-step process of
1172 * removing any related atoms from a slot's JSn_HEAD_NEXT (ringbuffer index 1),
1173 * should there have been a 'failure' on an atom in JSn_HEAD (ringbuffer index
1174 * 0). The first step is done in kbase_gpu_irq_evict().
1175 *
1176 * Note: 'STOPPED' atoms are considered 'failed', as they are in the HW, but
1177 * unlike other failure codes we _can_ re-run them.
1178 *
1179 * When the JSn_HEAD atom is considered to be 'failed', then this will dequeue
1180 * and return to the JS some (usually all) of the atoms evicted from the HW
1181 * during the kbase_gpu_irq_evict() for that JSn_HEAD atom. If it dequeues an
1182 * atom, that atom must not have been running or must already be evicted, as
1183 * otherwise we would be in the incorrect state of having an atom both running
1184 * on the HW and returned to the JS.
1185 */
1186
kbase_gpu_complete_hw(struct kbase_device * kbdev,unsigned int js,u32 completion_code,u64 job_tail,ktime_t * end_timestamp)1187 void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 completion_code,
1188 u64 job_tail, ktime_t *end_timestamp)
1189 {
1190 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
1191 struct kbase_context *kctx = NULL;
1192
1193 if (unlikely(!katom)) {
1194 dev_err(kbdev->dev, "Can't get a katom from js(%d)\n", js);
1195 return;
1196 }
1197
1198 kctx = katom->kctx;
1199
1200 dev_dbg(kbdev->dev,
1201 "Atom %pK completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n",
1202 (void *)katom, completion_code, job_tail, js);
1203
1204 lockdep_assert_held(&kbdev->hwaccess_lock);
1205
1206 /*
1207 * When a hard-stop is followed close after a soft-stop, the completion
1208 * code may be set to STOPPED, even though the job is terminated
1209 */
1210 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) {
1211 if (completion_code == BASE_JD_EVENT_STOPPED &&
1212 (katom->atom_flags &
1213 KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) {
1214 completion_code = BASE_JD_EVENT_TERMINATED;
1215 }
1216 }
1217
1218 if ((katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) &&
1219 completion_code != BASE_JD_EVENT_DONE &&
1220 !(completion_code & BASE_JD_SW_EVENT)) {
1221 /* When a job chain fails, on a T60x or when
1222 * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not
1223 * flushed. To prevent future evictions causing possible memory
1224 * corruption we need to flush the cache manually before any
1225 * affected memory gets reused.
1226 */
1227 katom->need_cache_flush_cores_retained = true;
1228 }
1229
1230 katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1231
1232 if (completion_code == BASE_JD_EVENT_STOPPED) {
1233 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1234 0);
1235
1236 /*
1237 * Dequeue next atom from ringbuffers on same slot if required.
1238 * This atom will already have been removed from the NEXT
1239 * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
1240 * the atoms on this slot are returned in the correct order.
1241 */
1242 if (next_katom &&
1243 kbase_js_atom_runs_before(kbdev, katom, next_katom, 0u)) {
1244 WARN_ON(next_katom->gpu_rb_state ==
1245 KBASE_ATOM_GPU_RB_SUBMITTED);
1246 kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1247 kbase_jm_return_atom_to_js(kbdev, next_katom);
1248 }
1249 } else if (completion_code != BASE_JD_EVENT_DONE) {
1250 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1251 unsigned int i;
1252
1253 if (!kbase_ctx_flag(katom->kctx, KCTX_DYING)) {
1254 dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)",
1255 js, completion_code,
1256 kbase_gpu_exception_name(
1257 completion_code));
1258
1259 }
1260
1261 #if KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR != 0
1262 KBASE_KTRACE_DUMP(kbdev);
1263 #endif
1264 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1265
1266 /*
1267 * Remove all atoms on the same context from ringbuffers. This
1268 * will not remove atoms that are already on the GPU, as these
1269 * are guaranteed not to have fail dependencies on the failed
1270 * atom.
1271 */
1272 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
1273 struct kbase_jd_atom *katom_idx0 =
1274 kbase_gpu_inspect(kbdev, i, 0);
1275 struct kbase_jd_atom *katom_idx1 =
1276 kbase_gpu_inspect(kbdev, i, 1);
1277
1278 if (katom_idx0 &&
1279 kbase_rb_atom_might_depend(katom, katom_idx0) &&
1280 katom_idx0->gpu_rb_state !=
1281 KBASE_ATOM_GPU_RB_SUBMITTED) {
1282 /* Dequeue katom_idx0 from ringbuffer */
1283 kbase_gpu_dequeue_atom(kbdev, i, end_timestamp);
1284
1285 if (katom_idx1 && kbase_rb_atom_might_depend(
1286 katom, katom_idx1) &&
1287 katom_idx0->gpu_rb_state !=
1288 KBASE_ATOM_GPU_RB_SUBMITTED) {
1289 /* Dequeue katom_idx1 from ringbuffer */
1290 kbase_gpu_dequeue_atom(kbdev, i,
1291 end_timestamp);
1292
1293 katom_idx1->event_code =
1294 BASE_JD_EVENT_STOPPED;
1295 kbase_jm_return_atom_to_js(kbdev,
1296 katom_idx1);
1297 }
1298 katom_idx0->event_code = BASE_JD_EVENT_STOPPED;
1299 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1300
1301 } else if (katom_idx1 && kbase_rb_atom_might_depend(
1302 katom, katom_idx1) &&
1303 katom_idx1->gpu_rb_state !=
1304 KBASE_ATOM_GPU_RB_SUBMITTED) {
1305 /* Can not dequeue this atom yet - will be
1306 * dequeued when atom at idx0 completes
1307 */
1308 katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
1309 kbase_gpu_mark_atom_for_return(kbdev,
1310 katom_idx1);
1311 }
1312 }
1313 }
1314
1315 KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, completion_code);
1316
1317 if (job_tail != 0 && job_tail != katom->jc) {
1318 /* Some of the job has been executed */
1319 dev_dbg(kbdev->dev,
1320 "Update job chain address of atom %pK to resume from 0x%llx\n",
1321 (void *)katom, job_tail);
1322
1323 katom->jc = job_tail;
1324 KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
1325 katom, job_tail, js);
1326 }
1327
1328 /* Only update the event code for jobs that weren't cancelled */
1329 if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
1330 katom->event_code = (enum base_jd_event_code)completion_code;
1331
1332 /* Complete the job, and start new ones
1333 *
1334 * Also defer remaining work onto the workqueue:
1335 * - Re-queue Soft-stopped jobs
1336 * - For any other jobs, queue the job back into the dependency system
1337 * - Schedule out the parent context if necessary, and schedule a new
1338 * one in.
1339 */
1340 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
1341 {
1342 /* The atom in the HEAD */
1343 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1344 0);
1345
1346 if (next_katom && next_katom->gpu_rb_state ==
1347 KBASE_ATOM_GPU_RB_SUBMITTED) {
1348 char js_string[16];
1349
1350 trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1351 js_string,
1352 sizeof(js_string)),
1353 ktime_to_ns(*end_timestamp),
1354 (u32)next_katom->kctx->id, 0,
1355 next_katom->work_id);
1356 } else {
1357 char js_string[16];
1358
1359 trace_gpu_sched_switch(kbasep_make_job_slot_string(js, js_string,
1360 sizeof(js_string)),
1361 ktime_to_ns(ktime_get_raw()), 0, 0, 0);
1362 }
1363 }
1364 #endif
1365
1366 if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)
1367 kbase_reset_gpu_silent(kbdev);
1368
1369 if (completion_code == BASE_JD_EVENT_STOPPED)
1370 katom = kbase_jm_return_atom_to_js(kbdev, katom);
1371 else
1372 katom = kbase_jm_complete(kbdev, katom, end_timestamp);
1373
1374 if (katom) {
1375 dev_dbg(kbdev->dev,
1376 "Cross-slot dependency %pK has become runnable.\n",
1377 (void *)katom);
1378
1379 /* Check if there are lower priority jobs to soft stop */
1380 kbase_job_slot_ctx_priority_check_locked(kctx, katom);
1381
1382 kbase_jm_try_kick(kbdev, 1 << katom->slot_nr);
1383 }
1384
1385 /* For partial shader core off L2 cache flush */
1386 kbase_pm_update_state(kbdev);
1387
1388 /* Job completion may have unblocked other atoms. Try to update all job
1389 * slots
1390 */
1391 kbase_backend_slot_update(kbdev);
1392 }
1393
kbase_backend_reset(struct kbase_device * kbdev,ktime_t * end_timestamp)1394 void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
1395 {
1396 unsigned int js;
1397
1398 lockdep_assert_held(&kbdev->hwaccess_lock);
1399
1400 /* Reset should always take the GPU out of protected mode */
1401 WARN_ON(kbase_gpu_in_protected_mode(kbdev));
1402
1403 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1404 int atom_idx = 0;
1405 int idx;
1406
1407 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1408 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1409 js, atom_idx);
1410 bool keep_in_jm_rb = false;
1411
1412 if (!katom)
1413 break;
1414 if (katom->protected_state.exit ==
1415 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) {
1416 /* protected mode sanity checks */
1417 WARN(kbase_jd_katom_is_protected(katom) !=
1418 kbase_gpu_in_protected_mode(kbdev),
1419 "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
1420 kbase_jd_katom_is_protected(katom),
1421 kbase_gpu_in_protected_mode(kbdev));
1422 WARN(!(kbase_jd_katom_is_protected(katom) && js == 0) &&
1423 kbase_jd_katom_is_protected(katom),
1424 "Protected atom on JS%u not supported", js);
1425 }
1426 if ((katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) &&
1427 !kbase_ctx_flag(katom->kctx, KCTX_DYING))
1428 keep_in_jm_rb = true;
1429
1430 kbase_gpu_release_atom(kbdev, katom, NULL);
1431
1432 /*
1433 * If the atom wasn't on HW when the reset was issued
1434 * then leave it in the RB and next time we're kicked
1435 * it will be processed again from the starting state.
1436 */
1437 if (keep_in_jm_rb) {
1438 katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
1439 /* As the atom was not removed, increment the
1440 * index so that we read the correct atom in the
1441 * next iteration.
1442 */
1443 atom_idx++;
1444 continue;
1445 }
1446
1447 /*
1448 * The atom was on the HW when the reset was issued
1449 * all we can do is fail the atom.
1450 */
1451 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1452 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1453 kbase_jm_complete(kbdev, katom, end_timestamp);
1454 }
1455
1456 /* Clear the slot's last katom submission kctx on reset */
1457 kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_NULL_TAG_VAL;
1458 }
1459
1460 /* Re-enable GPU hardware counters if we're resetting from protected
1461 * mode.
1462 */
1463 kbdev->protected_mode_hwcnt_desired = true;
1464 if (kbdev->protected_mode_hwcnt_disabled) {
1465 kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
1466 kbdev->protected_mode_hwcnt_disabled = false;
1467
1468 KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev, kbdev);
1469 }
1470
1471 kbdev->protected_mode_transition = false;
1472 kbase_pm_protected_override_disable(kbdev);
1473 }
1474
1475 /**
1476 * should_stop_next_atom - given a soft/hard stop action, determine if the next
1477 * atom on a slot should be stopped
1478 * @kbdev: kbase devices
1479 * @head_katom: atom currently in the JSn_HEAD
1480 * @next_katom: atom currently in the JSn_HEAD_NEXT
1481 * @action: JS_COMMAND_<...> action for soft/hard-stop
1482 *
1483 * This is used in cases where @head_katom is the target of the soft/hard-stop.
1484 * It only makes sense to call this when @head_katom and @next_katom are from
1485 * the same slot.
1486 *
1487 * Return: true if @next_katom should also be stopped with the given action,
1488 * false otherwise
1489 */
should_stop_next_atom(struct kbase_device * kbdev,const struct kbase_jd_atom * head_katom,const struct kbase_jd_atom * next_katom,u32 action)1490 static bool should_stop_next_atom(struct kbase_device *kbdev,
1491 const struct kbase_jd_atom *head_katom,
1492 const struct kbase_jd_atom *next_katom,
1493 u32 action)
1494 {
1495 bool ret = false;
1496 u32 hw_action = action & JS_COMMAND_MASK;
1497
1498 switch (hw_action) {
1499 case JS_COMMAND_SOFT_STOP:
1500 ret = kbase_js_atom_runs_before(kbdev, head_katom, next_katom,
1501 0u);
1502 break;
1503 case JS_COMMAND_HARD_STOP:
1504 /* Unlike soft-stop, a hard-stop targeting a particular atom
1505 * should not cause atoms from unrelated contexts to be
1506 * removed
1507 */
1508 ret = (head_katom->kctx == next_katom->kctx);
1509 break;
1510 default:
1511 /* Other stop actions are possible, but the driver should not
1512 * be generating them at this point in the call chain
1513 */
1514 WARN(1, "Unexpected stop action: 0x%.8x", hw_action);
1515 break;
1516 }
1517 return ret;
1518 }
1519
kbase_gpu_stop_atom(struct kbase_device * kbdev,unsigned int js,struct kbase_jd_atom * katom,u32 action)1520 static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, unsigned int js,
1521 struct kbase_jd_atom *katom, u32 action)
1522 {
1523 struct kbase_context *kctx = katom->kctx;
1524 u32 hw_action = action & JS_COMMAND_MASK;
1525
1526 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
1527 kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
1528 katom->core_req, katom);
1529 kbase_jsctx_slot_prio_blocked_set(kctx, js, katom->sched_priority);
1530 }
1531
kbase_gpu_remove_atom(struct kbase_device * kbdev,struct kbase_jd_atom * katom,u32 action,bool disjoint)1532 static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
1533 struct kbase_jd_atom *katom,
1534 u32 action,
1535 bool disjoint)
1536 {
1537 struct kbase_context *kctx = katom->kctx;
1538
1539 lockdep_assert_held(&kbdev->hwaccess_lock);
1540
1541 katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1542 kbase_gpu_mark_atom_for_return(kbdev, katom);
1543 kbase_jsctx_slot_prio_blocked_set(kctx, katom->slot_nr,
1544 katom->sched_priority);
1545
1546 if (disjoint)
1547 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
1548 katom);
1549 }
1550
should_stop_x_dep_slot(struct kbase_jd_atom * katom)1551 static int should_stop_x_dep_slot(struct kbase_jd_atom *katom)
1552 {
1553 if (katom->x_post_dep) {
1554 struct kbase_jd_atom *dep_atom = katom->x_post_dep;
1555
1556 if (dep_atom->gpu_rb_state !=
1557 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB &&
1558 dep_atom->gpu_rb_state !=
1559 KBASE_ATOM_GPU_RB_RETURN_TO_JS)
1560 return dep_atom->slot_nr;
1561 }
1562 return -1;
1563 }
1564
kbase_backend_soft_hard_stop_slot(struct kbase_device * kbdev,struct kbase_context * kctx,unsigned int js,struct kbase_jd_atom * katom,u32 action)1565 bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, struct kbase_context *kctx,
1566 unsigned int js, struct kbase_jd_atom *katom, u32 action)
1567 {
1568 struct kbase_jd_atom *katom_idx0;
1569 struct kbase_context *kctx_idx0 = NULL;
1570 struct kbase_jd_atom *katom_idx1;
1571 struct kbase_context *kctx_idx1 = NULL;
1572
1573 bool katom_idx0_valid, katom_idx1_valid;
1574
1575 bool ret = false;
1576
1577 int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
1578 int prio_idx0 = 0, prio_idx1 = 0;
1579
1580 lockdep_assert_held(&kbdev->hwaccess_lock);
1581
1582 katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
1583 katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
1584
1585 if (katom_idx0) {
1586 kctx_idx0 = katom_idx0->kctx;
1587 prio_idx0 = katom_idx0->sched_priority;
1588 }
1589 if (katom_idx1) {
1590 kctx_idx1 = katom_idx1->kctx;
1591 prio_idx1 = katom_idx1->sched_priority;
1592 }
1593
1594 if (katom) {
1595 katom_idx0_valid = (katom_idx0 == katom);
1596 if (katom_idx1)
1597 katom_idx1_valid = (katom_idx1 == katom);
1598 else
1599 katom_idx1_valid = false;
1600 } else {
1601 katom_idx0_valid = (katom_idx0 && (!kctx || kctx_idx0 == kctx));
1602 katom_idx1_valid = (katom_idx1 && (!kctx || kctx_idx1 == kctx));
1603 }
1604 /* If there's an atom in JSn_HEAD_NEXT that we haven't already decided
1605 * to stop, but we're stopping the JSn_HEAD atom, see if they are
1606 * related/ordered in some way that would require the same stop action
1607 */
1608 if (!katom_idx1_valid && katom_idx0_valid && katom_idx1)
1609 katom_idx1_valid = should_stop_next_atom(kbdev, katom_idx0,
1610 katom_idx1, action);
1611
1612 if (katom_idx0_valid)
1613 stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0);
1614 if (katom_idx1_valid)
1615 stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1);
1616
1617 if (katom_idx0_valid) {
1618 if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1619 /* Simple case - just dequeue and return */
1620 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1621 if (katom_idx1_valid) {
1622 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1623 katom_idx1->event_code =
1624 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1625 kbase_jm_return_atom_to_js(kbdev, katom_idx1);
1626 kbase_jsctx_slot_prio_blocked_set(kctx_idx1, js,
1627 prio_idx1);
1628 }
1629
1630 katom_idx0->event_code =
1631 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1632 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1633 kbase_jsctx_slot_prio_blocked_set(kctx_idx0, js,
1634 prio_idx0);
1635 } else {
1636 /* katom_idx0 is on GPU */
1637 if (katom_idx1_valid && katom_idx1->gpu_rb_state ==
1638 KBASE_ATOM_GPU_RB_SUBMITTED) {
1639 /* katom_idx0 and katom_idx1 are on GPU */
1640
1641 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1642 JS_COMMAND_NEXT)) == 0) {
1643 /* idx0 has already completed - stop
1644 * idx1 if needed
1645 */
1646 if (katom_idx1_valid) {
1647 kbase_gpu_stop_atom(kbdev, js,
1648 katom_idx1,
1649 action);
1650 ret = true;
1651 }
1652 } else {
1653 /* idx1 is in NEXT registers - attempt
1654 * to remove
1655 */
1656 kbase_reg_write(kbdev,
1657 JOB_SLOT_REG(js,
1658 JS_COMMAND_NEXT),
1659 JS_COMMAND_NOP);
1660
1661 if (kbase_reg_read(kbdev,
1662 JOB_SLOT_REG(js,
1663 JS_HEAD_NEXT_LO))
1664 != 0 ||
1665 kbase_reg_read(kbdev,
1666 JOB_SLOT_REG(js,
1667 JS_HEAD_NEXT_HI))
1668 != 0) {
1669 /* idx1 removed successfully,
1670 * will be handled in IRQ
1671 */
1672 kbase_gpu_remove_atom(kbdev,
1673 katom_idx1,
1674 action, true);
1675 /* Revert the last_context. */
1676 kbdev->hwaccess.backend.slot_rb[js]
1677 .last_kctx_tagged =
1678 SLOT_RB_TAG_KCTX(katom_idx0->kctx);
1679
1680 stop_x_dep_idx1 =
1681 should_stop_x_dep_slot(katom_idx1);
1682
1683 /* stop idx0 if still on GPU */
1684 kbase_gpu_stop_atom(kbdev, js,
1685 katom_idx0,
1686 action);
1687 ret = true;
1688 } else if (katom_idx1_valid) {
1689 /* idx0 has already completed,
1690 * stop idx1 if needed
1691 */
1692 kbase_gpu_stop_atom(kbdev, js,
1693 katom_idx1,
1694 action);
1695 ret = true;
1696 }
1697 }
1698 } else if (katom_idx1_valid) {
1699 /* idx1 not on GPU but must be dequeued*/
1700
1701 /* idx1 will be handled in IRQ */
1702 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1703 false);
1704 /* stop idx0 */
1705 /* This will be repeated for anything removed
1706 * from the next registers, since their normal
1707 * flow was also interrupted, and this function
1708 * might not enter disjoint state e.g. if we
1709 * don't actually do a hard stop on the head
1710 * atom
1711 */
1712 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1713 action);
1714 ret = true;
1715 } else {
1716 /* no atom in idx1 */
1717 /* just stop idx0 */
1718 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1719 action);
1720 ret = true;
1721 }
1722 }
1723 } else if (katom_idx1_valid) {
1724 if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1725 /* Mark for return */
1726 /* idx1 will be returned once idx0 completes */
1727 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1728 false);
1729 } else {
1730 /* idx1 is on GPU */
1731 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1732 JS_COMMAND_NEXT)) == 0) {
1733 /* idx0 has already completed - stop idx1 */
1734 kbase_gpu_stop_atom(kbdev, js, katom_idx1,
1735 action);
1736 ret = true;
1737 } else {
1738 /* idx1 is in NEXT registers - attempt to
1739 * remove
1740 */
1741 kbase_reg_write(kbdev, JOB_SLOT_REG(js,
1742 JS_COMMAND_NEXT),
1743 JS_COMMAND_NOP);
1744
1745 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1746 JS_HEAD_NEXT_LO)) != 0 ||
1747 kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1748 JS_HEAD_NEXT_HI)) != 0) {
1749 /* idx1 removed successfully, will be
1750 * handled in IRQ once idx0 completes
1751 */
1752 kbase_gpu_remove_atom(kbdev, katom_idx1,
1753 action,
1754 false);
1755 /* Revert the last_context, or mark as purged */
1756 kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged =
1757 kctx_idx0 ? SLOT_RB_TAG_KCTX(katom_idx0->kctx) :
1758 SLOT_RB_TAG_PURGED;
1759 } else {
1760 /* idx0 has already completed - stop
1761 * idx1
1762 */
1763 kbase_gpu_stop_atom(kbdev, js,
1764 katom_idx1,
1765 action);
1766 ret = true;
1767 }
1768 }
1769 }
1770 }
1771
1772
1773 if (stop_x_dep_idx0 != -1)
1774 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0,
1775 NULL, action);
1776
1777 if (stop_x_dep_idx1 != -1)
1778 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1,
1779 NULL, action);
1780
1781 return ret;
1782 }
1783
kbase_backend_cache_clean(struct kbase_device * kbdev,struct kbase_jd_atom * katom)1784 void kbase_backend_cache_clean(struct kbase_device *kbdev,
1785 struct kbase_jd_atom *katom)
1786 {
1787 if (katom->need_cache_flush_cores_retained) {
1788 kbase_gpu_start_cache_clean(kbdev,
1789 GPU_COMMAND_CACHE_CLN_INV_FULL);
1790 kbase_gpu_wait_cache_clean(kbdev);
1791
1792 katom->need_cache_flush_cores_retained = false;
1793 }
1794 }
1795
kbase_backend_complete_wq(struct kbase_device * kbdev,struct kbase_jd_atom * katom)1796 void kbase_backend_complete_wq(struct kbase_device *kbdev,
1797 struct kbase_jd_atom *katom)
1798 {
1799 /*
1800 * If cache flush required due to HW workaround then perform the flush
1801 * now
1802 */
1803 kbase_backend_cache_clean(kbdev, katom);
1804 }
1805
kbase_backend_complete_wq_post_sched(struct kbase_device * kbdev,base_jd_core_req core_req)1806 void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev,
1807 base_jd_core_req core_req)
1808 {
1809 if (!kbdev->pm.active_count) {
1810 kbase_pm_lock(kbdev);
1811 kbase_pm_update_active(kbdev);
1812 kbase_pm_unlock(kbdev);
1813 }
1814 }
1815
kbase_gpu_dump_slots(struct kbase_device * kbdev)1816 void kbase_gpu_dump_slots(struct kbase_device *kbdev)
1817 {
1818 unsigned long flags;
1819 unsigned int js;
1820
1821 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1822
1823 dev_info(kbdev->dev, "%s:\n", __func__);
1824
1825 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1826 int idx;
1827
1828 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1829 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1830 js,
1831 idx);
1832
1833 if (katom)
1834 dev_info(kbdev->dev, " js%u idx%d : katom=%pK gpu_rb_state=%d\n",
1835 js, idx, katom, katom->gpu_rb_state);
1836 else
1837 dev_info(kbdev->dev, " js%u idx%d : empty\n", js, idx);
1838 }
1839 }
1840
1841 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1842 }
1843
kbase_backend_slot_kctx_purge_locked(struct kbase_device * kbdev,struct kbase_context * kctx)1844 void kbase_backend_slot_kctx_purge_locked(struct kbase_device *kbdev, struct kbase_context *kctx)
1845 {
1846 unsigned int js;
1847 bool tracked = false;
1848
1849 lockdep_assert_held(&kbdev->hwaccess_lock);
1850
1851 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1852 u64 tagged_kctx = kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged;
1853
1854 if (tagged_kctx == SLOT_RB_TAG_KCTX(kctx)) {
1855 /* Marking the slot kctx tracking field is purged */
1856 kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_TAG_PURGED;
1857 tracked = true;
1858 }
1859 }
1860
1861 if (tracked) {
1862 /* The context had run some jobs before the purge, other slots
1863 * in SLOT_RB_NULL_TAG_VAL condition needs to be marked as
1864 * purged as well.
1865 */
1866 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1867 if (kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged ==
1868 SLOT_RB_NULL_TAG_VAL)
1869 kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged =
1870 SLOT_RB_TAG_PURGED;
1871 }
1872 }
1873 }
1874