1 /*
2 *
3 * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18
19 /*
20 * Register-based HW access backend specific APIs
21 */
22
23 #include <mali_kbase.h>
24 #include <mali_kbase_hwaccess_jm.h>
25 #include <mali_kbase_jm.h>
26 #include <mali_kbase_js.h>
27 #include <mali_kbase_tlstream.h>
28 #include <mali_kbase_10969_workaround.h>
29 #include <backend/gpu/mali_kbase_cache_policy_backend.h>
30 #include <backend/gpu/mali_kbase_device_internal.h>
31 #include <backend/gpu/mali_kbase_jm_internal.h>
32 #include <backend/gpu/mali_kbase_js_affinity.h>
33 #include <backend/gpu/mali_kbase_pm_internal.h>
34
35 /* Return whether the specified ringbuffer is empty. HW access lock must be
36 * held */
37 #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx)
38 /* Return number of atoms currently in the specified ringbuffer. HW access lock
39 * must be held */
40 #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx)
41
42 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
43 struct kbase_jd_atom *katom,
44 ktime_t *end_timestamp);
45
46 /**
47 * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer
48 * @kbdev: Device pointer
49 * @katom: Atom to enqueue
50 *
51 * Context: Caller must hold the HW access lock
52 */
kbase_gpu_enqueue_atom(struct kbase_device * kbdev,struct kbase_jd_atom * katom)53 static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev,
54 struct kbase_jd_atom *katom)
55 {
56 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr];
57
58 WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE);
59
60 lockdep_assert_held(&kbdev->hwaccess_lock);
61
62 rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom;
63 rb->write_idx++;
64
65 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
66 }
67
68 /**
69 * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once
70 * it has been completed
71 * @kbdev: Device pointer
72 * @js: Job slot to remove atom from
73 * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in
74 * which case current time will be used.
75 *
76 * Context: Caller must hold the HW access lock
77 *
78 * Return: Atom removed from ringbuffer
79 */
kbase_gpu_dequeue_atom(struct kbase_device * kbdev,int js,ktime_t * end_timestamp)80 static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev,
81 int js,
82 ktime_t *end_timestamp)
83 {
84 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
85 struct kbase_jd_atom *katom;
86
87 if (SLOT_RB_EMPTY(rb)) {
88 WARN(1, "GPU ringbuffer unexpectedly empty\n");
89 return NULL;
90 }
91
92 lockdep_assert_held(&kbdev->hwaccess_lock);
93
94 katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom;
95
96 kbase_gpu_release_atom(kbdev, katom, end_timestamp);
97
98 rb->read_idx++;
99
100 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB;
101
102 kbase_js_debug_log_current_affinities(kbdev);
103
104 return katom;
105 }
106
kbase_gpu_inspect(struct kbase_device * kbdev,int js,int idx)107 struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
108 int idx)
109 {
110 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
111
112 lockdep_assert_held(&kbdev->hwaccess_lock);
113
114 if ((SLOT_RB_ENTRIES(rb) - 1) < idx)
115 return NULL; /* idx out of range */
116
117 return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom;
118 }
119
kbase_backend_inspect_head(struct kbase_device * kbdev,int js)120 struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev,
121 int js)
122 {
123 return kbase_gpu_inspect(kbdev, js, 0);
124 }
125
kbase_backend_inspect_tail(struct kbase_device * kbdev,int js)126 struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev,
127 int js)
128 {
129 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
130
131 if (SLOT_RB_EMPTY(rb))
132 return NULL;
133
134 return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom;
135 }
136
137 /**
138 * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently
139 * on the GPU
140 * @kbdev: Device pointer
141 * @js: Job slot to inspect
142 *
143 * Return: true if there are atoms on the GPU for slot js,
144 * false otherwise
145 */
kbase_gpu_atoms_submitted(struct kbase_device * kbdev,int js)146 static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js)
147 {
148 int i;
149
150 lockdep_assert_held(&kbdev->hwaccess_lock);
151
152 for (i = 0; i < SLOT_RB_SIZE; i++) {
153 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
154
155 if (!katom)
156 return false;
157 if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED ||
158 katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY)
159 return true;
160 }
161
162 return false;
163 }
164
165 /**
166 * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms
167 * currently on the GPU
168 * @kbdev: Device pointer
169 *
170 * Return: true if there are any atoms on the GPU, false otherwise
171 */
kbase_gpu_atoms_submitted_any(struct kbase_device * kbdev)172 static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev)
173 {
174 int js;
175 int i;
176
177 lockdep_assert_held(&kbdev->hwaccess_lock);
178
179 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
180 for (i = 0; i < SLOT_RB_SIZE; i++) {
181 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
182
183 if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)
184 return true;
185 }
186 }
187 return false;
188 }
189
kbase_backend_nr_atoms_submitted(struct kbase_device * kbdev,int js)190 int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js)
191 {
192 int nr = 0;
193 int i;
194
195 lockdep_assert_held(&kbdev->hwaccess_lock);
196
197 for (i = 0; i < SLOT_RB_SIZE; i++) {
198 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
199
200 if (katom && (katom->gpu_rb_state ==
201 KBASE_ATOM_GPU_RB_SUBMITTED))
202 nr++;
203 }
204
205 return nr;
206 }
207
kbase_backend_nr_atoms_on_slot(struct kbase_device * kbdev,int js)208 int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js)
209 {
210 int nr = 0;
211 int i;
212
213 lockdep_assert_held(&kbdev->hwaccess_lock);
214
215 for (i = 0; i < SLOT_RB_SIZE; i++) {
216 if (kbase_gpu_inspect(kbdev, js, i))
217 nr++;
218 }
219
220 return nr;
221 }
222
kbase_gpu_nr_atoms_on_slot_min(struct kbase_device * kbdev,int js,enum kbase_atom_gpu_rb_state min_rb_state)223 static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js,
224 enum kbase_atom_gpu_rb_state min_rb_state)
225 {
226 int nr = 0;
227 int i;
228
229 lockdep_assert_held(&kbdev->hwaccess_lock);
230
231 for (i = 0; i < SLOT_RB_SIZE; i++) {
232 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
233
234 if (katom && (katom->gpu_rb_state >= min_rb_state))
235 nr++;
236 }
237
238 return nr;
239 }
240
241 /**
242 * check_secure_atom - Check if the given atom is in the given secure state and
243 * has a ringbuffer state of at least
244 * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
245 * @katom: Atom pointer
246 * @secure: Desired secure state
247 *
248 * Return: true if atom is in the given state, false otherwise
249 */
check_secure_atom(struct kbase_jd_atom * katom,bool secure)250 static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure)
251 {
252 if (katom->gpu_rb_state >=
253 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION &&
254 ((kbase_jd_katom_is_protected(katom) && secure) ||
255 (!kbase_jd_katom_is_protected(katom) && !secure)))
256 return true;
257
258 return false;
259 }
260
261 /**
262 * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given
263 * secure state in the ringbuffers of at least
264 * state
265 * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE
266 * @kbdev: Device pointer
267 * @secure: Desired secure state
268 *
269 * Return: true if any atoms are in the given state, false otherwise
270 */
kbase_gpu_check_secure_atoms(struct kbase_device * kbdev,bool secure)271 static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev,
272 bool secure)
273 {
274 int js, i;
275
276 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
277 for (i = 0; i < SLOT_RB_SIZE; i++) {
278 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
279 js, i);
280
281 if (katom) {
282 if (check_secure_atom(katom, secure))
283 return true;
284 }
285 }
286 }
287
288 return false;
289 }
290
kbase_backend_slot_free(struct kbase_device * kbdev,int js)291 int kbase_backend_slot_free(struct kbase_device *kbdev, int js)
292 {
293 if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) !=
294 KBASE_RESET_GPU_NOT_PENDING) {
295 /* The GPU is being reset - so prevent submission */
296 return 0;
297 }
298
299 return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js);
300 }
301
302
303 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
304 struct kbase_jd_atom *katom);
305
kbasep_js_job_check_ref_cores(struct kbase_device * kbdev,int js,struct kbase_jd_atom * katom)306 static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev,
307 int js,
308 struct kbase_jd_atom *katom)
309 {
310 /* The most recently checked affinity. Having this at this scope allows
311 * us to guarantee that we've checked the affinity in this function
312 * call.
313 */
314 u64 recently_chosen_affinity = 0;
315 bool chosen_affinity = false;
316 bool retry;
317
318 do {
319 retry = false;
320
321 /* NOTE: The following uses a number of FALLTHROUGHs to optimize
322 * the calls to this function. Ending of the function is
323 * indicated by BREAK OUT */
324 switch (katom->coreref_state) {
325 /* State when job is first attempted to be run */
326 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
327 KBASE_DEBUG_ASSERT(katom->affinity == 0);
328
329 /* Compute affinity */
330 if (false == kbase_js_choose_affinity(
331 &recently_chosen_affinity, kbdev, katom,
332 js)) {
333 /* No cores are currently available */
334 /* *** BREAK OUT: No state transition *** */
335 break;
336 }
337
338 chosen_affinity = true;
339
340 /* Request the cores */
341 kbase_pm_request_cores(kbdev,
342 katom->core_req & BASE_JD_REQ_T,
343 recently_chosen_affinity);
344
345 katom->affinity = recently_chosen_affinity;
346
347 /* Proceed to next state */
348 katom->coreref_state =
349 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
350
351 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
352 /* fallthrough */
353 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
354 {
355 enum kbase_pm_cores_ready cores_ready;
356
357 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
358 (katom->core_req & BASE_JD_REQ_T));
359
360 cores_ready = kbase_pm_register_inuse_cores(
361 kbdev,
362 katom->core_req & BASE_JD_REQ_T,
363 katom->affinity);
364 if (cores_ready == KBASE_NEW_AFFINITY) {
365 /* Affinity no longer valid - return to
366 * previous state */
367 kbasep_js_job_check_deref_cores(kbdev,
368 katom);
369 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
370 JS_CORE_REF_REGISTER_INUSE_FAILED,
371 katom->kctx, katom,
372 katom->jc, js,
373 (u32) katom->affinity);
374 /* *** BREAK OUT: Return to previous
375 * state, retry *** */
376 retry = true;
377 break;
378 }
379 if (cores_ready == KBASE_CORES_NOT_READY) {
380 /* Stay in this state and return, to
381 * retry at this state later */
382 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
383 JS_CORE_REF_REGISTER_INUSE_FAILED,
384 katom->kctx, katom,
385 katom->jc, js,
386 (u32) katom->affinity);
387 /* *** BREAK OUT: No state transition
388 * *** */
389 break;
390 }
391 /* Proceed to next state */
392 katom->coreref_state =
393 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
394 }
395
396 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
397 /* fallthrough */
398 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
399 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
400 (katom->core_req & BASE_JD_REQ_T));
401
402 /* Optimize out choosing the affinity twice in the same
403 * function call */
404 if (chosen_affinity == false) {
405 /* See if the affinity changed since a previous
406 * call. */
407 if (false == kbase_js_choose_affinity(
408 &recently_chosen_affinity,
409 kbdev, katom, js)) {
410 /* No cores are currently available */
411 kbasep_js_job_check_deref_cores(kbdev,
412 katom);
413 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
414 JS_CORE_REF_REQUEST_ON_RECHECK_FAILED,
415 katom->kctx, katom,
416 katom->jc, js,
417 (u32) recently_chosen_affinity);
418 /* *** BREAK OUT: Transition to lower
419 * state *** */
420 break;
421 }
422 chosen_affinity = true;
423 }
424
425 /* Now see if this requires a different set of cores */
426 if (recently_chosen_affinity != katom->affinity) {
427 enum kbase_pm_cores_ready cores_ready;
428
429 kbase_pm_request_cores(kbdev,
430 katom->core_req & BASE_JD_REQ_T,
431 recently_chosen_affinity);
432
433 /* Register new cores whilst we still hold the
434 * old ones, to minimize power transitions */
435 cores_ready =
436 kbase_pm_register_inuse_cores(kbdev,
437 katom->core_req & BASE_JD_REQ_T,
438 recently_chosen_affinity);
439 kbasep_js_job_check_deref_cores(kbdev, katom);
440
441 /* Fixup the state that was reduced by
442 * deref_cores: */
443 katom->coreref_state =
444 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
445 katom->affinity = recently_chosen_affinity;
446 if (cores_ready == KBASE_NEW_AFFINITY) {
447 /* Affinity no longer valid - return to
448 * previous state */
449 katom->coreref_state =
450 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
451
452 kbasep_js_job_check_deref_cores(kbdev,
453 katom);
454
455 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
456 JS_CORE_REF_REGISTER_INUSE_FAILED,
457 katom->kctx, katom,
458 katom->jc, js,
459 (u32) katom->affinity);
460 /* *** BREAK OUT: Return to previous
461 * state, retry *** */
462 retry = true;
463 break;
464 }
465 /* Now might be waiting for powerup again, with
466 * a new affinity */
467 if (cores_ready == KBASE_CORES_NOT_READY) {
468 /* Return to previous state */
469 katom->coreref_state =
470 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
471 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
472 JS_CORE_REF_REGISTER_ON_RECHECK_FAILED,
473 katom->kctx, katom,
474 katom->jc, js,
475 (u32) katom->affinity);
476 /* *** BREAK OUT: Transition to lower
477 * state *** */
478 break;
479 }
480 }
481 /* Proceed to next state */
482 katom->coreref_state =
483 KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS;
484
485 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
486 /* fallthrough */
487 case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS:
488 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
489 (katom->core_req & BASE_JD_REQ_T));
490 KBASE_DEBUG_ASSERT(katom->affinity ==
491 recently_chosen_affinity);
492
493 /* Note: this is where the caller must've taken the
494 * hwaccess_lock */
495
496 /* Check for affinity violations - if there are any,
497 * then we just ask the caller to requeue and try again
498 * later */
499 if (kbase_js_affinity_would_violate(kbdev, js,
500 katom->affinity) != false) {
501 /* Return to previous state */
502 katom->coreref_state =
503 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
504 /* *** BREAK OUT: Transition to lower state ***
505 */
506 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
507 JS_CORE_REF_AFFINITY_WOULD_VIOLATE,
508 katom->kctx, katom, katom->jc, js,
509 (u32) katom->affinity);
510 break;
511 }
512
513 /* No affinity violations would result, so the cores are
514 * ready */
515 katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY;
516 /* *** BREAK OUT: Cores Ready *** */
517 break;
518
519 default:
520 KBASE_DEBUG_ASSERT_MSG(false,
521 "Unhandled kbase_atom_coreref_state %d",
522 katom->coreref_state);
523 break;
524 }
525 } while (retry != false);
526
527 return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY);
528 }
529
kbasep_js_job_check_deref_cores(struct kbase_device * kbdev,struct kbase_jd_atom * katom)530 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
531 struct kbase_jd_atom *katom)
532 {
533 KBASE_DEBUG_ASSERT(kbdev != NULL);
534 KBASE_DEBUG_ASSERT(katom != NULL);
535
536 switch (katom->coreref_state) {
537 case KBASE_ATOM_COREREF_STATE_READY:
538 /* State where atom was submitted to the HW - just proceed to
539 * power-down */
540 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
541 (katom->core_req & BASE_JD_REQ_T));
542
543 /* fallthrough */
544
545 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
546 /* State where cores were registered */
547 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
548 (katom->core_req & BASE_JD_REQ_T));
549 kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
550 katom->affinity);
551
552 break;
553
554 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
555 /* State where cores were requested, but not registered */
556 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
557 (katom->core_req & BASE_JD_REQ_T));
558 kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
559 katom->affinity);
560 break;
561
562 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
563 /* Initial state - nothing required */
564 KBASE_DEBUG_ASSERT(katom->affinity == 0);
565 break;
566
567 default:
568 KBASE_DEBUG_ASSERT_MSG(false,
569 "Unhandled coreref_state: %d",
570 katom->coreref_state);
571 break;
572 }
573
574 katom->affinity = 0;
575 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
576 }
577
kbasep_js_job_check_deref_cores_nokatom(struct kbase_device * kbdev,base_jd_core_req core_req,u64 affinity,enum kbase_atom_coreref_state coreref_state)578 static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev,
579 base_jd_core_req core_req, u64 affinity,
580 enum kbase_atom_coreref_state coreref_state)
581 {
582 KBASE_DEBUG_ASSERT(kbdev != NULL);
583
584 switch (coreref_state) {
585 case KBASE_ATOM_COREREF_STATE_READY:
586 /* State where atom was submitted to the HW - just proceed to
587 * power-down */
588 KBASE_DEBUG_ASSERT(affinity != 0 ||
589 (core_req & BASE_JD_REQ_T));
590
591 /* fallthrough */
592
593 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
594 /* State where cores were registered */
595 KBASE_DEBUG_ASSERT(affinity != 0 ||
596 (core_req & BASE_JD_REQ_T));
597 kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T,
598 affinity);
599
600 break;
601
602 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
603 /* State where cores were requested, but not registered */
604 KBASE_DEBUG_ASSERT(affinity != 0 ||
605 (core_req & BASE_JD_REQ_T));
606 kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T,
607 affinity);
608 break;
609
610 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
611 /* Initial state - nothing required */
612 KBASE_DEBUG_ASSERT(affinity == 0);
613 break;
614
615 default:
616 KBASE_DEBUG_ASSERT_MSG(false,
617 "Unhandled coreref_state: %d",
618 coreref_state);
619 break;
620 }
621 }
622
kbase_gpu_release_atom(struct kbase_device * kbdev,struct kbase_jd_atom * katom,ktime_t * end_timestamp)623 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
624 struct kbase_jd_atom *katom,
625 ktime_t *end_timestamp)
626 {
627 struct kbase_context *kctx = katom->kctx;
628
629 switch (katom->gpu_rb_state) {
630 case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
631 /* Should be impossible */
632 WARN(1, "Attempting to release atom not in ringbuffer\n");
633 break;
634
635 case KBASE_ATOM_GPU_RB_SUBMITTED:
636 /* Inform power management at start/finish of atom so it can
637 * update its GPU utilisation metrics. Mark atom as not
638 * submitted beforehand. */
639 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
640 kbase_pm_metrics_update(kbdev, end_timestamp);
641
642 if (katom->core_req & BASE_JD_REQ_PERMON)
643 kbase_pm_release_gpu_cycle_counter_nolock(kbdev);
644 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
645
646 KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom,
647 &kbdev->gpu_props.props.raw_props.js_features
648 [katom->slot_nr]);
649 KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]);
650 KBASE_TLSTREAM_TL_NRET_CTX_LPU(kctx,
651 &kbdev->gpu_props.props.raw_props.js_features
652 [katom->slot_nr]);
653 /* fallthrough */
654 case KBASE_ATOM_GPU_RB_READY:
655 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
656 /* fallthrough */
657 case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
658 kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr,
659 katom->affinity);
660 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
661 /* fallthrough */
662 case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
663 break;
664
665 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION:
666 if (katom->protected_state.enter !=
667 KBASE_ATOM_ENTER_PROTECTED_CHECK ||
668 katom->protected_state.exit !=
669 KBASE_ATOM_EXIT_PROTECTED_CHECK)
670 kbdev->protected_mode_transition = false;
671
672 if (kbase_jd_katom_is_protected(katom) &&
673 (katom->protected_state.enter ==
674 KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) {
675 kbase_vinstr_resume(kbdev->vinstr_ctx);
676
677 /* Go back to configured model for IPA */
678 kbase_ipa_model_use_configured_locked(kbdev);
679 }
680
681
682 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
683 /* fallthrough */
684 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV:
685 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
686 /* fallthrough */
687 case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
688 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
689 /* fallthrough */
690 case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
691 break;
692 }
693
694 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
695 katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
696 }
697
kbase_gpu_mark_atom_for_return(struct kbase_device * kbdev,struct kbase_jd_atom * katom)698 static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev,
699 struct kbase_jd_atom *katom)
700 {
701 kbase_gpu_release_atom(kbdev, katom, NULL);
702 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS;
703 }
704
kbase_gpu_rmu_workaround(struct kbase_device * kbdev,int js)705 static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js)
706 {
707 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
708 bool slot_busy[3];
709
710 if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
711 return true;
712 slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0,
713 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
714 slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1,
715 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
716 slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2,
717 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
718
719 if ((js == 2 && !(slot_busy[0] || slot_busy[1])) ||
720 (js != 2 && !slot_busy[2]))
721 return true;
722
723 /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */
724 if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) ||
725 kbase_gpu_atoms_submitted(kbdev, 1) ||
726 backend->rmu_workaround_flag))
727 return false;
728
729 /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */
730 if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) ||
731 !backend->rmu_workaround_flag))
732 return false;
733
734 backend->rmu_workaround_flag = !backend->rmu_workaround_flag;
735
736 return true;
737 }
738
739 /**
740 * other_slots_busy - Determine if any job slots other than @js are currently
741 * running atoms
742 * @kbdev: Device pointer
743 * @js: Job slot
744 *
745 * Return: true if any slots other than @js are busy, false otherwise
746 */
other_slots_busy(struct kbase_device * kbdev,int js)747 static inline bool other_slots_busy(struct kbase_device *kbdev, int js)
748 {
749 int slot;
750
751 for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) {
752 if (slot == js)
753 continue;
754
755 if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot,
756 KBASE_ATOM_GPU_RB_SUBMITTED))
757 return true;
758 }
759
760 return false;
761 }
762
kbase_gpu_in_protected_mode(struct kbase_device * kbdev)763 static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev)
764 {
765 return kbdev->protected_mode;
766 }
767
kbase_gpu_protected_mode_enter(struct kbase_device * kbdev)768 static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev)
769 {
770 int err = -EINVAL;
771
772 lockdep_assert_held(&kbdev->hwaccess_lock);
773
774 WARN_ONCE(!kbdev->protected_ops,
775 "Cannot enter protected mode: protected callbacks not specified.\n");
776
777 /*
778 * When entering into protected mode, we must ensure that the
779 * GPU is not operating in coherent mode as well. This is to
780 * ensure that no protected memory can be leaked.
781 */
782 if (kbdev->system_coherency == COHERENCY_ACE)
783 kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE);
784
785 if (kbdev->protected_ops) {
786 /* Switch GPU to protected mode */
787 err = kbdev->protected_ops->protected_mode_enable(
788 kbdev->protected_dev);
789
790 if (err)
791 dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n",
792 err);
793 else
794 kbdev->protected_mode = true;
795 }
796
797 return err;
798 }
799
kbase_gpu_protected_mode_reset(struct kbase_device * kbdev)800 static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev)
801 {
802 lockdep_assert_held(&kbdev->hwaccess_lock);
803
804 WARN_ONCE(!kbdev->protected_ops,
805 "Cannot exit protected mode: protected callbacks not specified.\n");
806
807 if (!kbdev->protected_ops)
808 return -EINVAL;
809
810 /* The protected mode disable callback will be called as part of reset
811 */
812 kbase_reset_gpu_silent(kbdev);
813
814 return 0;
815 }
816
kbase_jm_enter_protected_mode(struct kbase_device * kbdev,struct kbase_jd_atom ** katom,int idx,int js)817 static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev,
818 struct kbase_jd_atom **katom, int idx, int js)
819 {
820 int err = 0;
821
822 switch (katom[idx]->protected_state.enter) {
823 case KBASE_ATOM_ENTER_PROTECTED_CHECK:
824 KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev);
825 /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV
826 * should ensure that we are not already transitiong, and that
827 * there are no atoms currently on the GPU. */
828 WARN_ON(kbdev->protected_mode_transition);
829 WARN_ON(kbase_gpu_atoms_submitted_any(kbdev));
830
831 kbdev->protected_mode_transition = true;
832 katom[idx]->protected_state.enter =
833 KBASE_ATOM_ENTER_PROTECTED_VINSTR;
834
835 /* ***TRANSITION TO HIGHER STATE*** */
836 /* fallthrough */
837 case KBASE_ATOM_ENTER_PROTECTED_VINSTR:
838 if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) {
839 /*
840 * We can't switch now because
841 * the vinstr core state switch
842 * is not done yet.
843 */
844 return -EAGAIN;
845 }
846
847 /* Use generic model for IPA in protected mode */
848 kbase_ipa_model_use_fallback_locked(kbdev);
849
850 /* Once reaching this point GPU must be
851 * switched to protected mode or vinstr
852 * re-enabled. */
853
854 /*
855 * Not in correct mode, begin protected mode switch.
856 * Entering protected mode requires us to power down the L2,
857 * and drop out of fully coherent mode.
858 */
859 katom[idx]->protected_state.enter =
860 KBASE_ATOM_ENTER_PROTECTED_IDLE_L2;
861
862 kbase_pm_update_cores_state_nolock(kbdev);
863
864 /* ***TRANSITION TO HIGHER STATE*** */
865 /* fallthrough */
866 case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2:
867 /* Avoid unnecessary waiting on non-ACE platforms. */
868 if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) {
869 if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) ||
870 kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) {
871 /*
872 * The L2 is still powered, wait for all the users to
873 * finish with it before doing the actual reset.
874 */
875 return -EAGAIN;
876 }
877 }
878
879 katom[idx]->protected_state.enter =
880 KBASE_ATOM_ENTER_PROTECTED_FINISHED;
881
882 /* ***TRANSITION TO HIGHER STATE*** */
883 /* fallthrough */
884 case KBASE_ATOM_ENTER_PROTECTED_FINISHED:
885
886 /* No jobs running, so we can switch GPU mode right now. */
887 err = kbase_gpu_protected_mode_enter(kbdev);
888
889 /*
890 * Regardless of result, we are no longer transitioning
891 * the GPU.
892 */
893 kbdev->protected_mode_transition = false;
894 KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev);
895 if (err) {
896 /*
897 * Failed to switch into protected mode, resume
898 * vinstr core and fail atom.
899 */
900 kbase_vinstr_resume(kbdev->vinstr_ctx);
901 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
902 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
903 /* Only return if head atom or previous atom
904 * already removed - as atoms must be returned
905 * in order. */
906 if (idx == 0 || katom[0]->gpu_rb_state ==
907 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
908 kbase_gpu_dequeue_atom(kbdev, js, NULL);
909 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
910 }
911
912 /* Go back to configured model for IPA */
913 kbase_ipa_model_use_configured_locked(kbdev);
914
915 return -EINVAL;
916 }
917
918 /* Protected mode sanity checks. */
919 KBASE_DEBUG_ASSERT_MSG(
920 kbase_jd_katom_is_protected(katom[idx]) ==
921 kbase_gpu_in_protected_mode(kbdev),
922 "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
923 kbase_jd_katom_is_protected(katom[idx]),
924 kbase_gpu_in_protected_mode(kbdev));
925 katom[idx]->gpu_rb_state =
926 KBASE_ATOM_GPU_RB_READY;
927 }
928
929 return 0;
930 }
931
kbase_jm_exit_protected_mode(struct kbase_device * kbdev,struct kbase_jd_atom ** katom,int idx,int js)932 static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev,
933 struct kbase_jd_atom **katom, int idx, int js)
934 {
935 int err = 0;
936
937
938 switch (katom[idx]->protected_state.exit) {
939 case KBASE_ATOM_EXIT_PROTECTED_CHECK:
940 KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev);
941 /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV
942 * should ensure that we are not already transitiong, and that
943 * there are no atoms currently on the GPU. */
944 WARN_ON(kbdev->protected_mode_transition);
945 WARN_ON(kbase_gpu_atoms_submitted_any(kbdev));
946
947 /*
948 * Exiting protected mode requires a reset, but first the L2
949 * needs to be powered down to ensure it's not active when the
950 * reset is issued.
951 */
952 katom[idx]->protected_state.exit =
953 KBASE_ATOM_EXIT_PROTECTED_IDLE_L2;
954
955 kbdev->protected_mode_transition = true;
956 kbase_pm_update_cores_state_nolock(kbdev);
957
958 /* ***TRANSITION TO HIGHER STATE*** */
959 /* fallthrough */
960 case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2:
961 if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) ||
962 kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) {
963 /*
964 * The L2 is still powered, wait for all the users to
965 * finish with it before doing the actual reset.
966 */
967 return -EAGAIN;
968 }
969 katom[idx]->protected_state.exit =
970 KBASE_ATOM_EXIT_PROTECTED_RESET;
971
972 /* ***TRANSITION TO HIGHER STATE*** */
973 /* fallthrough */
974 case KBASE_ATOM_EXIT_PROTECTED_RESET:
975 /* Issue the reset to the GPU */
976 err = kbase_gpu_protected_mode_reset(kbdev);
977
978 if (err) {
979 kbdev->protected_mode_transition = false;
980
981 /* Failed to exit protected mode, fail atom */
982 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
983 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
984 /* Only return if head atom or previous atom
985 * already removed - as atoms must be returned
986 * in order */
987 if (idx == 0 || katom[0]->gpu_rb_state ==
988 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
989 kbase_gpu_dequeue_atom(kbdev, js, NULL);
990 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
991 }
992
993 kbase_vinstr_resume(kbdev->vinstr_ctx);
994
995 /* Use generic model for IPA in protected mode */
996 kbase_ipa_model_use_fallback_locked(kbdev);
997
998 return -EINVAL;
999 }
1000
1001 katom[idx]->protected_state.exit =
1002 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT;
1003
1004 /* ***TRANSITION TO HIGHER STATE*** */
1005 /* fallthrough */
1006 case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT:
1007 /* A GPU reset is issued when exiting protected mode. Once the
1008 * reset is done all atoms' state will also be reset. For this
1009 * reason, if the atom is still in this state we can safely
1010 * say that the reset has not completed i.e., we have not
1011 * finished exiting protected mode yet.
1012 */
1013 return -EAGAIN;
1014 }
1015
1016 return 0;
1017 }
1018
kbase_backend_slot_update(struct kbase_device * kbdev)1019 void kbase_backend_slot_update(struct kbase_device *kbdev)
1020 {
1021 int js;
1022
1023 lockdep_assert_held(&kbdev->hwaccess_lock);
1024
1025 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1026 struct kbase_jd_atom *katom[2];
1027 int idx;
1028
1029 katom[0] = kbase_gpu_inspect(kbdev, js, 0);
1030 katom[1] = kbase_gpu_inspect(kbdev, js, 1);
1031 WARN_ON(katom[1] && !katom[0]);
1032
1033 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1034 bool cores_ready;
1035 int ret;
1036
1037 if (!katom[idx])
1038 continue;
1039
1040 switch (katom[idx]->gpu_rb_state) {
1041 case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
1042 /* Should be impossible */
1043 WARN(1, "Attempting to update atom not in ringbuffer\n");
1044 break;
1045
1046 case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
1047 if (katom[idx]->atom_flags &
1048 KBASE_KATOM_FLAG_X_DEP_BLOCKED)
1049 break;
1050
1051 katom[idx]->gpu_rb_state =
1052 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV;
1053
1054 /* ***TRANSITION TO HIGHER STATE*** */
1055 /* fallthrough */
1056 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV:
1057 if (kbase_gpu_check_secure_atoms(kbdev,
1058 !kbase_jd_katom_is_protected(
1059 katom[idx])))
1060 break;
1061
1062 if ((idx == 1) && (kbase_jd_katom_is_protected(
1063 katom[0]) !=
1064 kbase_jd_katom_is_protected(
1065 katom[1])))
1066 break;
1067
1068 if (kbdev->protected_mode_transition)
1069 break;
1070
1071 katom[idx]->gpu_rb_state =
1072 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION;
1073
1074 /* ***TRANSITION TO HIGHER STATE*** */
1075 /* fallthrough */
1076 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION:
1077
1078 /*
1079 * Exiting protected mode must be done before
1080 * the references on the cores are taken as
1081 * a power down the L2 is required which
1082 * can't happen after the references for this
1083 * atom are taken.
1084 */
1085
1086 if (!kbase_gpu_in_protected_mode(kbdev) &&
1087 kbase_jd_katom_is_protected(katom[idx])) {
1088 /* Atom needs to transition into protected mode. */
1089 ret = kbase_jm_enter_protected_mode(kbdev,
1090 katom, idx, js);
1091 if (ret)
1092 break;
1093 } else if (kbase_gpu_in_protected_mode(kbdev) &&
1094 !kbase_jd_katom_is_protected(katom[idx])) {
1095 /* Atom needs to transition out of protected mode. */
1096 ret = kbase_jm_exit_protected_mode(kbdev,
1097 katom, idx, js);
1098 if (ret)
1099 break;
1100 }
1101 katom[idx]->protected_state.exit =
1102 KBASE_ATOM_EXIT_PROTECTED_CHECK;
1103
1104 /* Atom needs no protected mode transition. */
1105
1106 katom[idx]->gpu_rb_state =
1107 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE;
1108
1109 /* ***TRANSITION TO HIGHER STATE*** */
1110 /* fallthrough */
1111 case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
1112 if (katom[idx]->will_fail_event_code) {
1113 kbase_gpu_mark_atom_for_return(kbdev,
1114 katom[idx]);
1115 /* Set EVENT_DONE so this atom will be
1116 completed, not unpulled. */
1117 katom[idx]->event_code =
1118 BASE_JD_EVENT_DONE;
1119 /* Only return if head atom or previous
1120 * atom already removed - as atoms must
1121 * be returned in order. */
1122 if (idx == 0 || katom[0]->gpu_rb_state ==
1123 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
1124 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1125 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
1126 }
1127 break;
1128 }
1129
1130 cores_ready =
1131 kbasep_js_job_check_ref_cores(kbdev, js,
1132 katom[idx]);
1133
1134 if (katom[idx]->event_code ==
1135 BASE_JD_EVENT_PM_EVENT) {
1136 katom[idx]->gpu_rb_state =
1137 KBASE_ATOM_GPU_RB_RETURN_TO_JS;
1138 break;
1139 }
1140
1141 if (!cores_ready)
1142 break;
1143
1144 kbase_js_affinity_retain_slot_cores(kbdev, js,
1145 katom[idx]->affinity);
1146 katom[idx]->gpu_rb_state =
1147 KBASE_ATOM_GPU_RB_WAITING_AFFINITY;
1148
1149 /* ***TRANSITION TO HIGHER STATE*** */
1150 /* fallthrough */
1151 case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
1152 if (!kbase_gpu_rmu_workaround(kbdev, js))
1153 break;
1154
1155 katom[idx]->gpu_rb_state =
1156 KBASE_ATOM_GPU_RB_READY;
1157
1158 /* ***TRANSITION TO HIGHER STATE*** */
1159 /* fallthrough */
1160 case KBASE_ATOM_GPU_RB_READY:
1161
1162 if (idx == 1) {
1163 /* Only submit if head atom or previous
1164 * atom already submitted */
1165 if ((katom[0]->gpu_rb_state !=
1166 KBASE_ATOM_GPU_RB_SUBMITTED &&
1167 katom[0]->gpu_rb_state !=
1168 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
1169 break;
1170
1171 /* If intra-slot serialization in use
1172 * then don't submit atom to NEXT slot
1173 */
1174 if (kbdev->serialize_jobs &
1175 KBASE_SERIALIZE_INTRA_SLOT)
1176 break;
1177 }
1178
1179 /* If inter-slot serialization in use then don't
1180 * submit atom if any other slots are in use */
1181 if ((kbdev->serialize_jobs &
1182 KBASE_SERIALIZE_INTER_SLOT) &&
1183 other_slots_busy(kbdev, js))
1184 break;
1185
1186 if ((kbdev->serialize_jobs &
1187 KBASE_SERIALIZE_RESET) &&
1188 kbase_reset_gpu_active(kbdev))
1189 break;
1190
1191 /* Check if this job needs the cycle counter
1192 * enabled before submission */
1193 if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
1194 kbase_pm_request_gpu_cycle_counter_l2_is_on(
1195 kbdev);
1196
1197 kbase_job_hw_submit(kbdev, katom[idx], js);
1198 katom[idx]->gpu_rb_state =
1199 KBASE_ATOM_GPU_RB_SUBMITTED;
1200
1201 /* Inform power management at start/finish of
1202 * atom so it can update its GPU utilisation
1203 * metrics. */
1204 kbase_pm_metrics_update(kbdev,
1205 &katom[idx]->start_timestamp);
1206
1207 /* ***TRANSITION TO HIGHER STATE*** */
1208 /* fallthrough */
1209 case KBASE_ATOM_GPU_RB_SUBMITTED:
1210 /* Atom submitted to HW, nothing else to do */
1211 break;
1212
1213 case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
1214 /* Only return if head atom or previous atom
1215 * already removed - as atoms must be returned
1216 * in order */
1217 if (idx == 0 || katom[0]->gpu_rb_state ==
1218 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
1219 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1220 kbase_jm_return_atom_to_js(kbdev,
1221 katom[idx]);
1222 }
1223 break;
1224 }
1225 }
1226 }
1227
1228 /* Warn if PRLAM-8987 affinity restrictions are violated */
1229 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
1230 WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) ||
1231 kbase_gpu_atoms_submitted(kbdev, 1)) &&
1232 kbase_gpu_atoms_submitted(kbdev, 2));
1233 }
1234
1235
kbase_backend_run_atom(struct kbase_device * kbdev,struct kbase_jd_atom * katom)1236 void kbase_backend_run_atom(struct kbase_device *kbdev,
1237 struct kbase_jd_atom *katom)
1238 {
1239 lockdep_assert_held(&kbdev->hwaccess_lock);
1240 kbase_gpu_enqueue_atom(kbdev, katom);
1241 kbase_backend_slot_update(kbdev);
1242 }
1243
1244 #define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \
1245 (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER))
1246
kbase_gpu_irq_evict(struct kbase_device * kbdev,int js)1247 bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
1248 {
1249 struct kbase_jd_atom *katom;
1250 struct kbase_jd_atom *next_katom;
1251
1252 lockdep_assert_held(&kbdev->hwaccess_lock);
1253
1254 katom = kbase_gpu_inspect(kbdev, js, 0);
1255 next_katom = kbase_gpu_inspect(kbdev, js, 1);
1256
1257 if (next_katom && katom->kctx == next_katom->kctx &&
1258 next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
1259 HAS_DEP(next_katom) &&
1260 (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL)
1261 != 0 ||
1262 kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL)
1263 != 0)) {
1264 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
1265 JS_COMMAND_NOP, NULL);
1266 next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
1267
1268 KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom,
1269 &kbdev->gpu_props.props.raw_props.js_features
1270 [katom->slot_nr]);
1271 KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as
1272 [katom->kctx->as_nr]);
1273 KBASE_TLSTREAM_TL_NRET_CTX_LPU(katom->kctx,
1274 &kbdev->gpu_props.props.raw_props.js_features
1275 [katom->slot_nr]);
1276
1277 return true;
1278 }
1279
1280 return false;
1281 }
1282
kbase_gpu_complete_hw(struct kbase_device * kbdev,int js,u32 completion_code,u64 job_tail,ktime_t * end_timestamp)1283 void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
1284 u32 completion_code,
1285 u64 job_tail,
1286 ktime_t *end_timestamp)
1287 {
1288 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
1289 struct kbase_context *kctx = katom->kctx;
1290
1291 lockdep_assert_held(&kbdev->hwaccess_lock);
1292
1293 /*
1294 * When a hard-stop is followed close after a soft-stop, the completion
1295 * code may be set to STOPPED, even though the job is terminated
1296 */
1297 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) {
1298 if (completion_code == BASE_JD_EVENT_STOPPED &&
1299 (katom->atom_flags &
1300 KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) {
1301 completion_code = BASE_JD_EVENT_TERMINATED;
1302 }
1303 }
1304
1305 if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) || (katom->core_req &
1306 BASE_JD_REQ_SKIP_CACHE_END)) &&
1307 completion_code != BASE_JD_EVENT_DONE &&
1308 !(completion_code & BASE_JD_SW_EVENT)) {
1309 /* When a job chain fails, on a T60x or when
1310 * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not
1311 * flushed. To prevent future evictions causing possible memory
1312 * corruption we need to flush the cache manually before any
1313 * affected memory gets reused. */
1314 katom->need_cache_flush_cores_retained = katom->affinity;
1315 kbase_pm_request_cores(kbdev, false, katom->affinity);
1316 } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
1317 if (kbdev->gpu_props.num_core_groups > 1 &&
1318 !(katom->affinity &
1319 kbdev->gpu_props.props.coherency_info.group[0].core_mask
1320 ) &&
1321 (katom->affinity &
1322 kbdev->gpu_props.props.coherency_info.group[1].core_mask
1323 )) {
1324 dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n");
1325 katom->need_cache_flush_cores_retained =
1326 katom->affinity;
1327 kbase_pm_request_cores(kbdev, false,
1328 katom->affinity);
1329 }
1330 }
1331
1332 katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1333 kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0);
1334
1335 if (completion_code == BASE_JD_EVENT_STOPPED) {
1336 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1337 0);
1338
1339 /*
1340 * Dequeue next atom from ringbuffers on same slot if required.
1341 * This atom will already have been removed from the NEXT
1342 * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
1343 * the atoms on this slot are returned in the correct order.
1344 */
1345 if (next_katom && katom->kctx == next_katom->kctx &&
1346 next_katom->sched_priority ==
1347 katom->sched_priority) {
1348 kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1349 kbase_jm_return_atom_to_js(kbdev, next_katom);
1350 }
1351 } else if (completion_code != BASE_JD_EVENT_DONE) {
1352 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1353 int i;
1354
1355 #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
1356 KBASE_TRACE_DUMP(kbdev);
1357 #endif
1358 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1359
1360 /*
1361 * Remove all atoms on the same context from ringbuffers. This
1362 * will not remove atoms that are already on the GPU, as these
1363 * are guaranteed not to have fail dependencies on the failed
1364 * atom.
1365 */
1366 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
1367 struct kbase_jd_atom *katom_idx0 =
1368 kbase_gpu_inspect(kbdev, i, 0);
1369 struct kbase_jd_atom *katom_idx1 =
1370 kbase_gpu_inspect(kbdev, i, 1);
1371
1372 if (katom_idx0 && katom_idx0->kctx == katom->kctx &&
1373 HAS_DEP(katom_idx0) &&
1374 katom_idx0->gpu_rb_state !=
1375 KBASE_ATOM_GPU_RB_SUBMITTED) {
1376 /* Dequeue katom_idx0 from ringbuffer */
1377 kbase_gpu_dequeue_atom(kbdev, i, end_timestamp);
1378
1379 if (katom_idx1 &&
1380 katom_idx1->kctx == katom->kctx
1381 && HAS_DEP(katom_idx1) &&
1382 katom_idx0->gpu_rb_state !=
1383 KBASE_ATOM_GPU_RB_SUBMITTED) {
1384 /* Dequeue katom_idx1 from ringbuffer */
1385 kbase_gpu_dequeue_atom(kbdev, i,
1386 end_timestamp);
1387
1388 katom_idx1->event_code =
1389 BASE_JD_EVENT_STOPPED;
1390 kbase_jm_return_atom_to_js(kbdev,
1391 katom_idx1);
1392 }
1393 katom_idx0->event_code = BASE_JD_EVENT_STOPPED;
1394 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1395
1396 } else if (katom_idx1 &&
1397 katom_idx1->kctx == katom->kctx &&
1398 HAS_DEP(katom_idx1) &&
1399 katom_idx1->gpu_rb_state !=
1400 KBASE_ATOM_GPU_RB_SUBMITTED) {
1401 /* Can not dequeue this atom yet - will be
1402 * dequeued when atom at idx0 completes */
1403 katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
1404 kbase_gpu_mark_atom_for_return(kbdev,
1405 katom_idx1);
1406 }
1407 }
1408 }
1409
1410 KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
1411 js, completion_code);
1412
1413 if (job_tail != 0 && job_tail != katom->jc) {
1414 bool was_updated = (job_tail != katom->jc);
1415
1416 /* Some of the job has been executed, so we update the job chain
1417 * address to where we should resume from */
1418 katom->jc = job_tail;
1419 if (was_updated)
1420 KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
1421 katom, job_tail, js);
1422 }
1423
1424 /* Only update the event code for jobs that weren't cancelled */
1425 if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
1426 katom->event_code = (base_jd_event_code)completion_code;
1427
1428 kbase_device_trace_register_access(kctx, REG_WRITE,
1429 JOB_CONTROL_REG(JOB_IRQ_CLEAR),
1430 1 << js);
1431
1432 /* Complete the job, and start new ones
1433 *
1434 * Also defer remaining work onto the workqueue:
1435 * - Re-queue Soft-stopped jobs
1436 * - For any other jobs, queue the job back into the dependency system
1437 * - Schedule out the parent context if necessary, and schedule a new
1438 * one in.
1439 */
1440 #ifdef CONFIG_GPU_TRACEPOINTS
1441 {
1442 /* The atom in the HEAD */
1443 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1444 0);
1445
1446 if (next_katom && next_katom->gpu_rb_state ==
1447 KBASE_ATOM_GPU_RB_SUBMITTED) {
1448 char js_string[16];
1449
1450 trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1451 js_string,
1452 sizeof(js_string)),
1453 ktime_to_ns(*end_timestamp),
1454 (u32)next_katom->kctx->id, 0,
1455 next_katom->work_id);
1456 kbdev->hwaccess.backend.slot_rb[js].last_context =
1457 next_katom->kctx;
1458 } else {
1459 char js_string[16];
1460
1461 trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1462 js_string,
1463 sizeof(js_string)),
1464 ktime_to_ns(ktime_get()), 0, 0,
1465 0);
1466 kbdev->hwaccess.backend.slot_rb[js].last_context = 0;
1467 }
1468 }
1469 #endif
1470
1471 if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)
1472 kbase_reset_gpu_silent(kbdev);
1473
1474 if (completion_code == BASE_JD_EVENT_STOPPED)
1475 katom = kbase_jm_return_atom_to_js(kbdev, katom);
1476 else
1477 katom = kbase_jm_complete(kbdev, katom, end_timestamp);
1478
1479 if (katom) {
1480 /* Cross-slot dependency has now become runnable. Try to submit
1481 * it. */
1482
1483 /* Check if there are lower priority jobs to soft stop */
1484 kbase_job_slot_ctx_priority_check_locked(kctx, katom);
1485
1486 kbase_jm_try_kick(kbdev, 1 << katom->slot_nr);
1487 }
1488
1489 /* Job completion may have unblocked other atoms. Try to update all job
1490 * slots */
1491 kbase_backend_slot_update(kbdev);
1492 }
1493
kbase_backend_reset(struct kbase_device * kbdev,ktime_t * end_timestamp)1494 void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
1495 {
1496 int js;
1497
1498 lockdep_assert_held(&kbdev->hwaccess_lock);
1499
1500 /* Reset should always take the GPU out of protected mode */
1501 WARN_ON(kbase_gpu_in_protected_mode(kbdev));
1502
1503 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1504 int atom_idx = 0;
1505 int idx;
1506
1507 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1508 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1509 js, atom_idx);
1510 bool keep_in_jm_rb = false;
1511
1512 if (!katom)
1513 break;
1514 if (katom->protected_state.exit ==
1515 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT)
1516 {
1517 KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev);
1518
1519 kbase_vinstr_resume(kbdev->vinstr_ctx);
1520
1521 /* protected mode sanity checks */
1522 KBASE_DEBUG_ASSERT_MSG(
1523 kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev),
1524 "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
1525 kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev));
1526 KBASE_DEBUG_ASSERT_MSG(
1527 (kbase_jd_katom_is_protected(katom) && js == 0) ||
1528 !kbase_jd_katom_is_protected(katom),
1529 "Protected atom on JS%d not supported", js);
1530 }
1531 if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED)
1532 keep_in_jm_rb = true;
1533
1534 kbase_gpu_release_atom(kbdev, katom, NULL);
1535
1536 /*
1537 * If the atom wasn't on HW when the reset was issued
1538 * then leave it in the RB and next time we're kicked
1539 * it will be processed again from the starting state.
1540 */
1541 if (keep_in_jm_rb) {
1542 kbasep_js_job_check_deref_cores(kbdev, katom);
1543 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1544 katom->affinity = 0;
1545 katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
1546 /* As the atom was not removed, increment the
1547 * index so that we read the correct atom in the
1548 * next iteration. */
1549 atom_idx++;
1550 continue;
1551 }
1552
1553 /*
1554 * The atom was on the HW when the reset was issued
1555 * all we can do is fail the atom.
1556 */
1557 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1558 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1559 kbase_jm_complete(kbdev, katom, end_timestamp);
1560 }
1561 }
1562
1563 kbdev->protected_mode_transition = false;
1564 }
1565
kbase_gpu_stop_atom(struct kbase_device * kbdev,int js,struct kbase_jd_atom * katom,u32 action)1566 static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev,
1567 int js,
1568 struct kbase_jd_atom *katom,
1569 u32 action)
1570 {
1571 u32 hw_action = action & JS_COMMAND_MASK;
1572
1573 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
1574 kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
1575 katom->core_req, katom);
1576 katom->kctx->blocked_js[js][katom->sched_priority] = true;
1577 }
1578
kbase_gpu_remove_atom(struct kbase_device * kbdev,struct kbase_jd_atom * katom,u32 action,bool disjoint)1579 static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
1580 struct kbase_jd_atom *katom,
1581 u32 action,
1582 bool disjoint)
1583 {
1584 katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1585 kbase_gpu_mark_atom_for_return(kbdev, katom);
1586 katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true;
1587
1588 if (disjoint)
1589 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
1590 katom);
1591 }
1592
should_stop_x_dep_slot(struct kbase_jd_atom * katom)1593 static int should_stop_x_dep_slot(struct kbase_jd_atom *katom)
1594 {
1595 if (katom->x_post_dep) {
1596 struct kbase_jd_atom *dep_atom = katom->x_post_dep;
1597
1598 if (dep_atom->gpu_rb_state !=
1599 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB &&
1600 dep_atom->gpu_rb_state !=
1601 KBASE_ATOM_GPU_RB_RETURN_TO_JS)
1602 return dep_atom->slot_nr;
1603 }
1604 return -1;
1605 }
1606
kbase_job_evicted(struct kbase_jd_atom * katom)1607 static void kbase_job_evicted(struct kbase_jd_atom *katom)
1608 {
1609 kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom,
1610 katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
1611 }
1612
kbase_backend_soft_hard_stop_slot(struct kbase_device * kbdev,struct kbase_context * kctx,int js,struct kbase_jd_atom * katom,u32 action)1613 bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
1614 struct kbase_context *kctx,
1615 int js,
1616 struct kbase_jd_atom *katom,
1617 u32 action)
1618 {
1619 struct kbase_jd_atom *katom_idx0;
1620 struct kbase_jd_atom *katom_idx1;
1621
1622 bool katom_idx0_valid, katom_idx1_valid;
1623
1624 bool ret = false;
1625
1626 int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
1627 int prio_idx0 = 0, prio_idx1 = 0;
1628
1629 lockdep_assert_held(&kbdev->hwaccess_lock);
1630
1631 katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
1632 katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
1633
1634 if (katom_idx0)
1635 prio_idx0 = katom_idx0->sched_priority;
1636 if (katom_idx1)
1637 prio_idx1 = katom_idx1->sched_priority;
1638
1639 if (katom) {
1640 katom_idx0_valid = (katom_idx0 == katom);
1641 /* If idx0 is to be removed and idx1 is on the same context,
1642 * then idx1 must also be removed otherwise the atoms might be
1643 * returned out of order */
1644 if (katom_idx1)
1645 katom_idx1_valid = (katom_idx1 == katom) ||
1646 (katom_idx0_valid &&
1647 (katom_idx0->kctx ==
1648 katom_idx1->kctx));
1649 else
1650 katom_idx1_valid = false;
1651 } else {
1652 katom_idx0_valid = (katom_idx0 &&
1653 (!kctx || katom_idx0->kctx == kctx));
1654 katom_idx1_valid = (katom_idx1 &&
1655 (!kctx || katom_idx1->kctx == kctx) &&
1656 prio_idx0 == prio_idx1);
1657 }
1658
1659 if (katom_idx0_valid)
1660 stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0);
1661 if (katom_idx1_valid)
1662 stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1);
1663
1664 if (katom_idx0_valid) {
1665 if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1666 /* Simple case - just dequeue and return */
1667 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1668 if (katom_idx1_valid) {
1669 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1670 katom_idx1->event_code =
1671 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1672 kbase_jm_return_atom_to_js(kbdev, katom_idx1);
1673 katom_idx1->kctx->blocked_js[js][prio_idx1] =
1674 true;
1675 }
1676
1677 katom_idx0->event_code =
1678 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1679 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1680 katom_idx0->kctx->blocked_js[js][prio_idx0] = true;
1681 } else {
1682 /* katom_idx0 is on GPU */
1683 if (katom_idx1 && katom_idx1->gpu_rb_state ==
1684 KBASE_ATOM_GPU_RB_SUBMITTED) {
1685 /* katom_idx0 and katom_idx1 are on GPU */
1686
1687 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1688 JS_COMMAND_NEXT), NULL) == 0) {
1689 /* idx0 has already completed - stop
1690 * idx1 if needed*/
1691 if (katom_idx1_valid) {
1692 kbase_gpu_stop_atom(kbdev, js,
1693 katom_idx1,
1694 action);
1695 ret = true;
1696 }
1697 } else {
1698 /* idx1 is in NEXT registers - attempt
1699 * to remove */
1700 kbase_reg_write(kbdev,
1701 JOB_SLOT_REG(js,
1702 JS_COMMAND_NEXT),
1703 JS_COMMAND_NOP, NULL);
1704
1705 if (kbase_reg_read(kbdev,
1706 JOB_SLOT_REG(js,
1707 JS_HEAD_NEXT_LO), NULL)
1708 != 0 ||
1709 kbase_reg_read(kbdev,
1710 JOB_SLOT_REG(js,
1711 JS_HEAD_NEXT_HI), NULL)
1712 != 0) {
1713 /* idx1 removed successfully,
1714 * will be handled in IRQ */
1715 kbase_job_evicted(katom_idx1);
1716 kbase_gpu_remove_atom(kbdev,
1717 katom_idx1,
1718 action, true);
1719 stop_x_dep_idx1 =
1720 should_stop_x_dep_slot(katom_idx1);
1721
1722 /* stop idx0 if still on GPU */
1723 kbase_gpu_stop_atom(kbdev, js,
1724 katom_idx0,
1725 action);
1726 ret = true;
1727 } else if (katom_idx1_valid) {
1728 /* idx0 has already completed,
1729 * stop idx1 if needed */
1730 kbase_gpu_stop_atom(kbdev, js,
1731 katom_idx1,
1732 action);
1733 ret = true;
1734 }
1735 }
1736 } else if (katom_idx1_valid) {
1737 /* idx1 not on GPU but must be dequeued*/
1738
1739 /* idx1 will be handled in IRQ */
1740 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1741 false);
1742 /* stop idx0 */
1743 /* This will be repeated for anything removed
1744 * from the next registers, since their normal
1745 * flow was also interrupted, and this function
1746 * might not enter disjoint state e.g. if we
1747 * don't actually do a hard stop on the head
1748 * atom */
1749 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1750 action);
1751 ret = true;
1752 } else {
1753 /* no atom in idx1 */
1754 /* just stop idx0 */
1755 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1756 action);
1757 ret = true;
1758 }
1759 }
1760 } else if (katom_idx1_valid) {
1761 if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1762 /* Mark for return */
1763 /* idx1 will be returned once idx0 completes */
1764 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1765 false);
1766 } else {
1767 /* idx1 is on GPU */
1768 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1769 JS_COMMAND_NEXT), NULL) == 0) {
1770 /* idx0 has already completed - stop idx1 */
1771 kbase_gpu_stop_atom(kbdev, js, katom_idx1,
1772 action);
1773 ret = true;
1774 } else {
1775 /* idx1 is in NEXT registers - attempt to
1776 * remove */
1777 kbase_reg_write(kbdev, JOB_SLOT_REG(js,
1778 JS_COMMAND_NEXT),
1779 JS_COMMAND_NOP, NULL);
1780
1781 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1782 JS_HEAD_NEXT_LO), NULL) != 0 ||
1783 kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1784 JS_HEAD_NEXT_HI), NULL) != 0) {
1785 /* idx1 removed successfully, will be
1786 * handled in IRQ once idx0 completes */
1787 kbase_job_evicted(katom_idx1);
1788 kbase_gpu_remove_atom(kbdev, katom_idx1,
1789 action,
1790 false);
1791 } else {
1792 /* idx0 has already completed - stop
1793 * idx1 */
1794 kbase_gpu_stop_atom(kbdev, js,
1795 katom_idx1,
1796 action);
1797 ret = true;
1798 }
1799 }
1800 }
1801 }
1802
1803
1804 if (stop_x_dep_idx0 != -1)
1805 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0,
1806 NULL, action);
1807
1808 if (stop_x_dep_idx1 != -1)
1809 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1,
1810 NULL, action);
1811
1812 return ret;
1813 }
1814
kbase_gpu_cacheclean(struct kbase_device * kbdev)1815 void kbase_gpu_cacheclean(struct kbase_device *kbdev)
1816 {
1817 /* Limit the number of loops to avoid a hang if the interrupt is missed
1818 */
1819 u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
1820
1821 mutex_lock(&kbdev->cacheclean_lock);
1822
1823 /* use GPU_COMMAND completion solution */
1824 /* clean & invalidate the caches */
1825 KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
1826 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1827 GPU_COMMAND_CLEAN_INV_CACHES, NULL);
1828
1829 /* wait for cache flush to complete before continuing */
1830 while (--max_loops &&
1831 (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1832 CLEAN_CACHES_COMPLETED) == 0)
1833 ;
1834
1835 /* clear the CLEAN_CACHES_COMPLETED irq */
1836 KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u,
1837 CLEAN_CACHES_COMPLETED);
1838 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
1839 CLEAN_CACHES_COMPLETED, NULL);
1840 KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state !=
1841 KBASE_INSTR_STATE_CLEANING,
1842 "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");
1843
1844 mutex_unlock(&kbdev->cacheclean_lock);
1845 }
1846
kbase_backend_cacheclean(struct kbase_device * kbdev,struct kbase_jd_atom * katom)1847 void kbase_backend_cacheclean(struct kbase_device *kbdev,
1848 struct kbase_jd_atom *katom)
1849 {
1850 if (katom->need_cache_flush_cores_retained) {
1851 unsigned long flags;
1852
1853 kbase_gpu_cacheclean(kbdev);
1854
1855 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1856 kbase_pm_unrequest_cores(kbdev, false,
1857 katom->need_cache_flush_cores_retained);
1858 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1859 katom->need_cache_flush_cores_retained = 0;
1860 }
1861 }
1862
kbase_backend_complete_wq(struct kbase_device * kbdev,struct kbase_jd_atom * katom)1863 void kbase_backend_complete_wq(struct kbase_device *kbdev,
1864 struct kbase_jd_atom *katom)
1865 {
1866 /*
1867 * If cache flush required due to HW workaround then perform the flush
1868 * now
1869 */
1870 kbase_backend_cacheclean(kbdev, katom);
1871
1872 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) &&
1873 (katom->core_req & BASE_JD_REQ_FS) &&
1874 katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT &&
1875 (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
1876 !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) {
1877 dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n");
1878 if (kbasep_10969_workaround_clamp_coordinates(katom)) {
1879 /* The job had a TILE_RANGE_FAULT after was soft-stopped
1880 * Due to an HW issue we try to execute the job again.
1881 */
1882 dev_dbg(kbdev->dev,
1883 "Clamping has been executed, try to rerun the job\n"
1884 );
1885 katom->event_code = BASE_JD_EVENT_STOPPED;
1886 katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;
1887 }
1888 }
1889
1890 /* Clear the coreref_state now - while check_deref_cores() may not have
1891 * been called yet, the caller will have taken a copy of this field. If
1892 * this is not done, then if the atom is re-scheduled (following a soft
1893 * stop) then the core reference would not be retaken. */
1894 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1895 katom->affinity = 0;
1896 }
1897
kbase_backend_complete_wq_post_sched(struct kbase_device * kbdev,base_jd_core_req core_req,u64 affinity,enum kbase_atom_coreref_state coreref_state)1898 void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev,
1899 base_jd_core_req core_req, u64 affinity,
1900 enum kbase_atom_coreref_state coreref_state)
1901 {
1902 unsigned long flags;
1903
1904 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1905 kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity,
1906 coreref_state);
1907 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1908
1909 if (!kbdev->pm.active_count) {
1910 mutex_lock(&kbdev->js_data.runpool_mutex);
1911 mutex_lock(&kbdev->pm.lock);
1912 kbase_pm_update_active(kbdev);
1913 mutex_unlock(&kbdev->pm.lock);
1914 mutex_unlock(&kbdev->js_data.runpool_mutex);
1915 }
1916 }
1917
kbase_gpu_dump_slots(struct kbase_device * kbdev)1918 void kbase_gpu_dump_slots(struct kbase_device *kbdev)
1919 {
1920 struct kbasep_js_device_data *js_devdata;
1921 unsigned long flags;
1922 int js;
1923
1924 js_devdata = &kbdev->js_data;
1925
1926 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1927
1928 dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n");
1929
1930 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1931 int idx;
1932
1933 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1934 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1935 js,
1936 idx);
1937
1938 if (katom)
1939 dev_info(kbdev->dev,
1940 " js%d idx%d : katom=%p gpu_rb_state=%d\n",
1941 js, idx, katom, katom->gpu_rb_state);
1942 else
1943 dev_info(kbdev->dev, " js%d idx%d : empty\n",
1944 js, idx);
1945 }
1946 }
1947
1948 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1949 }
1950
1951
1952
1953