1 /*
2 *
3 * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18 /*
19 * Base kernel job manager APIs
20 */
21
22 #include <mali_kbase.h>
23 #include <mali_kbase_config.h>
24 #include <mali_midg_regmap.h>
25 #if defined(CONFIG_MALI_GATOR_SUPPORT)
26 #include <mali_kbase_gator.h>
27 #endif
28 #include <mali_kbase_tlstream.h>
29 #include <mali_kbase_vinstr.h>
30 #include <mali_kbase_hw.h>
31 #include <mali_kbase_hwaccess_jm.h>
32 #include <mali_kbase_ctx_sched.h>
33 #include <backend/gpu/mali_kbase_device_internal.h>
34 #include <backend/gpu/mali_kbase_irq_internal.h>
35 #include <backend/gpu/mali_kbase_js_affinity.h>
36 #include <backend/gpu/mali_kbase_jm_internal.h>
37
38 #define beenthere(kctx, f, a...) \
39 dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
40
41 #if KBASE_GPU_RESET_EN
42 static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev);
43 static void kbasep_reset_timeout_worker(struct work_struct *data);
44 static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer);
45 #endif /* KBASE_GPU_RESET_EN */
46
kbasep_jm_is_js_free(struct kbase_device * kbdev,int js,struct kbase_context * kctx)47 static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
48 struct kbase_context *kctx)
49 {
50 return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx);
51 }
52
kbase_job_hw_submit(struct kbase_device * kbdev,struct kbase_jd_atom * katom,int js)53 void kbase_job_hw_submit(struct kbase_device *kbdev,
54 struct kbase_jd_atom *katom,
55 int js)
56 {
57 struct kbase_context *kctx;
58 u32 cfg;
59 u64 jc_head = katom->jc;
60
61 KBASE_DEBUG_ASSERT(kbdev);
62 KBASE_DEBUG_ASSERT(katom);
63
64 kctx = katom->kctx;
65
66 /* Command register must be available */
67 KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
68 /* Affinity is not violating */
69 kbase_js_debug_log_current_affinities(kbdev);
70 KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js,
71 katom->affinity));
72
73 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO),
74 jc_head & 0xFFFFFFFF, kctx);
75 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI),
76 jc_head >> 32, kctx);
77
78 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO),
79 katom->affinity & 0xFFFFFFFF, kctx);
80 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI),
81 katom->affinity >> 32, kctx);
82
83 /* start MMU, medium priority, cache clean/flush on end, clean/flush on
84 * start */
85 cfg = kctx->as_nr;
86
87 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION))
88 cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
89
90 #ifndef CONFIG_MALI_COH_GPU
91 if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START))
92 cfg |= JS_CONFIG_START_FLUSH_NO_ACTION;
93 else
94 cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
95
96 if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END))
97 cfg |= JS_CONFIG_END_FLUSH_NO_ACTION;
98 else
99 cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE;
100 #endif /* CONFIG_MALI_COH_GPU */
101
102 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10649))
103 cfg |= JS_CONFIG_START_MMU;
104
105 cfg |= JS_CONFIG_THREAD_PRI(8);
106
107 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) &&
108 (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED))
109 cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK;
110
111 if (kbase_hw_has_feature(kbdev,
112 BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
113 if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) {
114 cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
115 katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
116 kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
117 true;
118 } else {
119 katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
120 kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
121 false;
122 }
123 }
124
125 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx);
126
127 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION))
128 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT),
129 katom->flush_id, kctx);
130
131 /* Write an approximate start timestamp.
132 * It's approximate because there might be a job in the HEAD register.
133 */
134 katom->start_timestamp = ktime_get();
135
136 /* GO ! */
137 dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx",
138 katom, kctx, js, jc_head, katom->affinity);
139
140 KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js,
141 (u32) katom->affinity);
142
143 #if defined(CONFIG_MALI_GATOR_SUPPORT)
144 kbase_trace_mali_job_slots_event(
145 GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js),
146 kctx, kbase_jd_atom_id(kctx, katom));
147 #endif
148 KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(katom, jc_head,
149 katom->affinity, cfg);
150 KBASE_TLSTREAM_TL_RET_CTX_LPU(
151 kctx,
152 &kbdev->gpu_props.props.raw_props.js_features[
153 katom->slot_nr]);
154 KBASE_TLSTREAM_TL_RET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]);
155 KBASE_TLSTREAM_TL_RET_ATOM_LPU(
156 katom,
157 &kbdev->gpu_props.props.raw_props.js_features[js],
158 "ctx_nr,atom_nr");
159 #ifdef CONFIG_GPU_TRACEPOINTS
160 if (!kbase_backend_nr_atoms_submitted(kbdev, js)) {
161 /* If this is the only job on the slot, trace it as starting */
162 char js_string[16];
163
164 trace_gpu_sched_switch(
165 kbasep_make_job_slot_string(js, js_string,
166 sizeof(js_string)),
167 ktime_to_ns(katom->start_timestamp),
168 (u32)katom->kctx->id, 0, katom->work_id);
169 kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx;
170 }
171 #endif
172 kbase_timeline_job_slot_submit(kbdev, kctx, katom, js);
173
174 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
175 JS_COMMAND_START, katom->kctx);
176 }
177
178 /**
179 * kbasep_job_slot_update_head_start_timestamp - Update timestamp
180 * @kbdev: kbase device
181 * @js: job slot
182 * @end_timestamp: timestamp
183 *
184 * Update the start_timestamp of the job currently in the HEAD, based on the
185 * fact that we got an IRQ for the previous set of completed jobs.
186 *
187 * The estimate also takes into account the time the job was submitted, to
188 * work out the best estimate (which might still result in an over-estimate to
189 * the calculated time spent)
190 */
kbasep_job_slot_update_head_start_timestamp(struct kbase_device * kbdev,int js,ktime_t end_timestamp)191 static void kbasep_job_slot_update_head_start_timestamp(
192 struct kbase_device *kbdev,
193 int js,
194 ktime_t end_timestamp)
195 {
196 if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) {
197 struct kbase_jd_atom *katom;
198 ktime_t timestamp_diff;
199 /* The atom in the HEAD */
200 katom = kbase_gpu_inspect(kbdev, js, 0);
201
202 KBASE_DEBUG_ASSERT(katom != NULL);
203
204 timestamp_diff = ktime_sub(end_timestamp,
205 katom->start_timestamp);
206 if (ktime_to_ns(timestamp_diff) >= 0) {
207 /* Only update the timestamp if it's a better estimate
208 * than what's currently stored. This is because our
209 * estimate that accounts for the throttle time may be
210 * too much of an overestimate */
211 katom->start_timestamp = end_timestamp;
212 }
213 }
214 }
215
216 /**
217 * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline
218 * tracepoint
219 * @kbdev: kbase device
220 * @js: job slot
221 *
222 * Make a tracepoint call to the instrumentation module informing that
223 * softstop happened on given lpu (job slot).
224 */
kbasep_trace_tl_event_lpu_softstop(struct kbase_device * kbdev,int js)225 static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev,
226 int js)
227 {
228 KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(
229 &kbdev->gpu_props.props.raw_props.js_features[js]);
230 }
231
kbase_job_done(struct kbase_device * kbdev,u32 done)232 void kbase_job_done(struct kbase_device *kbdev, u32 done)
233 {
234 unsigned long flags;
235 int i;
236 u32 count = 0;
237 ktime_t end_timestamp = ktime_get();
238 struct kbasep_js_device_data *js_devdata;
239
240 KBASE_DEBUG_ASSERT(kbdev);
241 js_devdata = &kbdev->js_data;
242
243 KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done);
244
245 memset(&kbdev->slot_submit_count_irq[0], 0,
246 sizeof(kbdev->slot_submit_count_irq));
247
248 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
249
250 while (done) {
251 u32 failed = done >> 16;
252
253 /* treat failed slots as finished slots */
254 u32 finished = (done & 0xFFFF) | failed;
255
256 /* Note: This is inherently unfair, as we always check
257 * for lower numbered interrupts before the higher
258 * numbered ones.*/
259 i = ffs(finished) - 1;
260 KBASE_DEBUG_ASSERT(i >= 0);
261
262 do {
263 int nr_done;
264 u32 active;
265 u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */
266 u64 job_tail = 0;
267
268 if (failed & (1u << i)) {
269 /* read out the job slot status code if the job
270 * slot reported failure */
271 completion_code = kbase_reg_read(kbdev,
272 JOB_SLOT_REG(i, JS_STATUS), NULL);
273
274 switch (completion_code) {
275 case BASE_JD_EVENT_STOPPED:
276 #if defined(CONFIG_MALI_GATOR_SUPPORT)
277 kbase_trace_mali_job_slots_event(
278 GATOR_MAKE_EVENT(
279 GATOR_JOB_SLOT_SOFT_STOPPED, i),
280 NULL, 0);
281 #endif
282
283 kbasep_trace_tl_event_lpu_softstop(
284 kbdev, i);
285
286 /* Soft-stopped job - read the value of
287 * JS<n>_TAIL so that the job chain can
288 * be resumed */
289 job_tail = (u64)kbase_reg_read(kbdev,
290 JOB_SLOT_REG(i, JS_TAIL_LO),
291 NULL) |
292 ((u64)kbase_reg_read(kbdev,
293 JOB_SLOT_REG(i, JS_TAIL_HI),
294 NULL) << 32);
295 break;
296 case BASE_JD_EVENT_NOT_STARTED:
297 /* PRLAM-10673 can cause a TERMINATED
298 * job to come back as NOT_STARTED, but
299 * the error interrupt helps us detect
300 * it */
301 completion_code =
302 BASE_JD_EVENT_TERMINATED;
303 /* fall through */
304 default:
305 dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)",
306 i, completion_code,
307 kbase_exception_name
308 (kbdev,
309 completion_code));
310 }
311
312 kbase_gpu_irq_evict(kbdev, i);
313 }
314
315 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR),
316 done & ((1 << i) | (1 << (i + 16))),
317 NULL);
318 active = kbase_reg_read(kbdev,
319 JOB_CONTROL_REG(JOB_IRQ_JS_STATE),
320 NULL);
321
322 if (((active >> i) & 1) == 0 &&
323 (((done >> (i + 16)) & 1) == 0)) {
324 /* There is a potential race we must work
325 * around:
326 *
327 * 1. A job slot has a job in both current and
328 * next registers
329 * 2. The job in current completes
330 * successfully, the IRQ handler reads
331 * RAWSTAT and calls this function with the
332 * relevant bit set in "done"
333 * 3. The job in the next registers becomes the
334 * current job on the GPU
335 * 4. Sometime before the JOB_IRQ_CLEAR line
336 * above the job on the GPU _fails_
337 * 5. The IRQ_CLEAR clears the done bit but not
338 * the failed bit. This atomically sets
339 * JOB_IRQ_JS_STATE. However since both jobs
340 * have now completed the relevant bits for
341 * the slot are set to 0.
342 *
343 * If we now did nothing then we'd incorrectly
344 * assume that _both_ jobs had completed
345 * successfully (since we haven't yet observed
346 * the fail bit being set in RAWSTAT).
347 *
348 * So at this point if there are no active jobs
349 * left we check to see if RAWSTAT has a failure
350 * bit set for the job slot. If it does we know
351 * that there has been a new failure that we
352 * didn't previously know about, so we make sure
353 * that we record this in active (but we wait
354 * for the next loop to deal with it).
355 *
356 * If we were handling a job failure (i.e. done
357 * has the relevant high bit set) then we know
358 * that the value read back from
359 * JOB_IRQ_JS_STATE is the correct number of
360 * remaining jobs because the failed job will
361 * have prevented any futher jobs from starting
362 * execution.
363 */
364 u32 rawstat = kbase_reg_read(kbdev,
365 JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
366
367 if ((rawstat >> (i + 16)) & 1) {
368 /* There is a failed job that we've
369 * missed - add it back to active */
370 active |= (1u << i);
371 }
372 }
373
374 dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n",
375 completion_code);
376
377 nr_done = kbase_backend_nr_atoms_submitted(kbdev, i);
378 nr_done -= (active >> i) & 1;
379 nr_done -= (active >> (i + 16)) & 1;
380
381 if (nr_done <= 0) {
382 dev_warn(kbdev->dev, "Spurious interrupt on slot %d",
383 i);
384
385 goto spurious;
386 }
387
388 count += nr_done;
389
390 while (nr_done) {
391 if (nr_done == 1) {
392 kbase_gpu_complete_hw(kbdev, i,
393 completion_code,
394 job_tail,
395 &end_timestamp);
396 kbase_jm_try_kick_all(kbdev);
397 } else {
398 /* More than one job has completed.
399 * Since this is not the last job being
400 * reported this time it must have
401 * passed. This is because the hardware
402 * will not allow further jobs in a job
403 * slot to complete until the failed job
404 * is cleared from the IRQ status.
405 */
406 kbase_gpu_complete_hw(kbdev, i,
407 BASE_JD_EVENT_DONE,
408 0,
409 &end_timestamp);
410 }
411 nr_done--;
412 }
413 spurious:
414 done = kbase_reg_read(kbdev,
415 JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
416
417 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) {
418 /* Workaround for missing interrupt caused by
419 * PRLAM-10883 */
420 if (((active >> i) & 1) && (0 ==
421 kbase_reg_read(kbdev,
422 JOB_SLOT_REG(i,
423 JS_STATUS), NULL))) {
424 /* Force job slot to be processed again
425 */
426 done |= (1u << i);
427 }
428 }
429
430 failed = done >> 16;
431 finished = (done & 0xFFFF) | failed;
432 if (done)
433 end_timestamp = ktime_get();
434 } while (finished & (1 << i));
435
436 kbasep_job_slot_update_head_start_timestamp(kbdev, i,
437 end_timestamp);
438 }
439
440 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
441 #if KBASE_GPU_RESET_EN
442 if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
443 KBASE_RESET_GPU_COMMITTED) {
444 /* If we're trying to reset the GPU then we might be able to do
445 * it early (without waiting for a timeout) because some jobs
446 * have completed
447 */
448 kbasep_try_reset_gpu_early(kbdev);
449 }
450 #endif /* KBASE_GPU_RESET_EN */
451 KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
452 }
453 KBASE_EXPORT_TEST_API(kbase_job_done);
454
kbasep_soft_stop_allowed(struct kbase_device * kbdev,struct kbase_jd_atom * katom)455 static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev,
456 struct kbase_jd_atom *katom)
457 {
458 bool soft_stops_allowed = true;
459
460 if (kbase_jd_katom_is_protected(katom)) {
461 soft_stops_allowed = false;
462 } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) {
463 if ((katom->core_req & BASE_JD_REQ_T) != 0)
464 soft_stops_allowed = false;
465 }
466 return soft_stops_allowed;
467 }
468
kbasep_hard_stop_allowed(struct kbase_device * kbdev,base_jd_core_req core_reqs)469 static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev,
470 base_jd_core_req core_reqs)
471 {
472 bool hard_stops_allowed = true;
473
474 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) {
475 if ((core_reqs & BASE_JD_REQ_T) != 0)
476 hard_stops_allowed = false;
477 }
478 return hard_stops_allowed;
479 }
480
kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device * kbdev,int js,u32 action,base_jd_core_req core_reqs,struct kbase_jd_atom * target_katom)481 void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
482 int js,
483 u32 action,
484 base_jd_core_req core_reqs,
485 struct kbase_jd_atom *target_katom)
486 {
487 struct kbase_context *kctx = target_katom->kctx;
488 #if KBASE_TRACE_ENABLE
489 u32 status_reg_before;
490 u64 job_in_head_before;
491 u32 status_reg_after;
492
493 KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK)));
494
495 /* Check the head pointer */
496 job_in_head_before = ((u64) kbase_reg_read(kbdev,
497 JOB_SLOT_REG(js, JS_HEAD_LO), NULL))
498 | (((u64) kbase_reg_read(kbdev,
499 JOB_SLOT_REG(js, JS_HEAD_HI), NULL))
500 << 32);
501 status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS),
502 NULL);
503 #endif
504
505 if (action == JS_COMMAND_SOFT_STOP) {
506 bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev,
507 target_katom);
508
509 if (!soft_stop_allowed) {
510 #ifdef CONFIG_MALI_DEBUG
511 dev_dbg(kbdev->dev,
512 "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X",
513 (unsigned int)core_reqs);
514 #endif /* CONFIG_MALI_DEBUG */
515 return;
516 }
517
518 /* We are about to issue a soft stop, so mark the atom as having
519 * been soft stopped */
520 target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED;
521
522 /* Mark the point where we issue the soft-stop command */
523 KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(target_katom);
524
525 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) {
526 int i;
527
528 for (i = 0;
529 i < kbase_backend_nr_atoms_submitted(kbdev, js);
530 i++) {
531 struct kbase_jd_atom *katom;
532
533 katom = kbase_gpu_inspect(kbdev, js, i);
534
535 KBASE_DEBUG_ASSERT(katom);
536
537 /* For HW_ISSUE_8316, only 'bad' jobs attacking
538 * the system can cause this issue: normally,
539 * all memory should be allocated in multiples
540 * of 4 pages, and growable memory should be
541 * changed size in multiples of 4 pages.
542 *
543 * Whilst such 'bad' jobs can be cleared by a
544 * GPU reset, the locking up of a uTLB entry
545 * caused by the bad job could also stall other
546 * ASs, meaning that other ASs' jobs don't
547 * complete in the 'grace' period before the
548 * reset. We don't want to lose other ASs' jobs
549 * when they would normally complete fine, so we
550 * must 'poke' the MMU regularly to help other
551 * ASs complete */
552 kbase_as_poking_timer_retain_atom(
553 kbdev, katom->kctx, katom);
554 }
555 }
556
557 if (kbase_hw_has_feature(
558 kbdev,
559 BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
560 action = (target_katom->atom_flags &
561 KBASE_KATOM_FLAGS_JOBCHAIN) ?
562 JS_COMMAND_SOFT_STOP_1 :
563 JS_COMMAND_SOFT_STOP_0;
564 }
565 } else if (action == JS_COMMAND_HARD_STOP) {
566 bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev,
567 core_reqs);
568
569 if (!hard_stop_allowed) {
570 /* Jobs can be hard-stopped for the following reasons:
571 * * CFS decides the job has been running too long (and
572 * soft-stop has not occurred). In this case the GPU
573 * will be reset by CFS if the job remains on the
574 * GPU.
575 *
576 * * The context is destroyed, kbase_jd_zap_context
577 * will attempt to hard-stop the job. However it also
578 * has a watchdog which will cause the GPU to be
579 * reset if the job remains on the GPU.
580 *
581 * * An (unhandled) MMU fault occurred. As long as
582 * BASE_HW_ISSUE_8245 is defined then the GPU will be
583 * reset.
584 *
585 * All three cases result in the GPU being reset if the
586 * hard-stop fails, so it is safe to just return and
587 * ignore the hard-stop request.
588 */
589 dev_warn(kbdev->dev,
590 "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X",
591 (unsigned int)core_reqs);
592 return;
593 }
594 target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED;
595
596 if (kbase_hw_has_feature(
597 kbdev,
598 BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
599 action = (target_katom->atom_flags &
600 KBASE_KATOM_FLAGS_JOBCHAIN) ?
601 JS_COMMAND_HARD_STOP_1 :
602 JS_COMMAND_HARD_STOP_0;
603 }
604 }
605
606 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx);
607
608 #if KBASE_TRACE_ENABLE
609 status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS),
610 NULL);
611 if (status_reg_after == BASE_JD_EVENT_ACTIVE) {
612 struct kbase_jd_atom *head;
613 struct kbase_context *head_kctx;
614
615 head = kbase_gpu_inspect(kbdev, js, 0);
616 head_kctx = head->kctx;
617
618 if (status_reg_before == BASE_JD_EVENT_ACTIVE)
619 KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx,
620 head, job_in_head_before, js);
621 else
622 KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
623 0, js);
624
625 switch (action) {
626 case JS_COMMAND_SOFT_STOP:
627 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx,
628 head, head->jc, js);
629 break;
630 case JS_COMMAND_SOFT_STOP_0:
631 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx,
632 head, head->jc, js);
633 break;
634 case JS_COMMAND_SOFT_STOP_1:
635 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx,
636 head, head->jc, js);
637 break;
638 case JS_COMMAND_HARD_STOP:
639 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx,
640 head, head->jc, js);
641 break;
642 case JS_COMMAND_HARD_STOP_0:
643 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx,
644 head, head->jc, js);
645 break;
646 case JS_COMMAND_HARD_STOP_1:
647 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx,
648 head, head->jc, js);
649 break;
650 default:
651 BUG();
652 break;
653 }
654 } else {
655 if (status_reg_before == BASE_JD_EVENT_ACTIVE)
656 KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
657 job_in_head_before, js);
658 else
659 KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
660 0, js);
661
662 switch (action) {
663 case JS_COMMAND_SOFT_STOP:
664 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0,
665 js);
666 break;
667 case JS_COMMAND_SOFT_STOP_0:
668 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL,
669 0, js);
670 break;
671 case JS_COMMAND_SOFT_STOP_1:
672 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL,
673 0, js);
674 break;
675 case JS_COMMAND_HARD_STOP:
676 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0,
677 js);
678 break;
679 case JS_COMMAND_HARD_STOP_0:
680 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL,
681 0, js);
682 break;
683 case JS_COMMAND_HARD_STOP_1:
684 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL,
685 0, js);
686 break;
687 default:
688 BUG();
689 break;
690 }
691 }
692 #endif
693 }
694
kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context * kctx)695 void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx)
696 {
697 unsigned long flags;
698 struct kbase_device *kbdev;
699 struct kbasep_js_device_data *js_devdata;
700 int i;
701
702 KBASE_DEBUG_ASSERT(kctx != NULL);
703 kbdev = kctx->kbdev;
704 KBASE_DEBUG_ASSERT(kbdev != NULL);
705 js_devdata = &kbdev->js_data;
706
707 /* Cancel any remaining running jobs for this kctx */
708 mutex_lock(&kctx->jctx.lock);
709 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
710
711 /* Invalidate all jobs in context, to prevent re-submitting */
712 for (i = 0; i < BASE_JD_ATOM_COUNT; i++) {
713 if (!work_pending(&kctx->jctx.atoms[i].work))
714 kctx->jctx.atoms[i].event_code =
715 BASE_JD_EVENT_JOB_CANCELLED;
716 }
717
718 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
719 kbase_job_slot_hardstop(kctx, i, NULL);
720
721 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
722 mutex_unlock(&kctx->jctx.lock);
723 }
724
kbase_job_slot_ctx_priority_check_locked(struct kbase_context * kctx,struct kbase_jd_atom * target_katom)725 void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
726 struct kbase_jd_atom *target_katom)
727 {
728 struct kbase_device *kbdev;
729 int js = target_katom->slot_nr;
730 int priority = target_katom->sched_priority;
731 int i;
732 bool stop_sent = false;
733
734 KBASE_DEBUG_ASSERT(kctx != NULL);
735 kbdev = kctx->kbdev;
736 KBASE_DEBUG_ASSERT(kbdev != NULL);
737
738 lockdep_assert_held(&kbdev->hwaccess_lock);
739
740 for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
741 struct kbase_jd_atom *katom;
742
743 katom = kbase_gpu_inspect(kbdev, js, i);
744 if (!katom)
745 continue;
746
747 if (katom->kctx != kctx)
748 continue;
749
750 if (katom->sched_priority > priority) {
751 if (!stop_sent)
752 KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE(
753 target_katom);
754
755 kbase_job_slot_softstop(kbdev, js, katom);
756 stop_sent = true;
757 }
758 }
759 }
760
761 struct zap_reset_data {
762 /* The stages are:
763 * 1. The timer has never been called
764 * 2. The zap has timed out, all slots are soft-stopped - the GPU reset
765 * will happen. The GPU has been reset when
766 * kbdev->hwaccess.backend.reset_waitq is signalled
767 *
768 * (-1 - The timer has been cancelled)
769 */
770 int stage;
771 struct kbase_device *kbdev;
772 struct hrtimer timer;
773 spinlock_t lock; /* protects updates to stage member */
774 };
775
zap_timeout_callback(struct hrtimer * timer)776 static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer)
777 {
778 struct zap_reset_data *reset_data = container_of(timer,
779 struct zap_reset_data, timer);
780 struct kbase_device *kbdev = reset_data->kbdev;
781 unsigned long flags;
782
783 spin_lock_irqsave(&reset_data->lock, flags);
784
785 if (reset_data->stage == -1)
786 goto out;
787
788 #if KBASE_GPU_RESET_EN
789 if (kbase_prepare_to_reset_gpu(kbdev)) {
790 dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n",
791 ZAP_TIMEOUT);
792 kbase_reset_gpu(kbdev);
793 }
794 #endif /* KBASE_GPU_RESET_EN */
795 reset_data->stage = 2;
796
797 out:
798 spin_unlock_irqrestore(&reset_data->lock, flags);
799
800 return HRTIMER_NORESTART;
801 }
802
kbase_jm_wait_for_zero_jobs(struct kbase_context * kctx)803 void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx)
804 {
805 struct kbase_device *kbdev = kctx->kbdev;
806 struct zap_reset_data reset_data;
807 unsigned long flags;
808
809 hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC,
810 HRTIMER_MODE_REL);
811 reset_data.timer.function = zap_timeout_callback;
812
813 spin_lock_init(&reset_data.lock);
814
815 reset_data.kbdev = kbdev;
816 reset_data.stage = 1;
817
818 hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT),
819 HRTIMER_MODE_REL);
820
821 /* Wait for all jobs to finish, and for the context to be not-scheduled
822 * (due to kbase_job_zap_context(), we also guarentee it's not in the JS
823 * policy queue either */
824 wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0);
825 wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait,
826 !kbase_ctx_flag(kctx, KCTX_SCHEDULED));
827
828 spin_lock_irqsave(&reset_data.lock, flags);
829 if (reset_data.stage == 1) {
830 /* The timer hasn't run yet - so cancel it */
831 reset_data.stage = -1;
832 }
833 spin_unlock_irqrestore(&reset_data.lock, flags);
834
835 hrtimer_cancel(&reset_data.timer);
836
837 if (reset_data.stage == 2) {
838 /* The reset has already started.
839 * Wait for the reset to complete
840 */
841 wait_event(kbdev->hwaccess.backend.reset_wait,
842 atomic_read(&kbdev->hwaccess.backend.reset_gpu)
843 == KBASE_RESET_GPU_NOT_PENDING);
844 }
845 destroy_hrtimer_on_stack(&reset_data.timer);
846
847 dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx);
848
849 /* Ensure that the signallers of the waitqs have finished */
850 mutex_lock(&kctx->jctx.lock);
851 mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
852 mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
853 mutex_unlock(&kctx->jctx.lock);
854 }
855
kbase_backend_get_current_flush_id(struct kbase_device * kbdev)856 u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev)
857 {
858 u32 flush_id = 0;
859
860 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) {
861 mutex_lock(&kbdev->pm.lock);
862 if (kbdev->pm.backend.gpu_powered)
863 flush_id = kbase_reg_read(kbdev,
864 GPU_CONTROL_REG(LATEST_FLUSH), NULL);
865 mutex_unlock(&kbdev->pm.lock);
866 }
867
868 return flush_id;
869 }
870
kbase_job_slot_init(struct kbase_device * kbdev)871 int kbase_job_slot_init(struct kbase_device *kbdev)
872 {
873 #if KBASE_GPU_RESET_EN
874 kbdev->hwaccess.backend.reset_workq = alloc_workqueue(
875 "Mali reset workqueue", 0, 1);
876 if (NULL == kbdev->hwaccess.backend.reset_workq)
877 return -EINVAL;
878
879 KBASE_DEBUG_ASSERT(0 ==
880 object_is_on_stack(&kbdev->hwaccess.backend.reset_work));
881 INIT_WORK(&kbdev->hwaccess.backend.reset_work,
882 kbasep_reset_timeout_worker);
883
884 hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC,
885 HRTIMER_MODE_REL);
886 kbdev->hwaccess.backend.reset_timer.function =
887 kbasep_reset_timer_callback;
888 #endif
889
890 return 0;
891 }
892 KBASE_EXPORT_TEST_API(kbase_job_slot_init);
893
kbase_job_slot_halt(struct kbase_device * kbdev)894 void kbase_job_slot_halt(struct kbase_device *kbdev)
895 {
896 CSTD_UNUSED(kbdev);
897 }
898
kbase_job_slot_term(struct kbase_device * kbdev)899 void kbase_job_slot_term(struct kbase_device *kbdev)
900 {
901 #if KBASE_GPU_RESET_EN
902 destroy_workqueue(kbdev->hwaccess.backend.reset_workq);
903 #endif
904 }
905 KBASE_EXPORT_TEST_API(kbase_job_slot_term);
906
907 #if KBASE_GPU_RESET_EN
908 /**
909 * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot
910 * @kbdev: kbase device pointer
911 * @kctx: context to check against
912 * @js: slot to check
913 * @target_katom: An atom to check, or NULL if all atoms from @kctx on
914 * slot @js should be checked
915 *
916 * This checks are based upon parameters that would normally be passed to
917 * kbase_job_slot_hardstop().
918 *
919 * In the event of @target_katom being NULL, this will check the last jobs that
920 * are likely to be running on the slot to see if a) they belong to kctx, and
921 * so would be stopped, and b) whether they have AFBC
922 *
923 * In that case, It's guaranteed that a job currently executing on the HW with
924 * AFBC will be detected. However, this is a conservative check because it also
925 * detects jobs that have just completed too.
926 *
927 * Return: true when hard-stop _might_ stop an afbc atom, else false.
928 */
kbasep_check_for_afbc_on_slot(struct kbase_device * kbdev,struct kbase_context * kctx,int js,struct kbase_jd_atom * target_katom)929 static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev,
930 struct kbase_context *kctx, int js,
931 struct kbase_jd_atom *target_katom)
932 {
933 bool ret = false;
934 int i;
935
936 lockdep_assert_held(&kbdev->hwaccess_lock);
937
938 /* When we have an atom the decision can be made straight away. */
939 if (target_katom)
940 return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC);
941
942 /* Otherwise, we must chweck the hardware to see if it has atoms from
943 * this context with AFBC. */
944 for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
945 struct kbase_jd_atom *katom;
946
947 katom = kbase_gpu_inspect(kbdev, js, i);
948 if (!katom)
949 continue;
950
951 /* Ignore atoms from other contexts, they won't be stopped when
952 * we use this for checking if we should hard-stop them */
953 if (katom->kctx != kctx)
954 continue;
955
956 /* An atom on this slot and this context: check for AFBC */
957 if (katom->core_req & BASE_JD_REQ_FS_AFBC) {
958 ret = true;
959 break;
960 }
961 }
962
963 return ret;
964 }
965 #endif /* KBASE_GPU_RESET_EN */
966
967 /**
968 * kbase_job_slot_softstop_swflags - Soft-stop a job with flags
969 * @kbdev: The kbase device
970 * @js: The job slot to soft-stop
971 * @target_katom: The job that should be soft-stopped (or NULL for any job)
972 * @sw_flags: Flags to pass in about the soft-stop
973 *
974 * Context:
975 * The job slot lock must be held when calling this function.
976 * The job slot must not already be in the process of being soft-stopped.
977 *
978 * Soft-stop the specified job slot, with extra information about the stop
979 *
980 * Where possible any job in the next register is evicted before the soft-stop.
981 */
kbase_job_slot_softstop_swflags(struct kbase_device * kbdev,int js,struct kbase_jd_atom * target_katom,u32 sw_flags)982 void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
983 struct kbase_jd_atom *target_katom, u32 sw_flags)
984 {
985 KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK));
986 kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom,
987 JS_COMMAND_SOFT_STOP | sw_flags);
988 }
989
990 /**
991 * kbase_job_slot_softstop - Soft-stop the specified job slot
992 * @kbdev: The kbase device
993 * @js: The job slot to soft-stop
994 * @target_katom: The job that should be soft-stopped (or NULL for any job)
995 * Context:
996 * The job slot lock must be held when calling this function.
997 * The job slot must not already be in the process of being soft-stopped.
998 *
999 * Where possible any job in the next register is evicted before the soft-stop.
1000 */
kbase_job_slot_softstop(struct kbase_device * kbdev,int js,struct kbase_jd_atom * target_katom)1001 void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
1002 struct kbase_jd_atom *target_katom)
1003 {
1004 kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u);
1005 }
1006
1007 /**
1008 * kbase_job_slot_hardstop - Hard-stop the specified job slot
1009 * @kctx: The kbase context that contains the job(s) that should
1010 * be hard-stopped
1011 * @js: The job slot to hard-stop
1012 * @target_katom: The job that should be hard-stopped (or NULL for all
1013 * jobs from the context)
1014 * Context:
1015 * The job slot lock must be held when calling this function.
1016 */
kbase_job_slot_hardstop(struct kbase_context * kctx,int js,struct kbase_jd_atom * target_katom)1017 void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
1018 struct kbase_jd_atom *target_katom)
1019 {
1020 struct kbase_device *kbdev = kctx->kbdev;
1021 bool stopped;
1022 #if KBASE_GPU_RESET_EN
1023 /* We make the check for AFBC before evicting/stopping atoms. Note
1024 * that no other thread can modify the slots whilst we have the
1025 * hwaccess_lock. */
1026 int needs_workaround_for_afbc =
1027 kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542)
1028 && kbasep_check_for_afbc_on_slot(kbdev, kctx, js,
1029 target_katom);
1030 #endif
1031
1032 stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js,
1033 target_katom,
1034 JS_COMMAND_HARD_STOP);
1035 #if KBASE_GPU_RESET_EN
1036 if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) ||
1037 kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) ||
1038 needs_workaround_for_afbc)) {
1039 /* MIDBASE-2916 if a fragment job with AFBC encoding is
1040 * hardstopped, ensure to do a soft reset also in order to
1041 * clear the GPU status.
1042 * Workaround for HW issue 8401 has an issue,so after
1043 * hard-stopping just reset the GPU. This will ensure that the
1044 * jobs leave the GPU.*/
1045 if (kbase_prepare_to_reset_gpu_locked(kbdev)) {
1046 dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue");
1047 kbase_reset_gpu_locked(kbdev);
1048 }
1049 }
1050 #endif
1051 }
1052
1053 /**
1054 * kbase_job_check_enter_disjoint - potentiall enter disjoint mode
1055 * @kbdev: kbase device
1056 * @action: the event which has occurred
1057 * @core_reqs: core requirements of the atom
1058 * @target_katom: the atom which is being affected
1059 *
1060 * For a certain soft/hard-stop action, work out whether to enter disjoint
1061 * state.
1062 *
1063 * This does not register multiple disjoint events if the atom has already
1064 * started a disjoint period
1065 *
1066 * @core_reqs can be supplied as 0 if the atom had not started on the hardware
1067 * (and so a 'real' soft/hard-stop was not required, but it still interrupted
1068 * flow, perhaps on another context)
1069 *
1070 * kbase_job_check_leave_disjoint() should be used to end the disjoint
1071 * state when the soft/hard-stop action is complete
1072 */
kbase_job_check_enter_disjoint(struct kbase_device * kbdev,u32 action,base_jd_core_req core_reqs,struct kbase_jd_atom * target_katom)1073 void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
1074 base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom)
1075 {
1076 u32 hw_action = action & JS_COMMAND_MASK;
1077
1078 /* For hard-stop, don't enter if hard-stop not allowed */
1079 if (hw_action == JS_COMMAND_HARD_STOP &&
1080 !kbasep_hard_stop_allowed(kbdev, core_reqs))
1081 return;
1082
1083 /* For soft-stop, don't enter if soft-stop not allowed, or isn't
1084 * causing disjoint */
1085 if (hw_action == JS_COMMAND_SOFT_STOP &&
1086 !(kbasep_soft_stop_allowed(kbdev, target_katom) &&
1087 (action & JS_COMMAND_SW_CAUSES_DISJOINT)))
1088 return;
1089
1090 /* Nothing to do if already logged disjoint state on this atom */
1091 if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT)
1092 return;
1093
1094 target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT;
1095 kbase_disjoint_state_up(kbdev);
1096 }
1097
1098 /**
1099 * kbase_job_check_enter_disjoint - potentially leave disjoint state
1100 * @kbdev: kbase device
1101 * @target_katom: atom which is finishing
1102 *
1103 * Work out whether to leave disjoint state when finishing an atom that was
1104 * originated by kbase_job_check_enter_disjoint().
1105 */
kbase_job_check_leave_disjoint(struct kbase_device * kbdev,struct kbase_jd_atom * target_katom)1106 void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
1107 struct kbase_jd_atom *target_katom)
1108 {
1109 if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) {
1110 target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT;
1111 kbase_disjoint_state_down(kbdev);
1112 }
1113 }
1114
1115
1116 #if KBASE_GPU_RESET_EN
kbase_debug_dump_registers(struct kbase_device * kbdev)1117 static void kbase_debug_dump_registers(struct kbase_device *kbdev)
1118 {
1119 int i;
1120
1121 kbase_io_history_dump(kbdev);
1122
1123 dev_err(kbdev->dev, "Register state:");
1124 dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x",
1125 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL),
1126 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL));
1127 dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x",
1128 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL),
1129 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL));
1130 for (i = 0; i < 3; i++) {
1131 dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x",
1132 i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS),
1133 NULL),
1134 i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO),
1135 NULL));
1136 }
1137 dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x",
1138 kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL),
1139 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL));
1140 dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x",
1141 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL),
1142 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL),
1143 kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL));
1144 dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x",
1145 kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL),
1146 kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL));
1147 dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x",
1148 kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL),
1149 kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL));
1150 dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x",
1151 kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG), NULL),
1152 kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG), NULL));
1153 }
1154
kbasep_reset_timeout_worker(struct work_struct * data)1155 static void kbasep_reset_timeout_worker(struct work_struct *data)
1156 {
1157 unsigned long flags;
1158 struct kbase_device *kbdev;
1159 ktime_t end_timestamp = ktime_get();
1160 struct kbasep_js_device_data *js_devdata;
1161 bool try_schedule = false;
1162 bool silent = false;
1163 u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
1164
1165 KBASE_DEBUG_ASSERT(data);
1166
1167 kbdev = container_of(data, struct kbase_device,
1168 hwaccess.backend.reset_work);
1169
1170 KBASE_DEBUG_ASSERT(kbdev);
1171 js_devdata = &kbdev->js_data;
1172
1173 if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1174 KBASE_RESET_GPU_SILENT)
1175 silent = true;
1176
1177 KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
1178
1179 /* Suspend vinstr.
1180 * This call will block until vinstr is suspended. */
1181 kbase_vinstr_suspend(kbdev->vinstr_ctx);
1182
1183 /* Make sure the timer has completed - this cannot be done from
1184 * interrupt context, so this cannot be done within
1185 * kbasep_try_reset_gpu_early. */
1186 hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer);
1187
1188 if (kbase_pm_context_active_handle_suspend(kbdev,
1189 KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
1190 /* This would re-activate the GPU. Since it's already idle,
1191 * there's no need to reset it */
1192 atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1193 KBASE_RESET_GPU_NOT_PENDING);
1194 kbase_disjoint_state_down(kbdev);
1195 wake_up(&kbdev->hwaccess.backend.reset_wait);
1196 kbase_vinstr_resume(kbdev->vinstr_ctx);
1197 return;
1198 }
1199
1200 KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false);
1201
1202 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1203 spin_lock(&kbdev->hwaccess_lock);
1204 spin_lock(&kbdev->mmu_mask_change);
1205 /* We're about to flush out the IRQs and their bottom half's */
1206 kbdev->irq_reset_flush = true;
1207
1208 /* Disable IRQ to avoid IRQ handlers to kick in after releasing the
1209 * spinlock; this also clears any outstanding interrupts */
1210 kbase_pm_disable_interrupts_nolock(kbdev);
1211
1212 spin_unlock(&kbdev->mmu_mask_change);
1213 spin_unlock(&kbdev->hwaccess_lock);
1214 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1215
1216 /* Ensure that any IRQ handlers have finished
1217 * Must be done without any locks IRQ handlers will take */
1218 kbase_synchronize_irqs(kbdev);
1219
1220 /* Flush out any in-flight work items */
1221 kbase_flush_mmu_wqs(kbdev);
1222
1223 /* The flush has completed so reset the active indicator */
1224 kbdev->irq_reset_flush = false;
1225
1226 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) {
1227 /* Ensure that L2 is not transitioning when we send the reset
1228 * command */
1229 while (--max_loops && kbase_pm_get_trans_cores(kbdev,
1230 KBASE_PM_CORE_L2))
1231 ;
1232
1233 WARN(!max_loops, "L2 power transition timed out while trying to reset\n");
1234 }
1235
1236 mutex_lock(&kbdev->pm.lock);
1237 /* We hold the pm lock, so there ought to be a current policy */
1238 KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy);
1239
1240 /* All slot have been soft-stopped and we've waited
1241 * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we
1242 * assume that anything that is still left on the GPU is stuck there and
1243 * we'll kill it when we reset the GPU */
1244
1245 if (!silent)
1246 dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)",
1247 RESET_TIMEOUT);
1248
1249 /* Output the state of some interesting registers to help in the
1250 * debugging of GPU resets */
1251 if (!silent)
1252 kbase_debug_dump_registers(kbdev);
1253
1254 /* Complete any jobs that were still on the GPU */
1255 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1256 kbdev->protected_mode = false;
1257 kbase_backend_reset(kbdev, &end_timestamp);
1258 kbase_pm_metrics_update(kbdev, NULL);
1259 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1260
1261 /* Reset the GPU */
1262 kbase_pm_init_hw(kbdev, 0);
1263
1264 mutex_unlock(&kbdev->pm.lock);
1265
1266 mutex_lock(&js_devdata->runpool_mutex);
1267
1268 mutex_lock(&kbdev->mmu_hw_mutex);
1269 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1270 kbase_ctx_sched_restore_all_as(kbdev);
1271 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1272 mutex_unlock(&kbdev->mmu_hw_mutex);
1273
1274 kbase_pm_enable_interrupts(kbdev);
1275
1276 atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1277 KBASE_RESET_GPU_NOT_PENDING);
1278
1279 kbase_disjoint_state_down(kbdev);
1280
1281 wake_up(&kbdev->hwaccess.backend.reset_wait);
1282 if (!silent)
1283 dev_err(kbdev->dev, "Reset complete");
1284
1285 if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending)
1286 try_schedule = true;
1287
1288 mutex_unlock(&js_devdata->runpool_mutex);
1289
1290 mutex_lock(&kbdev->pm.lock);
1291
1292 /* Find out what cores are required now */
1293 kbase_pm_update_cores_state(kbdev);
1294
1295 /* Synchronously request and wait for those cores, because if
1296 * instrumentation is enabled it would need them immediately. */
1297 kbase_pm_check_transitions_sync(kbdev);
1298
1299 mutex_unlock(&kbdev->pm.lock);
1300
1301 /* Try submitting some jobs to restart processing */
1302 if (try_schedule) {
1303 KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u,
1304 0);
1305 kbase_js_sched_all(kbdev);
1306 }
1307
1308 /* Process any pending slot updates */
1309 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1310 kbase_backend_slot_update(kbdev);
1311 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1312
1313 kbase_pm_context_idle(kbdev);
1314
1315 /* Release vinstr */
1316 kbase_vinstr_resume(kbdev->vinstr_ctx);
1317
1318 KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
1319 }
1320
kbasep_reset_timer_callback(struct hrtimer * timer)1321 static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer)
1322 {
1323 struct kbase_device *kbdev = container_of(timer, struct kbase_device,
1324 hwaccess.backend.reset_timer);
1325
1326 KBASE_DEBUG_ASSERT(kbdev);
1327
1328 /* Reset still pending? */
1329 if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1330 KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) ==
1331 KBASE_RESET_GPU_COMMITTED)
1332 queue_work(kbdev->hwaccess.backend.reset_workq,
1333 &kbdev->hwaccess.backend.reset_work);
1334
1335 return HRTIMER_NORESTART;
1336 }
1337
1338 /*
1339 * If all jobs are evicted from the GPU then we can reset the GPU
1340 * immediately instead of waiting for the timeout to elapse
1341 */
1342
kbasep_try_reset_gpu_early_locked(struct kbase_device * kbdev)1343 static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev)
1344 {
1345 int i;
1346 int pending_jobs = 0;
1347
1348 KBASE_DEBUG_ASSERT(kbdev);
1349
1350 /* Count the number of jobs */
1351 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
1352 pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i);
1353
1354 if (pending_jobs > 0) {
1355 /* There are still jobs on the GPU - wait */
1356 return;
1357 }
1358
1359 /* To prevent getting incorrect registers when dumping failed job,
1360 * skip early reset.
1361 */
1362 if (kbdev->job_fault_debug != false)
1363 return;
1364
1365 /* Check that the reset has been committed to (i.e. kbase_reset_gpu has
1366 * been called), and that no other thread beat this thread to starting
1367 * the reset */
1368 if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1369 KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) !=
1370 KBASE_RESET_GPU_COMMITTED) {
1371 /* Reset has already occurred */
1372 return;
1373 }
1374
1375 queue_work(kbdev->hwaccess.backend.reset_workq,
1376 &kbdev->hwaccess.backend.reset_work);
1377 }
1378
kbasep_try_reset_gpu_early(struct kbase_device * kbdev)1379 static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev)
1380 {
1381 unsigned long flags;
1382 struct kbasep_js_device_data *js_devdata;
1383
1384 js_devdata = &kbdev->js_data;
1385 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1386 kbasep_try_reset_gpu_early_locked(kbdev);
1387 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1388 }
1389
1390 /**
1391 * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU
1392 * @kbdev: kbase device
1393 *
1394 * This function just soft-stops all the slots to ensure that as many jobs as
1395 * possible are saved.
1396 *
1397 * Return:
1398 * The function returns a boolean which should be interpreted as follows:
1399 * true - Prepared for reset, kbase_reset_gpu_locked should be called.
1400 * false - Another thread is performing a reset, kbase_reset_gpu should
1401 * not be called.
1402 */
kbase_prepare_to_reset_gpu_locked(struct kbase_device * kbdev)1403 bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev)
1404 {
1405 int i;
1406
1407 KBASE_DEBUG_ASSERT(kbdev);
1408
1409 if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1410 KBASE_RESET_GPU_NOT_PENDING,
1411 KBASE_RESET_GPU_PREPARED) !=
1412 KBASE_RESET_GPU_NOT_PENDING) {
1413 /* Some other thread is already resetting the GPU */
1414 return false;
1415 }
1416
1417 kbase_disjoint_state_up(kbdev);
1418
1419 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
1420 kbase_job_slot_softstop(kbdev, i, NULL);
1421
1422 return true;
1423 }
1424
kbase_prepare_to_reset_gpu(struct kbase_device * kbdev)1425 bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev)
1426 {
1427 unsigned long flags;
1428 bool ret;
1429 struct kbasep_js_device_data *js_devdata;
1430
1431 js_devdata = &kbdev->js_data;
1432 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1433 ret = kbase_prepare_to_reset_gpu_locked(kbdev);
1434 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1435
1436 return ret;
1437 }
1438 KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu);
1439
1440 /*
1441 * This function should be called after kbase_prepare_to_reset_gpu if it
1442 * returns true. It should never be called without a corresponding call to
1443 * kbase_prepare_to_reset_gpu.
1444 *
1445 * After this function is called (or not called if kbase_prepare_to_reset_gpu
1446 * returned false), the caller should wait for
1447 * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset
1448 * has completed.
1449 */
kbase_reset_gpu(struct kbase_device * kbdev)1450 void kbase_reset_gpu(struct kbase_device *kbdev)
1451 {
1452 KBASE_DEBUG_ASSERT(kbdev);
1453
1454 /* Note this is an assert/atomic_set because it is a software issue for
1455 * a race to be occuring here */
1456 KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1457 KBASE_RESET_GPU_PREPARED);
1458 atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1459 KBASE_RESET_GPU_COMMITTED);
1460
1461 dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1462 kbdev->reset_timeout_ms);
1463
1464 hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
1465 HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
1466 HRTIMER_MODE_REL);
1467
1468 /* Try resetting early */
1469 kbasep_try_reset_gpu_early(kbdev);
1470 }
1471 KBASE_EXPORT_TEST_API(kbase_reset_gpu);
1472
kbase_reset_gpu_locked(struct kbase_device * kbdev)1473 void kbase_reset_gpu_locked(struct kbase_device *kbdev)
1474 {
1475 KBASE_DEBUG_ASSERT(kbdev);
1476
1477 /* Note this is an assert/atomic_set because it is a software issue for
1478 * a race to be occuring here */
1479 KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1480 KBASE_RESET_GPU_PREPARED);
1481 atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1482 KBASE_RESET_GPU_COMMITTED);
1483
1484 dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1485 kbdev->reset_timeout_ms);
1486 hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
1487 HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
1488 HRTIMER_MODE_REL);
1489
1490 /* Try resetting early */
1491 kbasep_try_reset_gpu_early_locked(kbdev);
1492 }
1493
kbase_reset_gpu_silent(struct kbase_device * kbdev)1494 void kbase_reset_gpu_silent(struct kbase_device *kbdev)
1495 {
1496 if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1497 KBASE_RESET_GPU_NOT_PENDING,
1498 KBASE_RESET_GPU_SILENT) !=
1499 KBASE_RESET_GPU_NOT_PENDING) {
1500 /* Some other thread is already resetting the GPU */
1501 return;
1502 }
1503
1504 kbase_disjoint_state_up(kbdev);
1505
1506 queue_work(kbdev->hwaccess.backend.reset_workq,
1507 &kbdev->hwaccess.backend.reset_work);
1508 }
1509
kbase_reset_gpu_active(struct kbase_device * kbdev)1510 bool kbase_reset_gpu_active(struct kbase_device *kbdev)
1511 {
1512 if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1513 KBASE_RESET_GPU_NOT_PENDING)
1514 return false;
1515
1516 return true;
1517 }
1518 #endif /* KBASE_GPU_RESET_EN */
1519