1 /*
2 *
3 * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18 /*
19 * Power policy API implementations
20 */
21
22 #include <mali_kbase.h>
23 #include <mali_midg_regmap.h>
24 #include <mali_kbase_pm.h>
25 #include <mali_kbase_config_defaults.h>
26 #include <backend/gpu/mali_kbase_pm_internal.h>
27
28 static const struct kbase_pm_policy *const policy_list[] = {
29 #ifdef CONFIG_MALI_NO_MALI
30 &kbase_pm_always_on_policy_ops,
31 &kbase_pm_demand_policy_ops,
32 &kbase_pm_coarse_demand_policy_ops,
33 #if !MALI_CUSTOMER_RELEASE
34 &kbase_pm_demand_always_powered_policy_ops,
35 &kbase_pm_fast_start_policy_ops,
36 #endif
37 #else /* CONFIG_MALI_NO_MALI */
38 #if !PLATFORM_POWER_DOWN_ONLY
39 &kbase_pm_demand_policy_ops,
40 #endif /* !PLATFORM_POWER_DOWN_ONLY */
41 &kbase_pm_coarse_demand_policy_ops,
42 &kbase_pm_always_on_policy_ops,
43 #if !MALI_CUSTOMER_RELEASE
44 #if !PLATFORM_POWER_DOWN_ONLY
45 &kbase_pm_demand_always_powered_policy_ops,
46 &kbase_pm_fast_start_policy_ops,
47 #endif /* !PLATFORM_POWER_DOWN_ONLY */
48 #endif
49 #endif /* CONFIG_MALI_NO_MALI */
50 };
51
52 /* The number of policies available in the system.
53 * This is derived from the number of functions listed in policy_get_functions.
54 */
55 #define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
56
57
58 /* Function IDs for looking up Timeline Trace codes in
59 * kbase_pm_change_state_trace_code */
60 enum kbase_pm_func_id {
61 KBASE_PM_FUNC_ID_REQUEST_CORES_START,
62 KBASE_PM_FUNC_ID_REQUEST_CORES_END,
63 KBASE_PM_FUNC_ID_RELEASE_CORES_START,
64 KBASE_PM_FUNC_ID_RELEASE_CORES_END,
65 /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
66 * expect to hit it nor tend to hit it very much anyway. We can detect
67 * whether we need more instrumentation by a difference between
68 * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */
69
70 /* Must be the last */
71 KBASE_PM_FUNC_ID_COUNT
72 };
73
74
75 /* State changes during request/unrequest/release-ing cores */
76 enum {
77 KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
78 KBASE_PM_CHANGE_STATE_TILER = (1u << 1),
79
80 /* These two must be last */
81 KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER |
82 KBASE_PM_CHANGE_STATE_SHADER),
83 KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
84 };
85 typedef u32 kbase_pm_change_state;
86
87
88 #ifdef CONFIG_MALI_TRACE_TIMELINE
89 /* Timeline Trace code lookups for each function */
90 static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT]
91 [KBASE_PM_CHANGE_STATE_COUNT] = {
92 /* kbase_pm_request_cores */
93 [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
94 [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
95 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
96 [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
97 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
98 [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
99 KBASE_PM_CHANGE_STATE_TILER] =
100 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,
101
102 [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
103 [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
104 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
105 [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
106 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
107 [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
108 KBASE_PM_CHANGE_STATE_TILER] =
109 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,
110
111 /* kbase_pm_release_cores */
112 [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
113 [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
114 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
115 [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
116 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
117 [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
118 KBASE_PM_CHANGE_STATE_TILER] =
119 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,
120
121 [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
122 [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
123 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
124 [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
125 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
126 [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
127 KBASE_PM_CHANGE_STATE_TILER] =
128 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
129 };
130
kbase_timeline_pm_cores_func(struct kbase_device * kbdev,enum kbase_pm_func_id func_id,kbase_pm_change_state state)131 static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
132 enum kbase_pm_func_id func_id,
133 kbase_pm_change_state state)
134 {
135 int trace_code;
136
137 KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
138 KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) ==
139 state);
140
141 trace_code = kbase_pm_change_state_trace_code[func_id][state];
142 KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
143 }
144
145 #else /* CONFIG_MALI_TRACE_TIMELINE */
kbase_timeline_pm_cores_func(struct kbase_device * kbdev,enum kbase_pm_func_id func_id,kbase_pm_change_state state)146 static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
147 enum kbase_pm_func_id func_id, kbase_pm_change_state state)
148 {
149 }
150
151 #endif /* CONFIG_MALI_TRACE_TIMELINE */
152
153 /**
154 * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any
155 * requested shader cores
156 * @kbdev: Device pointer
157 */
kbasep_pm_do_poweroff_cores(struct kbase_device * kbdev)158 static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev)
159 {
160 u64 prev_shader_state = kbdev->pm.backend.desired_shader_state;
161 u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state;
162
163 lockdep_assert_held(&kbdev->hwaccess_lock);
164
165 kbdev->pm.backend.desired_shader_state &=
166 ~kbdev->pm.backend.shader_poweroff_pending;
167 kbdev->pm.backend.desired_tiler_state &=
168 ~kbdev->pm.backend.tiler_poweroff_pending;
169
170 kbdev->pm.backend.shader_poweroff_pending = 0;
171 kbdev->pm.backend.tiler_poweroff_pending = 0;
172
173 if (prev_shader_state != kbdev->pm.backend.desired_shader_state ||
174 prev_tiler_state !=
175 kbdev->pm.backend.desired_tiler_state ||
176 kbdev->pm.backend.ca_in_transition) {
177 bool cores_are_available;
178
179 KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
180 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
181 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
182 KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
183 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);
184
185 /* Don't need 'cores_are_available',
186 * because we don't return anything */
187 CSTD_UNUSED(cores_are_available);
188 }
189 }
190
191 static enum hrtimer_restart
kbasep_pm_do_gpu_poweroff_callback(struct hrtimer * timer)192 kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
193 {
194 struct kbase_device *kbdev;
195 unsigned long flags;
196
197 kbdev = container_of(timer, struct kbase_device,
198 pm.backend.gpu_poweroff_timer);
199
200 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
201
202 /* It is safe for this call to do nothing if the work item is already
203 * queued. The worker function will read the must up-to-date state of
204 * kbdev->pm.backend.gpu_poweroff_pending under lock.
205 *
206 * If a state change occurs while the worker function is processing,
207 * this call will succeed as a work item can be requeued once it has
208 * started processing.
209 */
210 if (kbdev->pm.backend.gpu_poweroff_pending)
211 queue_work(kbdev->pm.backend.gpu_poweroff_wq,
212 &kbdev->pm.backend.gpu_poweroff_work);
213
214 if (kbdev->pm.backend.shader_poweroff_pending ||
215 kbdev->pm.backend.tiler_poweroff_pending) {
216 kbdev->pm.backend.shader_poweroff_pending_time--;
217
218 KBASE_DEBUG_ASSERT(
219 kbdev->pm.backend.shader_poweroff_pending_time
220 >= 0);
221
222 if (!kbdev->pm.backend.shader_poweroff_pending_time)
223 kbasep_pm_do_poweroff_cores(kbdev);
224 }
225
226 if (kbdev->pm.backend.poweroff_timer_needed) {
227 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
228
229 hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);
230
231 return HRTIMER_RESTART;
232 }
233
234 kbdev->pm.backend.poweroff_timer_running = false;
235 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
236
237 return HRTIMER_NORESTART;
238 }
239
kbasep_pm_do_gpu_poweroff_wq(struct work_struct * data)240 static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
241 {
242 unsigned long flags;
243 struct kbase_device *kbdev;
244 bool do_poweroff = false;
245
246 kbdev = container_of(data, struct kbase_device,
247 pm.backend.gpu_poweroff_work);
248
249 mutex_lock(&kbdev->pm.lock);
250
251 if (kbdev->pm.backend.gpu_poweroff_pending == 0) {
252 mutex_unlock(&kbdev->pm.lock);
253 return;
254 }
255
256 kbdev->pm.backend.gpu_poweroff_pending--;
257
258 if (kbdev->pm.backend.gpu_poweroff_pending > 0) {
259 mutex_unlock(&kbdev->pm.lock);
260 return;
261 }
262
263 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0);
264
265 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
266
267 /* Only power off the GPU if a request is still pending */
268 if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev))
269 do_poweroff = true;
270
271 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
272
273 if (do_poweroff) {
274 kbdev->pm.backend.poweroff_timer_needed = false;
275 hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
276 kbdev->pm.backend.poweroff_timer_running = false;
277
278 /* Power off the GPU */
279 kbase_pm_do_poweroff(kbdev, false);
280 }
281
282 mutex_unlock(&kbdev->pm.lock);
283 }
284
kbase_pm_policy_init(struct kbase_device * kbdev)285 int kbase_pm_policy_init(struct kbase_device *kbdev)
286 {
287 struct workqueue_struct *wq;
288
289 wq = alloc_workqueue("kbase_pm_do_poweroff",
290 WQ_HIGHPRI | WQ_UNBOUND, 1);
291 if (!wq)
292 return -ENOMEM;
293
294 kbdev->pm.backend.gpu_poweroff_wq = wq;
295 INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work,
296 kbasep_pm_do_gpu_poweroff_wq);
297 hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer,
298 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
299 kbdev->pm.backend.gpu_poweroff_timer.function =
300 kbasep_pm_do_gpu_poweroff_callback;
301 kbdev->pm.backend.pm_current_policy = policy_list[0];
302 kbdev->pm.backend.pm_current_policy->init(kbdev);
303 kbdev->pm.gpu_poweroff_time =
304 HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS);
305 kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
306 kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU;
307
308 return 0;
309 }
310
kbase_pm_policy_term(struct kbase_device * kbdev)311 void kbase_pm_policy_term(struct kbase_device *kbdev)
312 {
313 kbdev->pm.backend.pm_current_policy->term(kbdev);
314 destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq);
315 }
316
kbase_pm_cancel_deferred_poweroff(struct kbase_device * kbdev)317 void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev)
318 {
319 unsigned long flags;
320
321 lockdep_assert_held(&kbdev->pm.lock);
322
323 kbdev->pm.backend.poweroff_timer_needed = false;
324 hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
325 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
326 kbdev->pm.backend.poweroff_timer_running = false;
327
328 /* If wq is already running but is held off by pm.lock, make sure it has
329 * no effect */
330 kbdev->pm.backend.gpu_poweroff_pending = 0;
331
332 kbdev->pm.backend.shader_poweroff_pending = 0;
333 kbdev->pm.backend.tiler_poweroff_pending = 0;
334 kbdev->pm.backend.shader_poweroff_pending_time = 0;
335
336 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
337 }
338
kbase_pm_update_active(struct kbase_device * kbdev)339 void kbase_pm_update_active(struct kbase_device *kbdev)
340 {
341 struct kbase_pm_device_data *pm = &kbdev->pm;
342 struct kbase_pm_backend_data *backend = &pm->backend;
343 unsigned long flags;
344 bool active;
345
346 lockdep_assert_held(&pm->lock);
347
348 /* pm_current_policy will never be NULL while pm.lock is held */
349 KBASE_DEBUG_ASSERT(backend->pm_current_policy);
350
351 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
352
353 active = backend->pm_current_policy->get_core_active(kbdev);
354
355 if (active) {
356 if (backend->gpu_poweroff_pending) {
357 /* Cancel any pending power off request */
358 backend->gpu_poweroff_pending = 0;
359
360 /* If a request was pending then the GPU was still
361 * powered, so no need to continue */
362 if (!kbdev->poweroff_pending) {
363 spin_unlock_irqrestore(&kbdev->hwaccess_lock,
364 flags);
365 return;
366 }
367 }
368
369 if (!backend->poweroff_timer_running && !backend->gpu_powered &&
370 (pm->poweroff_gpu_ticks ||
371 pm->poweroff_shader_ticks)) {
372 backend->poweroff_timer_needed = true;
373 backend->poweroff_timer_running = true;
374 hrtimer_start(&backend->gpu_poweroff_timer,
375 pm->gpu_poweroff_time,
376 HRTIMER_MODE_REL);
377 }
378
379 /* Power on the GPU and any cores requested by the policy */
380 if (pm->backend.poweroff_wait_in_progress) {
381 pm->backend.poweron_required = true;
382 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
383 } else {
384 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
385 kbase_pm_do_poweron(kbdev, false);
386 }
387 } else {
388 /* It is an error for the power policy to power off the GPU
389 * when there are contexts active */
390 KBASE_DEBUG_ASSERT(pm->active_count == 0);
391
392 if (backend->shader_poweroff_pending ||
393 backend->tiler_poweroff_pending) {
394 backend->shader_poweroff_pending = 0;
395 backend->tiler_poweroff_pending = 0;
396 backend->shader_poweroff_pending_time = 0;
397 }
398
399 /* Request power off */
400 if (pm->backend.gpu_powered) {
401 if (pm->poweroff_gpu_ticks) {
402 backend->gpu_poweroff_pending =
403 pm->poweroff_gpu_ticks;
404 backend->poweroff_timer_needed = true;
405 if (!backend->poweroff_timer_running) {
406 /* Start timer if not running (eg if
407 * power policy has been changed from
408 * always_on to something else). This
409 * will ensure the GPU is actually
410 * powered off */
411 backend->poweroff_timer_running
412 = true;
413 hrtimer_start(
414 &backend->gpu_poweroff_timer,
415 pm->gpu_poweroff_time,
416 HRTIMER_MODE_REL);
417 }
418 spin_unlock_irqrestore(&kbdev->hwaccess_lock,
419 flags);
420 } else {
421 spin_unlock_irqrestore(&kbdev->hwaccess_lock,
422 flags);
423
424 /* Power off the GPU immediately */
425 kbase_pm_do_poweroff(kbdev, false);
426 }
427 } else {
428 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
429 }
430 }
431 }
432
kbase_pm_update_cores_state_nolock(struct kbase_device * kbdev)433 void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
434 {
435 u64 desired_bitmap;
436 u64 desired_tiler_bitmap;
437 bool cores_are_available;
438 bool do_poweroff = false;
439
440 lockdep_assert_held(&kbdev->hwaccess_lock);
441
442 if (kbdev->pm.backend.pm_current_policy == NULL)
443 return;
444 if (kbdev->pm.backend.poweroff_wait_in_progress)
445 return;
446
447 if (kbdev->protected_mode_transition && !kbdev->shader_needed_bitmap &&
448 !kbdev->shader_inuse_bitmap && !kbdev->tiler_needed_cnt
449 && !kbdev->tiler_inuse_cnt) {
450 /* We are trying to change in/out of protected mode - force all
451 * cores off so that the L2 powers down */
452 desired_bitmap = 0;
453 desired_tiler_bitmap = 0;
454 } else {
455 desired_bitmap =
456 kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev);
457 desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
458
459 if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
460 desired_tiler_bitmap = 1;
461 else
462 desired_tiler_bitmap = 0;
463
464 if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) {
465 /* Unless XAFFINITY is supported, enable core 0 if tiler
466 * required, regardless of core availability */
467 if (kbdev->tiler_needed_cnt > 0 ||
468 kbdev->tiler_inuse_cnt > 0)
469 desired_bitmap |= 1;
470 }
471 }
472
473 if (kbdev->pm.backend.desired_shader_state != desired_bitmap)
474 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u,
475 (u32)desired_bitmap);
476 /* Are any cores being powered on? */
477 if (~kbdev->pm.backend.desired_shader_state & desired_bitmap ||
478 ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap ||
479 kbdev->pm.backend.ca_in_transition) {
480 /* Check if we are powering off any cores before updating shader
481 * state */
482 if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap ||
483 kbdev->pm.backend.desired_tiler_state &
484 ~desired_tiler_bitmap) {
485 /* Start timer to power off cores */
486 kbdev->pm.backend.shader_poweroff_pending |=
487 (kbdev->pm.backend.desired_shader_state &
488 ~desired_bitmap);
489 kbdev->pm.backend.tiler_poweroff_pending |=
490 (kbdev->pm.backend.desired_tiler_state &
491 ~desired_tiler_bitmap);
492
493 if (kbdev->pm.poweroff_shader_ticks &&
494 !kbdev->protected_mode_transition)
495 kbdev->pm.backend.shader_poweroff_pending_time =
496 kbdev->pm.poweroff_shader_ticks;
497 else
498 do_poweroff = true;
499 }
500
501 kbdev->pm.backend.desired_shader_state = desired_bitmap;
502 kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap;
503
504 /* If any cores are being powered on, transition immediately */
505 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
506 } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap ||
507 kbdev->pm.backend.desired_tiler_state &
508 ~desired_tiler_bitmap) {
509 /* Start timer to power off cores */
510 kbdev->pm.backend.shader_poweroff_pending |=
511 (kbdev->pm.backend.desired_shader_state &
512 ~desired_bitmap);
513 kbdev->pm.backend.tiler_poweroff_pending |=
514 (kbdev->pm.backend.desired_tiler_state &
515 ~desired_tiler_bitmap);
516 if (kbdev->pm.poweroff_shader_ticks &&
517 !kbdev->protected_mode_transition)
518 kbdev->pm.backend.shader_poweroff_pending_time =
519 kbdev->pm.poweroff_shader_ticks;
520 else
521 kbasep_pm_do_poweroff_cores(kbdev);
522 } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 &&
523 desired_tiler_bitmap != 0 &&
524 kbdev->pm.backend.poweroff_timer_needed) {
525 /* If power policy is keeping cores on despite there being no
526 * active contexts then disable poweroff timer as it isn't
527 * required.
528 * Only reset poweroff_timer_needed if we're not in the middle
529 * of the power off callback */
530 kbdev->pm.backend.poweroff_timer_needed = false;
531 }
532
533 /* Ensure timer does not power off wanted cores and make sure to power
534 * off unwanted cores */
535 if (kbdev->pm.backend.shader_poweroff_pending ||
536 kbdev->pm.backend.tiler_poweroff_pending) {
537 kbdev->pm.backend.shader_poweroff_pending &=
538 ~(kbdev->pm.backend.desired_shader_state &
539 desired_bitmap);
540 kbdev->pm.backend.tiler_poweroff_pending &=
541 ~(kbdev->pm.backend.desired_tiler_state &
542 desired_tiler_bitmap);
543
544 if (!kbdev->pm.backend.shader_poweroff_pending &&
545 !kbdev->pm.backend.tiler_poweroff_pending)
546 kbdev->pm.backend.shader_poweroff_pending_time = 0;
547 }
548
549 /* Shader poweroff is deferred to the end of the function, to eliminate
550 * issues caused by the core availability policy recursing into this
551 * function */
552 if (do_poweroff)
553 kbasep_pm_do_poweroff_cores(kbdev);
554
555 /* Don't need 'cores_are_available', because we don't return anything */
556 CSTD_UNUSED(cores_are_available);
557 }
558
kbase_pm_update_cores_state(struct kbase_device * kbdev)559 void kbase_pm_update_cores_state(struct kbase_device *kbdev)
560 {
561 unsigned long flags;
562
563 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
564
565 kbase_pm_update_cores_state_nolock(kbdev);
566
567 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
568 }
569
kbase_pm_list_policies(const struct kbase_pm_policy * const ** list)570 int kbase_pm_list_policies(const struct kbase_pm_policy * const **list)
571 {
572 if (!list)
573 return POLICY_COUNT;
574
575 *list = policy_list;
576
577 return POLICY_COUNT;
578 }
579
580 KBASE_EXPORT_TEST_API(kbase_pm_list_policies);
581
kbase_pm_get_policy(struct kbase_device * kbdev)582 const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev)
583 {
584 KBASE_DEBUG_ASSERT(kbdev != NULL);
585
586 return kbdev->pm.backend.pm_current_policy;
587 }
588
589 KBASE_EXPORT_TEST_API(kbase_pm_get_policy);
590
kbase_pm_set_policy(struct kbase_device * kbdev,const struct kbase_pm_policy * new_policy)591 void kbase_pm_set_policy(struct kbase_device *kbdev,
592 const struct kbase_pm_policy *new_policy)
593 {
594 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
595 const struct kbase_pm_policy *old_policy;
596 unsigned long flags;
597
598 KBASE_DEBUG_ASSERT(kbdev != NULL);
599 KBASE_DEBUG_ASSERT(new_policy != NULL);
600
601 KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
602
603 /* During a policy change we pretend the GPU is active */
604 /* A suspend won't happen here, because we're in a syscall from a
605 * userspace thread */
606 kbase_pm_context_active(kbdev);
607
608 mutex_lock(&js_devdata->runpool_mutex);
609 mutex_lock(&kbdev->pm.lock);
610
611 /* Remove the policy to prevent IRQ handlers from working on it */
612 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
613 old_policy = kbdev->pm.backend.pm_current_policy;
614 kbdev->pm.backend.pm_current_policy = NULL;
615 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
616
617 KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u,
618 old_policy->id);
619 if (old_policy->term)
620 old_policy->term(kbdev);
621
622 KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u,
623 new_policy->id);
624 if (new_policy->init)
625 new_policy->init(kbdev);
626
627 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
628 kbdev->pm.backend.pm_current_policy = new_policy;
629 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
630
631 /* If any core power state changes were previously attempted, but
632 * couldn't be made because the policy was changing (current_policy was
633 * NULL), then re-try them here. */
634 kbase_pm_update_active(kbdev);
635 kbase_pm_update_cores_state(kbdev);
636
637 mutex_unlock(&kbdev->pm.lock);
638 mutex_unlock(&js_devdata->runpool_mutex);
639
640 /* Now the policy change is finished, we release our fake context active
641 * reference */
642 kbase_pm_context_idle(kbdev);
643 }
644
645 KBASE_EXPORT_TEST_API(kbase_pm_set_policy);
646
647 /* Check whether a state change has finished, and trace it as completed */
648 static void
kbase_pm_trace_check_and_finish_state_change(struct kbase_device * kbdev)649 kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev)
650 {
651 if ((kbdev->shader_available_bitmap &
652 kbdev->pm.backend.desired_shader_state)
653 == kbdev->pm.backend.desired_shader_state &&
654 (kbdev->tiler_available_bitmap &
655 kbdev->pm.backend.desired_tiler_state)
656 == kbdev->pm.backend.desired_tiler_state)
657 kbase_timeline_pm_check_handle_event(kbdev,
658 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
659 }
660
kbase_pm_request_cores(struct kbase_device * kbdev,bool tiler_required,u64 shader_cores)661 void kbase_pm_request_cores(struct kbase_device *kbdev,
662 bool tiler_required, u64 shader_cores)
663 {
664 u64 cores;
665
666 kbase_pm_change_state change_gpu_state = 0u;
667
668 KBASE_DEBUG_ASSERT(kbdev != NULL);
669
670 lockdep_assert_held(&kbdev->hwaccess_lock);
671
672 cores = shader_cores;
673 while (cores) {
674 int bitnum = fls64(cores) - 1;
675 u64 bit = 1ULL << bitnum;
676
677 /* It should be almost impossible for this to overflow. It would
678 * require 2^32 atoms to request a particular core, which would
679 * require 2^24 contexts to submit. This would require an amount
680 * of memory that is impossible on a 32-bit system and extremely
681 * unlikely on a 64-bit system. */
682 int cnt = ++kbdev->shader_needed_cnt[bitnum];
683
684 if (1 == cnt) {
685 kbdev->shader_needed_bitmap |= bit;
686 change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
687 }
688
689 cores &= ~bit;
690 }
691
692 if (tiler_required) {
693 int cnt = ++kbdev->tiler_needed_cnt;
694
695 if (1 == cnt)
696 change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
697
698 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
699 }
700
701 if (change_gpu_state) {
702 KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL,
703 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
704
705 kbase_timeline_pm_cores_func(kbdev,
706 KBASE_PM_FUNC_ID_REQUEST_CORES_START,
707 change_gpu_state);
708 kbase_pm_update_cores_state_nolock(kbdev);
709 kbase_timeline_pm_cores_func(kbdev,
710 KBASE_PM_FUNC_ID_REQUEST_CORES_END,
711 change_gpu_state);
712 }
713 }
714
715 KBASE_EXPORT_TEST_API(kbase_pm_request_cores);
716
kbase_pm_unrequest_cores(struct kbase_device * kbdev,bool tiler_required,u64 shader_cores)717 void kbase_pm_unrequest_cores(struct kbase_device *kbdev,
718 bool tiler_required, u64 shader_cores)
719 {
720 kbase_pm_change_state change_gpu_state = 0u;
721
722 KBASE_DEBUG_ASSERT(kbdev != NULL);
723
724 lockdep_assert_held(&kbdev->hwaccess_lock);
725
726 while (shader_cores) {
727 int bitnum = fls64(shader_cores) - 1;
728 u64 bit = 1ULL << bitnum;
729 int cnt;
730
731 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
732
733 cnt = --kbdev->shader_needed_cnt[bitnum];
734
735 if (0 == cnt) {
736 kbdev->shader_needed_bitmap &= ~bit;
737
738 change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
739 }
740
741 shader_cores &= ~bit;
742 }
743
744 if (tiler_required) {
745 int cnt;
746
747 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
748
749 cnt = --kbdev->tiler_needed_cnt;
750
751 if (0 == cnt)
752 change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
753 }
754
755 if (change_gpu_state) {
756 KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL,
757 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
758
759 kbase_pm_update_cores_state_nolock(kbdev);
760
761 /* Trace that any state change effectively completes immediately
762 * - no-one will wait on the state change */
763 kbase_pm_trace_check_and_finish_state_change(kbdev);
764 }
765 }
766
767 KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores);
768
769 enum kbase_pm_cores_ready
kbase_pm_register_inuse_cores(struct kbase_device * kbdev,bool tiler_required,u64 shader_cores)770 kbase_pm_register_inuse_cores(struct kbase_device *kbdev,
771 bool tiler_required, u64 shader_cores)
772 {
773 u64 prev_shader_needed; /* Just for tracing */
774 u64 prev_shader_inuse; /* Just for tracing */
775
776 lockdep_assert_held(&kbdev->hwaccess_lock);
777
778 prev_shader_needed = kbdev->shader_needed_bitmap;
779 prev_shader_inuse = kbdev->shader_inuse_bitmap;
780
781 /* If desired_shader_state does not contain the requested cores, then
782 * power management is not attempting to powering those cores (most
783 * likely due to core availability policy) and a new job affinity must
784 * be chosen */
785 if ((kbdev->pm.backend.desired_shader_state & shader_cores) !=
786 shader_cores) {
787 return (kbdev->pm.backend.poweroff_wait_in_progress ||
788 kbdev->pm.backend.pm_current_policy == NULL) ?
789 KBASE_CORES_NOT_READY : KBASE_NEW_AFFINITY;
790 }
791
792 if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
793 (tiler_required && !kbdev->tiler_available_bitmap)) {
794 /* Trace ongoing core transition */
795 kbase_timeline_pm_l2_transition_start(kbdev);
796 return KBASE_CORES_NOT_READY;
797 }
798
799 /* If we started to trace a state change, then trace it has being
800 * finished by now, at the very latest */
801 kbase_pm_trace_check_and_finish_state_change(kbdev);
802 /* Trace core transition done */
803 kbase_timeline_pm_l2_transition_done(kbdev);
804
805 while (shader_cores) {
806 int bitnum = fls64(shader_cores) - 1;
807 u64 bit = 1ULL << bitnum;
808 int cnt;
809
810 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
811
812 cnt = --kbdev->shader_needed_cnt[bitnum];
813
814 if (0 == cnt)
815 kbdev->shader_needed_bitmap &= ~bit;
816
817 /* shader_inuse_cnt should not overflow because there can only
818 * be a very limited number of jobs on the h/w at one time */
819
820 kbdev->shader_inuse_cnt[bitnum]++;
821 kbdev->shader_inuse_bitmap |= bit;
822
823 shader_cores &= ~bit;
824 }
825
826 if (tiler_required) {
827 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
828
829 --kbdev->tiler_needed_cnt;
830
831 kbdev->tiler_inuse_cnt++;
832
833 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
834 }
835
836 if (prev_shader_needed != kbdev->shader_needed_bitmap)
837 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL,
838 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
839
840 if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
841 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL,
842 NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
843
844 return KBASE_CORES_READY;
845 }
846
847 KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores);
848
kbase_pm_release_cores(struct kbase_device * kbdev,bool tiler_required,u64 shader_cores)849 void kbase_pm_release_cores(struct kbase_device *kbdev,
850 bool tiler_required, u64 shader_cores)
851 {
852 kbase_pm_change_state change_gpu_state = 0u;
853
854 KBASE_DEBUG_ASSERT(kbdev != NULL);
855
856 lockdep_assert_held(&kbdev->hwaccess_lock);
857
858 while (shader_cores) {
859 int bitnum = fls64(shader_cores) - 1;
860 u64 bit = 1ULL << bitnum;
861 int cnt;
862
863 KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
864
865 cnt = --kbdev->shader_inuse_cnt[bitnum];
866
867 if (0 == cnt) {
868 kbdev->shader_inuse_bitmap &= ~bit;
869 change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
870 }
871
872 shader_cores &= ~bit;
873 }
874
875 if (tiler_required) {
876 int cnt;
877
878 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
879
880 cnt = --kbdev->tiler_inuse_cnt;
881
882 if (0 == cnt)
883 change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
884 }
885
886 if (change_gpu_state) {
887 KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL,
888 NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
889
890 kbase_timeline_pm_cores_func(kbdev,
891 KBASE_PM_FUNC_ID_RELEASE_CORES_START,
892 change_gpu_state);
893 kbase_pm_update_cores_state_nolock(kbdev);
894 kbase_timeline_pm_cores_func(kbdev,
895 KBASE_PM_FUNC_ID_RELEASE_CORES_END,
896 change_gpu_state);
897
898 /* Trace that any state change completed immediately */
899 kbase_pm_trace_check_and_finish_state_change(kbdev);
900 }
901 }
902
903 KBASE_EXPORT_TEST_API(kbase_pm_release_cores);
904
kbase_pm_request_cores_sync(struct kbase_device * kbdev,bool tiler_required,u64 shader_cores)905 void kbase_pm_request_cores_sync(struct kbase_device *kbdev,
906 bool tiler_required,
907 u64 shader_cores)
908 {
909 unsigned long flags;
910
911 kbase_pm_wait_for_poweroff_complete(kbdev);
912
913 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
914 kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
915 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
916
917 kbase_pm_check_transitions_sync(kbdev);
918 }
919
920 KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync);
921
kbase_pm_request_l2_caches(struct kbase_device * kbdev)922 void kbase_pm_request_l2_caches(struct kbase_device *kbdev)
923 {
924 unsigned long flags;
925 u32 prior_l2_users_count;
926
927 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
928
929 prior_l2_users_count = kbdev->l2_users_count++;
930
931 KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);
932
933 /* if the GPU is reset while the l2 is on, l2 will be off but
934 * prior_l2_users_count will be > 0. l2_available_bitmap will have been
935 * set to 0 though by kbase_pm_init_hw */
936 if (!prior_l2_users_count || !kbdev->l2_available_bitmap)
937 kbase_pm_check_transitions_nolock(kbdev);
938
939 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
940 wait_event(kbdev->pm.backend.l2_powered_wait,
941 kbdev->pm.backend.l2_powered == 1);
942
943 /* Trace that any state change completed immediately */
944 kbase_pm_trace_check_and_finish_state_change(kbdev);
945 }
946
947 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches);
948
kbase_pm_request_l2_caches_l2_is_on(struct kbase_device * kbdev)949 void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev)
950 {
951 lockdep_assert_held(&kbdev->hwaccess_lock);
952
953 kbdev->l2_users_count++;
954 }
955
956 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on);
957
kbase_pm_release_l2_caches(struct kbase_device * kbdev)958 void kbase_pm_release_l2_caches(struct kbase_device *kbdev)
959 {
960 lockdep_assert_held(&kbdev->hwaccess_lock);
961
962 KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);
963
964 --kbdev->l2_users_count;
965
966 if (!kbdev->l2_users_count) {
967 kbase_pm_check_transitions_nolock(kbdev);
968 /* Trace that any state change completed immediately */
969 kbase_pm_trace_check_and_finish_state_change(kbdev);
970 }
971 }
972
973 KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches);
974