1 /*
2 *
3 * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18
19
20 /*
21 * Base kernel Power Management hardware control
22 */
23
24 // #define ENABLE_DEBUG_LOG
25 #include "../../platform/rk/custom_log.h"
26
27 #include <mali_kbase.h>
28 #include <mali_kbase_config_defaults.h>
29 #include <mali_midg_regmap.h>
30 #if defined(CONFIG_MALI_GATOR_SUPPORT)
31 #include <mali_kbase_gator.h>
32 #endif
33 #include <mali_kbase_tlstream.h>
34 #include <mali_kbase_pm.h>
35 #include <mali_kbase_config_defaults.h>
36 #include <mali_kbase_smc.h>
37 #include <mali_kbase_hwaccess_jm.h>
38 #include <mali_kbase_ctx_sched.h>
39 #include <backend/gpu/mali_kbase_cache_policy_backend.h>
40 #include <backend/gpu/mali_kbase_device_internal.h>
41 #include <backend/gpu/mali_kbase_irq_internal.h>
42 #include <backend/gpu/mali_kbase_pm_internal.h>
43
44 #include <linux/of.h>
45
46 #if MALI_MOCK_TEST
47 #define MOCKABLE(function) function##_original
48 #else
49 #define MOCKABLE(function) function
50 #endif /* MALI_MOCK_TEST */
51
52 /**
53 * enum kbasep_pm_action - Actions that can be performed on a core.
54 *
55 * This enumeration is private to the file. Its values are set to allow
56 * core_type_to_reg() function, which decodes this enumeration, to be simpler
57 * and more efficient.
58 *
59 * @ACTION_PRESENT: The cores that are present
60 * @ACTION_READY: The cores that are ready
61 * @ACTION_PWRON: Power on the cores specified
62 * @ACTION_PWROFF: Power off the cores specified
63 * @ACTION_PWRTRANS: The cores that are transitioning
64 * @ACTION_PWRACTIVE: The cores that are active
65 */
66 enum kbasep_pm_action {
67 ACTION_PRESENT = 0,
68 ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO),
69 ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO),
70 ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO),
71 ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO),
72 ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO)
73 };
74
75 /*---------------------------------------------------------------------------*/
76
is_action_of_powering_off_l2(enum kbase_pm_core_type core_type,enum kbasep_pm_action active)77 static bool is_action_of_powering_off_l2(enum kbase_pm_core_type core_type,
78 enum kbasep_pm_action active)
79 {
80 return (KBASE_PM_CORE_L2 == core_type) && (ACTION_PWROFF == active);
81 }
82
is_action_of_powering_off_shader(enum kbase_pm_core_type core_type,enum kbasep_pm_action active)83 static bool is_action_of_powering_off_shader(enum kbase_pm_core_type core_type,
84 enum kbasep_pm_action active)
85 {
86 return (KBASE_PM_CORE_SHADER == core_type) && (ACTION_PWROFF == active);
87 }
88
is_action_of_powering_off_tiler(enum kbase_pm_core_type core_type,enum kbasep_pm_action active)89 static bool is_action_of_powering_off_tiler(enum kbase_pm_core_type core_type,
90 enum kbasep_pm_action active)
91 {
92 return (KBASE_PM_CORE_TILER == core_type) && (ACTION_PWROFF == active);
93 }
94
95 static u64 kbase_pm_get_state(
96 struct kbase_device *kbdev,
97 enum kbase_pm_core_type core_type,
98 enum kbasep_pm_action action);
99
100 /**
101 * core_type_to_reg - Decode a core type and action to a register.
102 *
103 * Given a core type (defined by kbase_pm_core_type) and an action (defined
104 * by kbasep_pm_action) this function will return the register offset that
105 * will perform the action on the core type. The register returned is the _LO
106 * register and an offset must be applied to use the _HI register.
107 *
108 * @core_type: The type of core
109 * @action: The type of action
110 *
111 * Return: The register offset of the _LO register that performs an action of
112 * type @action on a core of type @core_type.
113 */
core_type_to_reg(enum kbase_pm_core_type core_type,enum kbasep_pm_action action)114 static u32 core_type_to_reg(enum kbase_pm_core_type core_type,
115 enum kbasep_pm_action action)
116 {
117 #ifdef CONFIG_MALI_CORESTACK
118 if (core_type == KBASE_PM_CORE_STACK) {
119 switch (action) {
120 case ACTION_PRESENT:
121 return STACK_PRESENT_LO;
122 case ACTION_READY:
123 return STACK_READY_LO;
124 case ACTION_PWRON:
125 return STACK_PWRON_LO;
126 case ACTION_PWROFF:
127 return STACK_PWROFF_LO;
128 case ACTION_PWRTRANS:
129 return STACK_PWRTRANS_LO;
130 default:
131 BUG();
132 }
133 }
134 #endif /* CONFIG_MALI_CORESTACK */
135
136 return (u32)core_type + (u32)action;
137 }
138
139 #ifdef CONFIG_ARM64
mali_cci_flush_l2(struct kbase_device * kbdev)140 static void mali_cci_flush_l2(struct kbase_device *kbdev)
141 {
142 const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED;
143 u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
144 u32 raw;
145
146 /*
147 * Note that we don't take the cache flush mutex here since
148 * we expect to be the last user of the L2, all other L2 users
149 * would have dropped their references, to initiate L2 power
150 * down, L2 power down being the only valid place for this
151 * to be called from.
152 */
153
154 kbase_reg_write(kbdev,
155 GPU_CONTROL_REG(GPU_COMMAND),
156 GPU_COMMAND_CLEAN_INV_CACHES,
157 NULL);
158
159 raw = kbase_reg_read(kbdev,
160 GPU_CONTROL_REG(GPU_IRQ_RAWSTAT),
161 NULL);
162
163 /* Wait for cache flush to complete before continuing, exit on
164 * gpu resets or loop expiry. */
165 while (((raw & mask) == 0) && --loops) {
166 raw = kbase_reg_read(kbdev,
167 GPU_CONTROL_REG(GPU_IRQ_RAWSTAT),
168 NULL);
169 }
170 }
171 #endif
172
173 /**
174 * kbase_pm_invoke - Invokes an action on a core set
175 *
176 * This function performs the action given by @action on a set of cores of a
177 * type given by @core_type. It is a static function used by
178 * kbase_pm_transition_core_type()
179 *
180 * @kbdev: The kbase device structure of the device
181 * @core_type: The type of core that the action should be performed on
182 * @cores: A bit mask of cores to perform the action on (low 32 bits)
183 * @action: The action to perform on the cores
184 */
kbase_pm_invoke(struct kbase_device * kbdev,enum kbase_pm_core_type core_type,u64 cores,enum kbasep_pm_action action)185 static void kbase_pm_invoke(struct kbase_device *kbdev,
186 enum kbase_pm_core_type core_type,
187 u64 cores,
188 enum kbasep_pm_action action)
189 {
190 u32 reg;
191 u32 lo = cores & 0xFFFFFFFF;
192 u32 hi = (cores >> 32) & 0xFFFFFFFF;
193
194 lockdep_assert_held(&kbdev->hwaccess_lock);
195
196 /*-------------------------------------------------------*/
197
198 if ( is_action_of_powering_off_l2(core_type, action) ) {
199 D("not to power off l2 actually.");
200 return;
201 }
202 if ( is_action_of_powering_off_shader(core_type, action) ) {
203 D("not to power off shader actually. cores_lo : 0x%x, hi : 0x%x.",
204 lo,
205 hi);
206 return;
207 }
208 if ( is_action_of_powering_off_tiler(core_type, action) ) {
209 D("not to power off tiler actually.");
210 return;
211 }
212
213 /*-------------------------------------------------------*/
214
215 reg = core_type_to_reg(core_type, action);
216
217 KBASE_DEBUG_ASSERT(reg);
218 #if defined(CONFIG_MALI_GATOR_SUPPORT)
219 if (cores) {
220 if (action == ACTION_PWRON)
221 kbase_trace_mali_pm_power_on(core_type, cores);
222 else if (action == ACTION_PWROFF)
223 kbase_trace_mali_pm_power_off(core_type, cores);
224 }
225 #endif
226
227 if (cores) {
228 u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY);
229
230 if (action == ACTION_PWRON)
231 state |= cores;
232 else if (action == ACTION_PWROFF)
233 state &= ~cores;
234 KBASE_TLSTREAM_AUX_PM_STATE(core_type, state);
235 }
236
237 /* Tracing */
238 if (cores) {
239 if (action == ACTION_PWRON)
240 switch (core_type) {
241 case KBASE_PM_CORE_SHADER:
242 KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u,
243 lo);
244 break;
245 case KBASE_PM_CORE_TILER:
246 KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL,
247 NULL, 0u, lo);
248 break;
249 case KBASE_PM_CORE_L2:
250 KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL,
251 0u, lo);
252 break;
253 default:
254 break;
255 }
256 else if (action == ACTION_PWROFF)
257 switch (core_type) {
258 case KBASE_PM_CORE_SHADER:
259 KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL,
260 0u, lo);
261 break;
262 case KBASE_PM_CORE_TILER:
263 KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL,
264 NULL, 0u, lo);
265 break;
266 case KBASE_PM_CORE_L2:
267 KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL,
268 0u, lo);
269 /* disable snoops before L2 is turned off */
270 kbase_pm_cache_snoop_disable(kbdev);
271 break;
272 default:
273 break;
274 }
275 }
276
277 if (lo != 0)
278 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL);
279
280 if (hi != 0)
281 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL);
282 }
283
284 /**
285 * kbase_pm_get_state - Get information about a core set
286 *
287 * This function gets information (chosen by @action) about a set of cores of
288 * a type given by @core_type. It is a static function used by
289 * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and
290 * kbase_pm_get_ready_cores().
291 *
292 * @kbdev: The kbase device structure of the device
293 * @core_type: The type of core that the should be queried
294 * @action: The property of the cores to query
295 *
296 * Return: A bit mask specifying the state of the cores
297 */
kbase_pm_get_state(struct kbase_device * kbdev,enum kbase_pm_core_type core_type,enum kbasep_pm_action action)298 static u64 kbase_pm_get_state(struct kbase_device *kbdev,
299 enum kbase_pm_core_type core_type,
300 enum kbasep_pm_action action)
301 {
302 u32 reg;
303 u32 lo, hi;
304
305 reg = core_type_to_reg(core_type, action);
306
307 KBASE_DEBUG_ASSERT(reg);
308
309 lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL);
310 hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL);
311
312 return (((u64) hi) << 32) | ((u64) lo);
313 }
314
kbasep_pm_init_core_use_bitmaps(struct kbase_device * kbdev)315 void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev)
316 {
317 kbdev->shader_inuse_bitmap = 0;
318 kbdev->shader_needed_bitmap = 0;
319 kbdev->shader_available_bitmap = 0;
320 kbdev->tiler_available_bitmap = 0;
321 kbdev->l2_users_count = 0;
322 kbdev->l2_available_bitmap = 0;
323 kbdev->tiler_needed_cnt = 0;
324 kbdev->tiler_inuse_cnt = 0;
325
326 memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt));
327 }
328
329 /**
330 * kbase_pm_get_present_cores - Get the cores that are present
331 *
332 * @kbdev: Kbase device
333 * @type: The type of cores to query
334 *
335 * Return: Bitmask of the cores that are present
336 */
kbase_pm_get_present_cores(struct kbase_device * kbdev,enum kbase_pm_core_type type)337 u64 kbase_pm_get_present_cores(struct kbase_device *kbdev,
338 enum kbase_pm_core_type type)
339 {
340 KBASE_DEBUG_ASSERT(kbdev != NULL);
341
342 switch (type) {
343 case KBASE_PM_CORE_L2:
344 return kbdev->gpu_props.props.raw_props.l2_present;
345 case KBASE_PM_CORE_SHADER:
346 return kbdev->gpu_props.props.raw_props.shader_present;
347 case KBASE_PM_CORE_TILER:
348 return kbdev->gpu_props.props.raw_props.tiler_present;
349 #ifdef CONFIG_MALI_CORESTACK
350 case KBASE_PM_CORE_STACK:
351 return kbdev->gpu_props.props.raw_props.stack_present;
352 #endif /* CONFIG_MALI_CORESTACK */
353 default:
354 break;
355 }
356 KBASE_DEBUG_ASSERT(0);
357
358 return 0;
359 }
360
361 KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores);
362
363 /**
364 * kbase_pm_get_active_cores - Get the cores that are "active"
365 * (busy processing work)
366 *
367 * @kbdev: Kbase device
368 * @type: The type of cores to query
369 *
370 * Return: Bitmask of cores that are active
371 */
kbase_pm_get_active_cores(struct kbase_device * kbdev,enum kbase_pm_core_type type)372 u64 kbase_pm_get_active_cores(struct kbase_device *kbdev,
373 enum kbase_pm_core_type type)
374 {
375 return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE);
376 }
377
378 KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores);
379
380 /**
381 * kbase_pm_get_trans_cores - Get the cores that are transitioning between
382 * power states
383 *
384 * @kbdev: Kbase device
385 * @type: The type of cores to query
386 *
387 * Return: Bitmask of cores that are transitioning
388 */
kbase_pm_get_trans_cores(struct kbase_device * kbdev,enum kbase_pm_core_type type)389 u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev,
390 enum kbase_pm_core_type type)
391 {
392 return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS);
393 }
394
395 KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores);
396
397 /**
398 * kbase_pm_get_ready_cores - Get the cores that are powered on
399 *
400 * @kbdev: Kbase device
401 * @type: The type of cores to query
402 *
403 * Return: Bitmask of cores that are ready (powered on)
404 */
kbase_pm_get_ready_cores(struct kbase_device * kbdev,enum kbase_pm_core_type type)405 u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev,
406 enum kbase_pm_core_type type)
407 {
408 u64 result;
409
410 result = kbase_pm_get_state(kbdev, type, ACTION_READY);
411
412 switch (type) {
413 case KBASE_PM_CORE_SHADER:
414 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u,
415 (u32) result);
416 break;
417 case KBASE_PM_CORE_TILER:
418 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u,
419 (u32) result);
420 break;
421 case KBASE_PM_CORE_L2:
422 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u,
423 (u32) result);
424 break;
425 default:
426 break;
427 }
428
429 return result;
430 }
431
432 KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores);
433
434 /**
435 * kbase_pm_transition_core_type - Perform power transitions for a particular
436 * core type.
437 *
438 * This function will perform any available power transitions to make the actual
439 * hardware state closer to the desired state. If a core is currently
440 * transitioning then changes to the power state of that call cannot be made
441 * until the transition has finished. Cores which are not present in the
442 * hardware are ignored if they are specified in the desired_state bitmask,
443 * however the return value will always be 0 in this case.
444 *
445 * @kbdev: The kbase device
446 * @type: The core type to perform transitions for
447 * @desired_state: A bit mask of the desired state of the cores
448 * @in_use: A bit mask of the cores that are currently running
449 * jobs. These cores have to be kept powered up because
450 * there are jobs running (or about to run) on them.
451 * @available: Receives a bit mask of the cores that the job
452 * scheduler can use to submit jobs to. May be NULL if
453 * this is not needed.
454 * @powering_on: Bit mask to update with cores that are
455 * transitioning to a power-on state.
456 *
457 * Return: true if the desired state has been reached, false otherwise
458 */
kbase_pm_transition_core_type(struct kbase_device * kbdev,enum kbase_pm_core_type type,u64 desired_state,u64 in_use,u64 * const available,u64 * powering_on)459 static bool kbase_pm_transition_core_type(struct kbase_device *kbdev,
460 enum kbase_pm_core_type type,
461 u64 desired_state,
462 u64 in_use,
463 u64 * const available,
464 u64 *powering_on)
465 {
466 u64 present;
467 u64 ready;
468 u64 trans;
469 u64 powerup;
470 u64 powerdown;
471 u64 powering_on_trans;
472 u64 desired_state_in_use;
473
474 lockdep_assert_held(&kbdev->hwaccess_lock);
475
476 /* Get current state */
477 present = kbase_pm_get_present_cores(kbdev, type);
478 trans = kbase_pm_get_trans_cores(kbdev, type);
479 ready = kbase_pm_get_ready_cores(kbdev, type);
480 /* mask off ready from trans in case transitions finished between the
481 * register reads */
482 trans &= ~ready;
483
484 if (trans) /* Do not progress if any cores are transitioning */
485 return false;
486
487 powering_on_trans = trans & *powering_on;
488 *powering_on = powering_on_trans;
489
490 if (available != NULL)
491 *available = (ready | powering_on_trans) & desired_state;
492
493 /* Update desired state to include the in-use cores. These have to be
494 * kept powered up because there are jobs running or about to run on
495 * these cores
496 */
497 desired_state_in_use = desired_state | in_use;
498
499 /* Update state of whether l2 caches are powered */
500 if (type == KBASE_PM_CORE_L2) {
501 if ((ready == present) && (desired_state_in_use == ready) &&
502 (trans == 0)) {
503 /* All are ready, none will be turned off, and none are
504 * transitioning */
505 kbdev->pm.backend.l2_powered = 1;
506 /*
507 * Ensure snoops are enabled after L2 is powered up,
508 * note that kbase keeps track of the snoop state, so
509 * safe to repeatedly call.
510 */
511 kbase_pm_cache_snoop_enable(kbdev);
512 if (kbdev->l2_users_count > 0) {
513 /* Notify any registered l2 cache users
514 * (optimized out when no users waiting) */
515 wake_up(&kbdev->pm.backend.l2_powered_wait);
516 }
517 } else
518 kbdev->pm.backend.l2_powered = 0;
519 }
520
521 if (desired_state == ready && (trans == 0))
522 return true;
523
524 /* Restrict the cores to those that are actually present */
525 powerup = desired_state_in_use & present;
526 powerdown = (~desired_state_in_use) & present;
527
528 /* Restrict to cores that are not already in the desired state */
529 powerup &= ~ready;
530 powerdown &= ready;
531
532 /* Don't transition any cores that are already transitioning, except for
533 * Mali cores that support the following case:
534 *
535 * If the SHADER_PWRON or TILER_PWRON registers are written to turn on
536 * a core that is currently transitioning to power off, then this is
537 * remembered and the shader core is automatically powered up again once
538 * the original transition completes. Once the automatic power on is
539 * complete any job scheduled on the shader core should start.
540 */
541 powerdown &= ~trans;
542
543 if (kbase_hw_has_feature(kbdev,
544 BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS))
545 if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type)
546 trans = powering_on_trans; /* for exception cases, only
547 * mask off cores in power on
548 * transitions */
549
550 powerup &= ~trans;
551
552 /* Perform transitions if any */
553 kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON);
554 #if !PLATFORM_POWER_DOWN_ONLY
555 kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF);
556 #endif
557
558 /* Recalculate cores transitioning on, and re-evaluate our state */
559 powering_on_trans |= powerup;
560 *powering_on = powering_on_trans;
561 if (available != NULL)
562 *available = (ready | powering_on_trans) & desired_state;
563
564 return false;
565 }
566
567 KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type);
568
569 /**
570 * get_desired_cache_status - Determine which caches should be on for a
571 * particular core state
572 *
573 * This function takes a bit mask of the present caches and the cores (or
574 * caches) that are attached to the caches that will be powered. It then
575 * computes which caches should be turned on to allow the cores requested to be
576 * powered up.
577 *
578 * @present: The bit mask of present caches
579 * @cores_powered: A bit mask of cores (or L2 caches) that are desired to
580 * be powered
581 * @tilers_powered: The bit mask of tilers that are desired to be powered
582 *
583 * Return: A bit mask of the caches that should be turned on
584 */
get_desired_cache_status(u64 present,u64 cores_powered,u64 tilers_powered)585 static u64 get_desired_cache_status(u64 present, u64 cores_powered,
586 u64 tilers_powered)
587 {
588 u64 desired = 0;
589
590 while (present) {
591 /* Find out which is the highest set bit */
592 u64 bit = fls64(present) - 1;
593 u64 bit_mask = 1ull << bit;
594 /* Create a mask which has all bits from 'bit' upwards set */
595
596 u64 mask = ~(bit_mask - 1);
597
598 /* If there are any cores powered at this bit or above (that
599 * haven't previously been processed) then we need this core on
600 */
601 if (cores_powered & mask)
602 desired |= bit_mask;
603
604 /* Remove bits from cores_powered and present */
605 cores_powered &= ~mask;
606 present &= ~bit_mask;
607 }
608
609 /* Power up the required L2(s) for the tiler */
610 if (tilers_powered)
611 desired |= 1;
612
613 return desired;
614 }
615
616 KBASE_EXPORT_TEST_API(get_desired_cache_status);
617
618 #ifdef CONFIG_MALI_CORESTACK
kbase_pm_core_stack_mask(u64 cores)619 u64 kbase_pm_core_stack_mask(u64 cores)
620 {
621 u64 stack_mask = 0;
622 size_t const MAX_CORE_ID = 31;
623 size_t const NUM_CORES_PER_STACK = 4;
624 size_t i;
625
626 for (i = 0; i <= MAX_CORE_ID; ++i) {
627 if (test_bit(i, (unsigned long *)&cores)) {
628 /* Every core which ID >= 16 is filled to stacks 4-7
629 * instead of 0-3 */
630 size_t const stack_num = (i > 16) ?
631 (i % NUM_CORES_PER_STACK) + 4 :
632 (i % NUM_CORES_PER_STACK);
633 set_bit(stack_num, (unsigned long *)&stack_mask);
634 }
635 }
636
637 return stack_mask;
638 }
639 #endif /* CONFIG_MALI_CORESTACK */
640
641 bool
MOCKABLE(kbase_pm_check_transitions_nolock)642 MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev)
643 {
644 bool cores_are_available = false;
645 bool in_desired_state = true;
646 u64 desired_l2_state;
647 #ifdef CONFIG_MALI_CORESTACK
648 u64 desired_stack_state;
649 u64 stacks_powered;
650 #endif /* CONFIG_MALI_CORESTACK */
651 u64 cores_powered;
652 u64 tilers_powered;
653 u64 tiler_available_bitmap;
654 u64 tiler_transitioning_bitmap;
655 u64 shader_available_bitmap;
656 u64 shader_ready_bitmap;
657 u64 shader_transitioning_bitmap;
658 u64 l2_available_bitmap;
659 u64 prev_l2_available_bitmap;
660 u64 l2_inuse_bitmap;
661
662 KBASE_DEBUG_ASSERT(NULL != kbdev);
663 lockdep_assert_held(&kbdev->hwaccess_lock);
664
665 spin_lock(&kbdev->pm.backend.gpu_powered_lock);
666 if (kbdev->pm.backend.gpu_powered == false) {
667 spin_unlock(&kbdev->pm.backend.gpu_powered_lock);
668 if (kbdev->pm.backend.desired_shader_state == 0 &&
669 kbdev->pm.backend.desired_tiler_state == 0)
670 return true;
671 return false;
672 }
673
674 /* Trace that a change-state is being requested, and that it took
675 * (effectively) no time to start it. This is useful for counting how
676 * many state changes occurred, in a way that's backwards-compatible
677 * with processing the trace data */
678 kbase_timeline_pm_send_event(kbdev,
679 KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
680 kbase_timeline_pm_handle_event(kbdev,
681 KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
682
683 /* If any cores are already powered then, we must keep the caches on */
684 shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev,
685 KBASE_PM_CORE_SHADER);
686 cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
687 cores_powered |= kbdev->pm.backend.desired_shader_state;
688
689 #ifdef CONFIG_MALI_CORESTACK
690 /* Work out which core stacks want to be powered */
691 desired_stack_state = kbase_pm_core_stack_mask(cores_powered);
692 stacks_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK) |
693 desired_stack_state;
694 #endif /* CONFIG_MALI_CORESTACK */
695
696 /* Work out which tilers want to be powered */
697 tiler_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev,
698 KBASE_PM_CORE_TILER);
699 tilers_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER);
700 tilers_powered |= kbdev->pm.backend.desired_tiler_state;
701
702 /* If there are l2 cache users registered, keep all l2s powered even if
703 * all other cores are off. */
704 if (kbdev->l2_users_count > 0)
705 cores_powered |= kbdev->gpu_props.props.raw_props.l2_present;
706
707 desired_l2_state = get_desired_cache_status(
708 kbdev->gpu_props.props.raw_props.l2_present,
709 cores_powered, tilers_powered);
710
711 l2_inuse_bitmap = get_desired_cache_status(
712 kbdev->gpu_props.props.raw_props.l2_present,
713 cores_powered | shader_transitioning_bitmap,
714 tilers_powered | tiler_transitioning_bitmap);
715
716 #ifdef CONFIG_MALI_CORESTACK
717 if (stacks_powered)
718 desired_l2_state |= 1;
719 #endif /* CONFIG_MALI_CORESTACK */
720
721 /* If any l2 cache is on, then enable l2 #0, for use by job manager */
722 if (0 != desired_l2_state)
723 desired_l2_state |= 1;
724
725 prev_l2_available_bitmap = kbdev->l2_available_bitmap;
726 in_desired_state &= kbase_pm_transition_core_type(kbdev,
727 KBASE_PM_CORE_L2, desired_l2_state, l2_inuse_bitmap,
728 &l2_available_bitmap,
729 &kbdev->pm.backend.powering_on_l2_state);
730
731 if (kbdev->l2_available_bitmap != l2_available_bitmap)
732 KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap);
733
734 kbdev->l2_available_bitmap = l2_available_bitmap;
735
736
737 #ifdef CONFIG_MALI_CORESTACK
738 if (in_desired_state) {
739 in_desired_state &= kbase_pm_transition_core_type(kbdev,
740 KBASE_PM_CORE_STACK, desired_stack_state, 0,
741 &kbdev->stack_available_bitmap,
742 &kbdev->pm.backend.powering_on_stack_state);
743 }
744 #endif /* CONFIG_MALI_CORESTACK */
745
746 if (in_desired_state) {
747 in_desired_state &= kbase_pm_transition_core_type(kbdev,
748 KBASE_PM_CORE_TILER,
749 kbdev->pm.backend.desired_tiler_state,
750 0, &tiler_available_bitmap,
751 &kbdev->pm.backend.powering_on_tiler_state);
752 in_desired_state &= kbase_pm_transition_core_type(kbdev,
753 KBASE_PM_CORE_SHADER,
754 kbdev->pm.backend.desired_shader_state,
755 kbdev->shader_inuse_bitmap,
756 &shader_available_bitmap,
757 &kbdev->pm.backend.powering_on_shader_state);
758
759 if (kbdev->shader_available_bitmap != shader_available_bitmap) {
760 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
761 NULL, 0u,
762 (u32) shader_available_bitmap);
763 KBASE_TIMELINE_POWER_SHADER(kbdev,
764 shader_available_bitmap);
765 }
766
767 kbdev->shader_available_bitmap = shader_available_bitmap;
768
769 if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
770 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
771 NULL, NULL, 0u,
772 (u32) tiler_available_bitmap);
773 KBASE_TIMELINE_POWER_TILER(kbdev,
774 tiler_available_bitmap);
775 }
776
777 kbdev->tiler_available_bitmap = tiler_available_bitmap;
778
779 } else if ((l2_available_bitmap &
780 kbdev->gpu_props.props.raw_props.tiler_present) !=
781 kbdev->gpu_props.props.raw_props.tiler_present) {
782 tiler_available_bitmap = 0;
783
784 if (kbdev->tiler_available_bitmap != tiler_available_bitmap)
785 KBASE_TIMELINE_POWER_TILER(kbdev,
786 tiler_available_bitmap);
787
788 kbdev->tiler_available_bitmap = tiler_available_bitmap;
789 }
790
791 /* State updated for slow-path waiters */
792 kbdev->pm.backend.gpu_in_desired_state = in_desired_state;
793
794 shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev,
795 KBASE_PM_CORE_SHADER);
796 shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev,
797 KBASE_PM_CORE_SHADER);
798
799 /* Determine whether the cores are now available (even if the set of
800 * available cores is empty). Note that they can be available even if
801 * we've not finished transitioning to the desired state */
802 if ((kbdev->shader_available_bitmap &
803 kbdev->pm.backend.desired_shader_state)
804 == kbdev->pm.backend.desired_shader_state &&
805 (kbdev->tiler_available_bitmap &
806 kbdev->pm.backend.desired_tiler_state)
807 == kbdev->pm.backend.desired_tiler_state) {
808 cores_are_available = true;
809
810 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u,
811 (u32)(kbdev->shader_available_bitmap &
812 kbdev->pm.backend.desired_shader_state));
813 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u,
814 (u32)(kbdev->tiler_available_bitmap &
815 kbdev->pm.backend.desired_tiler_state));
816
817 /* Log timelining information about handling events that power
818 * up cores, to match up either with immediate submission either
819 * because cores already available, or from PM IRQ */
820 if (!in_desired_state)
821 kbase_timeline_pm_send_event(kbdev,
822 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
823 }
824
825 if (in_desired_state) {
826 KBASE_DEBUG_ASSERT(cores_are_available);
827
828 #if defined(CONFIG_MALI_GATOR_SUPPORT)
829 kbase_trace_mali_pm_status(KBASE_PM_CORE_L2,
830 kbase_pm_get_ready_cores(kbdev,
831 KBASE_PM_CORE_L2));
832 kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER,
833 kbase_pm_get_ready_cores(kbdev,
834 KBASE_PM_CORE_SHADER));
835 kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER,
836 kbase_pm_get_ready_cores(kbdev,
837 KBASE_PM_CORE_TILER));
838 #ifdef CONFIG_MALI_CORESTACK
839 kbase_trace_mali_pm_status(KBASE_PM_CORE_STACK,
840 kbase_pm_get_ready_cores(kbdev,
841 KBASE_PM_CORE_STACK));
842 #endif /* CONFIG_MALI_CORESTACK */
843 #endif
844
845 KBASE_TLSTREAM_AUX_PM_STATE(
846 KBASE_PM_CORE_L2,
847 kbase_pm_get_ready_cores(
848 kbdev, KBASE_PM_CORE_L2));
849 KBASE_TLSTREAM_AUX_PM_STATE(
850 KBASE_PM_CORE_SHADER,
851 kbase_pm_get_ready_cores(
852 kbdev, KBASE_PM_CORE_SHADER));
853 KBASE_TLSTREAM_AUX_PM_STATE(
854 KBASE_PM_CORE_TILER,
855 kbase_pm_get_ready_cores(
856 kbdev,
857 KBASE_PM_CORE_TILER));
858 #ifdef CONFIG_MALI_CORESTACK
859 KBASE_TLSTREAM_AUX_PM_STATE(
860 KBASE_PM_CORE_STACK,
861 kbase_pm_get_ready_cores(
862 kbdev,
863 KBASE_PM_CORE_STACK));
864 #endif /* CONFIG_MALI_CORESTACK */
865
866 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL,
867 kbdev->pm.backend.gpu_in_desired_state,
868 (u32)kbdev->pm.backend.desired_shader_state);
869 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u,
870 (u32)kbdev->pm.backend.desired_tiler_state);
871
872 /* Log timelining information for synchronous waiters */
873 kbase_timeline_pm_send_event(kbdev,
874 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
875 /* Wake slow-path waiters. Job scheduler does not use this. */
876 KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
877
878 wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
879 }
880
881 spin_unlock(&kbdev->pm.backend.gpu_powered_lock);
882
883 /* kbase_pm_ca_update_core_status can cause one-level recursion into
884 * this function, so it must only be called once all changes to kbdev
885 * have been committed, and after the gpu_powered_lock has been
886 * dropped. */
887 if (kbdev->shader_ready_bitmap != shader_ready_bitmap ||
888 kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) {
889 kbdev->shader_ready_bitmap = shader_ready_bitmap;
890 kbdev->shader_transitioning_bitmap =
891 shader_transitioning_bitmap;
892
893 kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap,
894 shader_transitioning_bitmap);
895 }
896
897 /* The core availability policy is not allowed to keep core group 0
898 * turned off (unless it was changing the l2 power state) */
899 if (!((shader_ready_bitmap | shader_transitioning_bitmap) &
900 kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
901 (prev_l2_available_bitmap == desired_l2_state) &&
902 !(kbase_pm_ca_get_core_mask(kbdev) &
903 kbdev->gpu_props.props.coherency_info.group[0].core_mask))
904 BUG();
905
906 /* The core availability policy is allowed to keep core group 1 off,
907 * but all jobs specifically targeting CG1 must fail */
908 if (!((shader_ready_bitmap | shader_transitioning_bitmap) &
909 kbdev->gpu_props.props.coherency_info.group[1].core_mask) &&
910 !(kbase_pm_ca_get_core_mask(kbdev) &
911 kbdev->gpu_props.props.coherency_info.group[1].core_mask))
912 kbdev->pm.backend.cg1_disabled = true;
913 else
914 kbdev->pm.backend.cg1_disabled = false;
915
916 return cores_are_available;
917 }
918 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock);
919
920 /* Timeout for kbase_pm_check_transitions_sync when wait_event_killable has
921 * aborted due to a fatal signal. If the time spent waiting has exceeded this
922 * threshold then there is most likely a hardware issue. */
923 #define PM_TIMEOUT (5*HZ) /* 5s */
924
kbase_pm_check_transitions_sync(struct kbase_device * kbdev)925 void kbase_pm_check_transitions_sync(struct kbase_device *kbdev)
926 {
927 unsigned long flags;
928 unsigned long timeout;
929 bool cores_are_available;
930 int ret;
931
932 /* Force the transition to be checked and reported - the cores may be
933 * 'available' (for job submission) but not fully powered up. */
934 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
935
936 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
937
938 /* Don't need 'cores_are_available', because we don't return anything */
939 CSTD_UNUSED(cores_are_available);
940 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
941
942 timeout = jiffies + PM_TIMEOUT;
943
944 /* Wait for cores */
945 ret = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait,
946 kbdev->pm.backend.gpu_in_desired_state);
947
948 if (ret < 0 && time_after(jiffies, timeout)) {
949 dev_err(kbdev->dev, "Power transition timed out unexpectedly\n");
950 dev_err(kbdev->dev, "Desired state :\n");
951 dev_err(kbdev->dev, "\tShader=%016llx\n",
952 kbdev->pm.backend.desired_shader_state);
953 dev_err(kbdev->dev, "\tTiler =%016llx\n",
954 kbdev->pm.backend.desired_tiler_state);
955 dev_err(kbdev->dev, "Current state :\n");
956 dev_err(kbdev->dev, "\tShader=%08x%08x\n",
957 kbase_reg_read(kbdev,
958 GPU_CONTROL_REG(SHADER_READY_HI), NULL),
959 kbase_reg_read(kbdev,
960 GPU_CONTROL_REG(SHADER_READY_LO),
961 NULL));
962 dev_err(kbdev->dev, "\tTiler =%08x%08x\n",
963 kbase_reg_read(kbdev,
964 GPU_CONTROL_REG(TILER_READY_HI), NULL),
965 kbase_reg_read(kbdev,
966 GPU_CONTROL_REG(TILER_READY_LO), NULL));
967 dev_err(kbdev->dev, "\tL2 =%08x%08x\n",
968 kbase_reg_read(kbdev,
969 GPU_CONTROL_REG(L2_READY_HI), NULL),
970 kbase_reg_read(kbdev,
971 GPU_CONTROL_REG(L2_READY_LO), NULL));
972 dev_err(kbdev->dev, "Cores transitioning :\n");
973 dev_err(kbdev->dev, "\tShader=%08x%08x\n",
974 kbase_reg_read(kbdev, GPU_CONTROL_REG(
975 SHADER_PWRTRANS_HI), NULL),
976 kbase_reg_read(kbdev, GPU_CONTROL_REG(
977 SHADER_PWRTRANS_LO), NULL));
978 dev_err(kbdev->dev, "\tTiler =%08x%08x\n",
979 kbase_reg_read(kbdev, GPU_CONTROL_REG(
980 TILER_PWRTRANS_HI), NULL),
981 kbase_reg_read(kbdev, GPU_CONTROL_REG(
982 TILER_PWRTRANS_LO), NULL));
983 dev_err(kbdev->dev, "\tL2 =%08x%08x\n",
984 kbase_reg_read(kbdev, GPU_CONTROL_REG(
985 L2_PWRTRANS_HI), NULL),
986 kbase_reg_read(kbdev, GPU_CONTROL_REG(
987 L2_PWRTRANS_LO), NULL));
988 #if KBASE_GPU_RESET_EN
989 dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n");
990 if (kbase_prepare_to_reset_gpu(kbdev))
991 kbase_reset_gpu(kbdev);
992 #endif /* KBASE_GPU_RESET_EN */
993 } else {
994 /* Log timelining information that a change in state has
995 * completed */
996 kbase_timeline_pm_handle_event(kbdev,
997 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
998 }
999 }
1000 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync);
1001
kbase_pm_enable_interrupts(struct kbase_device * kbdev)1002 void kbase_pm_enable_interrupts(struct kbase_device *kbdev)
1003 {
1004 unsigned long flags;
1005
1006 KBASE_DEBUG_ASSERT(NULL != kbdev);
1007 /*
1008 * Clear all interrupts,
1009 * and unmask them all.
1010 */
1011 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1012 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
1013 NULL);
1014 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL,
1015 NULL);
1016 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1017
1018 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
1019 NULL);
1020 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL);
1021
1022 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
1023 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL);
1024 }
1025
1026 KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts);
1027
kbase_pm_disable_interrupts_nolock(struct kbase_device * kbdev)1028 void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev)
1029 {
1030 KBASE_DEBUG_ASSERT(NULL != kbdev);
1031 /*
1032 * Mask all interrupts,
1033 * and clear them all.
1034 */
1035 lockdep_assert_held(&kbdev->hwaccess_lock);
1036
1037 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL);
1038 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
1039 NULL);
1040 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL);
1041 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
1042 NULL);
1043
1044 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
1045 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
1046 }
1047
kbase_pm_disable_interrupts(struct kbase_device * kbdev)1048 void kbase_pm_disable_interrupts(struct kbase_device *kbdev)
1049 {
1050 unsigned long flags;
1051
1052 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1053 kbase_pm_disable_interrupts_nolock(kbdev);
1054 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1055 }
1056
1057 KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts);
1058
1059
1060 /*
1061 * pmu layout:
1062 * 0x0000: PMU TAG (RO) (0xCAFECAFE)
1063 * 0x0004: PMU VERSION ID (RO) (0x00000000)
1064 * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE)
1065 */
kbase_pm_clock_on(struct kbase_device * kbdev,bool is_resume)1066 void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
1067 {
1068 bool reset_required = is_resume;
1069 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1070 unsigned long flags;
1071
1072 KBASE_DEBUG_ASSERT(NULL != kbdev);
1073 lockdep_assert_held(&js_devdata->runpool_mutex);
1074 lockdep_assert_held(&kbdev->pm.lock);
1075
1076 if (kbdev->pm.backend.gpu_powered) {
1077 /* Already turned on */
1078 if (kbdev->poweroff_pending)
1079 kbase_pm_enable_interrupts(kbdev);
1080 kbdev->poweroff_pending = false;
1081 KBASE_DEBUG_ASSERT(!is_resume);
1082 return;
1083 }
1084
1085 kbdev->poweroff_pending = false;
1086
1087 KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
1088
1089 if (is_resume && kbdev->pm.backend.callback_power_resume) {
1090 kbdev->pm.backend.callback_power_resume(kbdev);
1091 return;
1092 } else if (kbdev->pm.backend.callback_power_on) {
1093 kbdev->pm.backend.callback_power_on(kbdev);
1094 /* If your platform properly keeps the GPU state you may use the
1095 * return value of the callback_power_on function to
1096 * conditionally reset the GPU on power up. Currently we are
1097 * conservative and always reset the GPU. */
1098 reset_required = true;
1099 }
1100
1101 spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
1102 kbdev->pm.backend.gpu_powered = true;
1103 spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
1104
1105 if (reset_required) {
1106 /* GPU state was lost, reset GPU to ensure it is in a
1107 * consistent state */
1108 kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS);
1109 }
1110
1111 mutex_lock(&kbdev->mmu_hw_mutex);
1112 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1113 kbase_ctx_sched_restore_all_as(kbdev);
1114 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1115 mutex_unlock(&kbdev->mmu_hw_mutex);
1116
1117 /* Lastly, enable the interrupts */
1118 kbase_pm_enable_interrupts(kbdev);
1119 }
1120
1121 KBASE_EXPORT_TEST_API(kbase_pm_clock_on);
1122
kbase_pm_clock_off(struct kbase_device * kbdev,bool is_suspend)1123 bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend)
1124 {
1125 unsigned long flags;
1126
1127 KBASE_DEBUG_ASSERT(NULL != kbdev);
1128 lockdep_assert_held(&kbdev->pm.lock);
1129
1130 /* ASSERT that the cores should now be unavailable. No lock needed. */
1131 KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u);
1132
1133 kbdev->poweroff_pending = true;
1134
1135 if (!kbdev->pm.backend.gpu_powered) {
1136 /* Already turned off */
1137 if (is_suspend && kbdev->pm.backend.callback_power_suspend)
1138 kbdev->pm.backend.callback_power_suspend(kbdev);
1139 return true;
1140 }
1141
1142 KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
1143
1144 /* Disable interrupts. This also clears any outstanding interrupts */
1145 kbase_pm_disable_interrupts(kbdev);
1146 /* Ensure that any IRQ handlers have finished */
1147 kbase_synchronize_irqs(kbdev);
1148
1149 spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
1150
1151 if (atomic_read(&kbdev->faults_pending)) {
1152 /* Page/bus faults are still being processed. The GPU can not
1153 * be powered off until they have completed */
1154 spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
1155 flags);
1156 return false;
1157 }
1158
1159 kbase_pm_cache_snoop_disable(kbdev);
1160
1161 /* The GPU power may be turned off from this point */
1162 kbdev->pm.backend.gpu_powered = false;
1163 spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
1164
1165 if (is_suspend && kbdev->pm.backend.callback_power_suspend)
1166 kbdev->pm.backend.callback_power_suspend(kbdev);
1167 else if (kbdev->pm.backend.callback_power_off)
1168 kbdev->pm.backend.callback_power_off(kbdev);
1169 return true;
1170 }
1171
1172 KBASE_EXPORT_TEST_API(kbase_pm_clock_off);
1173
1174 struct kbasep_reset_timeout_data {
1175 struct hrtimer timer;
1176 bool timed_out;
1177 struct kbase_device *kbdev;
1178 };
1179
kbase_pm_reset_done(struct kbase_device * kbdev)1180 void kbase_pm_reset_done(struct kbase_device *kbdev)
1181 {
1182 KBASE_DEBUG_ASSERT(kbdev != NULL);
1183 kbdev->pm.backend.reset_done = true;
1184 wake_up(&kbdev->pm.backend.reset_done_wait);
1185 }
1186
1187 /**
1188 * kbase_pm_wait_for_reset - Wait for a reset to happen
1189 *
1190 * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state.
1191 *
1192 * @kbdev: Kbase device
1193 */
kbase_pm_wait_for_reset(struct kbase_device * kbdev)1194 static void kbase_pm_wait_for_reset(struct kbase_device *kbdev)
1195 {
1196 lockdep_assert_held(&kbdev->pm.lock);
1197
1198 wait_event(kbdev->pm.backend.reset_done_wait,
1199 (kbdev->pm.backend.reset_done));
1200 kbdev->pm.backend.reset_done = false;
1201 }
1202
1203 KBASE_EXPORT_TEST_API(kbase_pm_reset_done);
1204
kbasep_reset_timeout(struct hrtimer * timer)1205 static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
1206 {
1207 struct kbasep_reset_timeout_data *rtdata =
1208 container_of(timer, struct kbasep_reset_timeout_data, timer);
1209
1210 rtdata->timed_out = 1;
1211
1212 /* Set the wait queue to wake up kbase_pm_init_hw even though the reset
1213 * hasn't completed */
1214 kbase_pm_reset_done(rtdata->kbdev);
1215
1216 return HRTIMER_NORESTART;
1217 }
1218
kbase_pm_hw_issues_detect(struct kbase_device * kbdev)1219 static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
1220 {
1221 struct device_node *np = kbdev->dev->of_node;
1222 u32 jm_values[4];
1223 const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
1224 const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
1225 GPU_ID_VERSION_PRODUCT_ID_SHIFT;
1226 const u32 major = (gpu_id & GPU_ID_VERSION_MAJOR) >>
1227 GPU_ID_VERSION_MAJOR_SHIFT;
1228
1229 kbdev->hw_quirks_sc = 0;
1230
1231 /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443.
1232 * and needed due to MIDGLES-3539. See PRLAM-11035 */
1233 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) ||
1234 kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035))
1235 kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE;
1236
1237 /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327.
1238 */
1239 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327))
1240 kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD;
1241
1242 #ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY
1243 /* Enable alternative hardware counter selection if configured. */
1244 if (!GPU_ID_IS_NEW_FORMAT(prod_id))
1245 kbdev->hw_quirks_sc |= SC_ALT_COUNTERS;
1246 #endif
1247
1248 /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */
1249 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797))
1250 kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS;
1251
1252 if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) {
1253 if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */
1254 kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE;
1255 else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */
1256 kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES;
1257 }
1258
1259 if (!kbdev->hw_quirks_sc)
1260 kbdev->hw_quirks_sc = kbase_reg_read(kbdev,
1261 GPU_CONTROL_REG(SHADER_CONFIG), NULL);
1262
1263 kbdev->hw_quirks_tiler = kbase_reg_read(kbdev,
1264 GPU_CONTROL_REG(TILER_CONFIG), NULL);
1265
1266 /* Set tiler clock gate override if required */
1267 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953))
1268 kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
1269
1270 /* Limit the GPU bus bandwidth if the platform needs this. */
1271 kbdev->hw_quirks_mmu = kbase_reg_read(kbdev,
1272 GPU_CONTROL_REG(L2_MMU_CONFIG), NULL);
1273
1274 /* Limit read ID width for AXI */
1275 kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS);
1276 kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) <<
1277 L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT;
1278
1279 /* Limit write ID width for AXI */
1280 kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
1281 kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) <<
1282 L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT;
1283
1284 if (kbdev->system_coherency == COHERENCY_ACE) {
1285 /* Allow memory configuration disparity to be ignored, we
1286 * optimize the use of shared memory and thus we expect
1287 * some disparity in the memory configuration */
1288 kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY;
1289 }
1290
1291 kbdev->hw_quirks_jm = 0;
1292 /* Only for T86x/T88x-based products after r2p0 */
1293 if (prod_id >= 0x860 && prod_id <= 0x880 && major >= 2) {
1294
1295 if (of_property_read_u32_array(np,
1296 "jm_config",
1297 &jm_values[0],
1298 ARRAY_SIZE(jm_values))) {
1299 /* Entry not in device tree, use defaults */
1300 jm_values[0] = 0;
1301 jm_values[1] = 0;
1302 jm_values[2] = 0;
1303 jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT;
1304 }
1305
1306 /* Limit throttle limit to 6 bits*/
1307 if (jm_values[3] > JM_MAX_JOB_THROTTLE_LIMIT) {
1308 dev_dbg(kbdev->dev, "JOB_THROTTLE_LIMIT supplied in device tree is too large. Limiting to MAX (63).");
1309 jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT;
1310 }
1311
1312 /* Aggregate to one integer. */
1313 kbdev->hw_quirks_jm |= (jm_values[0] ?
1314 JM_TIMESTAMP_OVERRIDE : 0);
1315 kbdev->hw_quirks_jm |= (jm_values[1] ?
1316 JM_CLOCK_GATE_OVERRIDE : 0);
1317 kbdev->hw_quirks_jm |= (jm_values[2] ?
1318 JM_JOB_THROTTLE_ENABLE : 0);
1319 kbdev->hw_quirks_jm |= (jm_values[3] <<
1320 JM_JOB_THROTTLE_LIMIT_SHIFT);
1321
1322 } else if (GPU_ID_IS_NEW_FORMAT(prod_id) &&
1323 (GPU_ID2_MODEL_MATCH_VALUE(prod_id) ==
1324 GPU_ID2_PRODUCT_TMIX)) {
1325 /* Only for tMIx */
1326 u32 coherency_features;
1327
1328 coherency_features = kbase_reg_read(kbdev,
1329 GPU_CONTROL_REG(COHERENCY_FEATURES), NULL);
1330
1331 /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly
1332 * documented for tMIx so force correct value here.
1333 */
1334 if (coherency_features ==
1335 COHERENCY_FEATURE_BIT(COHERENCY_ACE)) {
1336 kbdev->hw_quirks_jm |=
1337 (COHERENCY_ACE_LITE | COHERENCY_ACE) <<
1338 JM_FORCE_COHERENCY_FEATURES_SHIFT;
1339 }
1340 }
1341
1342 if (!kbdev->hw_quirks_jm)
1343 kbdev->hw_quirks_jm = kbase_reg_read(kbdev,
1344 GPU_CONTROL_REG(JM_CONFIG), NULL);
1345
1346 #ifdef CONFIG_MALI_CORESTACK
1347 #define MANUAL_POWER_CONTROL ((u32)(1 << 8))
1348 kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL;
1349 #endif /* CONFIG_MALI_CORESTACK */
1350 }
1351
kbase_pm_hw_issues_apply(struct kbase_device * kbdev)1352 static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev)
1353 {
1354 kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG),
1355 kbdev->hw_quirks_sc, NULL);
1356
1357 kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG),
1358 kbdev->hw_quirks_tiler, NULL);
1359
1360 kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG),
1361 kbdev->hw_quirks_mmu, NULL);
1362
1363 kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG),
1364 kbdev->hw_quirks_jm, NULL);
1365
1366 }
1367
kbase_pm_cache_snoop_enable(struct kbase_device * kbdev)1368 void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev)
1369 {
1370 if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) &&
1371 !kbdev->cci_snoop_enabled) {
1372 #ifdef CONFIG_ARM64
1373 if (kbdev->snoop_enable_smc != 0)
1374 kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0);
1375 #endif /* CONFIG_ARM64 */
1376 dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n");
1377 kbdev->cci_snoop_enabled = true;
1378 }
1379 }
1380
kbase_pm_cache_snoop_disable(struct kbase_device * kbdev)1381 void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev)
1382 {
1383 if (kbdev->cci_snoop_enabled) {
1384 #ifdef CONFIG_ARM64
1385 if (kbdev->snoop_disable_smc != 0) {
1386 mali_cci_flush_l2(kbdev);
1387 kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0);
1388 }
1389 #endif /* CONFIG_ARM64 */
1390 dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n");
1391 kbdev->cci_snoop_enabled = false;
1392 }
1393 }
1394
kbase_pm_do_reset(struct kbase_device * kbdev)1395 static int kbase_pm_do_reset(struct kbase_device *kbdev)
1396 {
1397 struct kbasep_reset_timeout_data rtdata;
1398
1399 KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
1400
1401 KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev);
1402
1403 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1404 GPU_COMMAND_SOFT_RESET, NULL);
1405
1406 /* Unmask the reset complete interrupt only */
1407 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED,
1408 NULL);
1409
1410 /* Initialize a structure for tracking the status of the reset */
1411 rtdata.kbdev = kbdev;
1412 rtdata.timed_out = 0;
1413
1414 /* Create a timer to use as a timeout on the reset */
1415 hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1416 rtdata.timer.function = kbasep_reset_timeout;
1417
1418 hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
1419 HRTIMER_MODE_REL);
1420
1421 /* Wait for the RESET_COMPLETED interrupt to be raised */
1422 kbase_pm_wait_for_reset(kbdev);
1423
1424 if (rtdata.timed_out == 0) {
1425 /* GPU has been reset */
1426 hrtimer_cancel(&rtdata.timer);
1427 destroy_hrtimer_on_stack(&rtdata.timer);
1428 return 0;
1429 }
1430
1431 /* No interrupt has been received - check if the RAWSTAT register says
1432 * the reset has completed */
1433 if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1434 RESET_COMPLETED) {
1435 /* The interrupt is set in the RAWSTAT; this suggests that the
1436 * interrupts are not getting to the CPU */
1437 dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
1438 /* If interrupts aren't working we can't continue. */
1439 destroy_hrtimer_on_stack(&rtdata.timer);
1440 return -EINVAL;
1441 }
1442
1443 /* The GPU doesn't seem to be responding to the reset so try a hard
1444 * reset */
1445 dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n",
1446 RESET_TIMEOUT);
1447 KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
1448 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1449 GPU_COMMAND_HARD_RESET, NULL);
1450
1451 /* Restart the timer to wait for the hard reset to complete */
1452 rtdata.timed_out = 0;
1453
1454 hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
1455 HRTIMER_MODE_REL);
1456
1457 /* Wait for the RESET_COMPLETED interrupt to be raised */
1458 kbase_pm_wait_for_reset(kbdev);
1459
1460 if (rtdata.timed_out == 0) {
1461 /* GPU has been reset */
1462 hrtimer_cancel(&rtdata.timer);
1463 destroy_hrtimer_on_stack(&rtdata.timer);
1464 return 0;
1465 }
1466
1467 destroy_hrtimer_on_stack(&rtdata.timer);
1468
1469 dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n",
1470 RESET_TIMEOUT);
1471
1472 return -EINVAL;
1473 }
1474
kbasep_protected_mode_enable(struct protected_mode_device * pdev)1475 static int kbasep_protected_mode_enable(struct protected_mode_device *pdev)
1476 {
1477 struct kbase_device *kbdev = pdev->data;
1478
1479 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1480 GPU_COMMAND_SET_PROTECTED_MODE, NULL);
1481 return 0;
1482 }
1483
kbasep_protected_mode_disable(struct protected_mode_device * pdev)1484 static int kbasep_protected_mode_disable(struct protected_mode_device *pdev)
1485 {
1486 struct kbase_device *kbdev = pdev->data;
1487
1488 lockdep_assert_held(&kbdev->pm.lock);
1489
1490 return kbase_pm_do_reset(kbdev);
1491 }
1492
1493 struct protected_mode_ops kbase_native_protected_ops = {
1494 .protected_mode_enable = kbasep_protected_mode_enable,
1495 .protected_mode_disable = kbasep_protected_mode_disable
1496 };
1497
kbase_pm_init_hw(struct kbase_device * kbdev,unsigned int flags)1498 int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
1499 {
1500 unsigned long irq_flags;
1501 int err;
1502 bool resume_vinstr = false;
1503
1504 KBASE_DEBUG_ASSERT(NULL != kbdev);
1505 lockdep_assert_held(&kbdev->pm.lock);
1506
1507 /* Ensure the clock is on before attempting to access the hardware */
1508 if (!kbdev->pm.backend.gpu_powered) {
1509 if (kbdev->pm.backend.callback_power_on)
1510 kbdev->pm.backend.callback_power_on(kbdev);
1511
1512 spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock,
1513 irq_flags);
1514 kbdev->pm.backend.gpu_powered = true;
1515 spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
1516 irq_flags);
1517 }
1518
1519 /* Ensure interrupts are off to begin with, this also clears any
1520 * outstanding interrupts */
1521 kbase_pm_disable_interrupts(kbdev);
1522 /* Ensure cache snoops are disabled before reset. */
1523 kbase_pm_cache_snoop_disable(kbdev);
1524 /* Prepare for the soft-reset */
1525 kbdev->pm.backend.reset_done = false;
1526
1527 /* The cores should be made unavailable due to the reset */
1528 spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
1529 if (kbdev->shader_available_bitmap != 0u)
1530 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
1531 NULL, 0u, (u32)0u);
1532 if (kbdev->tiler_available_bitmap != 0u)
1533 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
1534 NULL, NULL, 0u, (u32)0u);
1535 kbdev->shader_available_bitmap = 0u;
1536 kbdev->tiler_available_bitmap = 0u;
1537 kbdev->l2_available_bitmap = 0u;
1538 spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
1539
1540 /* Soft reset the GPU */
1541 if (kbdev->protected_mode_support)
1542 err = kbdev->protected_ops->protected_mode_disable(
1543 kbdev->protected_dev);
1544 else
1545 err = kbase_pm_do_reset(kbdev);
1546
1547 spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
1548 if (kbdev->protected_mode)
1549 resume_vinstr = true;
1550 kbdev->protected_mode = false;
1551 kbase_ipa_model_use_configured_locked(kbdev);
1552
1553 spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
1554
1555 if (err)
1556 goto exit;
1557
1558 if (flags & PM_HW_ISSUES_DETECT)
1559 kbase_pm_hw_issues_detect(kbdev);
1560
1561 kbase_pm_hw_issues_apply(kbdev);
1562 kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency);
1563
1564 /* Sanity check protected mode was left after reset */
1565 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) {
1566 u32 gpu_status = kbase_reg_read(kbdev,
1567 GPU_CONTROL_REG(GPU_STATUS), NULL);
1568
1569 WARN_ON(gpu_status & GPU_STATUS_PROTECTED_MODE_ACTIVE);
1570 }
1571
1572 /* If cycle counter was in use re-enable it, enable_irqs will only be
1573 * false when called from kbase_pm_powerup */
1574 if (kbdev->pm.backend.gpu_cycle_counter_requests &&
1575 (flags & PM_ENABLE_IRQS)) {
1576 /* enable interrupts as the L2 may have to be powered on */
1577 kbase_pm_enable_interrupts(kbdev);
1578 kbase_pm_request_l2_caches(kbdev);
1579
1580 /* Re-enable the counters if we need to */
1581 spin_lock_irqsave(
1582 &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1583 irq_flags);
1584 if (kbdev->pm.backend.gpu_cycle_counter_requests)
1585 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1586 GPU_COMMAND_CYCLE_COUNT_START, NULL);
1587 spin_unlock_irqrestore(
1588 &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1589 irq_flags);
1590
1591 spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
1592 kbase_pm_release_l2_caches(kbdev);
1593 spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
1594
1595 kbase_pm_disable_interrupts(kbdev);
1596 }
1597
1598 if (flags & PM_ENABLE_IRQS)
1599 kbase_pm_enable_interrupts(kbdev);
1600
1601 exit:
1602 /* If GPU is leaving protected mode resume vinstr operation. */
1603 if (kbdev->vinstr_ctx && resume_vinstr)
1604 kbase_vinstr_resume(kbdev->vinstr_ctx);
1605
1606 return err;
1607 }
1608
1609 /**
1610 * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters
1611 *
1612 * Increase the count of cycle counter users and turn the cycle counters on if
1613 * they were previously off
1614 *
1615 * This function is designed to be called by
1616 * kbase_pm_request_gpu_cycle_counter() or
1617 * kbase_pm_request_gpu_cycle_counter_l2_is_on() only
1618 *
1619 * When this function is called the l2 cache must be on and the l2 cache users
1620 * count must have been incremented by a call to (
1621 * kbase_pm_request_l2_caches() or kbase_pm_request_l2_caches_l2_on() )
1622 *
1623 * @kbdev: The kbase device structure of the device
1624 */
1625 static void
kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device * kbdev)1626 kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
1627 {
1628 unsigned long flags;
1629
1630 spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1631 flags);
1632
1633 ++kbdev->pm.backend.gpu_cycle_counter_requests;
1634
1635 if (1 == kbdev->pm.backend.gpu_cycle_counter_requests)
1636 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1637 GPU_COMMAND_CYCLE_COUNT_START, NULL);
1638
1639 spin_unlock_irqrestore(
1640 &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1641 flags);
1642 }
1643
kbase_pm_request_gpu_cycle_counter(struct kbase_device * kbdev)1644 void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev)
1645 {
1646 KBASE_DEBUG_ASSERT(kbdev != NULL);
1647
1648 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
1649
1650 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
1651 INT_MAX);
1652
1653 kbase_pm_request_l2_caches(kbdev);
1654
1655 kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
1656 }
1657
1658 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter);
1659
kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device * kbdev)1660 void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev)
1661 {
1662 KBASE_DEBUG_ASSERT(kbdev != NULL);
1663
1664 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
1665
1666 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
1667 INT_MAX);
1668
1669 kbase_pm_request_l2_caches_l2_is_on(kbdev);
1670
1671 kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
1672 }
1673
1674 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on);
1675
kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device * kbdev)1676 void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev)
1677 {
1678 unsigned long flags;
1679
1680 KBASE_DEBUG_ASSERT(kbdev != NULL);
1681
1682 lockdep_assert_held(&kbdev->hwaccess_lock);
1683
1684 spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1685 flags);
1686
1687 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0);
1688
1689 --kbdev->pm.backend.gpu_cycle_counter_requests;
1690
1691 if (0 == kbdev->pm.backend.gpu_cycle_counter_requests)
1692 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1693 GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
1694
1695 spin_unlock_irqrestore(
1696 &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1697 flags);
1698
1699 kbase_pm_release_l2_caches(kbdev);
1700 }
1701
kbase_pm_release_gpu_cycle_counter(struct kbase_device * kbdev)1702 void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev)
1703 {
1704 unsigned long flags;
1705
1706 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1707
1708 kbase_pm_release_gpu_cycle_counter_nolock(kbdev);
1709
1710 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1711 }
1712
1713 KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter);
1714