1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /*
23 * Base kernel Power Management hardware control
24 */
25
26 #include <mali_kbase.h>
27 #include <mali_kbase_config_defaults.h>
28 #include <gpu/mali_kbase_gpu_regmap.h>
29 #include <tl/mali_kbase_tracepoints.h>
30 #include <mali_kbase_pm.h>
31 #include <mali_kbase_config_defaults.h>
32 #include <mali_kbase_smc.h>
33
34 #if MALI_USE_CSF
35 #include <csf/ipa_control/mali_kbase_csf_ipa_control.h>
36 #else
37 #include <mali_kbase_hwaccess_jm.h>
38 #endif /* !MALI_USE_CSF */
39
40 #include <mali_kbase_reset_gpu.h>
41 #include <mali_kbase_ctx_sched.h>
42 #include <hwcnt/mali_kbase_hwcnt_context.h>
43 #include <mali_kbase_pbha.h>
44 #include <backend/gpu/mali_kbase_cache_policy_backend.h>
45 #include <device/mali_kbase_device.h>
46 #include <backend/gpu/mali_kbase_irq_internal.h>
47 #include <backend/gpu/mali_kbase_pm_internal.h>
48 #include <backend/gpu/mali_kbase_l2_mmu_config.h>
49 #include <mali_kbase_dummy_job_wa.h>
50 #ifdef CONFIG_MALI_ARBITER_SUPPORT
51 #include <arbiter/mali_kbase_arbiter_pm.h>
52 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
53 #if MALI_USE_CSF
54 #include <csf/ipa_control/mali_kbase_csf_ipa_control.h>
55 #endif
56
57 #if MALI_USE_CSF
58 #include <linux/delay.h>
59 #endif
60
61 #include <linux/of.h>
62
63 #ifdef CONFIG_MALI_CORESTACK
64 bool corestack_driver_control = true;
65 #else
66 bool corestack_driver_control; /* Default value of 0/false */
67 #endif
68 module_param(corestack_driver_control, bool, 0444);
69 MODULE_PARM_DESC(corestack_driver_control,
70 "Let the driver power on/off the GPU core stack independently "
71 "without involving the Power Domain Controller. This should "
72 "only be enabled on platforms for which integration of the PDC "
73 "to the Mali GPU is known to be problematic.");
74 KBASE_EXPORT_TEST_API(corestack_driver_control);
75
76 /**
77 * enum kbasep_pm_action - Actions that can be performed on a core.
78 *
79 * @ACTION_PRESENT: The cores that are present
80 * @ACTION_READY: The cores that are ready
81 * @ACTION_PWRON: Power on the cores specified
82 * @ACTION_PWROFF: Power off the cores specified
83 * @ACTION_PWRTRANS: The cores that are transitioning
84 * @ACTION_PWRACTIVE: The cores that are active
85 *
86 * This enumeration is private to the file. Its values are set to allow
87 * core_type_to_reg() function, which decodes this enumeration, to be simpler
88 * and more efficient.
89 */
90 enum kbasep_pm_action {
91 ACTION_PRESENT = 0,
92 ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO),
93 ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO),
94 ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO),
95 ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO),
96 ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO)
97 };
98
99 static u64 kbase_pm_get_state(
100 struct kbase_device *kbdev,
101 enum kbase_pm_core_type core_type,
102 enum kbasep_pm_action action);
103
104 static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev);
105
106 #if MALI_USE_CSF
kbase_pm_is_mcu_desired(struct kbase_device * kbdev)107 bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev)
108 {
109 lockdep_assert_held(&kbdev->hwaccess_lock);
110
111 if (unlikely(!kbdev->csf.firmware_inited))
112 return false;
113
114 if (kbdev->csf.scheduler.pm_active_count &&
115 kbdev->pm.backend.mcu_desired)
116 return true;
117
118 #ifdef KBASE_PM_RUNTIME
119 if (kbdev->pm.backend.gpu_wakeup_override)
120 return true;
121 #endif
122
123 /* MCU is supposed to be ON, only when scheduler.pm_active_count is
124 * non zero. But for always_on policy, the MCU needs to be kept on,
125 * unless policy changing transition needs it off.
126 */
127
128 return (kbdev->pm.backend.mcu_desired &&
129 kbase_pm_no_mcu_core_pwroff(kbdev) &&
130 !kbdev->pm.backend.policy_change_clamp_state_to_off);
131 }
132 #endif
133
kbase_pm_is_l2_desired(struct kbase_device * kbdev)134 bool kbase_pm_is_l2_desired(struct kbase_device *kbdev)
135 {
136 #if !MALI_USE_CSF
137 if (kbdev->pm.backend.protected_entry_transition_override)
138 return false;
139
140 if (kbdev->pm.backend.protected_transition_override &&
141 kbdev->pm.backend.protected_l2_override)
142 return true;
143
144 if (kbdev->pm.backend.protected_transition_override &&
145 !kbdev->pm.backend.shaders_desired)
146 return false;
147 #else
148 if (unlikely(kbdev->pm.backend.policy_change_clamp_state_to_off))
149 return false;
150
151 /* Power up the L2 cache only when MCU is desired */
152 if (likely(kbdev->csf.firmware_inited))
153 return kbase_pm_is_mcu_desired(kbdev);
154 #endif
155
156 return kbdev->pm.backend.l2_desired;
157 }
158
159 #if !MALI_USE_CSF
kbase_pm_protected_override_enable(struct kbase_device * kbdev)160 void kbase_pm_protected_override_enable(struct kbase_device *kbdev)
161 {
162 lockdep_assert_held(&kbdev->hwaccess_lock);
163
164 kbdev->pm.backend.protected_transition_override = true;
165 }
kbase_pm_protected_override_disable(struct kbase_device * kbdev)166 void kbase_pm_protected_override_disable(struct kbase_device *kbdev)
167 {
168 lockdep_assert_held(&kbdev->hwaccess_lock);
169
170 kbdev->pm.backend.protected_transition_override = false;
171 }
172
kbase_pm_protected_entry_override_enable(struct kbase_device * kbdev)173 int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev)
174 {
175 lockdep_assert_held(&kbdev->hwaccess_lock);
176
177 WARN_ON(!kbdev->protected_mode_transition);
178
179 if (kbdev->pm.backend.l2_always_on &&
180 (kbdev->system_coherency == COHERENCY_ACE)) {
181 WARN_ON(kbdev->pm.backend.protected_entry_transition_override);
182
183 /*
184 * If there is already a GPU reset pending then wait for it to
185 * complete before initiating a special reset for protected
186 * mode entry.
187 */
188 if (kbase_reset_gpu_silent(kbdev))
189 return -EAGAIN;
190
191 kbdev->pm.backend.protected_entry_transition_override = true;
192 }
193
194 return 0;
195 }
196
kbase_pm_protected_entry_override_disable(struct kbase_device * kbdev)197 void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev)
198 {
199 lockdep_assert_held(&kbdev->hwaccess_lock);
200
201 WARN_ON(!kbdev->protected_mode_transition);
202
203 if (kbdev->pm.backend.l2_always_on &&
204 (kbdev->system_coherency == COHERENCY_ACE)) {
205 WARN_ON(!kbdev->pm.backend.protected_entry_transition_override);
206
207 kbdev->pm.backend.protected_entry_transition_override = false;
208 }
209 }
210
kbase_pm_protected_l2_override(struct kbase_device * kbdev,bool override)211 void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override)
212 {
213 lockdep_assert_held(&kbdev->hwaccess_lock);
214
215 if (override) {
216 kbdev->pm.backend.protected_l2_override++;
217 WARN_ON(kbdev->pm.backend.protected_l2_override <= 0);
218 } else {
219 kbdev->pm.backend.protected_l2_override--;
220 WARN_ON(kbdev->pm.backend.protected_l2_override < 0);
221 }
222
223 kbase_pm_update_state(kbdev);
224 }
225 #endif
226
227 /**
228 * core_type_to_reg - Decode a core type and action to a register.
229 *
230 * @core_type: The type of core
231 * @action: The type of action
232 *
233 * Given a core type (defined by kbase_pm_core_type) and an action (defined
234 * by kbasep_pm_action) this function will return the register offset that
235 * will perform the action on the core type. The register returned is the _LO
236 * register and an offset must be applied to use the _HI register.
237 *
238 * Return: The register offset of the _LO register that performs an action of
239 * type @action on a core of type @core_type.
240 */
core_type_to_reg(enum kbase_pm_core_type core_type,enum kbasep_pm_action action)241 static u32 core_type_to_reg(enum kbase_pm_core_type core_type,
242 enum kbasep_pm_action action)
243 {
244 if (corestack_driver_control) {
245 if (core_type == KBASE_PM_CORE_STACK) {
246 switch (action) {
247 case ACTION_PRESENT:
248 return STACK_PRESENT_LO;
249 case ACTION_READY:
250 return STACK_READY_LO;
251 case ACTION_PWRON:
252 return STACK_PWRON_LO;
253 case ACTION_PWROFF:
254 return STACK_PWROFF_LO;
255 case ACTION_PWRTRANS:
256 return STACK_PWRTRANS_LO;
257 default:
258 WARN(1, "Invalid action for core type\n");
259 }
260 }
261 }
262
263 return (u32)core_type + (u32)action;
264 }
265
266 #if IS_ENABLED(CONFIG_ARM64)
mali_cci_flush_l2(struct kbase_device * kbdev)267 static void mali_cci_flush_l2(struct kbase_device *kbdev)
268 {
269 const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED;
270 u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
271 u32 raw;
272
273 /*
274 * Note that we don't take the cache flush mutex here since
275 * we expect to be the last user of the L2, all other L2 users
276 * would have dropped their references, to initiate L2 power
277 * down, L2 power down being the only valid place for this
278 * to be called from.
279 */
280
281 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
282 GPU_COMMAND_CACHE_CLN_INV_L2);
283
284 raw = kbase_reg_read(kbdev,
285 GPU_CONTROL_REG(GPU_IRQ_RAWSTAT));
286
287 /* Wait for cache flush to complete before continuing, exit on
288 * gpu resets or loop expiry.
289 */
290 while (((raw & mask) == 0) && --loops) {
291 raw = kbase_reg_read(kbdev,
292 GPU_CONTROL_REG(GPU_IRQ_RAWSTAT));
293 }
294 }
295 #endif
296
297 /**
298 * kbase_pm_invoke - Invokes an action on a core set
299 *
300 * @kbdev: The kbase device structure of the device
301 * @core_type: The type of core that the action should be performed on
302 * @cores: A bit mask of cores to perform the action on (low 32 bits)
303 * @action: The action to perform on the cores
304 *
305 * This function performs the action given by @action on a set of cores of a
306 * type given by @core_type. It is a static function used by
307 * kbase_pm_transition_core_type()
308 */
kbase_pm_invoke(struct kbase_device * kbdev,enum kbase_pm_core_type core_type,u64 cores,enum kbasep_pm_action action)309 static void kbase_pm_invoke(struct kbase_device *kbdev,
310 enum kbase_pm_core_type core_type,
311 u64 cores,
312 enum kbasep_pm_action action)
313 {
314 u32 reg;
315 u32 lo = cores & 0xFFFFFFFF;
316 u32 hi = (cores >> 32) & 0xFFFFFFFF;
317
318 lockdep_assert_held(&kbdev->hwaccess_lock);
319
320 reg = core_type_to_reg(core_type, action);
321
322 KBASE_DEBUG_ASSERT(reg);
323
324 if (cores) {
325 u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY);
326
327 if (action == ACTION_PWRON)
328 state |= cores;
329 else if (action == ACTION_PWROFF)
330 state &= ~cores;
331 KBASE_TLSTREAM_AUX_PM_STATE(kbdev, core_type, state);
332 }
333
334 /* Tracing */
335 if (cores) {
336 if (action == ACTION_PWRON)
337 switch (core_type) {
338 case KBASE_PM_CORE_SHADER:
339 KBASE_KTRACE_ADD(kbdev, PM_PWRON, NULL, cores);
340 break;
341 case KBASE_PM_CORE_TILER:
342 KBASE_KTRACE_ADD(kbdev, PM_PWRON_TILER, NULL, cores);
343 break;
344 case KBASE_PM_CORE_L2:
345 KBASE_KTRACE_ADD(kbdev, PM_PWRON_L2, NULL, cores);
346 break;
347 default:
348 break;
349 }
350 else if (action == ACTION_PWROFF)
351 switch (core_type) {
352 case KBASE_PM_CORE_SHADER:
353 KBASE_KTRACE_ADD(kbdev, PM_PWROFF, NULL, cores);
354 break;
355 case KBASE_PM_CORE_TILER:
356 KBASE_KTRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, cores);
357 break;
358 case KBASE_PM_CORE_L2:
359 KBASE_KTRACE_ADD(kbdev, PM_PWROFF_L2, NULL, cores);
360 /* disable snoops before L2 is turned off */
361 kbase_pm_cache_snoop_disable(kbdev);
362 break;
363 default:
364 break;
365 }
366 }
367
368 if (kbase_dummy_job_wa_enabled(kbdev) &&
369 action == ACTION_PWRON &&
370 core_type == KBASE_PM_CORE_SHADER &&
371 !(kbdev->dummy_job_wa.flags &
372 KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER)) {
373 kbase_dummy_job_wa_execute(kbdev, cores);
374 } else {
375 if (lo != 0)
376 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo);
377 if (hi != 0)
378 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi);
379 }
380 }
381
382 /**
383 * kbase_pm_get_state - Get information about a core set
384 *
385 * @kbdev: The kbase device structure of the device
386 * @core_type: The type of core that the should be queried
387 * @action: The property of the cores to query
388 *
389 * This function gets information (chosen by @action) about a set of cores of
390 * a type given by @core_type. It is a static function used by
391 * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and
392 * kbase_pm_get_ready_cores().
393 *
394 * Return: A bit mask specifying the state of the cores
395 */
kbase_pm_get_state(struct kbase_device * kbdev,enum kbase_pm_core_type core_type,enum kbasep_pm_action action)396 static u64 kbase_pm_get_state(struct kbase_device *kbdev,
397 enum kbase_pm_core_type core_type,
398 enum kbasep_pm_action action)
399 {
400 u32 reg;
401 u32 lo, hi;
402
403 reg = core_type_to_reg(core_type, action);
404
405 KBASE_DEBUG_ASSERT(reg);
406
407 lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg));
408 hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4));
409
410 return (((u64) hi) << 32) | ((u64) lo);
411 }
412
413 /**
414 * kbase_pm_get_present_cores - Get the cores that are present
415 *
416 * @kbdev: Kbase device
417 * @type: The type of cores to query
418 *
419 * Return: Bitmask of the cores that are present
420 */
kbase_pm_get_present_cores(struct kbase_device * kbdev,enum kbase_pm_core_type type)421 u64 kbase_pm_get_present_cores(struct kbase_device *kbdev,
422 enum kbase_pm_core_type type)
423 {
424 KBASE_DEBUG_ASSERT(kbdev != NULL);
425
426 switch (type) {
427 case KBASE_PM_CORE_L2:
428 return kbdev->gpu_props.curr_config.l2_present;
429 case KBASE_PM_CORE_SHADER:
430 return kbdev->gpu_props.curr_config.shader_present;
431 case KBASE_PM_CORE_TILER:
432 return kbdev->gpu_props.props.raw_props.tiler_present;
433 case KBASE_PM_CORE_STACK:
434 return kbdev->gpu_props.props.raw_props.stack_present;
435 default:
436 break;
437 }
438 KBASE_DEBUG_ASSERT(0);
439
440 return 0;
441 }
442
443 KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores);
444
445 /**
446 * kbase_pm_get_active_cores - Get the cores that are "active"
447 * (busy processing work)
448 *
449 * @kbdev: Kbase device
450 * @type: The type of cores to query
451 *
452 * Return: Bitmask of cores that are active
453 */
kbase_pm_get_active_cores(struct kbase_device * kbdev,enum kbase_pm_core_type type)454 u64 kbase_pm_get_active_cores(struct kbase_device *kbdev,
455 enum kbase_pm_core_type type)
456 {
457 return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE);
458 }
459
460 KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores);
461
462 /**
463 * kbase_pm_get_trans_cores - Get the cores that are transitioning between
464 * power states
465 *
466 * @kbdev: Kbase device
467 * @type: The type of cores to query
468 *
469 * Return: Bitmask of cores that are transitioning
470 */
kbase_pm_get_trans_cores(struct kbase_device * kbdev,enum kbase_pm_core_type type)471 u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev,
472 enum kbase_pm_core_type type)
473 {
474 return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS);
475 }
476
477 KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores);
478
479 /**
480 * kbase_pm_get_ready_cores - Get the cores that are powered on
481 *
482 * @kbdev: Kbase device
483 * @type: The type of cores to query
484 *
485 * Return: Bitmask of cores that are ready (powered on)
486 */
kbase_pm_get_ready_cores(struct kbase_device * kbdev,enum kbase_pm_core_type type)487 u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev,
488 enum kbase_pm_core_type type)
489 {
490 u64 result;
491
492 result = kbase_pm_get_state(kbdev, type, ACTION_READY);
493
494 switch (type) {
495 case KBASE_PM_CORE_SHADER:
496 KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED, NULL, result);
497 break;
498 case KBASE_PM_CORE_TILER:
499 KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, result);
500 break;
501 case KBASE_PM_CORE_L2:
502 KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, result);
503 break;
504 default:
505 break;
506 }
507
508 return result;
509 }
510
511 KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores);
512
kbase_pm_trigger_hwcnt_disable(struct kbase_device * kbdev)513 static void kbase_pm_trigger_hwcnt_disable(struct kbase_device *kbdev)
514 {
515 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
516
517 lockdep_assert_held(&kbdev->hwaccess_lock);
518
519 /* See if we can get away with disabling hwcnt
520 * atomically, otherwise kick off a worker.
521 */
522 if (kbase_hwcnt_context_disable_atomic(kbdev->hwcnt_gpu_ctx)) {
523 backend->hwcnt_disabled = true;
524
525 } else {
526 kbase_hwcnt_context_queue_work(kbdev->hwcnt_gpu_ctx,
527 &backend->hwcnt_disable_work);
528 }
529 }
530
kbase_pm_l2_config_override(struct kbase_device * kbdev)531 static void kbase_pm_l2_config_override(struct kbase_device *kbdev)
532 {
533 u32 val;
534
535 /*
536 * Skip if it is not supported
537 */
538 if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG))
539 return;
540
541 #if MALI_USE_CSF
542 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) {
543 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG));
544 kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_CONFIG),
545 L2_CONFIG_PBHA_HWU_SET(val, kbdev->pbha_propagate_bits));
546 }
547 #endif /* MALI_USE_CSF */
548
549 /*
550 * Skip if size and hash are not given explicitly,
551 * which means default values are used.
552 */
553 if ((kbdev->l2_size_override == 0) && (kbdev->l2_hash_override == 0) &&
554 (!kbdev->l2_hash_values_override))
555 return;
556
557 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG));
558
559 if (kbdev->l2_size_override) {
560 val &= ~L2_CONFIG_SIZE_MASK;
561 val |= (kbdev->l2_size_override << L2_CONFIG_SIZE_SHIFT);
562 }
563
564 if (kbdev->l2_hash_override) {
565 WARN_ON(kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH));
566 val &= ~L2_CONFIG_HASH_MASK;
567 val |= (kbdev->l2_hash_override << L2_CONFIG_HASH_SHIFT);
568 } else if (kbdev->l2_hash_values_override) {
569 int i;
570
571 WARN_ON(!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH));
572 val &= ~L2_CONFIG_ASN_HASH_ENABLE_MASK;
573 val |= (0x1 << L2_CONFIG_ASN_HASH_ENABLE_SHIFT);
574
575 for (i = 0; i < ASN_HASH_COUNT; i++) {
576 dev_dbg(kbdev->dev, "Program 0x%x to ASN_HASH[%d]\n",
577 kbdev->l2_hash_values[i], i);
578 kbase_reg_write(kbdev, GPU_CONTROL_REG(ASN_HASH(i)),
579 kbdev->l2_hash_values[i]);
580 }
581 }
582
583 dev_dbg(kbdev->dev, "Program 0x%x to L2_CONFIG\n", val);
584 kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_CONFIG), val);
585 }
586
kbase_pm_control_gpu_clock(struct kbase_device * kbdev)587 static void kbase_pm_control_gpu_clock(struct kbase_device *kbdev)
588 {
589 struct kbase_pm_backend_data *const backend = &kbdev->pm.backend;
590
591 lockdep_assert_held(&kbdev->hwaccess_lock);
592
593 queue_work(system_wq, &backend->gpu_clock_control_work);
594 }
595
596 #if MALI_USE_CSF
kbase_mcu_state_to_string(enum kbase_mcu_state state)597 static const char *kbase_mcu_state_to_string(enum kbase_mcu_state state)
598 {
599 const char *const strings[] = {
600 #define KBASEP_MCU_STATE(n) #n,
601 #include "mali_kbase_pm_mcu_states.h"
602 #undef KBASEP_MCU_STATE
603 };
604 if (WARN_ON((size_t)state >= ARRAY_SIZE(strings)))
605 return "Bad MCU state";
606 else
607 return strings[state];
608 }
609
610 static
kbase_ktrace_log_mcu_state(struct kbase_device * kbdev,enum kbase_mcu_state state)611 void kbase_ktrace_log_mcu_state(struct kbase_device *kbdev, enum kbase_mcu_state state)
612 {
613 #if KBASE_KTRACE_ENABLE
614 switch (state) {
615 #define KBASEP_MCU_STATE(n) \
616 case KBASE_MCU_ ## n: \
617 KBASE_KTRACE_ADD(kbdev, PM_MCU_ ## n, NULL, state); \
618 break;
619 #include "mali_kbase_pm_mcu_states.h"
620 #undef KBASEP_MCU_STATE
621 }
622 #endif
623 }
624
kbase_pm_handle_mcu_core_attr_update(struct kbase_device * kbdev)625 static inline bool kbase_pm_handle_mcu_core_attr_update(struct kbase_device *kbdev)
626 {
627 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
628 bool timer_update;
629 bool core_mask_update;
630
631 lockdep_assert_held(&kbdev->hwaccess_lock);
632
633 WARN_ON(backend->mcu_state != KBASE_MCU_ON);
634
635 /* This function is only for cases where the MCU managing Cores, if
636 * the firmware mode is with host control, do nothing here.
637 */
638 if (unlikely(kbdev->csf.firmware_hctl_core_pwr))
639 return false;
640
641 core_mask_update =
642 backend->shaders_avail != backend->shaders_desired_mask;
643
644 timer_update = kbdev->csf.mcu_core_pwroff_dur_count !=
645 kbdev->csf.mcu_core_pwroff_reg_shadow;
646
647 if (core_mask_update || timer_update)
648 kbase_csf_firmware_update_core_attr(kbdev, timer_update,
649 core_mask_update, backend->shaders_desired_mask);
650
651 return (core_mask_update || timer_update);
652 }
653
kbase_pm_is_mcu_inactive(struct kbase_device * kbdev,enum kbase_mcu_state state)654 bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev,
655 enum kbase_mcu_state state)
656 {
657 lockdep_assert_held(&kbdev->hwaccess_lock);
658
659 return ((state == KBASE_MCU_OFF) || (state == KBASE_MCU_IN_SLEEP));
660 }
661
662 #ifdef KBASE_PM_RUNTIME
663 /**
664 * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on
665 * MCU side
666 *
667 * @kbdev: Pointer to the device.
668 *
669 * This function is called to re-enable the Doorbell notification on MCU side
670 * when MCU needs to beome active again.
671 */
kbase_pm_enable_mcu_db_notification(struct kbase_device * kbdev)672 static void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev)
673 {
674 u32 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_CONTROL));
675
676 lockdep_assert_held(&kbdev->hwaccess_lock);
677
678 val &= ~MCU_CNTRL_DOORBELL_DISABLE_MASK;
679 kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), val);
680 }
681
682 /**
683 * wait_mcu_as_inactive - Wait for AS used by MCU FW to get configured
684 *
685 * @kbdev: Pointer to the device.
686 *
687 * This function is called to wait for the AS used by MCU FW to get configured
688 * before DB notification on MCU is enabled, as a workaround for HW issue.
689 */
wait_mcu_as_inactive(struct kbase_device * kbdev)690 static void wait_mcu_as_inactive(struct kbase_device *kbdev)
691 {
692 unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
693
694 lockdep_assert_held(&kbdev->hwaccess_lock);
695
696 if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_2716))
697 return;
698
699 /* Wait for the AS_ACTIVE_INT bit to become 0 for the AS used by MCU FW */
700 while (--max_loops &&
701 kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)) &
702 AS_STATUS_AS_ACTIVE_INT)
703 ;
704
705 if (!WARN_ON_ONCE(max_loops == 0))
706 return;
707
708 dev_err(kbdev->dev, "AS_ACTIVE_INT bit stuck for AS %d used by MCU FW", MCU_AS_NR);
709
710 if (kbase_prepare_to_reset_gpu(kbdev, 0))
711 kbase_reset_gpu(kbdev);
712 }
713 #endif
714
715 /**
716 * kbasep_pm_toggle_power_interrupt - Toggles the IRQ mask for power interrupts
717 * from the firmware
718 *
719 * @kbdev: Pointer to the device
720 * @enable: boolean indicating to enable interrupts or not
721 *
722 * The POWER_CHANGED_ALL interrupt can be disabled after L2 has been turned on
723 * when FW is controlling the power for the shader cores. Correspondingly, the
724 * interrupts can be re-enabled after the MCU has been disabled before the
725 * power down of L2.
726 */
kbasep_pm_toggle_power_interrupt(struct kbase_device * kbdev,bool enable)727 static void kbasep_pm_toggle_power_interrupt(struct kbase_device *kbdev, bool enable)
728 {
729 u32 irq_mask;
730
731 lockdep_assert_held(&kbdev->hwaccess_lock);
732
733 irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
734
735 if (enable) {
736 irq_mask |= POWER_CHANGED_ALL;
737 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), POWER_CHANGED_ALL);
738 } else {
739 irq_mask &= ~POWER_CHANGED_ALL;
740 }
741
742 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask);
743 }
744
kbase_pm_mcu_update_state(struct kbase_device * kbdev)745 static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
746 {
747 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
748 enum kbase_mcu_state prev_state;
749
750 lockdep_assert_held(&kbdev->hwaccess_lock);
751
752 /*
753 * Initial load of firmware should have been done to
754 * exercise the MCU state machine.
755 */
756 if (unlikely(!kbdev->csf.firmware_inited)) {
757 WARN_ON(backend->mcu_state != KBASE_MCU_OFF);
758 return 0;
759 }
760
761 do {
762 u64 shaders_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER);
763 u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
764
765 /* mask off ready from trans in case transitions finished
766 * between the register reads
767 */
768 shaders_trans &= ~shaders_ready;
769
770 prev_state = backend->mcu_state;
771
772 switch (backend->mcu_state) {
773 case KBASE_MCU_OFF:
774 if (kbase_pm_is_mcu_desired(kbdev) &&
775 !backend->policy_change_clamp_state_to_off &&
776 backend->l2_state == KBASE_L2_ON) {
777 kbase_csf_firmware_trigger_reload(kbdev);
778 backend->mcu_state = KBASE_MCU_PEND_ON_RELOAD;
779 }
780 break;
781
782 case KBASE_MCU_PEND_ON_RELOAD:
783 if (kbdev->csf.firmware_reloaded) {
784 backend->shaders_desired_mask =
785 kbase_pm_ca_get_core_mask(kbdev);
786 kbase_csf_firmware_global_reinit(kbdev,
787 backend->shaders_desired_mask);
788 if (!kbdev->csf.firmware_hctl_core_pwr)
789 kbasep_pm_toggle_power_interrupt(kbdev, false);
790 backend->mcu_state =
791 KBASE_MCU_ON_GLB_REINIT_PEND;
792 }
793 break;
794
795 case KBASE_MCU_ON_GLB_REINIT_PEND:
796 if (kbase_csf_firmware_global_reinit_complete(kbdev)) {
797 backend->shaders_avail =
798 backend->shaders_desired_mask;
799 backend->pm_shaders_core_mask = 0;
800 if (kbdev->csf.firmware_hctl_core_pwr) {
801 kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
802 backend->shaders_avail, ACTION_PWRON);
803 backend->mcu_state =
804 KBASE_MCU_HCTL_SHADERS_PEND_ON;
805 } else
806 backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
807 #if IS_ENABLED(CONFIG_MALI_CORESIGHT)
808 if (kbase_debug_coresight_csf_state_check(
809 kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED)) {
810 kbase_debug_coresight_csf_state_request(
811 kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED);
812 backend->mcu_state = KBASE_MCU_CORESIGHT_ENABLE;
813 } else if (kbase_debug_coresight_csf_state_check(
814 kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED)) {
815 backend->mcu_state = KBASE_MCU_CORESIGHT_ENABLE;
816 }
817 #endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
818 }
819 break;
820
821 case KBASE_MCU_HCTL_SHADERS_PEND_ON:
822 if (!shaders_trans &&
823 shaders_ready == backend->shaders_avail) {
824 /* Cores now stable, notify MCU the stable mask */
825 kbase_csf_firmware_update_core_attr(kbdev,
826 false, true, shaders_ready);
827
828 backend->pm_shaders_core_mask = shaders_ready;
829 backend->mcu_state =
830 KBASE_MCU_HCTL_CORES_NOTIFY_PEND;
831 }
832 break;
833
834 case KBASE_MCU_HCTL_CORES_NOTIFY_PEND:
835 /* Wait for the acknowledgement */
836 if (kbase_csf_firmware_core_attr_updated(kbdev))
837 backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
838 break;
839
840 case KBASE_MCU_ON_HWCNT_ENABLE:
841 backend->hwcnt_desired = true;
842 if (backend->hwcnt_disabled) {
843 unsigned long flags;
844
845 kbase_csf_scheduler_spin_lock(kbdev, &flags);
846 kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
847 kbase_csf_scheduler_spin_unlock(kbdev, flags);
848 backend->hwcnt_disabled = false;
849 }
850 backend->mcu_state = KBASE_MCU_ON;
851 break;
852
853 case KBASE_MCU_ON:
854 backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev);
855
856 if (!kbase_pm_is_mcu_desired(kbdev))
857 backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE;
858 else if (kbdev->csf.firmware_hctl_core_pwr) {
859 /* Host control scale up/down cores as needed */
860 if (backend->shaders_desired_mask != shaders_ready) {
861 backend->hwcnt_desired = false;
862 if (!backend->hwcnt_disabled)
863 kbase_pm_trigger_hwcnt_disable(kbdev);
864 backend->mcu_state =
865 KBASE_MCU_HCTL_MCU_ON_RECHECK;
866 }
867 } else if (kbase_pm_handle_mcu_core_attr_update(kbdev))
868 backend->mcu_state = KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND;
869 #if IS_ENABLED(CONFIG_MALI_CORESIGHT)
870 else if (kbdev->csf.coresight.disable_on_pmode_enter) {
871 kbase_debug_coresight_csf_state_request(
872 kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED);
873 backend->mcu_state = KBASE_MCU_ON_PMODE_ENTER_CORESIGHT_DISABLE;
874 } else if (kbdev->csf.coresight.enable_on_pmode_exit) {
875 kbase_debug_coresight_csf_state_request(
876 kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED);
877 backend->mcu_state = KBASE_MCU_ON_PMODE_EXIT_CORESIGHT_ENABLE;
878 }
879 #endif
880 break;
881
882 case KBASE_MCU_HCTL_MCU_ON_RECHECK:
883 backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev);
884
885 if (!backend->hwcnt_disabled) {
886 /* Wait for being disabled */
887 ;
888 } else if (!kbase_pm_is_mcu_desired(kbdev)) {
889 /* Converging to MCU powering down flow */
890 backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE;
891 } else if (backend->shaders_desired_mask & ~shaders_ready) {
892 /* set cores ready but not available to
893 * meet SHADERS_PEND_ON check pass
894 */
895 backend->shaders_avail =
896 (backend->shaders_desired_mask | shaders_ready);
897
898 kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
899 backend->shaders_avail & ~shaders_ready,
900 ACTION_PWRON);
901 backend->mcu_state =
902 KBASE_MCU_HCTL_SHADERS_PEND_ON;
903
904 } else if (~backend->shaders_desired_mask & shaders_ready) {
905 kbase_csf_firmware_update_core_attr(kbdev, false, true,
906 backend->shaders_desired_mask);
907 backend->mcu_state = KBASE_MCU_HCTL_CORES_DOWN_SCALE_NOTIFY_PEND;
908 } else {
909 backend->mcu_state =
910 KBASE_MCU_HCTL_SHADERS_PEND_ON;
911 }
912 break;
913
914 case KBASE_MCU_HCTL_CORES_DOWN_SCALE_NOTIFY_PEND:
915 if (kbase_csf_firmware_core_attr_updated(kbdev)) {
916 /* wait in queue until cores idle */
917 queue_work(backend->core_idle_wq, &backend->core_idle_work);
918 backend->mcu_state = KBASE_MCU_HCTL_CORE_INACTIVE_PEND;
919 }
920 break;
921
922 case KBASE_MCU_HCTL_CORE_INACTIVE_PEND:
923 {
924 u64 active_cores = kbase_pm_get_active_cores(
925 kbdev,
926 KBASE_PM_CORE_SHADER);
927 u64 cores_to_disable = shaders_ready &
928 ~backend->shaders_desired_mask;
929
930 if (!(cores_to_disable & active_cores)) {
931 kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
932 cores_to_disable,
933 ACTION_PWROFF);
934 backend->shaders_avail = backend->shaders_desired_mask;
935 backend->mcu_state = KBASE_MCU_HCTL_SHADERS_CORE_OFF_PEND;
936 }
937 }
938 break;
939
940 case KBASE_MCU_HCTL_SHADERS_CORE_OFF_PEND:
941 if (!shaders_trans && shaders_ready == backend->shaders_avail) {
942 /* Cores now stable */
943 backend->pm_shaders_core_mask = shaders_ready;
944 backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
945 }
946 break;
947
948 case KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND:
949 if (kbase_csf_firmware_core_attr_updated(kbdev)) {
950 backend->shaders_avail = backend->shaders_desired_mask;
951 backend->mcu_state = KBASE_MCU_ON;
952 }
953 break;
954
955 case KBASE_MCU_ON_HWCNT_DISABLE:
956 if (kbase_pm_is_mcu_desired(kbdev)) {
957 backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
958 break;
959 }
960
961 backend->hwcnt_desired = false;
962 if (!backend->hwcnt_disabled)
963 kbase_pm_trigger_hwcnt_disable(kbdev);
964
965
966 if (backend->hwcnt_disabled) {
967 #ifdef KBASE_PM_RUNTIME
968 if (backend->gpu_sleep_mode_active)
969 backend->mcu_state = KBASE_MCU_ON_SLEEP_INITIATE;
970 else {
971 #endif
972 backend->mcu_state = KBASE_MCU_ON_HALT;
973 #if IS_ENABLED(CONFIG_MALI_CORESIGHT)
974 kbase_debug_coresight_csf_state_request(
975 kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED);
976 backend->mcu_state = KBASE_MCU_CORESIGHT_DISABLE;
977 #endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
978 }
979 }
980 break;
981
982 #if IS_ENABLED(CONFIG_MALI_CORESIGHT)
983 case KBASE_MCU_ON_PMODE_ENTER_CORESIGHT_DISABLE:
984 if (kbase_debug_coresight_csf_state_check(
985 kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED)) {
986 backend->mcu_state = KBASE_MCU_ON;
987 kbdev->csf.coresight.disable_on_pmode_enter = false;
988 }
989 break;
990 case KBASE_MCU_ON_PMODE_EXIT_CORESIGHT_ENABLE:
991 if (kbase_debug_coresight_csf_state_check(
992 kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED)) {
993 backend->mcu_state = KBASE_MCU_ON;
994 kbdev->csf.coresight.enable_on_pmode_exit = false;
995 }
996 break;
997 case KBASE_MCU_CORESIGHT_DISABLE:
998 if (kbase_debug_coresight_csf_state_check(
999 kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED))
1000 backend->mcu_state = KBASE_MCU_ON_HALT;
1001 break;
1002
1003 case KBASE_MCU_CORESIGHT_ENABLE:
1004 if (kbase_debug_coresight_csf_state_check(
1005 kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED))
1006 backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
1007 break;
1008 #endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
1009
1010 case KBASE_MCU_ON_HALT:
1011 if (!kbase_pm_is_mcu_desired(kbdev)) {
1012 kbase_csf_firmware_trigger_mcu_halt(kbdev);
1013 backend->mcu_state = KBASE_MCU_ON_PEND_HALT;
1014 } else
1015 backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
1016 break;
1017
1018 case KBASE_MCU_ON_PEND_HALT:
1019 if (kbase_csf_firmware_mcu_halted(kbdev)) {
1020 KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_HALTED, NULL,
1021 kbase_csf_ktrace_gpu_cycle_cnt(kbdev));
1022 if (kbdev->csf.firmware_hctl_core_pwr)
1023 backend->mcu_state =
1024 KBASE_MCU_HCTL_SHADERS_READY_OFF;
1025 else
1026 backend->mcu_state = KBASE_MCU_POWER_DOWN;
1027 }
1028 break;
1029
1030 case KBASE_MCU_HCTL_SHADERS_READY_OFF:
1031 kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
1032 shaders_ready, ACTION_PWROFF);
1033 backend->mcu_state =
1034 KBASE_MCU_HCTL_SHADERS_PEND_OFF;
1035 break;
1036
1037 case KBASE_MCU_HCTL_SHADERS_PEND_OFF:
1038 if (!shaders_trans && !shaders_ready) {
1039 backend->pm_shaders_core_mask = 0;
1040 backend->mcu_state = KBASE_MCU_POWER_DOWN;
1041 }
1042 break;
1043
1044 case KBASE_MCU_POWER_DOWN:
1045 kbase_csf_firmware_disable_mcu(kbdev);
1046 backend->mcu_state = KBASE_MCU_PEND_OFF;
1047 break;
1048
1049 case KBASE_MCU_PEND_OFF:
1050 /* wait synchronously for the MCU to get disabled */
1051 kbase_csf_firmware_disable_mcu_wait(kbdev);
1052 if (!kbdev->csf.firmware_hctl_core_pwr)
1053 kbasep_pm_toggle_power_interrupt(kbdev, true);
1054 backend->mcu_state = KBASE_MCU_OFF;
1055 break;
1056 #ifdef KBASE_PM_RUNTIME
1057 case KBASE_MCU_ON_SLEEP_INITIATE:
1058 if (!kbase_pm_is_mcu_desired(kbdev)) {
1059 kbase_csf_firmware_trigger_mcu_sleep(kbdev);
1060 backend->mcu_state = KBASE_MCU_ON_PEND_SLEEP;
1061 } else
1062 backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
1063 break;
1064
1065 case KBASE_MCU_ON_PEND_SLEEP:
1066 if (kbase_csf_firmware_is_mcu_in_sleep(kbdev)) {
1067 KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_SLEEP, NULL,
1068 kbase_csf_ktrace_gpu_cycle_cnt(kbdev));
1069 backend->mcu_state = KBASE_MCU_IN_SLEEP;
1070 kbase_pm_enable_db_mirror_interrupt(kbdev);
1071 kbase_csf_scheduler_reval_idleness_post_sleep(kbdev);
1072 /* Enable PM interrupt, after MCU has been put
1073 * to sleep, for the power down of L2.
1074 */
1075 if (!kbdev->csf.firmware_hctl_core_pwr)
1076 kbasep_pm_toggle_power_interrupt(kbdev, true);
1077 }
1078 break;
1079
1080 case KBASE_MCU_IN_SLEEP:
1081 if (kbase_pm_is_mcu_desired(kbdev) &&
1082 backend->l2_state == KBASE_L2_ON) {
1083 wait_mcu_as_inactive(kbdev);
1084 KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_WAKEUP(
1085 kbdev, kbase_backend_get_cycle_cnt(kbdev));
1086 kbase_pm_enable_mcu_db_notification(kbdev);
1087 kbase_pm_disable_db_mirror_interrupt(kbdev);
1088 /* Disable PM interrupt after L2 has been
1089 * powered up for the wakeup of MCU.
1090 */
1091 if (!kbdev->csf.firmware_hctl_core_pwr)
1092 kbasep_pm_toggle_power_interrupt(kbdev, false);
1093 backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
1094 kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
1095 }
1096 break;
1097 #endif
1098 case KBASE_MCU_RESET_WAIT:
1099 /* Reset complete */
1100 if (!backend->in_reset)
1101 backend->mcu_state = KBASE_MCU_OFF;
1102
1103 #if IS_ENABLED(CONFIG_MALI_CORESIGHT)
1104 kbdev->csf.coresight.disable_on_pmode_enter = false;
1105 kbdev->csf.coresight.enable_on_pmode_exit = false;
1106 #endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
1107 break;
1108
1109 default:
1110 WARN(1, "Invalid state in mcu_state: %d",
1111 backend->mcu_state);
1112 }
1113
1114 if (backend->mcu_state != prev_state) {
1115 dev_dbg(kbdev->dev, "MCU state transition: %s to %s\n",
1116 kbase_mcu_state_to_string(prev_state),
1117 kbase_mcu_state_to_string(backend->mcu_state));
1118 kbase_ktrace_log_mcu_state(kbdev, backend->mcu_state);
1119 }
1120
1121 } while (backend->mcu_state != prev_state);
1122
1123 return 0;
1124 }
1125
core_idle_worker(struct work_struct * work)1126 static void core_idle_worker(struct work_struct *work)
1127 {
1128 struct kbase_device *kbdev =
1129 container_of(work, struct kbase_device, pm.backend.core_idle_work);
1130 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
1131 unsigned long flags;
1132
1133 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1134 while (backend->gpu_powered && (backend->mcu_state == KBASE_MCU_HCTL_CORE_INACTIVE_PEND)) {
1135 const unsigned int core_inactive_wait_ms = 1;
1136 u64 active_cores = kbase_pm_get_active_cores(kbdev, KBASE_PM_CORE_SHADER);
1137 u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
1138 u64 cores_to_disable = shaders_ready & ~backend->shaders_desired_mask;
1139
1140 if (!(cores_to_disable & active_cores)) {
1141 kbase_pm_update_state(kbdev);
1142 break;
1143 }
1144
1145 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1146 msleep(core_inactive_wait_ms);
1147 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1148 }
1149
1150 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1151 }
1152 #endif
1153
kbase_l2_core_state_to_string(enum kbase_l2_core_state state)1154 static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state)
1155 {
1156 const char *const strings[] = {
1157 #define KBASEP_L2_STATE(n) #n,
1158 #include "mali_kbase_pm_l2_states.h"
1159 #undef KBASEP_L2_STATE
1160 };
1161 if (WARN_ON((size_t)state >= ARRAY_SIZE(strings)))
1162 return "Bad level 2 cache state";
1163 else
1164 return strings[state];
1165 }
1166
1167 static
kbase_ktrace_log_l2_core_state(struct kbase_device * kbdev,enum kbase_l2_core_state state)1168 void kbase_ktrace_log_l2_core_state(struct kbase_device *kbdev, enum kbase_l2_core_state state)
1169 {
1170 #if KBASE_KTRACE_ENABLE
1171 switch (state) {
1172 #define KBASEP_L2_STATE(n) \
1173 case KBASE_L2_ ## n: \
1174 KBASE_KTRACE_ADD(kbdev, PM_L2_ ## n, NULL, state); \
1175 break;
1176 #include "mali_kbase_pm_l2_states.h"
1177 #undef KBASEP_L2_STATE
1178 }
1179 #endif
1180 }
1181
1182 #if !MALI_USE_CSF
1183 /* On powering on the L2, the tracked kctx becomes stale and can be cleared.
1184 * This enables the backend to spare the START_FLUSH.INV_SHADER_OTHER
1185 * operation on the first submitted katom after the L2 powering on.
1186 */
kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device * kbdev)1187 static void kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device *kbdev)
1188 {
1189 int js;
1190
1191 lockdep_assert_held(&kbdev->hwaccess_lock);
1192
1193 /* Clear the slots' last katom submission kctx */
1194 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
1195 kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_NULL_TAG_VAL;
1196 }
1197 #endif
1198
can_power_down_l2(struct kbase_device * kbdev)1199 static bool can_power_down_l2(struct kbase_device *kbdev)
1200 {
1201 #if MALI_USE_CSF
1202 /* Due to the HW issue GPU2019-3878, need to prevent L2 power off
1203 * whilst MMU command is in progress.
1204 * Also defer the power-down if MMU is in process of page migration.
1205 */
1206 return !kbdev->mmu_hw_operation_in_progress && !kbdev->mmu_page_migrate_in_progress;
1207 #else
1208 return !kbdev->mmu_page_migrate_in_progress;
1209 #endif
1210 }
1211
can_power_up_l2(struct kbase_device * kbdev)1212 static bool can_power_up_l2(struct kbase_device *kbdev)
1213 {
1214 lockdep_assert_held(&kbdev->hwaccess_lock);
1215
1216 /* Avoiding l2 transition if MMU is undergoing page migration */
1217 return !kbdev->mmu_page_migrate_in_progress;
1218 }
1219
need_tiler_control(struct kbase_device * kbdev)1220 static bool need_tiler_control(struct kbase_device *kbdev)
1221 {
1222 #if MALI_USE_CSF
1223 if (kbase_pm_no_mcu_core_pwroff(kbdev))
1224 return true;
1225 else
1226 return false;
1227 #else
1228 return true;
1229 #endif
1230 }
1231
kbase_pm_l2_update_state(struct kbase_device * kbdev)1232 static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
1233 {
1234 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
1235 u64 l2_present = kbdev->gpu_props.curr_config.l2_present;
1236 u64 tiler_present = kbdev->gpu_props.props.raw_props.tiler_present;
1237 bool l2_power_up_done;
1238 enum kbase_l2_core_state prev_state;
1239
1240 lockdep_assert_held(&kbdev->hwaccess_lock);
1241
1242 do {
1243 /* Get current state */
1244 u64 l2_trans = kbase_pm_get_trans_cores(kbdev,
1245 KBASE_PM_CORE_L2);
1246 u64 l2_ready = kbase_pm_get_ready_cores(kbdev,
1247 KBASE_PM_CORE_L2);
1248
1249 #ifdef CONFIG_MALI_ARBITER_SUPPORT
1250 /*
1251 * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
1252 * are vulnerable to corruption if gpu is lost
1253 */
1254 if (kbase_is_gpu_removed(kbdev) || kbase_pm_is_gpu_lost(kbdev)) {
1255 backend->shaders_state =
1256 KBASE_SHADERS_OFF_CORESTACK_OFF;
1257 backend->hwcnt_desired = false;
1258 if (!backend->hwcnt_disabled) {
1259 /* Don't progress until hw counters are disabled
1260 * This may involve waiting for a worker to complete.
1261 * The HW counters backend disable code checks for the
1262 * GPU removed case and will error out without touching
1263 * the hardware. This step is needed to keep the HW
1264 * counters in a consistent state after a GPU lost.
1265 */
1266 backend->l2_state =
1267 KBASE_L2_ON_HWCNT_DISABLE;
1268 KBASE_KTRACE_ADD(kbdev, PM_L2_ON_HWCNT_DISABLE, NULL,
1269 backend->l2_state);
1270 kbase_pm_trigger_hwcnt_disable(kbdev);
1271 }
1272
1273 if (backend->hwcnt_disabled) {
1274 backend->l2_state = KBASE_L2_OFF;
1275 KBASE_KTRACE_ADD(kbdev, PM_L2_OFF, NULL, backend->l2_state);
1276 dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n");
1277 }
1278 break;
1279 }
1280 #endif
1281
1282 /* mask off ready from trans in case transitions finished
1283 * between the register reads
1284 */
1285 l2_trans &= ~l2_ready;
1286
1287 prev_state = backend->l2_state;
1288
1289 switch (backend->l2_state) {
1290 case KBASE_L2_OFF:
1291 if (kbase_pm_is_l2_desired(kbdev) && can_power_up_l2(kbdev)) {
1292 #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
1293 /* Enable HW timer of IPA control before
1294 * L2 cache is powered-up.
1295 */
1296 kbase_ipa_control_handle_gpu_sleep_exit(kbdev);
1297 #endif
1298 /*
1299 * Set the desired config for L2 before
1300 * powering it on
1301 */
1302 kbase_pm_l2_config_override(kbdev);
1303 kbase_pbha_write_settings(kbdev);
1304
1305 /* If Host is controlling the power for shader
1306 * cores, then it also needs to control the
1307 * power for Tiler.
1308 * Powering on the tiler will also power the
1309 * L2 cache.
1310 */
1311 if (need_tiler_control(kbdev)) {
1312 kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, tiler_present,
1313 ACTION_PWRON);
1314 } else {
1315 kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present,
1316 ACTION_PWRON);
1317 }
1318 #if !MALI_USE_CSF
1319 /* If we have more than one L2 cache then we
1320 * must power them on explicitly.
1321 */
1322 if (l2_present != 1)
1323 kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
1324 l2_present & ~1,
1325 ACTION_PWRON);
1326 /* Clear backend slot submission kctx */
1327 kbase_pm_l2_clear_backend_slot_submit_kctx(kbdev);
1328 #endif
1329 backend->l2_state = KBASE_L2_PEND_ON;
1330 }
1331 break;
1332
1333 case KBASE_L2_PEND_ON:
1334 l2_power_up_done = false;
1335 if (!l2_trans && l2_ready == l2_present) {
1336 if (need_tiler_control(kbdev)) {
1337 u64 tiler_trans = kbase_pm_get_trans_cores(
1338 kbdev, KBASE_PM_CORE_TILER);
1339 u64 tiler_ready = kbase_pm_get_ready_cores(
1340 kbdev, KBASE_PM_CORE_TILER);
1341 tiler_trans &= ~tiler_ready;
1342
1343 if (!tiler_trans && tiler_ready == tiler_present) {
1344 KBASE_KTRACE_ADD(kbdev,
1345 PM_CORES_CHANGE_AVAILABLE_TILER,
1346 NULL, tiler_ready);
1347 l2_power_up_done = true;
1348 }
1349 } else {
1350 KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL,
1351 l2_ready);
1352 l2_power_up_done = true;
1353 }
1354 }
1355 if (l2_power_up_done) {
1356 /*
1357 * Ensure snoops are enabled after L2 is powered
1358 * up. Note that kbase keeps track of the snoop
1359 * state, so safe to repeatedly call.
1360 */
1361 kbase_pm_cache_snoop_enable(kbdev);
1362
1363 /* With the L2 enabled, we can now enable
1364 * hardware counters.
1365 */
1366 if (kbdev->pm.backend.gpu_clock_slow_down_wa)
1367 backend->l2_state =
1368 KBASE_L2_RESTORE_CLOCKS;
1369 else
1370 backend->l2_state =
1371 KBASE_L2_ON_HWCNT_ENABLE;
1372
1373 /* Now that the L2 is on, the shaders can start
1374 * powering on if they're required. The obvious
1375 * way to do this would be to call
1376 * kbase_pm_shaders_update_state() here.
1377 * However, that would make the two state
1378 * machines mutually recursive, as the opposite
1379 * would be needed for powering down. Instead,
1380 * callers of this function should use the
1381 * kbase_pm_update_state() wrapper, which will
1382 * call the shader state machine immediately
1383 * after the L2 (for power up), or
1384 * automatically re-invoke the L2 state machine
1385 * when the shaders power down.
1386 */
1387 }
1388 break;
1389
1390 case KBASE_L2_RESTORE_CLOCKS:
1391 /* We always assume only GPUs being affected by
1392 * BASE_HW_ISSUE_GPU2017_1336 fall into this state
1393 */
1394 WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa);
1395
1396 /* If L2 not needed, we need to make sure cancellation
1397 * of any previously issued work to restore GPU clock.
1398 * For it, move to KBASE_L2_SLOW_DOWN_CLOCKS state.
1399 */
1400 if (!kbase_pm_is_l2_desired(kbdev)) {
1401 backend->l2_state = KBASE_L2_SLOW_DOWN_CLOCKS;
1402 break;
1403 }
1404
1405 backend->gpu_clock_slow_down_desired = false;
1406 if (backend->gpu_clock_slowed_down)
1407 kbase_pm_control_gpu_clock(kbdev);
1408 else
1409 backend->l2_state = KBASE_L2_ON_HWCNT_ENABLE;
1410 break;
1411
1412 case KBASE_L2_ON_HWCNT_ENABLE:
1413 #if !MALI_USE_CSF
1414 backend->hwcnt_desired = true;
1415 if (backend->hwcnt_disabled) {
1416 kbase_hwcnt_context_enable(
1417 kbdev->hwcnt_gpu_ctx);
1418 backend->hwcnt_disabled = false;
1419 }
1420 #endif
1421 backend->l2_state = KBASE_L2_ON;
1422 break;
1423
1424 case KBASE_L2_ON:
1425 if (!kbase_pm_is_l2_desired(kbdev)) {
1426 #if !MALI_USE_CSF
1427 /* Do not power off L2 until the shaders and
1428 * core stacks are off.
1429 */
1430 if (backend->shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF)
1431 break;
1432 #else
1433 /* Do not power off L2 until the MCU has been stopped */
1434 if ((backend->mcu_state != KBASE_MCU_OFF) &&
1435 (backend->mcu_state != KBASE_MCU_IN_SLEEP))
1436 break;
1437 #endif
1438
1439 /* We need to make sure hardware counters are
1440 * disabled before powering down the L2, to
1441 * prevent loss of data.
1442 *
1443 * We waited until after the cores were powered
1444 * down to prevent ping-ponging between hwcnt
1445 * enabled and disabled, which would have
1446 * happened if userspace submitted more work
1447 * while we were trying to power down.
1448 */
1449 backend->l2_state = KBASE_L2_ON_HWCNT_DISABLE;
1450 }
1451 break;
1452
1453 case KBASE_L2_ON_HWCNT_DISABLE:
1454 #if !MALI_USE_CSF
1455 /* If the L2 became desired while we were waiting on the
1456 * worker to do the actual hwcnt disable (which might
1457 * happen if some work was submitted immediately after
1458 * the shaders powered off), then we need to early-out
1459 * of this state and re-enable hwcnt.
1460 *
1461 * If we get lucky, the hwcnt disable might not have
1462 * actually started yet, and the logic in the hwcnt
1463 * enable state will prevent the worker from
1464 * performing the disable entirely, preventing loss of
1465 * any hardware counter data.
1466 *
1467 * If the hwcnt disable has started, then we'll lose
1468 * a tiny amount of hardware counter data between the
1469 * disable and the re-enable occurring.
1470 *
1471 * This loss of data is preferable to the alternative,
1472 * which is to block the shader cores from doing any
1473 * work until we're sure hwcnt has been re-enabled.
1474 */
1475 if (kbase_pm_is_l2_desired(kbdev)) {
1476 backend->l2_state = KBASE_L2_ON_HWCNT_ENABLE;
1477 break;
1478 }
1479
1480 backend->hwcnt_desired = false;
1481 if (!backend->hwcnt_disabled)
1482 kbase_pm_trigger_hwcnt_disable(kbdev);
1483 #endif
1484
1485 if (backend->hwcnt_disabled) {
1486 if (kbdev->pm.backend.gpu_clock_slow_down_wa)
1487 backend->l2_state =
1488 KBASE_L2_SLOW_DOWN_CLOCKS;
1489 else
1490 backend->l2_state = KBASE_L2_POWER_DOWN;
1491 }
1492 break;
1493
1494 case KBASE_L2_SLOW_DOWN_CLOCKS:
1495 /* We always assume only GPUs being affected by
1496 * BASE_HW_ISSUE_GPU2017_1336 fall into this state
1497 */
1498 WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa);
1499
1500 /* L2 needs to be powered up. And we need to make sure
1501 * cancellation of any previously issued work to slow
1502 * down GPU clock. For it, we move to the state,
1503 * KBASE_L2_RESTORE_CLOCKS.
1504 */
1505 if (kbase_pm_is_l2_desired(kbdev)) {
1506 backend->l2_state = KBASE_L2_RESTORE_CLOCKS;
1507 break;
1508 }
1509
1510 backend->gpu_clock_slow_down_desired = true;
1511 if (!backend->gpu_clock_slowed_down)
1512 kbase_pm_control_gpu_clock(kbdev);
1513 else
1514 backend->l2_state = KBASE_L2_POWER_DOWN;
1515
1516 break;
1517
1518 case KBASE_L2_POWER_DOWN:
1519 if (kbase_pm_is_l2_desired(kbdev))
1520 backend->l2_state = KBASE_L2_PEND_ON;
1521 else if (can_power_down_l2(kbdev)) {
1522 if (!backend->l2_always_on)
1523 /* Powering off the L2 will also power off the
1524 * tiler.
1525 */
1526 kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
1527 l2_present,
1528 ACTION_PWROFF);
1529 else
1530 /* If L2 cache is powered then we must flush it
1531 * before we power off the GPU. Normally this
1532 * would have been handled when the L2 was
1533 * powered off.
1534 */
1535 kbase_gpu_start_cache_clean_nolock(
1536 kbdev, GPU_COMMAND_CACHE_CLN_INV_L2);
1537 #if !MALI_USE_CSF
1538 KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u);
1539 #else
1540 KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, 0u);
1541 #endif
1542 backend->l2_state = KBASE_L2_PEND_OFF;
1543 }
1544 break;
1545
1546 case KBASE_L2_PEND_OFF:
1547 if (!backend->l2_always_on) {
1548 /* We only need to check the L2 here - if the L2
1549 * is off then the tiler is definitely also off.
1550 */
1551 if (!l2_trans && !l2_ready) {
1552 #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
1553 /* Allow clock gating within the GPU and prevent it
1554 * from being seen as active during sleep.
1555 */
1556 kbase_ipa_control_handle_gpu_sleep_enter(kbdev);
1557 #endif
1558 /* L2 is now powered off */
1559 backend->l2_state = KBASE_L2_OFF;
1560 }
1561 } else {
1562 if (!kbdev->cache_clean_in_progress) {
1563 #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
1564 /* Allow clock gating within the GPU and prevent it
1565 * from being seen as active during sleep.
1566 */
1567 kbase_ipa_control_handle_gpu_sleep_enter(kbdev);
1568 #endif
1569 backend->l2_state = KBASE_L2_OFF;
1570 }
1571 }
1572 break;
1573
1574 case KBASE_L2_RESET_WAIT:
1575 /* Reset complete */
1576 if (!backend->in_reset)
1577 backend->l2_state = KBASE_L2_OFF;
1578 break;
1579
1580 default:
1581 WARN(1, "Invalid state in l2_state: %d",
1582 backend->l2_state);
1583 }
1584
1585 if (backend->l2_state != prev_state) {
1586 dev_dbg(kbdev->dev, "L2 state transition: %s to %s\n",
1587 kbase_l2_core_state_to_string(prev_state),
1588 kbase_l2_core_state_to_string(
1589 backend->l2_state));
1590 kbase_ktrace_log_l2_core_state(kbdev, backend->l2_state);
1591 }
1592
1593 } while (backend->l2_state != prev_state);
1594
1595 if (kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off &&
1596 backend->l2_state == KBASE_L2_OFF) {
1597 kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = false;
1598 queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq,
1599 &kbdev->pm.backend.gpu_poweroff_wait_work);
1600 }
1601
1602 return 0;
1603 }
1604
shader_poweroff_timer_stop_callback(struct work_struct * data)1605 static void shader_poweroff_timer_stop_callback(struct work_struct *data)
1606 {
1607 unsigned long flags;
1608 struct kbasep_pm_tick_timer_state *stt = container_of(data,
1609 struct kbasep_pm_tick_timer_state, work);
1610 struct kbase_device *kbdev = container_of(stt, struct kbase_device,
1611 pm.backend.shader_tick_timer);
1612
1613 hrtimer_cancel(&stt->timer);
1614
1615 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1616
1617 stt->cancel_queued = false;
1618 if (kbdev->pm.backend.gpu_powered)
1619 kbase_pm_update_state(kbdev);
1620
1621 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1622 }
1623
1624 /**
1625 * shader_poweroff_timer_queue_cancel - cancel the shader poweroff tick timer
1626 * @kbdev: pointer to kbase device
1627 *
1628 * Synchronization between the shader state machine and the timer thread is
1629 * difficult. This is because situations may arise where the state machine
1630 * wants to start the timer, but the callback is already running, and has
1631 * already passed the point at which it checks whether it is required, and so
1632 * cancels itself, even though the state machine may have just tried to call
1633 * hrtimer_start.
1634 *
1635 * This cannot be stopped by holding hwaccess_lock in the timer thread,
1636 * because there are still infinitesimally small sections at the start and end
1637 * of the callback where the lock is not held.
1638 *
1639 * Instead, a new state is added to the shader state machine,
1640 * KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF. This is used to guarantee
1641 * that when the shaders are switched off, the timer has definitely been
1642 * cancelled. As a result, when KBASE_SHADERS_ON_CORESTACK_ON is left and the
1643 * timer is started, it is guaranteed that either the timer is already running
1644 * (from an availability change or cancelled timer), or hrtimer_start will
1645 * succeed. It is critical to avoid ending up in
1646 * KBASE_SHADERS_WAIT_OFF_CORESTACK_ON without the timer running, or it could
1647 * hang there forever.
1648 */
shader_poweroff_timer_queue_cancel(struct kbase_device * kbdev)1649 static void shader_poweroff_timer_queue_cancel(struct kbase_device *kbdev)
1650 {
1651 struct kbasep_pm_tick_timer_state *stt =
1652 &kbdev->pm.backend.shader_tick_timer;
1653
1654 lockdep_assert_held(&kbdev->hwaccess_lock);
1655
1656 stt->needed = false;
1657
1658 if (hrtimer_active(&stt->timer) && !stt->cancel_queued) {
1659 stt->cancel_queued = true;
1660 queue_work(stt->wq, &stt->work);
1661 }
1662 }
1663
1664 #if !MALI_USE_CSF
kbase_shader_core_state_to_string(enum kbase_shader_core_state state)1665 static const char *kbase_shader_core_state_to_string(
1666 enum kbase_shader_core_state state)
1667 {
1668 const char *const strings[] = {
1669 #define KBASEP_SHADER_STATE(n) #n,
1670 #include "mali_kbase_pm_shader_states.h"
1671 #undef KBASEP_SHADER_STATE
1672 };
1673 if (WARN_ON((size_t)state >= ARRAY_SIZE(strings)))
1674 return "Bad shader core state";
1675 else
1676 return strings[state];
1677 }
1678
kbase_pm_shaders_update_state(struct kbase_device * kbdev)1679 static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
1680 {
1681 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
1682 struct kbasep_pm_tick_timer_state *stt =
1683 &kbdev->pm.backend.shader_tick_timer;
1684 enum kbase_shader_core_state prev_state;
1685 u64 stacks_avail = 0;
1686
1687 lockdep_assert_held(&kbdev->hwaccess_lock);
1688
1689 if (corestack_driver_control)
1690 /* Always power on all the corestacks. Disabling certain
1691 * corestacks when their respective shaders are not in the
1692 * available bitmap is not currently supported.
1693 */
1694 stacks_avail = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_STACK);
1695
1696 do {
1697 u64 shaders_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER);
1698 u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
1699 u64 stacks_trans = 0;
1700 u64 stacks_ready = 0;
1701
1702 if (corestack_driver_control) {
1703 stacks_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_STACK);
1704 stacks_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK);
1705 }
1706
1707 /*
1708 * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
1709 * are vulnerable to corruption if gpu is lost
1710 */
1711 if (kbase_is_gpu_removed(kbdev)
1712 #ifdef CONFIG_MALI_ARBITER_SUPPORT
1713 || kbase_pm_is_gpu_lost(kbdev)) {
1714 #else
1715 ) {
1716 #endif
1717 backend->shaders_state =
1718 KBASE_SHADERS_OFF_CORESTACK_OFF;
1719 dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n");
1720 break;
1721 }
1722
1723 /* mask off ready from trans in case transitions finished
1724 * between the register reads
1725 */
1726 shaders_trans &= ~shaders_ready;
1727 stacks_trans &= ~stacks_ready;
1728
1729 prev_state = backend->shaders_state;
1730
1731 switch (backend->shaders_state) {
1732 case KBASE_SHADERS_OFF_CORESTACK_OFF:
1733 /* Ignore changes to the shader core availability
1734 * except at certain points where we can handle it,
1735 * i.e. off and SHADERS_ON_CORESTACK_ON.
1736 */
1737 backend->shaders_desired_mask =
1738 kbase_pm_ca_get_core_mask(kbdev);
1739 backend->pm_shaders_core_mask = 0;
1740
1741 if (backend->shaders_desired &&
1742 backend->l2_state == KBASE_L2_ON) {
1743 if (backend->hwcnt_desired &&
1744 !backend->hwcnt_disabled) {
1745 /* Trigger a hwcounter dump */
1746 backend->hwcnt_desired = false;
1747 kbase_pm_trigger_hwcnt_disable(kbdev);
1748 }
1749
1750 if (backend->hwcnt_disabled) {
1751 if (corestack_driver_control) {
1752 kbase_pm_invoke(kbdev,
1753 KBASE_PM_CORE_STACK,
1754 stacks_avail,
1755 ACTION_PWRON);
1756 }
1757 backend->shaders_state =
1758 KBASE_SHADERS_OFF_CORESTACK_PEND_ON;
1759 }
1760 }
1761 break;
1762
1763 case KBASE_SHADERS_OFF_CORESTACK_PEND_ON:
1764 if (!stacks_trans && stacks_ready == stacks_avail) {
1765 backend->shaders_avail =
1766 backend->shaders_desired_mask;
1767 kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
1768 backend->shaders_avail, ACTION_PWRON);
1769
1770 if (backend->pm_current_policy &&
1771 backend->pm_current_policy->handle_event)
1772 backend->pm_current_policy->handle_event(
1773 kbdev,
1774 KBASE_PM_POLICY_EVENT_POWER_ON);
1775
1776 backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON;
1777 }
1778 break;
1779
1780 case KBASE_SHADERS_PEND_ON_CORESTACK_ON:
1781 if (!shaders_trans && shaders_ready == backend->shaders_avail) {
1782 KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, shaders_ready);
1783 backend->pm_shaders_core_mask = shaders_ready;
1784 backend->hwcnt_desired = true;
1785 if (backend->hwcnt_disabled) {
1786 #if MALI_USE_CSF
1787 unsigned long flags;
1788
1789 kbase_csf_scheduler_spin_lock(kbdev,
1790 &flags);
1791 #endif
1792 kbase_hwcnt_context_enable(
1793 kbdev->hwcnt_gpu_ctx);
1794 #if MALI_USE_CSF
1795 kbase_csf_scheduler_spin_unlock(kbdev,
1796 flags);
1797 #endif
1798 backend->hwcnt_disabled = false;
1799 }
1800
1801 backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON;
1802 }
1803 break;
1804
1805 case KBASE_SHADERS_ON_CORESTACK_ON:
1806 backend->shaders_desired_mask =
1807 kbase_pm_ca_get_core_mask(kbdev);
1808
1809 /* If shaders to change state, trigger a counter dump */
1810 if (!backend->shaders_desired ||
1811 (backend->shaders_desired_mask != shaders_ready)) {
1812 backend->hwcnt_desired = false;
1813 if (!backend->hwcnt_disabled)
1814 kbase_pm_trigger_hwcnt_disable(kbdev);
1815 backend->shaders_state =
1816 KBASE_SHADERS_ON_CORESTACK_ON_RECHECK;
1817 }
1818 break;
1819
1820 case KBASE_SHADERS_ON_CORESTACK_ON_RECHECK:
1821 backend->shaders_desired_mask =
1822 kbase_pm_ca_get_core_mask(kbdev);
1823
1824 if (!backend->hwcnt_disabled) {
1825 /* Wait for being disabled */
1826 ;
1827 } else if (!backend->shaders_desired) {
1828 if (backend->pm_current_policy &&
1829 backend->pm_current_policy->handle_event)
1830 backend->pm_current_policy->handle_event(
1831 kbdev,
1832 KBASE_PM_POLICY_EVENT_IDLE);
1833
1834 if (kbdev->pm.backend.protected_transition_override ||
1835 #ifdef CONFIG_MALI_ARBITER_SUPPORT
1836 kbase_pm_is_suspending(kbdev) ||
1837 kbase_pm_is_gpu_lost(kbdev) ||
1838 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
1839 !stt->configured_ticks ||
1840 WARN_ON(stt->cancel_queued)) {
1841 backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
1842 } else {
1843 stt->remaining_ticks = stt->configured_ticks;
1844 stt->needed = true;
1845
1846 /* The shader hysteresis timer is not
1847 * done the obvious way, which would be
1848 * to start an hrtimer when the shader
1849 * power off is requested. Instead,
1850 * use a 'tick' timer, and set the
1851 * remaining number of ticks on a power
1852 * off request. This avoids the
1853 * latency of starting, then
1854 * immediately cancelling an hrtimer
1855 * when the shaders are re-requested
1856 * before the timeout expires.
1857 */
1858 if (!hrtimer_active(&stt->timer))
1859 hrtimer_start(&stt->timer,
1860 stt->configured_interval,
1861 HRTIMER_MODE_REL);
1862
1863 backend->shaders_state = KBASE_SHADERS_WAIT_OFF_CORESTACK_ON;
1864 }
1865 } else if (backend->shaders_desired_mask & ~shaders_ready) {
1866 /* set cores ready but not available to
1867 * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON
1868 * check pass
1869 */
1870 backend->shaders_avail =
1871 (backend->shaders_desired_mask | shaders_ready);
1872
1873 kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
1874 backend->shaders_avail & ~shaders_ready,
1875 ACTION_PWRON);
1876 backend->shaders_state =
1877 KBASE_SHADERS_PEND_ON_CORESTACK_ON;
1878 } else if (shaders_ready & ~backend->shaders_desired_mask) {
1879 backend->shaders_state =
1880 KBASE_SHADERS_WAIT_GPU_IDLE;
1881 } else {
1882 backend->shaders_state =
1883 KBASE_SHADERS_PEND_ON_CORESTACK_ON;
1884 }
1885 break;
1886
1887 case KBASE_SHADERS_WAIT_OFF_CORESTACK_ON:
1888 if (WARN_ON(!hrtimer_active(&stt->timer))) {
1889 stt->remaining_ticks = 0;
1890 backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
1891 }
1892
1893 if (backend->shaders_desired) {
1894 if (backend->pm_current_policy &&
1895 backend->pm_current_policy->handle_event)
1896 backend->pm_current_policy->handle_event(
1897 kbdev,
1898 KBASE_PM_POLICY_EVENT_TIMER_HIT);
1899
1900 stt->remaining_ticks = 0;
1901 backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK;
1902 } else if (stt->remaining_ticks == 0) {
1903 if (backend->pm_current_policy &&
1904 backend->pm_current_policy->handle_event)
1905 backend->pm_current_policy->handle_event(
1906 kbdev,
1907 KBASE_PM_POLICY_EVENT_TIMER_MISS);
1908
1909 backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
1910 #ifdef CONFIG_MALI_ARBITER_SUPPORT
1911 } else if (kbase_pm_is_suspending(kbdev) ||
1912 kbase_pm_is_gpu_lost(kbdev)) {
1913 backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
1914 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
1915 }
1916 break;
1917
1918 case KBASE_SHADERS_WAIT_GPU_IDLE:
1919 /* If partial shader core off need to wait the job in
1920 * running and next register finished then flush L2
1921 * or it might hit GPU2017-861
1922 */
1923 if (!kbase_gpu_atoms_submitted_any(kbdev)) {
1924 backend->partial_shaderoff = true;
1925 backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
1926 }
1927 break;
1928
1929 case KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON:
1930 if (!backend->partial_shaderoff)
1931 shader_poweroff_timer_queue_cancel(kbdev);
1932
1933 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) {
1934 kbase_gpu_start_cache_clean_nolock(
1935 kbdev, GPU_COMMAND_CACHE_CLN_INV_L2);
1936 backend->shaders_state =
1937 KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON;
1938 } else {
1939 backend->shaders_state =
1940 KBASE_SHADERS_READY_OFF_CORESTACK_ON;
1941 }
1942 break;
1943
1944 case KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON:
1945 if (!kbdev->cache_clean_in_progress)
1946 backend->shaders_state =
1947 KBASE_SHADERS_READY_OFF_CORESTACK_ON;
1948
1949 break;
1950
1951 case KBASE_SHADERS_READY_OFF_CORESTACK_ON:
1952 if (backend->partial_shaderoff) {
1953 backend->partial_shaderoff = false;
1954 /* remove cores available but not ready to
1955 * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON
1956 * check pass
1957 */
1958
1959 /* shaders_desired_mask shall be a subset of
1960 * shaders_ready
1961 */
1962 WARN_ON(backend->shaders_desired_mask & ~shaders_ready);
1963 WARN_ON(!(backend->shaders_desired_mask & shaders_ready));
1964
1965 backend->shaders_avail =
1966 backend->shaders_desired_mask;
1967 kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
1968 shaders_ready & ~backend->shaders_avail, ACTION_PWROFF);
1969 backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON;
1970 KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, (shaders_ready & ~backend->shaders_avail));
1971 } else {
1972 kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
1973 shaders_ready, ACTION_PWROFF);
1974
1975 KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u);
1976
1977 backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON;
1978 }
1979 break;
1980
1981 case KBASE_SHADERS_PEND_OFF_CORESTACK_ON:
1982 if (!shaders_trans && !shaders_ready) {
1983 if (corestack_driver_control)
1984 kbase_pm_invoke(kbdev, KBASE_PM_CORE_STACK,
1985 stacks_avail, ACTION_PWROFF);
1986
1987 backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_PEND_OFF;
1988 }
1989 break;
1990
1991 case KBASE_SHADERS_OFF_CORESTACK_PEND_OFF:
1992 if (!stacks_trans && !stacks_ready) {
1993 /* On powered off, re-enable the hwcnt */
1994 backend->pm_shaders_core_mask = 0;
1995 backend->hwcnt_desired = true;
1996 if (backend->hwcnt_disabled) {
1997 #if MALI_USE_CSF
1998 unsigned long flags;
1999
2000 kbase_csf_scheduler_spin_lock(kbdev,
2001 &flags);
2002 #endif
2003 kbase_hwcnt_context_enable(
2004 kbdev->hwcnt_gpu_ctx);
2005 #if MALI_USE_CSF
2006 kbase_csf_scheduler_spin_unlock(kbdev,
2007 flags);
2008 #endif
2009 backend->hwcnt_disabled = false;
2010 }
2011 backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF;
2012 }
2013 break;
2014
2015 case KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF:
2016 if (!hrtimer_active(&stt->timer) && !stt->cancel_queued)
2017 backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF;
2018 break;
2019
2020 case KBASE_SHADERS_RESET_WAIT:
2021 /* Reset complete */
2022 if (!backend->in_reset)
2023 backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF;
2024 break;
2025 }
2026
2027 if (backend->shaders_state != prev_state)
2028 dev_dbg(kbdev->dev, "Shader state transition: %s to %s\n",
2029 kbase_shader_core_state_to_string(prev_state),
2030 kbase_shader_core_state_to_string(
2031 backend->shaders_state));
2032
2033 } while (backend->shaders_state != prev_state);
2034
2035 return 0;
2036 }
2037 #endif /* !MALI_USE_CSF */
2038
2039 static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev)
2040 {
2041 bool in_desired_state = true;
2042
2043 lockdep_assert_held(&kbdev->hwaccess_lock);
2044
2045 in_desired_state = kbase_pm_l2_is_in_desired_state(kbdev);
2046
2047 #if !MALI_USE_CSF
2048 if (kbdev->pm.backend.shaders_desired &&
2049 kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON)
2050 in_desired_state = false;
2051 else if (!kbdev->pm.backend.shaders_desired &&
2052 kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF)
2053 in_desired_state = false;
2054 #else
2055 in_desired_state &= kbase_pm_mcu_is_in_desired_state(kbdev);
2056 #endif
2057
2058 return in_desired_state;
2059 }
2060
2061 static bool kbase_pm_is_in_desired_state(struct kbase_device *kbdev)
2062 {
2063 bool in_desired_state;
2064 unsigned long flags;
2065
2066 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2067 in_desired_state = kbase_pm_is_in_desired_state_nolock(kbdev);
2068 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2069
2070 return in_desired_state;
2071 }
2072
2073 static bool kbase_pm_is_in_desired_state_with_l2_powered(
2074 struct kbase_device *kbdev)
2075 {
2076 bool in_desired_state = false;
2077 unsigned long flags;
2078
2079 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2080 if (kbase_pm_is_in_desired_state_nolock(kbdev) &&
2081 (kbdev->pm.backend.l2_state == KBASE_L2_ON))
2082 in_desired_state = true;
2083 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2084
2085 return in_desired_state;
2086 }
2087
2088 static void kbase_pm_trace_power_state(struct kbase_device *kbdev)
2089 {
2090 lockdep_assert_held(&kbdev->hwaccess_lock);
2091
2092 KBASE_TLSTREAM_AUX_PM_STATE(
2093 kbdev,
2094 KBASE_PM_CORE_L2,
2095 kbase_pm_get_ready_cores(
2096 kbdev, KBASE_PM_CORE_L2));
2097 KBASE_TLSTREAM_AUX_PM_STATE(
2098 kbdev,
2099 KBASE_PM_CORE_SHADER,
2100 kbase_pm_get_ready_cores(
2101 kbdev, KBASE_PM_CORE_SHADER));
2102 KBASE_TLSTREAM_AUX_PM_STATE(
2103 kbdev,
2104 KBASE_PM_CORE_TILER,
2105 kbase_pm_get_ready_cores(
2106 kbdev,
2107 KBASE_PM_CORE_TILER));
2108
2109 if (corestack_driver_control)
2110 KBASE_TLSTREAM_AUX_PM_STATE(
2111 kbdev,
2112 KBASE_PM_CORE_STACK,
2113 kbase_pm_get_ready_cores(
2114 kbdev,
2115 KBASE_PM_CORE_STACK));
2116 }
2117
2118 void kbase_pm_update_state(struct kbase_device *kbdev)
2119 {
2120 #if !MALI_USE_CSF
2121 enum kbase_shader_core_state prev_shaders_state =
2122 kbdev->pm.backend.shaders_state;
2123 #else
2124 enum kbase_mcu_state prev_mcu_state = kbdev->pm.backend.mcu_state;
2125 #endif
2126
2127 lockdep_assert_held(&kbdev->hwaccess_lock);
2128
2129 if (!kbdev->pm.backend.gpu_ready)
2130 return; /* Do nothing if the GPU is not ready */
2131
2132 if (kbase_pm_l2_update_state(kbdev))
2133 return;
2134
2135 #if !MALI_USE_CSF
2136 if (kbase_pm_shaders_update_state(kbdev))
2137 return;
2138
2139 /* If the shaders just turned off, re-invoke the L2 state machine, in
2140 * case it was waiting for the shaders to turn off before powering down
2141 * the L2.
2142 */
2143 if (prev_shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF &&
2144 kbdev->pm.backend.shaders_state ==
2145 KBASE_SHADERS_OFF_CORESTACK_OFF) {
2146 if (kbase_pm_l2_update_state(kbdev))
2147 return;
2148 }
2149 #else
2150 if (kbase_pm_mcu_update_state(kbdev))
2151 return;
2152
2153 if (!kbase_pm_is_mcu_inactive(kbdev, prev_mcu_state) &&
2154 kbase_pm_is_mcu_inactive(kbdev, kbdev->pm.backend.mcu_state)) {
2155 if (kbase_pm_l2_update_state(kbdev))
2156 return;
2157 }
2158 #endif
2159
2160 if (kbase_pm_is_in_desired_state_nolock(kbdev)) {
2161 KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL,
2162 kbdev->pm.backend.shaders_avail);
2163
2164 kbase_pm_trace_power_state(kbdev);
2165
2166 KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0);
2167 wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
2168 }
2169 }
2170
2171 static enum hrtimer_restart
2172 shader_tick_timer_callback(struct hrtimer *timer)
2173 {
2174 struct kbasep_pm_tick_timer_state *stt = container_of(timer,
2175 struct kbasep_pm_tick_timer_state, timer);
2176 struct kbase_device *kbdev = container_of(stt, struct kbase_device,
2177 pm.backend.shader_tick_timer);
2178 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
2179 unsigned long flags;
2180 enum hrtimer_restart restart = HRTIMER_NORESTART;
2181
2182 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2183
2184 if (stt->remaining_ticks &&
2185 backend->shaders_state == KBASE_SHADERS_WAIT_OFF_CORESTACK_ON) {
2186 stt->remaining_ticks--;
2187
2188 /* If the remaining ticks just changed from 1 to 0, invoke the
2189 * PM state machine to power off the shader cores.
2190 */
2191 if (!stt->remaining_ticks && !backend->shaders_desired)
2192 kbase_pm_update_state(kbdev);
2193 }
2194
2195 if (stt->needed) {
2196 hrtimer_forward_now(timer, stt->configured_interval);
2197 restart = HRTIMER_RESTART;
2198 }
2199
2200 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2201
2202 return restart;
2203 }
2204
2205 int kbase_pm_state_machine_init(struct kbase_device *kbdev)
2206 {
2207 struct kbasep_pm_tick_timer_state *stt = &kbdev->pm.backend.shader_tick_timer;
2208
2209 stt->wq = alloc_workqueue("kbase_pm_shader_poweroff", WQ_HIGHPRI | WQ_UNBOUND, 1);
2210 if (!stt->wq)
2211 return -ENOMEM;
2212
2213 INIT_WORK(&stt->work, shader_poweroff_timer_stop_callback);
2214
2215 stt->needed = false;
2216 hrtimer_init(&stt->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2217 stt->timer.function = shader_tick_timer_callback;
2218 stt->configured_interval = HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS);
2219 stt->default_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
2220 stt->configured_ticks = stt->default_ticks;
2221
2222 #if MALI_USE_CSF
2223 kbdev->pm.backend.core_idle_wq = alloc_workqueue("coreoff_wq", WQ_HIGHPRI | WQ_UNBOUND, 1);
2224 if (!kbdev->pm.backend.core_idle_wq) {
2225 destroy_workqueue(stt->wq);
2226 return -ENOMEM;
2227 }
2228
2229 INIT_WORK(&kbdev->pm.backend.core_idle_work, core_idle_worker);
2230 #endif
2231
2232 return 0;
2233 }
2234
2235 void kbase_pm_state_machine_term(struct kbase_device *kbdev)
2236 {
2237 #if MALI_USE_CSF
2238 destroy_workqueue(kbdev->pm.backend.core_idle_wq);
2239 #endif
2240 hrtimer_cancel(&kbdev->pm.backend.shader_tick_timer.timer);
2241 destroy_workqueue(kbdev->pm.backend.shader_tick_timer.wq);
2242 }
2243
2244 void kbase_pm_reset_start_locked(struct kbase_device *kbdev)
2245 {
2246 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
2247
2248 lockdep_assert_held(&kbdev->hwaccess_lock);
2249
2250 backend->in_reset = true;
2251 backend->l2_state = KBASE_L2_RESET_WAIT;
2252 KBASE_KTRACE_ADD(kbdev, PM_L2_RESET_WAIT, NULL, backend->l2_state);
2253 #if !MALI_USE_CSF
2254 backend->shaders_state = KBASE_SHADERS_RESET_WAIT;
2255 #else
2256 /* MCU state machine is exercised only after the initial load/boot
2257 * of the firmware.
2258 */
2259 if (likely(kbdev->csf.firmware_inited)) {
2260 backend->mcu_state = KBASE_MCU_RESET_WAIT;
2261 KBASE_KTRACE_ADD(kbdev, PM_MCU_RESET_WAIT, NULL, backend->mcu_state);
2262 #ifdef KBASE_PM_RUNTIME
2263 backend->exit_gpu_sleep_mode = true;
2264 #endif
2265 kbdev->csf.firmware_reload_needed = true;
2266 } else {
2267 WARN_ON(backend->mcu_state != KBASE_MCU_OFF);
2268 }
2269 #endif
2270
2271 /* We're in a reset, so hwcnt will have been synchronously disabled by
2272 * this function's caller as part of the reset process. We therefore
2273 * know that any call to kbase_hwcnt_context_disable_atomic, if
2274 * required to sync the hwcnt refcount with our internal state, is
2275 * guaranteed to succeed.
2276 */
2277 backend->hwcnt_desired = false;
2278 if (!backend->hwcnt_disabled) {
2279 WARN_ON(!kbase_hwcnt_context_disable_atomic(
2280 kbdev->hwcnt_gpu_ctx));
2281 backend->hwcnt_disabled = true;
2282 }
2283
2284 shader_poweroff_timer_queue_cancel(kbdev);
2285 }
2286
2287 void kbase_pm_reset_complete(struct kbase_device *kbdev)
2288 {
2289 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
2290 unsigned long flags;
2291
2292 WARN_ON(!kbase_reset_gpu_is_active(kbdev));
2293 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2294
2295 /* As GPU has just been reset, that results in implicit flush of L2
2296 * cache, can safely mark the pending cache flush operation (if there
2297 * was any) as complete and unblock the waiter.
2298 * No work can be submitted whilst GPU reset is ongoing.
2299 */
2300 kbase_gpu_cache_clean_wait_complete(kbdev);
2301 backend->in_reset = false;
2302 #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
2303 backend->gpu_wakeup_override = false;
2304 #endif
2305 kbase_pm_update_state(kbdev);
2306
2307 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2308 }
2309
2310 #if !MALI_USE_CSF
2311 /* Timeout in milliseconds for GPU Power Management to reach the desired
2312 * Shader and L2 state. If the time spent waiting has exceeded this threshold
2313 * then there is most likely a hardware issue.
2314 */
2315 #define PM_TIMEOUT_MS (5000) /* 5s */
2316 #endif
2317
2318 static void kbase_pm_timed_out(struct kbase_device *kbdev)
2319 {
2320 unsigned long flags;
2321
2322 dev_err(kbdev->dev, "Power transition timed out unexpectedly\n");
2323 #if !MALI_USE_CSF
2324 CSTD_UNUSED(flags);
2325 dev_err(kbdev->dev, "Desired state :\n");
2326 dev_err(kbdev->dev, "\tShader=%016llx\n",
2327 kbdev->pm.backend.shaders_desired ? kbdev->pm.backend.shaders_avail : 0);
2328 #else
2329 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2330 dev_err(kbdev->dev, "\tMCU desired = %d\n",
2331 kbase_pm_is_mcu_desired(kbdev));
2332 dev_err(kbdev->dev, "\tMCU sw state = %d\n",
2333 kbdev->pm.backend.mcu_state);
2334 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2335 #endif
2336 dev_err(kbdev->dev, "Current state :\n");
2337 dev_err(kbdev->dev, "\tShader=%08x%08x\n",
2338 kbase_reg_read(kbdev,
2339 GPU_CONTROL_REG(SHADER_READY_HI)),
2340 kbase_reg_read(kbdev,
2341 GPU_CONTROL_REG(SHADER_READY_LO)));
2342 dev_err(kbdev->dev, "\tTiler =%08x%08x\n",
2343 kbase_reg_read(kbdev,
2344 GPU_CONTROL_REG(TILER_READY_HI)),
2345 kbase_reg_read(kbdev,
2346 GPU_CONTROL_REG(TILER_READY_LO)));
2347 dev_err(kbdev->dev, "\tL2 =%08x%08x\n",
2348 kbase_reg_read(kbdev,
2349 GPU_CONTROL_REG(L2_READY_HI)),
2350 kbase_reg_read(kbdev,
2351 GPU_CONTROL_REG(L2_READY_LO)));
2352 #if MALI_USE_CSF
2353 dev_err(kbdev->dev, "\tMCU status = %d\n",
2354 kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS)));
2355 #endif
2356 dev_err(kbdev->dev, "Cores transitioning :\n");
2357 dev_err(kbdev->dev, "\tShader=%08x%08x\n",
2358 kbase_reg_read(kbdev, GPU_CONTROL_REG(
2359 SHADER_PWRTRANS_HI)),
2360 kbase_reg_read(kbdev, GPU_CONTROL_REG(
2361 SHADER_PWRTRANS_LO)));
2362 dev_err(kbdev->dev, "\tTiler =%08x%08x\n",
2363 kbase_reg_read(kbdev, GPU_CONTROL_REG(
2364 TILER_PWRTRANS_HI)),
2365 kbase_reg_read(kbdev, GPU_CONTROL_REG(
2366 TILER_PWRTRANS_LO)));
2367 dev_err(kbdev->dev, "\tL2 =%08x%08x\n",
2368 kbase_reg_read(kbdev, GPU_CONTROL_REG(
2369 L2_PWRTRANS_HI)),
2370 kbase_reg_read(kbdev, GPU_CONTROL_REG(
2371 L2_PWRTRANS_LO)));
2372
2373 dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n");
2374 if (kbase_prepare_to_reset_gpu(kbdev,
2375 RESET_FLAGS_HWC_UNRECOVERABLE_ERROR))
2376 kbase_reset_gpu(kbdev);
2377 }
2378
2379 int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev)
2380 {
2381 unsigned long flags;
2382 unsigned long timeout;
2383 long remaining;
2384 int err = 0;
2385
2386 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2387 kbase_pm_update_state(kbdev);
2388 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2389
2390 #if MALI_USE_CSF
2391 timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
2392 #else
2393 timeout = msecs_to_jiffies(PM_TIMEOUT_MS);
2394 #endif
2395
2396 /* Wait for cores */
2397 #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE
2398 remaining = wait_event_killable_timeout(kbdev->pm.backend.gpu_in_desired_state_wait,
2399 kbase_pm_is_in_desired_state_with_l2_powered(kbdev),
2400 timeout);
2401 #else
2402 remaining = wait_event_timeout(
2403 kbdev->pm.backend.gpu_in_desired_state_wait,
2404 kbase_pm_is_in_desired_state_with_l2_powered(kbdev), timeout);
2405 #endif
2406
2407 if (!remaining) {
2408 kbase_pm_timed_out(kbdev);
2409 err = -ETIMEDOUT;
2410 } else if (remaining < 0) {
2411 dev_info(
2412 kbdev->dev,
2413 "Wait for desired PM state with L2 powered got interrupted");
2414 err = (int)remaining;
2415 }
2416
2417 return err;
2418 }
2419
2420 int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev)
2421 {
2422 unsigned long flags;
2423 long remaining;
2424 #if MALI_USE_CSF
2425 long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
2426 #else
2427 long timeout = msecs_to_jiffies(PM_TIMEOUT_MS);
2428 #endif
2429 int err = 0;
2430
2431 /* Let the state machine latch the most recent desired state. */
2432 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2433 kbase_pm_update_state(kbdev);
2434 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2435
2436 /* Wait for cores */
2437 #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE
2438 remaining = wait_event_killable_timeout(
2439 kbdev->pm.backend.gpu_in_desired_state_wait,
2440 kbase_pm_is_in_desired_state(kbdev), timeout);
2441 #else
2442 remaining = wait_event_timeout(
2443 kbdev->pm.backend.gpu_in_desired_state_wait,
2444 kbase_pm_is_in_desired_state(kbdev), timeout);
2445 #endif
2446
2447 if (!remaining) {
2448 kbase_pm_timed_out(kbdev);
2449 err = -ETIMEDOUT;
2450 } else if (remaining < 0) {
2451 dev_info(kbdev->dev,
2452 "Wait for desired PM state got interrupted");
2453 err = (int)remaining;
2454 }
2455
2456 return err;
2457 }
2458 KBASE_EXPORT_TEST_API(kbase_pm_wait_for_desired_state);
2459
2460 #if MALI_USE_CSF
2461 /**
2462 * core_mask_update_done - Check if downscaling of shader cores is done
2463 *
2464 * @kbdev: The kbase device structure for the device.
2465 *
2466 * This function checks if the downscaling of cores is effectively complete.
2467 *
2468 * Return: true if the downscale is done.
2469 */
2470 static bool core_mask_update_done(struct kbase_device *kbdev)
2471 {
2472 bool update_done = false;
2473 unsigned long flags;
2474
2475 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2476 /* If MCU is in stable ON state then it implies that the downscale
2477 * request had completed.
2478 * If MCU is not active then it implies all cores are off, so can
2479 * consider the downscale request as complete.
2480 */
2481 if ((kbdev->pm.backend.mcu_state == KBASE_MCU_ON) ||
2482 kbase_pm_is_mcu_inactive(kbdev, kbdev->pm.backend.mcu_state))
2483 update_done = true;
2484 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2485
2486 return update_done;
2487 }
2488
2489 int kbase_pm_wait_for_cores_down_scale(struct kbase_device *kbdev)
2490 {
2491 long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
2492 long remaining;
2493 int err = 0;
2494
2495 /* Wait for core mask update to complete */
2496 #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE
2497 remaining = wait_event_killable_timeout(
2498 kbdev->pm.backend.gpu_in_desired_state_wait,
2499 core_mask_update_done(kbdev), timeout);
2500 #else
2501 remaining = wait_event_timeout(
2502 kbdev->pm.backend.gpu_in_desired_state_wait,
2503 core_mask_update_done(kbdev), timeout);
2504 #endif
2505
2506 if (!remaining) {
2507 kbase_pm_timed_out(kbdev);
2508 err = -ETIMEDOUT;
2509 } else if (remaining < 0) {
2510 dev_info(
2511 kbdev->dev,
2512 "Wait for cores down scaling got interrupted");
2513 err = (int)remaining;
2514 }
2515
2516 return err;
2517 }
2518 #endif
2519
2520 void kbase_pm_enable_interrupts(struct kbase_device *kbdev)
2521 {
2522 unsigned long flags;
2523
2524 KBASE_DEBUG_ASSERT(kbdev != NULL);
2525 /*
2526 * Clear all interrupts,
2527 * and unmask them all.
2528 */
2529 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2530 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL);
2531 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL);
2532 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2533
2534 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF);
2535 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF);
2536
2537 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF);
2538 #if MALI_USE_CSF
2539 /* Enable only the Page fault bits part */
2540 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFF);
2541 #else
2542 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF);
2543 #endif
2544 }
2545
2546 KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts);
2547
2548 void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev)
2549 {
2550 KBASE_DEBUG_ASSERT(kbdev != NULL);
2551 /*
2552 * Mask all interrupts,
2553 * and clear them all.
2554 */
2555 lockdep_assert_held(&kbdev->hwaccess_lock);
2556
2557 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0);
2558 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL);
2559 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0);
2560 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF);
2561
2562 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0);
2563 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF);
2564 }
2565
2566 void kbase_pm_disable_interrupts(struct kbase_device *kbdev)
2567 {
2568 unsigned long flags;
2569
2570 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2571 kbase_pm_disable_interrupts_nolock(kbdev);
2572 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2573 }
2574
2575 KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts);
2576
2577 #if MALI_USE_CSF
2578 /**
2579 * update_user_reg_page_mapping - Update the mapping for USER Register page
2580 *
2581 * @kbdev: The kbase device structure for the device.
2582 *
2583 * This function must be called to unmap the dummy or real page from USER Register page
2584 * mapping whenever GPU is powered up or down. The dummy or real page would get
2585 * appropriately mapped in when Userspace reads the LATEST_FLUSH value.
2586 */
2587 static void update_user_reg_page_mapping(struct kbase_device *kbdev)
2588 {
2589 struct kbase_context *kctx, *n;
2590
2591 lockdep_assert_held(&kbdev->pm.lock);
2592
2593 mutex_lock(&kbdev->csf.reg_lock);
2594 list_for_each_entry_safe(kctx, n, &kbdev->csf.user_reg.list, csf.user_reg.link) {
2595 /* This would zap the PTE corresponding to the mapping of User
2596 * Register page of the kbase context. The mapping will be reestablished
2597 * when the context (user process) needs to access to the page.
2598 */
2599 unmap_mapping_range(kbdev->csf.user_reg.filp->f_inode->i_mapping,
2600 kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE, 1);
2601 list_del_init(&kctx->csf.user_reg.link);
2602 dev_dbg(kbdev->dev, "Updated USER Reg page mapping of ctx %d_%d", kctx->tgid,
2603 kctx->id);
2604 }
2605 mutex_unlock(&kbdev->csf.reg_lock);
2606 }
2607 #endif
2608
2609 /*
2610 * pmu layout:
2611 * 0x0000: PMU TAG (RO) (0xCAFECAFE)
2612 * 0x0004: PMU VERSION ID (RO) (0x00000000)
2613 * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE)
2614 */
2615 void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
2616 {
2617 struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
2618 bool reset_required = is_resume;
2619 unsigned long flags;
2620
2621 KBASE_DEBUG_ASSERT(kbdev != NULL);
2622 #if !MALI_USE_CSF
2623 lockdep_assert_held(&kbdev->js_data.runpool_mutex);
2624 #endif /* !MALI_USE_CSF */
2625 lockdep_assert_held(&kbdev->pm.lock);
2626
2627 #ifdef CONFIG_MALI_ARBITER_SUPPORT
2628 if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) {
2629 dev_err(kbdev->dev,
2630 "%s: Cannot power up while GPU lost", __func__);
2631 return;
2632 }
2633 #endif
2634
2635 if (backend->gpu_powered) {
2636 #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
2637 if (backend->gpu_idled) {
2638 backend->callback_power_runtime_gpu_active(kbdev);
2639 backend->gpu_idled = false;
2640 }
2641 #endif
2642 /* Already turned on */
2643 if (kbdev->poweroff_pending)
2644 kbase_pm_enable_interrupts(kbdev);
2645 kbdev->poweroff_pending = false;
2646 KBASE_DEBUG_ASSERT(!is_resume);
2647 return;
2648 }
2649
2650 kbdev->poweroff_pending = false;
2651
2652 KBASE_KTRACE_ADD(kbdev, PM_GPU_ON, NULL, 0u);
2653
2654 if (is_resume && backend->callback_power_resume) {
2655 backend->callback_power_resume(kbdev);
2656 return;
2657 } else if (backend->callback_power_on) {
2658 reset_required = backend->callback_power_on(kbdev);
2659 }
2660
2661 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2662 backend->gpu_powered = true;
2663 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2664
2665 #if MALI_USE_CSF
2666 /* GPU has been turned on, can switch to actual register page */
2667 update_user_reg_page_mapping(kbdev);
2668 #endif
2669
2670
2671 if (reset_required) {
2672 /* GPU state was lost, reset GPU to ensure it is in a
2673 * consistent state
2674 */
2675 kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS);
2676 }
2677 #ifdef CONFIG_MALI_ARBITER_SUPPORT
2678 else {
2679 if (kbdev->arb.arb_if) {
2680 struct kbase_arbiter_vm_state *arb_vm_state =
2681 kbdev->pm.arb_vm_state;
2682
2683 /* In the case that the GPU has just been granted by
2684 * the Arbiter, a reset will have already been done.
2685 * However, it is still necessary to initialize the GPU.
2686 */
2687 if (arb_vm_state->vm_arb_starting)
2688 kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS |
2689 PM_NO_RESET);
2690 }
2691 }
2692 /*
2693 * This point means that the GPU trasitioned to ON. So there is a chance
2694 * that a repartitioning occurred. In this case the current config
2695 * should be read again.
2696 */
2697 kbase_gpuprops_get_curr_config_props(kbdev,
2698 &kbdev->gpu_props.curr_config);
2699 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
2700
2701 mutex_lock(&kbdev->mmu_hw_mutex);
2702 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2703 kbase_ctx_sched_restore_all_as(kbdev);
2704 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2705 mutex_unlock(&kbdev->mmu_hw_mutex);
2706
2707 if (kbdev->dummy_job_wa.flags &
2708 KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) {
2709 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2710 kbase_dummy_job_wa_execute(kbdev,
2711 kbase_pm_get_present_cores(kbdev,
2712 KBASE_PM_CORE_SHADER));
2713 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2714 }
2715
2716 /* Enable the interrupts */
2717 kbase_pm_enable_interrupts(kbdev);
2718
2719 /* Turn on the L2 caches */
2720 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2721 backend->gpu_ready = true;
2722 backend->l2_desired = true;
2723 #if MALI_USE_CSF
2724 if (reset_required) {
2725 /* GPU reset was done after the power on, so send the post
2726 * reset event instead. This is okay as GPU power off event
2727 * is same as pre GPU reset event.
2728 */
2729 kbase_ipa_control_handle_gpu_reset_post(kbdev);
2730 } else {
2731 kbase_ipa_control_handle_gpu_power_on(kbdev);
2732 }
2733 #endif
2734 kbase_pm_update_state(kbdev);
2735 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2736
2737 #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
2738 /* GPU is now powered up. Invoke the GPU active callback as GPU idle
2739 * callback would have been invoked before the power down.
2740 */
2741 if (backend->gpu_idled) {
2742 backend->callback_power_runtime_gpu_active(kbdev);
2743 backend->gpu_idled = false;
2744 }
2745 #endif
2746 }
2747
2748 KBASE_EXPORT_TEST_API(kbase_pm_clock_on);
2749
2750 bool kbase_pm_clock_off(struct kbase_device *kbdev)
2751 {
2752 unsigned long flags;
2753
2754 KBASE_DEBUG_ASSERT(kbdev != NULL);
2755 lockdep_assert_held(&kbdev->pm.lock);
2756
2757 /* ASSERT that the cores should now be unavailable. No lock needed. */
2758 WARN_ON(kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF);
2759
2760 kbdev->poweroff_pending = true;
2761
2762 if (!kbdev->pm.backend.gpu_powered) {
2763 /* Already turned off */
2764 return true;
2765 }
2766
2767 KBASE_KTRACE_ADD(kbdev, PM_GPU_OFF, NULL, 0u);
2768
2769 /* Disable interrupts. This also clears any outstanding interrupts */
2770 kbase_pm_disable_interrupts(kbdev);
2771 /* Ensure that any IRQ handlers have finished */
2772 kbase_synchronize_irqs(kbdev);
2773
2774 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2775
2776 if (atomic_read(&kbdev->faults_pending)) {
2777 /* Page/bus faults are still being processed. The GPU can not
2778 * be powered off until they have completed
2779 */
2780 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2781 return false;
2782 }
2783
2784 kbase_pm_cache_snoop_disable(kbdev);
2785 #if MALI_USE_CSF
2786 kbase_ipa_control_handle_gpu_power_off(kbdev);
2787 #endif
2788
2789 if (kbase_is_gpu_removed(kbdev)
2790 #ifdef CONFIG_MALI_ARBITER_SUPPORT
2791 || kbase_pm_is_gpu_lost(kbdev)) {
2792 #else
2793 ) {
2794 #endif
2795 /* Ensure we unblock any threads that are stuck waiting
2796 * for the GPU
2797 */
2798 kbase_gpu_cache_clean_wait_complete(kbdev);
2799 }
2800
2801 kbdev->pm.backend.gpu_ready = false;
2802
2803 /* The GPU power may be turned off from this point */
2804 kbdev->pm.backend.gpu_powered = false;
2805
2806 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2807
2808 #if MALI_USE_CSF
2809 /* GPU is about to be turned off, switch to dummy page */
2810 update_user_reg_page_mapping(kbdev);
2811 #endif
2812
2813 #ifdef CONFIG_MALI_ARBITER_SUPPORT
2814 kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT);
2815 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
2816
2817 if (kbdev->pm.backend.callback_power_off)
2818 kbdev->pm.backend.callback_power_off(kbdev);
2819 return true;
2820 }
2821
2822 KBASE_EXPORT_TEST_API(kbase_pm_clock_off);
2823
2824 struct kbasep_reset_timeout_data {
2825 struct hrtimer timer;
2826 bool timed_out;
2827 struct kbase_device *kbdev;
2828 };
2829
2830 void kbase_pm_reset_done(struct kbase_device *kbdev)
2831 {
2832 KBASE_DEBUG_ASSERT(kbdev != NULL);
2833 kbdev->pm.backend.reset_done = true;
2834 wake_up(&kbdev->pm.backend.reset_done_wait);
2835 }
2836
2837 /**
2838 * kbase_pm_wait_for_reset - Wait for a reset to happen
2839 *
2840 * @kbdev: Kbase device
2841 *
2842 * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state.
2843 */
2844 static void kbase_pm_wait_for_reset(struct kbase_device *kbdev)
2845 {
2846 lockdep_assert_held(&kbdev->pm.lock);
2847
2848 wait_event(kbdev->pm.backend.reset_done_wait,
2849 (kbdev->pm.backend.reset_done));
2850 kbdev->pm.backend.reset_done = false;
2851 }
2852
2853 KBASE_EXPORT_TEST_API(kbase_pm_reset_done);
2854
2855 static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
2856 {
2857 struct kbasep_reset_timeout_data *rtdata =
2858 container_of(timer, struct kbasep_reset_timeout_data, timer);
2859
2860 rtdata->timed_out = true;
2861
2862 /* Set the wait queue to wake up kbase_pm_init_hw even though the reset
2863 * hasn't completed
2864 */
2865 kbase_pm_reset_done(rtdata->kbdev);
2866
2867 return HRTIMER_NORESTART;
2868 }
2869
2870 static int kbase_set_gpu_quirks(struct kbase_device *kbdev, const u32 prod_id)
2871 {
2872 #if MALI_USE_CSF
2873 kbdev->hw_quirks_gpu =
2874 kbase_reg_read(kbdev, GPU_CONTROL_REG(CSF_CONFIG));
2875 #else
2876 u32 hw_quirks_gpu = kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG));
2877
2878 if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == GPU_ID2_PRODUCT_TMIX) {
2879 /* Only for tMIx */
2880 u32 coherency_features;
2881
2882 coherency_features = kbase_reg_read(kbdev,
2883 GPU_CONTROL_REG(COHERENCY_FEATURES));
2884
2885 /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly
2886 * documented for tMIx so force correct value here.
2887 */
2888 if (coherency_features ==
2889 COHERENCY_FEATURE_BIT(COHERENCY_ACE)) {
2890 hw_quirks_gpu |= (COHERENCY_ACE_LITE | COHERENCY_ACE)
2891 << JM_FORCE_COHERENCY_FEATURES_SHIFT;
2892 }
2893 }
2894
2895 if (kbase_is_gpu_removed(kbdev))
2896 return -EIO;
2897
2898 kbdev->hw_quirks_gpu = hw_quirks_gpu;
2899
2900 #endif /* !MALI_USE_CSF */
2901 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) {
2902 int default_idvs_group_size = 0xF;
2903 u32 group_size = 0;
2904
2905 if (of_property_read_u32(kbdev->dev->of_node, "idvs-group-size",
2906 &group_size))
2907 group_size = default_idvs_group_size;
2908
2909 if (group_size > IDVS_GROUP_MAX_SIZE) {
2910 dev_err(kbdev->dev,
2911 "idvs-group-size of %d is too large. Maximum value is %d",
2912 group_size, IDVS_GROUP_MAX_SIZE);
2913 group_size = default_idvs_group_size;
2914 }
2915
2916 kbdev->hw_quirks_gpu |= group_size << IDVS_GROUP_SIZE_SHIFT;
2917 }
2918
2919 #define MANUAL_POWER_CONTROL ((u32)(1 << 8))
2920 if (corestack_driver_control)
2921 kbdev->hw_quirks_gpu |= MANUAL_POWER_CONTROL;
2922
2923 return 0;
2924 }
2925
2926 static int kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id)
2927 {
2928 u32 hw_quirks_sc = kbase_reg_read(kbdev,
2929 GPU_CONTROL_REG(SHADER_CONFIG));
2930
2931 if (kbase_is_gpu_removed(kbdev))
2932 return -EIO;
2933
2934 if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */
2935 hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE;
2936 else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */
2937 hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES;
2938
2939 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162))
2940 hw_quirks_sc |= SC_VAR_ALGORITHM;
2941
2942 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING))
2943 hw_quirks_sc |= SC_TLS_HASH_ENABLE;
2944
2945 kbdev->hw_quirks_sc = hw_quirks_sc;
2946
2947 return 0;
2948 }
2949
2950 static int kbase_set_tiler_quirks(struct kbase_device *kbdev)
2951 {
2952 u32 hw_quirks_tiler = kbase_reg_read(kbdev,
2953 GPU_CONTROL_REG(TILER_CONFIG));
2954
2955 if (kbase_is_gpu_removed(kbdev))
2956 return -EIO;
2957
2958 /* Set tiler clock gate override if required */
2959 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953))
2960 hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
2961
2962 kbdev->hw_quirks_tiler = hw_quirks_tiler;
2963
2964 return 0;
2965 }
2966
2967 static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
2968 {
2969 struct device_node *np = kbdev->dev->of_node;
2970 const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
2971 const u32 prod_id =
2972 (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> KBASE_GPU_ID_VERSION_PRODUCT_ID_SHIFT;
2973 int error = 0;
2974
2975 kbdev->hw_quirks_gpu = 0;
2976 kbdev->hw_quirks_sc = 0;
2977 kbdev->hw_quirks_tiler = 0;
2978 kbdev->hw_quirks_mmu = 0;
2979
2980 if (!of_property_read_u32(np, "quirks_gpu", &kbdev->hw_quirks_gpu)) {
2981 dev_info(kbdev->dev,
2982 "Found quirks_gpu = [0x%x] in Devicetree\n",
2983 kbdev->hw_quirks_gpu);
2984 } else {
2985 error = kbase_set_gpu_quirks(kbdev, prod_id);
2986 if (error)
2987 return error;
2988 }
2989
2990 if (!of_property_read_u32(np, "quirks_sc",
2991 &kbdev->hw_quirks_sc)) {
2992 dev_info(kbdev->dev,
2993 "Found quirks_sc = [0x%x] in Devicetree\n",
2994 kbdev->hw_quirks_sc);
2995 } else {
2996 error = kbase_set_sc_quirks(kbdev, prod_id);
2997 if (error)
2998 return error;
2999 }
3000
3001 if (!of_property_read_u32(np, "quirks_tiler",
3002 &kbdev->hw_quirks_tiler)) {
3003 dev_info(kbdev->dev,
3004 "Found quirks_tiler = [0x%x] in Devicetree\n",
3005 kbdev->hw_quirks_tiler);
3006 } else {
3007 error = kbase_set_tiler_quirks(kbdev);
3008 if (error)
3009 return error;
3010 }
3011
3012 if (!of_property_read_u32(np, "quirks_mmu",
3013 &kbdev->hw_quirks_mmu)) {
3014 dev_info(kbdev->dev,
3015 "Found quirks_mmu = [0x%x] in Devicetree\n",
3016 kbdev->hw_quirks_mmu);
3017 } else {
3018 error = kbase_set_mmu_quirks(kbdev);
3019 }
3020
3021 return error;
3022 }
3023
3024 static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev)
3025 {
3026 kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG),
3027 kbdev->hw_quirks_sc);
3028
3029 kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG),
3030 kbdev->hw_quirks_tiler);
3031
3032 kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG),
3033 kbdev->hw_quirks_mmu);
3034 #if MALI_USE_CSF
3035 kbase_reg_write(kbdev, GPU_CONTROL_REG(CSF_CONFIG),
3036 kbdev->hw_quirks_gpu);
3037 #else
3038 kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG),
3039 kbdev->hw_quirks_gpu);
3040 #endif
3041 }
3042
3043 void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev)
3044 {
3045 if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) &&
3046 !kbdev->cci_snoop_enabled) {
3047 #if IS_ENABLED(CONFIG_ARM64)
3048 if (kbdev->snoop_enable_smc != 0)
3049 kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0);
3050 #endif /* CONFIG_ARM64 */
3051 dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n");
3052 kbdev->cci_snoop_enabled = true;
3053 }
3054 }
3055
3056 void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev)
3057 {
3058 if (kbdev->cci_snoop_enabled) {
3059 #if IS_ENABLED(CONFIG_ARM64)
3060 if (kbdev->snoop_disable_smc != 0) {
3061 mali_cci_flush_l2(kbdev);
3062 kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0);
3063 }
3064 #endif /* CONFIG_ARM64 */
3065 dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n");
3066 kbdev->cci_snoop_enabled = false;
3067 }
3068 }
3069
3070 #if !MALI_USE_CSF
3071 static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev)
3072 {
3073 unsigned long irq_flags;
3074
3075 spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
3076 kbdev->protected_mode_hwcnt_desired = true;
3077 if (kbdev->protected_mode_hwcnt_disabled) {
3078 kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
3079 kbdev->protected_mode_hwcnt_disabled = false;
3080 }
3081 spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
3082 }
3083 #endif
3084
3085 static int kbase_pm_do_reset(struct kbase_device *kbdev)
3086 {
3087 struct kbasep_reset_timeout_data rtdata;
3088 int ret;
3089
3090 KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0);
3091
3092 KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev);
3093
3094 if (kbdev->pm.backend.callback_soft_reset) {
3095 ret = kbdev->pm.backend.callback_soft_reset(kbdev);
3096 if (ret < 0)
3097 return ret;
3098 else if (ret > 0)
3099 return 0;
3100 } else {
3101 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
3102 GPU_COMMAND_SOFT_RESET);
3103 }
3104
3105 /* Unmask the reset complete interrupt only */
3106 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED);
3107
3108 /* Initialize a structure for tracking the status of the reset */
3109 rtdata.kbdev = kbdev;
3110 rtdata.timed_out = false;
3111
3112 /* Create a timer to use as a timeout on the reset */
3113 hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3114 rtdata.timer.function = kbasep_reset_timeout;
3115
3116 hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
3117 HRTIMER_MODE_REL);
3118
3119 /* Wait for the RESET_COMPLETED interrupt to be raised */
3120 kbase_pm_wait_for_reset(kbdev);
3121
3122 if (!rtdata.timed_out) {
3123 /* GPU has been reset */
3124 hrtimer_cancel(&rtdata.timer);
3125 destroy_hrtimer_on_stack(&rtdata.timer);
3126 return 0;
3127 }
3128
3129 /* No interrupt has been received - check if the RAWSTAT register says
3130 * the reset has completed
3131 */
3132 if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) &
3133 RESET_COMPLETED)) {
3134 /* The interrupt is set in the RAWSTAT; this suggests that the
3135 * interrupts are not getting to the CPU
3136 */
3137 dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
3138 /* If interrupts aren't working we can't continue. */
3139 destroy_hrtimer_on_stack(&rtdata.timer);
3140 return -EINVAL;
3141 }
3142
3143 if (kbase_is_gpu_removed(kbdev)) {
3144 dev_dbg(kbdev->dev, "GPU has been removed, reset no longer needed.\n");
3145 destroy_hrtimer_on_stack(&rtdata.timer);
3146 return -EINVAL;
3147 }
3148
3149 /* The GPU doesn't seem to be responding to the reset so try a hard
3150 * reset, but only when NOT in arbitration mode.
3151 */
3152 #ifdef CONFIG_MALI_ARBITER_SUPPORT
3153 if (!kbdev->arb.arb_if) {
3154 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
3155 dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n",
3156 RESET_TIMEOUT);
3157 KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0);
3158 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
3159 GPU_COMMAND_HARD_RESET);
3160
3161 /* Restart the timer to wait for the hard reset to complete */
3162 rtdata.timed_out = false;
3163
3164 hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
3165 HRTIMER_MODE_REL);
3166
3167 /* Wait for the RESET_COMPLETED interrupt to be raised */
3168 kbase_pm_wait_for_reset(kbdev);
3169
3170 if (!rtdata.timed_out) {
3171 /* GPU has been reset */
3172 hrtimer_cancel(&rtdata.timer);
3173 destroy_hrtimer_on_stack(&rtdata.timer);
3174 return 0;
3175 }
3176
3177 destroy_hrtimer_on_stack(&rtdata.timer);
3178
3179 dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n",
3180 RESET_TIMEOUT);
3181 #ifdef CONFIG_MALI_ARBITER_SUPPORT
3182 }
3183 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
3184
3185 return -EINVAL;
3186 }
3187
3188 int kbase_pm_protected_mode_enable(struct kbase_device *const kbdev)
3189 {
3190 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
3191 GPU_COMMAND_SET_PROTECTED_MODE);
3192 return 0;
3193 }
3194
3195 int kbase_pm_protected_mode_disable(struct kbase_device *const kbdev)
3196 {
3197 lockdep_assert_held(&kbdev->pm.lock);
3198
3199 return kbase_pm_do_reset(kbdev);
3200 }
3201
3202 int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
3203 {
3204 unsigned long irq_flags;
3205 int err = 0;
3206
3207 KBASE_DEBUG_ASSERT(kbdev != NULL);
3208 lockdep_assert_held(&kbdev->pm.lock);
3209
3210 /* Ensure the clock is on before attempting to access the hardware */
3211 if (!kbdev->pm.backend.gpu_powered) {
3212 if (kbdev->pm.backend.callback_power_on)
3213 kbdev->pm.backend.callback_power_on(kbdev);
3214
3215 kbdev->pm.backend.gpu_powered = true;
3216 }
3217
3218 /* Ensure interrupts are off to begin with, this also clears any
3219 * outstanding interrupts
3220 */
3221 kbase_pm_disable_interrupts(kbdev);
3222 /* Ensure cache snoops are disabled before reset. */
3223 kbase_pm_cache_snoop_disable(kbdev);
3224 /* Prepare for the soft-reset */
3225 kbdev->pm.backend.reset_done = false;
3226
3227 /* The cores should be made unavailable due to the reset */
3228 spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
3229 if (kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF)
3230 KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u);
3231 spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
3232
3233 /* Soft reset the GPU */
3234 #ifdef CONFIG_MALI_ARBITER_SUPPORT
3235 if (!(flags & PM_NO_RESET))
3236 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
3237 err = kbdev->protected_ops->protected_mode_disable(
3238 kbdev->protected_dev);
3239
3240 spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
3241 #if MALI_USE_CSF
3242 if (kbdev->protected_mode) {
3243 unsigned long flags;
3244
3245 kbase_ipa_control_protm_exited(kbdev);
3246
3247 kbase_csf_scheduler_spin_lock(kbdev, &flags);
3248 kbase_hwcnt_backend_csf_protm_exited(&kbdev->hwcnt_gpu_iface);
3249 kbase_csf_scheduler_spin_unlock(kbdev, flags);
3250 }
3251 #endif
3252 kbdev->protected_mode = false;
3253 spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
3254
3255 if (err)
3256 goto exit;
3257
3258 if (flags & PM_HW_ISSUES_DETECT) {
3259 err = kbase_pm_hw_issues_detect(kbdev);
3260 if (err)
3261 goto exit;
3262 }
3263
3264 kbase_pm_hw_issues_apply(kbdev);
3265 kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency);
3266
3267 /* Sanity check protected mode was left after reset */
3268 WARN_ON(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
3269 GPU_STATUS_PROTECTED_MODE_ACTIVE);
3270
3271 /* If cycle counter was in use re-enable it, enable_irqs will only be
3272 * false when called from kbase_pm_powerup
3273 */
3274 if (kbdev->pm.backend.gpu_cycle_counter_requests &&
3275 (flags & PM_ENABLE_IRQS)) {
3276 kbase_pm_enable_interrupts(kbdev);
3277
3278 /* Re-enable the counters if we need to */
3279 spin_lock_irqsave(
3280 &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
3281 irq_flags);
3282 if (kbdev->pm.backend.gpu_cycle_counter_requests)
3283 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
3284 GPU_COMMAND_CYCLE_COUNT_START);
3285 spin_unlock_irqrestore(
3286 &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
3287 irq_flags);
3288
3289 kbase_pm_disable_interrupts(kbdev);
3290 }
3291
3292 if (flags & PM_ENABLE_IRQS)
3293 kbase_pm_enable_interrupts(kbdev);
3294
3295 exit:
3296 #if !MALI_USE_CSF
3297 if (!kbdev->pm.backend.protected_entry_transition_override) {
3298 /* Re-enable GPU hardware counters if we're resetting from
3299 * protected mode.
3300 */
3301 reenable_protected_mode_hwcnt(kbdev);
3302 }
3303 #endif
3304
3305 return err;
3306 }
3307
3308 /**
3309 * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters
3310 * @kbdev: The kbase device structure of the device
3311 *
3312 * Increase the count of cycle counter users and turn the cycle counters on if
3313 * they were previously off
3314 *
3315 * This function is designed to be called by
3316 * kbase_pm_request_gpu_cycle_counter() or
3317 * kbase_pm_request_gpu_cycle_counter_l2_is_on() only
3318 *
3319 * When this function is called the l2 cache must be on - i.e., the GPU must be
3320 * on.
3321 */
3322 static void
3323 kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
3324 {
3325 unsigned long flags;
3326
3327 spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
3328 flags);
3329 ++kbdev->pm.backend.gpu_cycle_counter_requests;
3330
3331 if (kbdev->pm.backend.gpu_cycle_counter_requests == 1)
3332 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
3333 GPU_COMMAND_CYCLE_COUNT_START);
3334 else {
3335 /* This might happen after GPU reset.
3336 * Then counter needs to be kicked.
3337 */
3338 #if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
3339 if (!(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
3340 GPU_STATUS_CYCLE_COUNT_ACTIVE)) {
3341 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
3342 GPU_COMMAND_CYCLE_COUNT_START);
3343 }
3344 #endif
3345 }
3346
3347 spin_unlock_irqrestore(
3348 &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
3349 flags);
3350 }
3351
3352 void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev)
3353 {
3354 KBASE_DEBUG_ASSERT(kbdev != NULL);
3355
3356 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
3357
3358 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
3359 INT_MAX);
3360
3361 kbase_pm_wait_for_l2_powered(kbdev);
3362
3363 kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
3364 }
3365
3366 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter);
3367
3368 void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev)
3369 {
3370 KBASE_DEBUG_ASSERT(kbdev != NULL);
3371
3372 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
3373
3374 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
3375 INT_MAX);
3376
3377 kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
3378 }
3379
3380 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on);
3381
3382 void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev)
3383 {
3384 unsigned long flags;
3385
3386 KBASE_DEBUG_ASSERT(kbdev != NULL);
3387
3388 lockdep_assert_held(&kbdev->hwaccess_lock);
3389
3390 spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
3391 flags);
3392
3393 KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0);
3394
3395 --kbdev->pm.backend.gpu_cycle_counter_requests;
3396
3397 if (kbdev->pm.backend.gpu_cycle_counter_requests == 0)
3398 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
3399 GPU_COMMAND_CYCLE_COUNT_STOP);
3400
3401 spin_unlock_irqrestore(
3402 &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
3403 flags);
3404 }
3405
3406 void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev)
3407 {
3408 unsigned long flags;
3409
3410 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
3411
3412 kbase_pm_release_gpu_cycle_counter_nolock(kbdev);
3413
3414 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
3415 }
3416
3417 KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter);
3418