1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 #include <mali_kbase.h>
23 #include <mali_kbase_config_defaults.h>
24 #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
25 #include "mali_kbase_csf_ipa_control.h"
26
27 /*
28 * Status flags from the STATUS register of the IPA Control interface.
29 */
30 #define STATUS_COMMAND_ACTIVE ((u32)1 << 0)
31 #define STATUS_PROTECTED_MODE ((u32)1 << 8)
32 #define STATUS_RESET ((u32)1 << 9)
33 #define STATUS_TIMER_ENABLED ((u32)1 << 31)
34
35 /*
36 * Commands for the COMMAND register of the IPA Control interface.
37 */
38 #define COMMAND_APPLY ((u32)1)
39 #define COMMAND_SAMPLE ((u32)3)
40 #define COMMAND_PROTECTED_ACK ((u32)4)
41 #define COMMAND_RESET_ACK ((u32)5)
42
43 /*
44 * Number of timer events per second.
45 */
46 #define TIMER_EVENTS_PER_SECOND ((u32)1000 / IPA_CONTROL_TIMER_DEFAULT_VALUE_MS)
47
48 /*
49 * Maximum number of loops polling the GPU before we assume the GPU has hung.
50 */
51 #define IPA_INACTIVE_MAX_LOOPS (8000000U)
52
53 /*
54 * Number of bits used to configure a performance counter in SELECT registers.
55 */
56 #define IPA_CONTROL_SELECT_BITS_PER_CNT ((u64)8)
57
58 /*
59 * Maximum value of a performance counter.
60 */
61 #define MAX_PRFCNT_VALUE (((u64)1 << 48) - 1)
62
63 /**
64 * struct kbase_ipa_control_listener_data - Data for the GPU clock frequency
65 * listener
66 *
67 * @listener: GPU clock frequency listener.
68 * @kbdev: Pointer to kbase device.
69 */
70 struct kbase_ipa_control_listener_data {
71 struct kbase_clk_rate_listener listener;
72 struct kbase_device *kbdev;
73 };
74
timer_value(u32 gpu_rate)75 static u32 timer_value(u32 gpu_rate)
76 {
77 return gpu_rate / TIMER_EVENTS_PER_SECOND;
78 }
79
wait_status(struct kbase_device * kbdev,u32 flags)80 static int wait_status(struct kbase_device *kbdev, u32 flags)
81 {
82 unsigned int max_loops = IPA_INACTIVE_MAX_LOOPS;
83 u32 status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS));
84
85 /*
86 * Wait for the STATUS register to indicate that flags have been
87 * cleared, in case a transition is pending.
88 */
89 while (--max_loops && (status & flags))
90 status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS));
91 if (max_loops == 0) {
92 dev_err(kbdev->dev, "IPA_CONTROL STATUS register stuck");
93 return -EBUSY;
94 }
95
96 return 0;
97 }
98
apply_select_config(struct kbase_device * kbdev,u64 * select)99 static int apply_select_config(struct kbase_device *kbdev, u64 *select)
100 {
101 int ret;
102
103 u32 select_cshw_lo = (u32)(select[KBASE_IPA_CORE_TYPE_CSHW] & U32_MAX);
104 u32 select_cshw_hi =
105 (u32)((select[KBASE_IPA_CORE_TYPE_CSHW] >> 32) & U32_MAX);
106 u32 select_memsys_lo =
107 (u32)(select[KBASE_IPA_CORE_TYPE_MEMSYS] & U32_MAX);
108 u32 select_memsys_hi =
109 (u32)((select[KBASE_IPA_CORE_TYPE_MEMSYS] >> 32) & U32_MAX);
110 u32 select_tiler_lo =
111 (u32)(select[KBASE_IPA_CORE_TYPE_TILER] & U32_MAX);
112 u32 select_tiler_hi =
113 (u32)((select[KBASE_IPA_CORE_TYPE_TILER] >> 32) & U32_MAX);
114 u32 select_shader_lo =
115 (u32)(select[KBASE_IPA_CORE_TYPE_SHADER] & U32_MAX);
116 u32 select_shader_hi =
117 (u32)((select[KBASE_IPA_CORE_TYPE_SHADER] >> 32) & U32_MAX);
118
119 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_LO), select_cshw_lo);
120 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_HI), select_cshw_hi);
121 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_MEMSYS_LO),
122 select_memsys_lo);
123 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_MEMSYS_HI),
124 select_memsys_hi);
125 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_TILER_LO),
126 select_tiler_lo);
127 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_TILER_HI),
128 select_tiler_hi);
129 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_SHADER_LO),
130 select_shader_lo);
131 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_SHADER_HI),
132 select_shader_hi);
133
134 ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
135
136 if (!ret) {
137 kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_APPLY);
138 ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
139 } else {
140 dev_err(kbdev->dev, "Wait for the pending command failed");
141 }
142
143 return ret;
144 }
145
read_value_cnt(struct kbase_device * kbdev,u8 type,int select_idx)146 static u64 read_value_cnt(struct kbase_device *kbdev, u8 type, int select_idx)
147 {
148 u32 value_lo, value_hi;
149
150 switch (type) {
151 case KBASE_IPA_CORE_TYPE_CSHW:
152 value_lo = kbase_reg_read(
153 kbdev, IPA_CONTROL_REG(VALUE_CSHW_REG_LO(select_idx)));
154 value_hi = kbase_reg_read(
155 kbdev, IPA_CONTROL_REG(VALUE_CSHW_REG_HI(select_idx)));
156 break;
157 case KBASE_IPA_CORE_TYPE_MEMSYS:
158 value_lo = kbase_reg_read(
159 kbdev,
160 IPA_CONTROL_REG(VALUE_MEMSYS_REG_LO(select_idx)));
161 value_hi = kbase_reg_read(
162 kbdev,
163 IPA_CONTROL_REG(VALUE_MEMSYS_REG_HI(select_idx)));
164 break;
165 case KBASE_IPA_CORE_TYPE_TILER:
166 value_lo = kbase_reg_read(
167 kbdev, IPA_CONTROL_REG(VALUE_TILER_REG_LO(select_idx)));
168 value_hi = kbase_reg_read(
169 kbdev, IPA_CONTROL_REG(VALUE_TILER_REG_HI(select_idx)));
170 break;
171 case KBASE_IPA_CORE_TYPE_SHADER:
172 value_lo = kbase_reg_read(
173 kbdev,
174 IPA_CONTROL_REG(VALUE_SHADER_REG_LO(select_idx)));
175 value_hi = kbase_reg_read(
176 kbdev,
177 IPA_CONTROL_REG(VALUE_SHADER_REG_HI(select_idx)));
178 break;
179 default:
180 WARN(1, "Unknown core type: %u\n", type);
181 value_lo = value_hi = 0;
182 break;
183 }
184
185 return (((u64)value_hi << 32) | value_lo);
186 }
187
build_select_config(struct kbase_ipa_control * ipa_ctrl,u64 * select_config)188 static void build_select_config(struct kbase_ipa_control *ipa_ctrl,
189 u64 *select_config)
190 {
191 size_t i;
192
193 for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++) {
194 size_t j;
195
196 select_config[i] = 0ULL;
197
198 for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) {
199 struct kbase_ipa_control_prfcnt_config *prfcnt_config =
200 &ipa_ctrl->blocks[i].select[j];
201
202 select_config[i] |=
203 ((u64)prfcnt_config->idx
204 << (IPA_CONTROL_SELECT_BITS_PER_CNT * j));
205 }
206 }
207 }
208
update_select_registers(struct kbase_device * kbdev)209 static int update_select_registers(struct kbase_device *kbdev)
210 {
211 u64 select_config[KBASE_IPA_CORE_TYPE_NUM];
212
213 lockdep_assert_held(&kbdev->csf.ipa_control.lock);
214
215 build_select_config(&kbdev->csf.ipa_control, select_config);
216
217 return apply_select_config(kbdev, select_config);
218 }
219
calc_prfcnt_delta(struct kbase_device * kbdev,struct kbase_ipa_control_prfcnt * prfcnt,bool gpu_ready)220 static inline void calc_prfcnt_delta(struct kbase_device *kbdev,
221 struct kbase_ipa_control_prfcnt *prfcnt,
222 bool gpu_ready)
223 {
224 u64 delta_value, raw_value;
225
226 if (gpu_ready)
227 raw_value = read_value_cnt(kbdev, (u8)prfcnt->type,
228 prfcnt->select_idx);
229 else
230 raw_value = prfcnt->latest_raw_value;
231
232 if (raw_value < prfcnt->latest_raw_value) {
233 delta_value = (MAX_PRFCNT_VALUE - prfcnt->latest_raw_value) +
234 raw_value;
235 } else {
236 delta_value = raw_value - prfcnt->latest_raw_value;
237 }
238
239 delta_value *= prfcnt->scaling_factor;
240
241 if (kbdev->csf.ipa_control.cur_gpu_rate == 0) {
242 static bool warned;
243
244 if (!warned) {
245 dev_warn(kbdev->dev, "%s: GPU freq is unexpectedly 0", __func__);
246 warned = true;
247 }
248 } else if (prfcnt->gpu_norm)
249 delta_value = div_u64(delta_value, kbdev->csf.ipa_control.cur_gpu_rate);
250
251 prfcnt->latest_raw_value = raw_value;
252
253 /* Accumulate the difference */
254 prfcnt->accumulated_diff += delta_value;
255 }
256
257 /**
258 * kbase_ipa_control_rate_change_notify - GPU frequency change callback
259 *
260 * @listener: Clock frequency change listener.
261 * @clk_index: Index of the clock for which the change has occurred.
262 * @clk_rate_hz: Clock frequency(Hz).
263 *
264 * This callback notifies kbase_ipa_control about GPU frequency changes.
265 * Only top-level clock changes are meaningful. GPU frequency updates
266 * affect all performance counters which require GPU normalization
267 * in every session.
268 */
269 static void
kbase_ipa_control_rate_change_notify(struct kbase_clk_rate_listener * listener,u32 clk_index,u32 clk_rate_hz)270 kbase_ipa_control_rate_change_notify(struct kbase_clk_rate_listener *listener,
271 u32 clk_index, u32 clk_rate_hz)
272 {
273 if ((clk_index == KBASE_CLOCK_DOMAIN_TOP) && (clk_rate_hz != 0)) {
274 size_t i;
275 unsigned long flags;
276 struct kbase_ipa_control_listener_data *listener_data =
277 container_of(listener,
278 struct kbase_ipa_control_listener_data,
279 listener);
280 struct kbase_device *kbdev = listener_data->kbdev;
281 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
282
283 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
284
285 if (!kbdev->pm.backend.gpu_ready) {
286 dev_err(kbdev->dev,
287 "%s: GPU frequency cannot change while GPU is off",
288 __func__);
289 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
290 return;
291 }
292
293 /* Interrupts are already disabled and interrupt state is also saved */
294 spin_lock(&ipa_ctrl->lock);
295
296 for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
297 struct kbase_ipa_control_session *session = &ipa_ctrl->sessions[i];
298
299 if (session->active) {
300 size_t j;
301
302 for (j = 0; j < session->num_prfcnts; j++) {
303 struct kbase_ipa_control_prfcnt *prfcnt =
304 &session->prfcnts[j];
305
306 if (prfcnt->gpu_norm)
307 calc_prfcnt_delta(kbdev, prfcnt, true);
308 }
309 }
310 }
311
312 ipa_ctrl->cur_gpu_rate = clk_rate_hz;
313
314 /* Update the timer for automatic sampling if active sessions
315 * are present. Counters have already been manually sampled.
316 */
317 if (ipa_ctrl->num_active_sessions > 0) {
318 kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER),
319 timer_value(ipa_ctrl->cur_gpu_rate));
320 }
321
322 spin_unlock(&ipa_ctrl->lock);
323
324 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
325 }
326 }
327
kbase_ipa_control_init(struct kbase_device * kbdev)328 void kbase_ipa_control_init(struct kbase_device *kbdev)
329 {
330 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
331 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
332 struct kbase_ipa_control_listener_data *listener_data;
333 size_t i, j;
334
335 for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++) {
336 for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) {
337 ipa_ctrl->blocks[i].select[j].idx = 0;
338 ipa_ctrl->blocks[i].select[j].refcount = 0;
339 }
340 ipa_ctrl->blocks[i].num_available_counters =
341 KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS;
342 }
343
344 spin_lock_init(&ipa_ctrl->lock);
345 ipa_ctrl->num_active_sessions = 0;
346 for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++)
347 ipa_ctrl->sessions[i].active = false;
348
349 listener_data = kmalloc(sizeof(struct kbase_ipa_control_listener_data),
350 GFP_KERNEL);
351 if (listener_data) {
352 listener_data->listener.notify =
353 kbase_ipa_control_rate_change_notify;
354 listener_data->kbdev = kbdev;
355 ipa_ctrl->rtm_listener_data = listener_data;
356 }
357
358 spin_lock(&clk_rtm->lock);
359 if (clk_rtm->clks[KBASE_CLOCK_DOMAIN_TOP])
360 ipa_ctrl->cur_gpu_rate =
361 clk_rtm->clks[KBASE_CLOCK_DOMAIN_TOP]->clock_val;
362 if (listener_data)
363 kbase_clk_rate_trace_manager_subscribe_no_lock(
364 clk_rtm, &listener_data->listener);
365 spin_unlock(&clk_rtm->lock);
366 }
367 KBASE_EXPORT_TEST_API(kbase_ipa_control_init);
368
kbase_ipa_control_term(struct kbase_device * kbdev)369 void kbase_ipa_control_term(struct kbase_device *kbdev)
370 {
371 unsigned long flags;
372 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
373 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
374 struct kbase_ipa_control_listener_data *listener_data =
375 ipa_ctrl->rtm_listener_data;
376
377 WARN_ON(ipa_ctrl->num_active_sessions);
378
379 if (listener_data)
380 kbase_clk_rate_trace_manager_unsubscribe(clk_rtm, &listener_data->listener);
381 kfree(ipa_ctrl->rtm_listener_data);
382
383 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
384 if (kbdev->pm.backend.gpu_powered)
385 kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), 0);
386 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
387 }
388 KBASE_EXPORT_TEST_API(kbase_ipa_control_term);
389
390 /** session_read_raw_values - Read latest raw values for a sessions
391 * @kbdev: Pointer to kbase device.
392 * @session: Pointer to the session whose performance counters shall be read.
393 *
394 * Read and update the latest raw values of all the performance counters
395 * belonging to a given session.
396 */
session_read_raw_values(struct kbase_device * kbdev,struct kbase_ipa_control_session * session)397 static void session_read_raw_values(struct kbase_device *kbdev,
398 struct kbase_ipa_control_session *session)
399 {
400 size_t i;
401
402 lockdep_assert_held(&kbdev->csf.ipa_control.lock);
403
404 for (i = 0; i < session->num_prfcnts; i++) {
405 struct kbase_ipa_control_prfcnt *prfcnt = &session->prfcnts[i];
406 u64 raw_value = read_value_cnt(kbdev, (u8)prfcnt->type,
407 prfcnt->select_idx);
408
409 prfcnt->latest_raw_value = raw_value;
410 }
411 }
412
413 /** session_gpu_start - Start one or all sessions
414 * @kbdev: Pointer to kbase device.
415 * @ipa_ctrl: Pointer to IPA_CONTROL descriptor.
416 * @session: Pointer to the session to initialize, or NULL to initialize
417 * all sessions.
418 *
419 * This function starts one or all sessions by capturing a manual sample,
420 * reading the latest raw value of performance counters and possibly enabling
421 * the timer for automatic sampling if necessary.
422 *
423 * If a single session is given, it is assumed to be active, regardless of
424 * the number of active sessions. The number of performance counters belonging
425 * to the session shall be set in advance.
426 *
427 * If no session is given, the function shall start all sessions.
428 * The function does nothing if there are no active sessions.
429 *
430 * Return: 0 on success, or error code on failure.
431 */
session_gpu_start(struct kbase_device * kbdev,struct kbase_ipa_control * ipa_ctrl,struct kbase_ipa_control_session * session)432 static int session_gpu_start(struct kbase_device *kbdev,
433 struct kbase_ipa_control *ipa_ctrl,
434 struct kbase_ipa_control_session *session)
435 {
436 bool first_start =
437 (session != NULL) && (ipa_ctrl->num_active_sessions == 0);
438 int ret = 0;
439
440 lockdep_assert_held(&kbdev->csf.ipa_control.lock);
441
442 /*
443 * Exit immediately if the caller intends to start all sessions
444 * but there are no active sessions. It's important that no operation
445 * is done on the IPA_CONTROL interface in that case.
446 */
447 if (!session && ipa_ctrl->num_active_sessions == 0)
448 return ret;
449
450 /*
451 * Take a manual sample unconditionally if the caller intends
452 * to start all sessions. Otherwise, only take a manual sample
453 * if this is the first session to be initialized, for accumulator
454 * registers are empty and no timer has been configured for automatic
455 * sampling.
456 */
457 if (!session || first_start) {
458 kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND),
459 COMMAND_SAMPLE);
460 ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
461 if (ret)
462 dev_err(kbdev->dev, "%s: failed to sample new counters",
463 __func__);
464 kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER),
465 timer_value(ipa_ctrl->cur_gpu_rate));
466 }
467
468 /*
469 * Read current raw value to start the session.
470 * This is necessary to put the first query in condition
471 * to generate a correct value by calculating the difference
472 * from the beginning of the session. This consideration
473 * is true regardless of the number of sessions the caller
474 * intends to start.
475 */
476 if (!ret) {
477 if (session) {
478 /* On starting a session, value read is required for
479 * IPA power model's calculation initialization.
480 */
481 session_read_raw_values(kbdev, session);
482 } else {
483 size_t session_idx;
484
485 for (session_idx = 0;
486 session_idx < KBASE_IPA_CONTROL_MAX_SESSIONS;
487 session_idx++) {
488 struct kbase_ipa_control_session *session_to_check = &ipa_ctrl->sessions[session_idx];
489
490 if (session_to_check->active)
491 session_read_raw_values(kbdev, session_to_check);
492 }
493 }
494 }
495
496 return ret;
497 }
498
kbase_ipa_control_register(struct kbase_device * kbdev,const struct kbase_ipa_control_perf_counter * perf_counters,size_t num_counters,void ** client)499 int kbase_ipa_control_register(
500 struct kbase_device *kbdev,
501 const struct kbase_ipa_control_perf_counter *perf_counters,
502 size_t num_counters, void **client)
503 {
504 int ret = 0;
505 size_t i, session_idx, req_counters[KBASE_IPA_CORE_TYPE_NUM];
506 bool already_configured[KBASE_IPA_CONTROL_MAX_COUNTERS];
507 bool new_config = false;
508 struct kbase_ipa_control *ipa_ctrl;
509 struct kbase_ipa_control_session *session = NULL;
510 unsigned long flags;
511
512 if (WARN_ON(unlikely(kbdev == NULL)))
513 return -ENODEV;
514
515 if (WARN_ON(perf_counters == NULL) || WARN_ON(client == NULL) ||
516 WARN_ON(num_counters > KBASE_IPA_CONTROL_MAX_COUNTERS)) {
517 dev_err(kbdev->dev, "%s: wrong input arguments", __func__);
518 return -EINVAL;
519 }
520
521 kbase_pm_context_active(kbdev);
522
523 ipa_ctrl = &kbdev->csf.ipa_control;
524 spin_lock_irqsave(&ipa_ctrl->lock, flags);
525
526 if (ipa_ctrl->num_active_sessions == KBASE_IPA_CONTROL_MAX_SESSIONS) {
527 dev_err(kbdev->dev, "%s: too many sessions", __func__);
528 ret = -EBUSY;
529 goto exit;
530 }
531
532 for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++)
533 req_counters[i] = 0;
534
535 /*
536 * Count how many counters would need to be configured in order to
537 * satisfy the request. Requested counters which happen to be already
538 * configured can be skipped.
539 */
540 for (i = 0; i < num_counters; i++) {
541 size_t j;
542 enum kbase_ipa_core_type type = perf_counters[i].type;
543 u8 idx = perf_counters[i].idx;
544
545 if ((type >= KBASE_IPA_CORE_TYPE_NUM) ||
546 (idx >= KBASE_IPA_CONTROL_CNT_MAX_IDX)) {
547 dev_err(kbdev->dev,
548 "%s: invalid requested type %u and/or index %u",
549 __func__, type, idx);
550 ret = -EINVAL;
551 goto exit;
552 }
553
554 for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) {
555 struct kbase_ipa_control_prfcnt_config *prfcnt_config =
556 &ipa_ctrl->blocks[type].select[j];
557
558 if (prfcnt_config->refcount > 0) {
559 if (prfcnt_config->idx == idx) {
560 already_configured[i] = true;
561 break;
562 }
563 }
564 }
565
566 if (j == KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS) {
567 already_configured[i] = false;
568 req_counters[type]++;
569 new_config = true;
570 }
571 }
572
573 for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++)
574 if (req_counters[i] >
575 ipa_ctrl->blocks[i].num_available_counters) {
576 dev_err(kbdev->dev,
577 "%s: more counters (%zu) than available (%zu) have been requested for type %zu",
578 __func__, req_counters[i],
579 ipa_ctrl->blocks[i].num_available_counters, i);
580 ret = -EINVAL;
581 goto exit;
582 }
583
584 /*
585 * The request has been validated.
586 * Firstly, find an available session and then set up the initial state
587 * of the session and update the configuration of performance counters
588 * in the internal state of kbase_ipa_control.
589 */
590 for (session_idx = 0; session_idx < KBASE_IPA_CONTROL_MAX_SESSIONS;
591 session_idx++) {
592 if (!ipa_ctrl->sessions[session_idx].active) {
593 session = &ipa_ctrl->sessions[session_idx];
594 break;
595 }
596 }
597
598 if (!session) {
599 dev_err(kbdev->dev, "%s: wrong or corrupt session state",
600 __func__);
601 ret = -EBUSY;
602 goto exit;
603 }
604
605 for (i = 0; i < num_counters; i++) {
606 struct kbase_ipa_control_prfcnt_config *prfcnt_config;
607 size_t j;
608 u8 type = perf_counters[i].type;
609 u8 idx = perf_counters[i].idx;
610
611 for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) {
612 prfcnt_config = &ipa_ctrl->blocks[type].select[j];
613
614 if (already_configured[i]) {
615 if ((prfcnt_config->refcount > 0) &&
616 (prfcnt_config->idx == idx)) {
617 break;
618 }
619 } else {
620 if (prfcnt_config->refcount == 0)
621 break;
622 }
623 }
624
625 if (WARN_ON((prfcnt_config->refcount > 0 &&
626 prfcnt_config->idx != idx) ||
627 (j == KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS))) {
628 dev_err(kbdev->dev,
629 "%s: invalid internal state: counter already configured or no counter available to configure",
630 __func__);
631 ret = -EBUSY;
632 goto exit;
633 }
634
635 if (prfcnt_config->refcount == 0) {
636 prfcnt_config->idx = idx;
637 ipa_ctrl->blocks[type].num_available_counters--;
638 }
639
640 session->prfcnts[i].accumulated_diff = 0;
641 session->prfcnts[i].type = type;
642 session->prfcnts[i].select_idx = j;
643 session->prfcnts[i].scaling_factor =
644 perf_counters[i].scaling_factor;
645 session->prfcnts[i].gpu_norm = perf_counters[i].gpu_norm;
646
647 /* Reports to this client for GPU time spent in protected mode
648 * should begin from the point of registration.
649 */
650 session->last_query_time = ktime_get_raw_ns();
651
652 /* Initially, no time has been spent in protected mode */
653 session->protm_time = 0;
654
655 prfcnt_config->refcount++;
656 }
657
658 /*
659 * Apply new configuration, if necessary.
660 * As a temporary solution, make sure that the GPU is on
661 * before applying the new configuration.
662 */
663 if (new_config) {
664 ret = update_select_registers(kbdev);
665 if (ret)
666 dev_err(kbdev->dev,
667 "%s: failed to apply new SELECT configuration",
668 __func__);
669 }
670
671 if (!ret) {
672 session->num_prfcnts = num_counters;
673 ret = session_gpu_start(kbdev, ipa_ctrl, session);
674 }
675
676 if (!ret) {
677 session->active = true;
678 ipa_ctrl->num_active_sessions++;
679 *client = session;
680 }
681
682 exit:
683 spin_unlock_irqrestore(&ipa_ctrl->lock, flags);
684 kbase_pm_context_idle(kbdev);
685 return ret;
686 }
687 KBASE_EXPORT_TEST_API(kbase_ipa_control_register);
688
kbase_ipa_control_unregister(struct kbase_device * kbdev,const void * client)689 int kbase_ipa_control_unregister(struct kbase_device *kbdev, const void *client)
690 {
691 struct kbase_ipa_control *ipa_ctrl;
692 struct kbase_ipa_control_session *session;
693 int ret = 0;
694 size_t i;
695 unsigned long flags;
696 bool new_config = false, valid_session = false;
697
698 if (WARN_ON(unlikely(kbdev == NULL)))
699 return -ENODEV;
700
701 if (WARN_ON(client == NULL)) {
702 dev_err(kbdev->dev, "%s: wrong input arguments", __func__);
703 return -EINVAL;
704 }
705
706 kbase_pm_context_active(kbdev);
707
708 ipa_ctrl = &kbdev->csf.ipa_control;
709 session = (struct kbase_ipa_control_session *)client;
710
711 spin_lock_irqsave(&ipa_ctrl->lock, flags);
712
713 for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
714 if (session == &ipa_ctrl->sessions[i]) {
715 valid_session = true;
716 break;
717 }
718 }
719
720 if (!valid_session) {
721 dev_err(kbdev->dev, "%s: invalid session handle", __func__);
722 ret = -EINVAL;
723 goto exit;
724 }
725
726 if (ipa_ctrl->num_active_sessions == 0) {
727 dev_err(kbdev->dev, "%s: no active sessions found", __func__);
728 ret = -EINVAL;
729 goto exit;
730 }
731
732 if (!session->active) {
733 dev_err(kbdev->dev, "%s: session is already inactive",
734 __func__);
735 ret = -EINVAL;
736 goto exit;
737 }
738
739 for (i = 0; i < session->num_prfcnts; i++) {
740 struct kbase_ipa_control_prfcnt_config *prfcnt_config;
741 u8 type = session->prfcnts[i].type;
742 u8 idx = session->prfcnts[i].select_idx;
743
744 prfcnt_config = &ipa_ctrl->blocks[type].select[idx];
745
746 if (!WARN_ON(prfcnt_config->refcount == 0)) {
747 prfcnt_config->refcount--;
748 if (prfcnt_config->refcount == 0) {
749 new_config = true;
750 ipa_ctrl->blocks[type].num_available_counters++;
751 }
752 }
753 }
754
755 if (new_config) {
756 ret = update_select_registers(kbdev);
757 if (ret)
758 dev_err(kbdev->dev,
759 "%s: failed to apply SELECT configuration",
760 __func__);
761 }
762
763 session->num_prfcnts = 0;
764 session->active = false;
765 ipa_ctrl->num_active_sessions--;
766
767 exit:
768 spin_unlock_irqrestore(&ipa_ctrl->lock, flags);
769 kbase_pm_context_idle(kbdev);
770 return ret;
771 }
772 KBASE_EXPORT_TEST_API(kbase_ipa_control_unregister);
773
kbase_ipa_control_query(struct kbase_device * kbdev,const void * client,u64 * values,size_t num_values,u64 * protected_time)774 int kbase_ipa_control_query(struct kbase_device *kbdev, const void *client,
775 u64 *values, size_t num_values, u64 *protected_time)
776 {
777 struct kbase_ipa_control *ipa_ctrl;
778 struct kbase_ipa_control_session *session;
779 size_t i;
780 unsigned long flags;
781 bool gpu_ready;
782
783 if (WARN_ON(unlikely(kbdev == NULL)))
784 return -ENODEV;
785
786 if (WARN_ON(client == NULL) || WARN_ON(values == NULL)) {
787 dev_err(kbdev->dev, "%s: wrong input arguments", __func__);
788 return -EINVAL;
789 }
790
791 ipa_ctrl = &kbdev->csf.ipa_control;
792 session = (struct kbase_ipa_control_session *)client;
793
794 if (!session->active) {
795 dev_err(kbdev->dev,
796 "%s: attempt to query inactive session", __func__);
797 return -EINVAL;
798 }
799
800 if (WARN_ON(num_values < session->num_prfcnts)) {
801 dev_err(kbdev->dev,
802 "%s: not enough space (%zu) to return all counter values (%zu)",
803 __func__, num_values, session->num_prfcnts);
804 return -EINVAL;
805 }
806
807 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
808 gpu_ready = kbdev->pm.backend.gpu_ready;
809
810 for (i = 0; i < session->num_prfcnts; i++) {
811 struct kbase_ipa_control_prfcnt *prfcnt = &session->prfcnts[i];
812
813 calc_prfcnt_delta(kbdev, prfcnt, gpu_ready);
814 /* Return all the accumulated difference */
815 values[i] = prfcnt->accumulated_diff;
816 prfcnt->accumulated_diff = 0;
817 }
818
819 if (protected_time) {
820 u64 time_now = ktime_get_raw_ns();
821
822 /* This is the amount of protected-mode time spent prior to
823 * the current protm period.
824 */
825 *protected_time = session->protm_time;
826
827 if (kbdev->protected_mode) {
828 *protected_time +=
829 time_now - MAX(session->last_query_time,
830 ipa_ctrl->protm_start);
831 }
832 session->last_query_time = time_now;
833 session->protm_time = 0;
834 }
835
836 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
837
838 for (i = session->num_prfcnts; i < num_values; i++)
839 values[i] = 0;
840
841 return 0;
842 }
843 KBASE_EXPORT_TEST_API(kbase_ipa_control_query);
844
kbase_ipa_control_handle_gpu_power_off(struct kbase_device * kbdev)845 void kbase_ipa_control_handle_gpu_power_off(struct kbase_device *kbdev)
846 {
847 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
848 size_t session_idx;
849 int ret;
850
851 lockdep_assert_held(&kbdev->hwaccess_lock);
852
853 /* GPU should still be ready for use when this function gets called */
854 WARN_ON(!kbdev->pm.backend.gpu_ready);
855
856 /* Interrupts are already disabled and interrupt state is also saved */
857 spin_lock(&ipa_ctrl->lock);
858
859 /* First disable the automatic sampling through TIMER */
860 kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), 0);
861 ret = wait_status(kbdev, STATUS_TIMER_ENABLED);
862 if (ret) {
863 dev_err(kbdev->dev,
864 "Wait for disabling of IPA control timer failed: %d",
865 ret);
866 }
867
868 /* Now issue the manual SAMPLE command */
869 kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_SAMPLE);
870 ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
871 if (ret) {
872 dev_err(kbdev->dev,
873 "Wait for the completion of manual sample failed: %d",
874 ret);
875 }
876
877 for (session_idx = 0; session_idx < KBASE_IPA_CONTROL_MAX_SESSIONS;
878 session_idx++) {
879
880 struct kbase_ipa_control_session *session =
881 &ipa_ctrl->sessions[session_idx];
882
883 if (session->active) {
884 size_t i;
885
886 for (i = 0; i < session->num_prfcnts; i++) {
887 struct kbase_ipa_control_prfcnt *prfcnt =
888 &session->prfcnts[i];
889
890 calc_prfcnt_delta(kbdev, prfcnt, true);
891 }
892 }
893 }
894 spin_unlock(&ipa_ctrl->lock);
895 }
896
kbase_ipa_control_handle_gpu_power_on(struct kbase_device * kbdev)897 void kbase_ipa_control_handle_gpu_power_on(struct kbase_device *kbdev)
898 {
899 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
900 int ret;
901
902 lockdep_assert_held(&kbdev->hwaccess_lock);
903
904 /* GPU should have become ready for use when this function gets called */
905 WARN_ON(!kbdev->pm.backend.gpu_ready);
906
907 /* Interrupts are already disabled and interrupt state is also saved */
908 spin_lock(&ipa_ctrl->lock);
909
910 ret = update_select_registers(kbdev);
911 if (ret) {
912 dev_err(kbdev->dev,
913 "Failed to reconfigure the select registers: %d", ret);
914 }
915
916 /* Accumulator registers would not contain any sample after GPU power
917 * cycle if the timer has not been enabled first. Initialize all sessions.
918 */
919 ret = session_gpu_start(kbdev, ipa_ctrl, NULL);
920
921 spin_unlock(&ipa_ctrl->lock);
922 }
923
kbase_ipa_control_handle_gpu_reset_pre(struct kbase_device * kbdev)924 void kbase_ipa_control_handle_gpu_reset_pre(struct kbase_device *kbdev)
925 {
926 /* A soft reset is treated as a power down */
927 kbase_ipa_control_handle_gpu_power_off(kbdev);
928 }
929 KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_reset_pre);
930
kbase_ipa_control_handle_gpu_reset_post(struct kbase_device * kbdev)931 void kbase_ipa_control_handle_gpu_reset_post(struct kbase_device *kbdev)
932 {
933 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
934 int ret;
935 u32 status;
936
937 lockdep_assert_held(&kbdev->hwaccess_lock);
938
939 /* GPU should have become ready for use when this function gets called */
940 WARN_ON(!kbdev->pm.backend.gpu_ready);
941
942 /* Interrupts are already disabled and interrupt state is also saved */
943 spin_lock(&ipa_ctrl->lock);
944
945 /* Check the status reset bit is set before acknowledging it */
946 status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS));
947 if (status & STATUS_RESET) {
948 /* Acknowledge the reset command */
949 kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_RESET_ACK);
950 ret = wait_status(kbdev, STATUS_RESET);
951 if (ret) {
952 dev_err(kbdev->dev,
953 "Wait for the reset ack command failed: %d",
954 ret);
955 }
956 }
957
958 spin_unlock(&ipa_ctrl->lock);
959
960 kbase_ipa_control_handle_gpu_power_on(kbdev);
961 }
962 KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_reset_post);
963
964 #ifdef KBASE_PM_RUNTIME
kbase_ipa_control_handle_gpu_sleep_enter(struct kbase_device * kbdev)965 void kbase_ipa_control_handle_gpu_sleep_enter(struct kbase_device *kbdev)
966 {
967 lockdep_assert_held(&kbdev->hwaccess_lock);
968
969 if (kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP) {
970 /* GPU Sleep is treated as a power down */
971 kbase_ipa_control_handle_gpu_power_off(kbdev);
972
973 /* SELECT_CSHW register needs to be cleared to prevent any
974 * IPA control message to be sent to the top level GPU HWCNT.
975 */
976 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_LO), 0);
977 kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_HI), 0);
978
979 /* No need to issue the APPLY command here */
980 }
981 }
982 KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_sleep_enter);
983
kbase_ipa_control_handle_gpu_sleep_exit(struct kbase_device * kbdev)984 void kbase_ipa_control_handle_gpu_sleep_exit(struct kbase_device *kbdev)
985 {
986 lockdep_assert_held(&kbdev->hwaccess_lock);
987
988 if (kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP) {
989 /* To keep things simple, currently exit from
990 * GPU Sleep is treated as a power on event where
991 * all 4 SELECT registers are reconfigured.
992 * On exit from sleep, reconfiguration is needed
993 * only for the SELECT_CSHW register.
994 */
995 kbase_ipa_control_handle_gpu_power_on(kbdev);
996 }
997 }
998 KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_sleep_exit);
999 #endif
1000
1001 #if MALI_UNIT_TEST
kbase_ipa_control_rate_change_notify_test(struct kbase_device * kbdev,u32 clk_index,u32 clk_rate_hz)1002 void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev,
1003 u32 clk_index, u32 clk_rate_hz)
1004 {
1005 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
1006 struct kbase_ipa_control_listener_data *listener_data =
1007 ipa_ctrl->rtm_listener_data;
1008
1009 kbase_ipa_control_rate_change_notify(&listener_data->listener,
1010 clk_index, clk_rate_hz);
1011 }
1012 KBASE_EXPORT_TEST_API(kbase_ipa_control_rate_change_notify_test);
1013 #endif
1014
kbase_ipa_control_protm_entered(struct kbase_device * kbdev)1015 void kbase_ipa_control_protm_entered(struct kbase_device *kbdev)
1016 {
1017 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
1018
1019 lockdep_assert_held(&kbdev->hwaccess_lock);
1020 ipa_ctrl->protm_start = ktime_get_raw_ns();
1021 }
1022
kbase_ipa_control_protm_exited(struct kbase_device * kbdev)1023 void kbase_ipa_control_protm_exited(struct kbase_device *kbdev)
1024 {
1025 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
1026 size_t i;
1027 u64 time_now = ktime_get_raw_ns();
1028 u32 status;
1029
1030 lockdep_assert_held(&kbdev->hwaccess_lock);
1031
1032 for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
1033
1034 struct kbase_ipa_control_session *session =
1035 &ipa_ctrl->sessions[i];
1036
1037 if (session->active) {
1038 u64 protm_time = time_now - MAX(session->last_query_time,
1039 ipa_ctrl->protm_start);
1040
1041 session->protm_time += protm_time;
1042 }
1043 }
1044
1045 /* Acknowledge the protected_mode bit in the IPA_CONTROL STATUS
1046 * register
1047 */
1048 status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS));
1049 if (status & STATUS_PROTECTED_MODE) {
1050 int ret;
1051
1052 /* Acknowledge the protm command */
1053 kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND),
1054 COMMAND_PROTECTED_ACK);
1055 ret = wait_status(kbdev, STATUS_PROTECTED_MODE);
1056 if (ret) {
1057 dev_err(kbdev->dev,
1058 "Wait for the protm ack command failed: %d",
1059 ret);
1060 }
1061 }
1062 }
1063
1064