1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2014-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /*
23 * GPU backend instrumentation APIs.
24 */
25
26 #include <mali_kbase.h>
27 #include <gpu/mali_kbase_gpu_regmap.h>
28 #include <mali_kbase_hwaccess_instr.h>
29 #include <device/mali_kbase_device.h>
30 #include <backend/gpu/mali_kbase_instr_internal.h>
31
wait_prfcnt_ready(struct kbase_device * kbdev)32 static int wait_prfcnt_ready(struct kbase_device *kbdev)
33 {
34 u32 loops;
35
36 for (loops = 0; loops < KBASE_PRFCNT_ACTIVE_MAX_LOOPS; loops++) {
37 const u32 prfcnt_active = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
38 GPU_STATUS_PRFCNT_ACTIVE;
39 if (!prfcnt_active)
40 return 0;
41 }
42
43 dev_err(kbdev->dev, "PRFCNT_ACTIVE bit stuck\n");
44 return -EBUSY;
45 }
46
kbase_instr_hwcnt_enable_internal(struct kbase_device * kbdev,struct kbase_context * kctx,struct kbase_instr_hwcnt_enable * enable)47 int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev,
48 struct kbase_context *kctx,
49 struct kbase_instr_hwcnt_enable *enable)
50 {
51 unsigned long flags;
52 int err = -EINVAL;
53 u32 irq_mask;
54 u32 prfcnt_config;
55
56 lockdep_assert_held(&kbdev->hwaccess_lock);
57
58 /* alignment failure */
59 if ((enable->dump_buffer == 0ULL) || (enable->dump_buffer & (2048 - 1)))
60 return err;
61
62 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
63
64 if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) {
65 /* Instrumentation is already enabled */
66 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
67 return err;
68 }
69
70 if (kbase_is_gpu_removed(kbdev)) {
71 /* GPU has been removed by Arbiter */
72 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
73 return err;
74 }
75
76 /* Enable interrupt */
77 irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
78 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask |
79 PRFCNT_SAMPLE_COMPLETED);
80
81 /* In use, this context is the owner */
82 kbdev->hwcnt.kctx = kctx;
83 /* Remember the dump address so we can reprogram it later */
84 kbdev->hwcnt.addr = enable->dump_buffer;
85 kbdev->hwcnt.addr_bytes = enable->dump_buffer_bytes;
86
87 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
88
89 /* Configure */
90 prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT;
91 #ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
92 prfcnt_config |= kbdev->hwcnt.backend.override_counter_set
93 << PRFCNT_CONFIG_SETSELECT_SHIFT;
94 #else
95 prfcnt_config |= enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT;
96 #endif
97
98 /* Wait until prfcnt config register can be written */
99 err = wait_prfcnt_ready(kbdev);
100 if (err)
101 return err;
102
103 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
104 prfcnt_config | PRFCNT_CONFIG_MODE_OFF);
105
106 /* Wait until prfcnt is disabled before writing configuration registers */
107 err = wait_prfcnt_ready(kbdev);
108 if (err)
109 return err;
110
111 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
112 enable->dump_buffer & 0xFFFFFFFF);
113 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
114 enable->dump_buffer >> 32);
115
116 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN),
117 enable->fe_bm);
118
119 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN),
120 enable->shader_bm);
121 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN),
122 enable->mmu_l2_bm);
123
124 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
125 enable->tiler_bm);
126
127 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
128 prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL);
129
130 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
131
132 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
133 kbdev->hwcnt.backend.triggered = 1;
134 wake_up(&kbdev->hwcnt.backend.wait);
135
136 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
137
138 dev_dbg(kbdev->dev, "HW counters dumping set-up for context %pK", kctx);
139 return 0;
140 }
141
kbasep_instr_hwc_disable_hw_prfcnt(struct kbase_device * kbdev)142 static void kbasep_instr_hwc_disable_hw_prfcnt(struct kbase_device *kbdev)
143 {
144 u32 irq_mask;
145
146 lockdep_assert_held(&kbdev->hwaccess_lock);
147 lockdep_assert_held(&kbdev->hwcnt.lock);
148
149 if (kbase_is_gpu_removed(kbdev))
150 /* GPU has been removed by Arbiter */
151 return;
152
153 /* Disable interrupt */
154 irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
155
156 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~PRFCNT_SAMPLE_COMPLETED);
157
158 /* Wait until prfcnt config register can be written, then disable the counters.
159 * Return value is ignored as we are disabling anyway.
160 */
161 wait_prfcnt_ready(kbdev);
162 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0);
163
164 kbdev->hwcnt.kctx = NULL;
165 kbdev->hwcnt.addr = 0ULL;
166 kbdev->hwcnt.addr_bytes = 0ULL;
167 }
168
kbase_instr_hwcnt_disable_internal(struct kbase_context * kctx)169 int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx)
170 {
171 unsigned long flags, pm_flags;
172 struct kbase_device *kbdev = kctx->kbdev;
173
174 while (1) {
175 spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags);
176 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
177
178 if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) {
179 /* Instrumentation is in unrecoverable error state,
180 * there is nothing for us to do.
181 */
182 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
183 spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
184 /* Already disabled, return no error. */
185 return 0;
186 }
187
188 if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) {
189 /* Instrumentation is not enabled */
190 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
191 spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
192 return -EINVAL;
193 }
194
195 if (kbdev->hwcnt.kctx != kctx) {
196 /* Instrumentation has been setup for another context */
197 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
198 spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
199 return -EINVAL;
200 }
201
202 if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE)
203 break;
204
205 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
206 spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
207
208 /* Ongoing dump/setup - wait for its completion */
209 wait_event(kbdev->hwcnt.backend.wait,
210 kbdev->hwcnt.backend.triggered != 0);
211 }
212
213 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
214 kbdev->hwcnt.backend.triggered = 0;
215
216 kbasep_instr_hwc_disable_hw_prfcnt(kbdev);
217
218 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
219 spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
220
221 dev_dbg(kbdev->dev, "HW counters dumping disabled for context %pK",
222 kctx);
223
224 return 0;
225 }
226
kbase_instr_hwcnt_request_dump(struct kbase_context * kctx)227 int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx)
228 {
229 unsigned long flags;
230 int err = -EINVAL;
231 struct kbase_device *kbdev = kctx->kbdev;
232
233 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
234
235 if (kbdev->hwcnt.kctx != kctx) {
236 /* The instrumentation has been setup for another context */
237 goto unlock;
238 }
239
240 if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) {
241 /* HW counters are disabled or another dump is ongoing, or we're
242 * resetting, or we are in unrecoverable error state.
243 */
244 goto unlock;
245 }
246
247 if (kbase_is_gpu_removed(kbdev)) {
248 /* GPU has been removed by Arbiter */
249 goto unlock;
250 }
251
252 kbdev->hwcnt.backend.triggered = 0;
253
254 /* Mark that we're dumping - the PF handler can signal that we faulted
255 */
256 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING;
257
258 /* Wait until prfcnt is ready to request dump */
259 err = wait_prfcnt_ready(kbdev);
260 if (err)
261 goto unlock;
262
263 /* Reconfigure the dump address */
264 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
265 kbdev->hwcnt.addr & 0xFFFFFFFF);
266 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
267 kbdev->hwcnt.addr >> 32);
268
269 /* Start dumping */
270 KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL,
271 kbdev->hwcnt.addr);
272
273 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
274 GPU_COMMAND_PRFCNT_SAMPLE);
275
276 dev_dbg(kbdev->dev, "HW counters dumping done for context %pK", kctx);
277
278 unlock:
279 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
280 return err;
281 }
282 KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump);
283
kbase_instr_hwcnt_dump_complete(struct kbase_context * kctx,bool * const success)284 bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx,
285 bool * const success)
286 {
287 unsigned long flags;
288 bool complete = false;
289 struct kbase_device *kbdev = kctx->kbdev;
290
291 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
292
293 if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) {
294 *success = true;
295 complete = true;
296 } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
297 *success = false;
298 complete = true;
299 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
300 }
301
302 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
303
304 return complete;
305 }
306 KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete);
307
kbase_instr_hwcnt_sample_done(struct kbase_device * kbdev)308 void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev)
309 {
310 unsigned long flags;
311
312 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
313
314 /* If the state is in unrecoverable error, we already wake_up the waiter
315 * and don't need to do any action when sample is done.
316 */
317
318 if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
319 kbdev->hwcnt.backend.triggered = 1;
320 wake_up(&kbdev->hwcnt.backend.wait);
321 } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) {
322 /* All finished and idle */
323 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
324 kbdev->hwcnt.backend.triggered = 1;
325 wake_up(&kbdev->hwcnt.backend.wait);
326 }
327
328 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
329 }
330
kbase_instr_hwcnt_wait_for_dump(struct kbase_context * kctx)331 int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx)
332 {
333 struct kbase_device *kbdev = kctx->kbdev;
334 unsigned long flags;
335 int err;
336
337 /* Wait for dump & cache clean to complete */
338 wait_event(kbdev->hwcnt.backend.wait,
339 kbdev->hwcnt.backend.triggered != 0);
340
341 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
342
343 if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
344 err = -EINVAL;
345 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
346 } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) {
347 err = -EIO;
348 } else {
349 /* Dump done */
350 KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
351 KBASE_INSTR_STATE_IDLE);
352 err = 0;
353 }
354
355 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
356
357 return err;
358 }
359
kbase_instr_hwcnt_clear(struct kbase_context * kctx)360 int kbase_instr_hwcnt_clear(struct kbase_context *kctx)
361 {
362 unsigned long flags;
363 int err = -EINVAL;
364 struct kbase_device *kbdev = kctx->kbdev;
365
366 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
367
368 /* Check it's the context previously set up and we're not in IDLE
369 * state.
370 */
371 if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state !=
372 KBASE_INSTR_STATE_IDLE)
373 goto unlock;
374
375 if (kbase_is_gpu_removed(kbdev)) {
376 /* GPU has been removed by Arbiter */
377 goto unlock;
378 }
379
380 /* Wait until prfcnt is ready to clear */
381 err = wait_prfcnt_ready(kbdev);
382 if (err)
383 goto unlock;
384
385 /* Clear the counters */
386 KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0);
387 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
388 GPU_COMMAND_PRFCNT_CLEAR);
389
390 unlock:
391 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
392 return err;
393 }
394 KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear);
395
kbase_instr_hwcnt_on_unrecoverable_error(struct kbase_device * kbdev)396 void kbase_instr_hwcnt_on_unrecoverable_error(struct kbase_device *kbdev)
397 {
398 unsigned long flags;
399
400 lockdep_assert_held(&kbdev->hwaccess_lock);
401
402 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
403
404 /* If we already in unrecoverable error state, early return. */
405 if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) {
406 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
407 return;
408 }
409
410 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR;
411
412 /* Need to disable HW if it's not disabled yet. */
413 if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED)
414 kbasep_instr_hwc_disable_hw_prfcnt(kbdev);
415
416 /* Wake up any waiters. */
417 kbdev->hwcnt.backend.triggered = 1;
418 wake_up(&kbdev->hwcnt.backend.wait);
419
420 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
421 }
422 KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_on_unrecoverable_error);
423
kbase_instr_hwcnt_on_before_reset(struct kbase_device * kbdev)424 void kbase_instr_hwcnt_on_before_reset(struct kbase_device *kbdev)
425 {
426 unsigned long flags;
427
428 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
429
430 /* A reset is the only way to exit the unrecoverable error state */
431 if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR)
432 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
433
434 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
435 }
436 KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_on_before_reset);
437
kbase_instr_backend_init(struct kbase_device * kbdev)438 int kbase_instr_backend_init(struct kbase_device *kbdev)
439 {
440 spin_lock_init(&kbdev->hwcnt.lock);
441
442 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
443
444 init_waitqueue_head(&kbdev->hwcnt.backend.wait);
445
446 kbdev->hwcnt.backend.triggered = 0;
447
448 #ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
449 /* Use the build time option for the override default. */
450 #if defined(CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY)
451 kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_SECONDARY;
452 #elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY)
453 kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_TERTIARY;
454 #else
455 /* Default to primary */
456 kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_PRIMARY;
457 #endif
458 #endif
459 return 0;
460 }
461
kbase_instr_backend_term(struct kbase_device * kbdev)462 void kbase_instr_backend_term(struct kbase_device *kbdev)
463 {
464 CSTD_UNUSED(kbdev);
465 }
466
467 #ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
kbase_instr_backend_debugfs_init(struct kbase_device * kbdev)468 void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev)
469 {
470 /* No validation is done on the debugfs input. Invalid input could cause
471 * performance counter errors. This is acceptable since this is a debug
472 * only feature and users should know what they are doing.
473 *
474 * Valid inputs are the values accepted bythe SET_SELECT bits of the
475 * PRFCNT_CONFIG register as defined in the architecture specification.
476 */
477 debugfs_create_u8("hwcnt_set_select", 0644,
478 kbdev->mali_debugfs_directory,
479 (u8 *)&kbdev->hwcnt.backend.override_counter_set);
480 }
481 #endif
482