1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2018-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 #include "hwcnt/backend/mali_kbase_hwcnt_backend_jm.h"
23 #include "hwcnt/mali_kbase_hwcnt_gpu.h"
24 #include "hwcnt/mali_kbase_hwcnt_types.h"
25 #include "mali_kbase.h"
26 #include "backend/gpu/mali_kbase_pm_ca.h"
27 #include "mali_kbase_hwaccess_instr.h"
28 #include "mali_kbase_hwaccess_time.h"
29 #include "mali_kbase_ccswe.h"
30 #include "backend/gpu/mali_kbase_model_linux.h"
31 #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
32
33 #include "backend/gpu/mali_kbase_pm_internal.h"
34
35 /**
36 * struct kbase_hwcnt_backend_jm_info - Information used to create an instance
37 * of a JM hardware counter backend.
38 * @kbdev: KBase device.
39 * @counter_set: The performance counter set to use.
40 * @metadata: Hardware counter metadata.
41 * @dump_bytes: Bytes of GPU memory required to perform a
42 * hardware counter dump.
43 * @hwcnt_gpu_info: Hardware counter block information.
44 */
45 struct kbase_hwcnt_backend_jm_info {
46 struct kbase_device *kbdev;
47 enum kbase_hwcnt_set counter_set;
48 const struct kbase_hwcnt_metadata *metadata;
49 size_t dump_bytes;
50 struct kbase_hwcnt_gpu_info hwcnt_gpu_info;
51 };
52
53 /**
54 * struct kbase_hwcnt_jm_physical_layout - HWC sample memory physical layout
55 * information.
56 * @fe_cnt: Front end block count.
57 * @tiler_cnt: Tiler block count.
58 * @mmu_l2_cnt: Memory system(MMU and L2 cache) block count.
59 * @shader_cnt: Shader Core block count.
60 * @block_cnt: Total block count (sum of all other block counts).
61 * @shader_avail_mask: Bitmap of all shader cores in the system.
62 * @enable_mask_offset: Offset in array elements of enable mask in each block
63 * starting from the beginning of block.
64 * @headers_per_block: Header size per block.
65 * @counters_per_block: Counters size per block.
66 * @values_per_block: Total size per block.
67 */
68 struct kbase_hwcnt_jm_physical_layout {
69 u8 fe_cnt;
70 u8 tiler_cnt;
71 u8 mmu_l2_cnt;
72 u8 shader_cnt;
73 u8 block_cnt;
74 u64 shader_avail_mask;
75 size_t enable_mask_offset;
76 size_t headers_per_block;
77 size_t counters_per_block;
78 size_t values_per_block;
79 };
80
81 /**
82 * struct kbase_hwcnt_backend_jm - Instance of a JM hardware counter backend.
83 * @info: Info used to create the backend.
84 * @kctx: KBase context used for GPU memory allocation and
85 * counter dumping.
86 * @gpu_dump_va: GPU hardware counter dump buffer virtual address.
87 * @cpu_dump_va: CPU mapping of gpu_dump_va.
88 * @vmap: Dump buffer vmap.
89 * @to_user_buf: HWC sample buffer for client user, size
90 * metadata.dump_buf_bytes.
91 * @enabled: True if dumping has been enabled, else false.
92 * @pm_core_mask: PM state sync-ed shaders core mask for the enabled
93 * dumping.
94 * @curr_config: Current allocated hardware resources to correctly map the
95 * source raw dump buffer to the destination dump buffer.
96 * @clk_enable_map: The enable map specifying enabled clock domains.
97 * @cycle_count_elapsed:
98 * Cycle count elapsed for a given sample period.
99 * The top clock cycle, index 0, is read directly from
100 * hardware, but the other clock domains need to be
101 * calculated with software estimation.
102 * @prev_cycle_count: Previous cycle count to calculate the cycle count for
103 * sample period.
104 * @rate_listener: Clock rate listener callback state.
105 * @ccswe_shader_cores: Shader cores cycle count software estimator.
106 * @phys_layout: Physical memory layout information of HWC sample buffer.
107 */
108 struct kbase_hwcnt_backend_jm {
109 const struct kbase_hwcnt_backend_jm_info *info;
110 struct kbase_context *kctx;
111 u64 gpu_dump_va;
112 void *cpu_dump_va;
113 struct kbase_vmap_struct *vmap;
114 u64 *to_user_buf;
115 bool enabled;
116 u64 pm_core_mask;
117 struct kbase_hwcnt_curr_config curr_config;
118 u64 clk_enable_map;
119 u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS];
120 u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS];
121 struct kbase_clk_rate_listener rate_listener;
122 struct kbase_ccswe ccswe_shader_cores;
123 struct kbase_hwcnt_jm_physical_layout phys_layout;
124 };
125
126 /**
127 * kbasep_hwcnt_backend_jm_gpu_info_init() - Initialise an info structure used
128 * to create the hwcnt metadata.
129 * @kbdev: Non-NULL pointer to kbase device.
130 * @info: Non-NULL pointer to data structure to be filled in.
131 *
132 * The initialised info struct will only be valid for use while kbdev is valid.
133 *
134 * Return: 0 on success, else error code.
135 */
kbasep_hwcnt_backend_jm_gpu_info_init(struct kbase_device * kbdev,struct kbase_hwcnt_gpu_info * info)136 static int kbasep_hwcnt_backend_jm_gpu_info_init(struct kbase_device *kbdev,
137 struct kbase_hwcnt_gpu_info *info)
138 {
139 size_t clk;
140
141 if (!kbdev || !info)
142 return -EINVAL;
143
144 #if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
145 info->l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS;
146 info->core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1;
147 info->prfcnt_values_per_block = KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK;
148 #else /* CONFIG_MALI_BIFROST_NO_MALI */
149 {
150 const struct base_gpu_props *props = &kbdev->gpu_props.props;
151 const size_t l2_count = props->l2_props.num_l2_slices;
152 const size_t core_mask = props->coherency_info.group[0].core_mask;
153
154 info->l2_count = l2_count;
155 info->core_mask = core_mask;
156 info->prfcnt_values_per_block = KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK;
157 }
158 #endif /* CONFIG_MALI_BIFROST_NO_MALI */
159
160 /* Determine the number of available clock domains. */
161 for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) {
162 if (kbdev->pm.clk_rtm.clks[clk] == NULL)
163 break;
164 }
165 info->clk_cnt = clk;
166
167 return 0;
168 }
169
kbasep_hwcnt_backend_jm_init_layout(const struct kbase_hwcnt_gpu_info * gpu_info,struct kbase_hwcnt_jm_physical_layout * phys_layout)170 static void kbasep_hwcnt_backend_jm_init_layout(const struct kbase_hwcnt_gpu_info *gpu_info,
171 struct kbase_hwcnt_jm_physical_layout *phys_layout)
172 {
173 u8 shader_core_cnt;
174
175 WARN_ON(!gpu_info);
176 WARN_ON(!phys_layout);
177
178 shader_core_cnt = fls64(gpu_info->core_mask);
179
180 *phys_layout = (struct kbase_hwcnt_jm_physical_layout){
181 .fe_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT,
182 .tiler_cnt = KBASE_HWCNT_V5_TILER_BLOCK_COUNT,
183 .mmu_l2_cnt = gpu_info->l2_count,
184 .shader_cnt = shader_core_cnt,
185 .block_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT + KBASE_HWCNT_V5_TILER_BLOCK_COUNT +
186 gpu_info->l2_count + shader_core_cnt,
187 .shader_avail_mask = gpu_info->core_mask,
188 .headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK,
189 .values_per_block = gpu_info->prfcnt_values_per_block,
190 .counters_per_block =
191 gpu_info->prfcnt_values_per_block - KBASE_HWCNT_V5_HEADERS_PER_BLOCK,
192 .enable_mask_offset = KBASE_HWCNT_V5_PRFCNT_EN_HEADER,
193 };
194 }
195
196 static void
kbasep_hwcnt_backend_jm_dump_sample(const struct kbase_hwcnt_backend_jm * const backend_jm)197 kbasep_hwcnt_backend_jm_dump_sample(const struct kbase_hwcnt_backend_jm *const backend_jm)
198 {
199 size_t block_idx;
200 const u32 *new_sample_buf = backend_jm->cpu_dump_va;
201 const u32 *new_block = new_sample_buf;
202 u64 *dst_buf = backend_jm->to_user_buf;
203 u64 *dst_block = dst_buf;
204 const size_t values_per_block = backend_jm->phys_layout.values_per_block;
205 const size_t dump_bytes = backend_jm->info->dump_bytes;
206
207 for (block_idx = 0; block_idx < backend_jm->phys_layout.block_cnt; block_idx++) {
208 size_t ctr_idx;
209
210 for (ctr_idx = 0; ctr_idx < values_per_block; ctr_idx++)
211 dst_block[ctr_idx] = new_block[ctr_idx];
212
213 new_block += values_per_block;
214 dst_block += values_per_block;
215 }
216
217 WARN_ON(new_block != new_sample_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES));
218 WARN_ON(dst_block != dst_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES));
219 }
220
221 /**
222 * kbasep_hwcnt_backend_jm_on_freq_change() - On freq change callback
223 *
224 * @rate_listener: Callback state
225 * @clk_index: Clock index
226 * @clk_rate_hz: Clock frequency(hz)
227 */
kbasep_hwcnt_backend_jm_on_freq_change(struct kbase_clk_rate_listener * rate_listener,u32 clk_index,u32 clk_rate_hz)228 static void kbasep_hwcnt_backend_jm_on_freq_change(struct kbase_clk_rate_listener *rate_listener,
229 u32 clk_index, u32 clk_rate_hz)
230 {
231 struct kbase_hwcnt_backend_jm *backend_jm =
232 container_of(rate_listener, struct kbase_hwcnt_backend_jm, rate_listener);
233 u64 timestamp_ns;
234
235 if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES)
236 return;
237
238 timestamp_ns = ktime_get_raw_ns();
239 kbase_ccswe_freq_change(&backend_jm->ccswe_shader_cores, timestamp_ns, clk_rate_hz);
240 }
241
242 /**
243 * kbasep_hwcnt_backend_jm_cc_enable() - Enable cycle count tracking
244 *
245 * @backend_jm: Non-NULL pointer to backend.
246 * @enable_map: Non-NULL pointer to enable map specifying enabled counters.
247 * @timestamp_ns: Timestamp(ns) when HWCNT were enabled.
248 */
kbasep_hwcnt_backend_jm_cc_enable(struct kbase_hwcnt_backend_jm * backend_jm,const struct kbase_hwcnt_enable_map * enable_map,u64 timestamp_ns)249 static void kbasep_hwcnt_backend_jm_cc_enable(struct kbase_hwcnt_backend_jm *backend_jm,
250 const struct kbase_hwcnt_enable_map *enable_map,
251 u64 timestamp_ns)
252 {
253 struct kbase_device *kbdev = backend_jm->kctx->kbdev;
254 u64 clk_enable_map = enable_map->clk_enable_map;
255 u64 cycle_count;
256
257 if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) {
258 /* turn on the cycle counter */
259 kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev);
260 /* Read cycle count for top clock domain. */
261 kbase_backend_get_gpu_time_norequest(kbdev, &cycle_count, NULL, NULL);
262
263 backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_TOP] = cycle_count;
264 }
265
266 if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) {
267 /* software estimation for non-top clock domains */
268 struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm;
269 const struct kbase_clk_data *clk_data = rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES];
270 u32 cur_freq;
271 unsigned long flags;
272
273 spin_lock_irqsave(&rtm->lock, flags);
274
275 cur_freq = (u32)clk_data->clock_val;
276 kbase_ccswe_reset(&backend_jm->ccswe_shader_cores);
277 kbase_ccswe_freq_change(&backend_jm->ccswe_shader_cores, timestamp_ns, cur_freq);
278
279 kbase_clk_rate_trace_manager_subscribe_no_lock(rtm, &backend_jm->rate_listener);
280
281 spin_unlock_irqrestore(&rtm->lock, flags);
282
283 /* ccswe was reset. The estimated cycle is zero. */
284 backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_SHADER_CORES] = 0;
285 }
286
287 /* Keep clk_enable_map for dump_request. */
288 backend_jm->clk_enable_map = clk_enable_map;
289 }
290
291 /**
292 * kbasep_hwcnt_backend_jm_cc_disable() - Disable cycle count tracking
293 *
294 * @backend_jm: Non-NULL pointer to backend.
295 */
kbasep_hwcnt_backend_jm_cc_disable(struct kbase_hwcnt_backend_jm * backend_jm)296 static void kbasep_hwcnt_backend_jm_cc_disable(struct kbase_hwcnt_backend_jm *backend_jm)
297 {
298 struct kbase_device *kbdev = backend_jm->kctx->kbdev;
299 struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm;
300 u64 clk_enable_map = backend_jm->clk_enable_map;
301
302 if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) {
303 /* turn off the cycle counter */
304 kbase_pm_release_gpu_cycle_counter(kbdev);
305 }
306
307 if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) {
308 kbase_clk_rate_trace_manager_unsubscribe(rtm, &backend_jm->rate_listener);
309 }
310 }
311
312 /**
313 * kbasep_hwcnt_gpu_update_curr_config() - Update the destination buffer with
314 * current config information.
315 * @kbdev: Non-NULL pointer to kbase device.
316 * @curr_config: Non-NULL pointer to return the current configuration of
317 * hardware allocated to the GPU.
318 *
319 * The current configuration information is used for architectures where the
320 * max_config interface is available from the Arbiter. In this case the current
321 * allocated hardware is not always the same, so the current config information
322 * is used to correctly map the current allocated resources to the memory layout
323 * that is copied to the user space.
324 *
325 * Return: 0 on success, else error code.
326 */
kbasep_hwcnt_gpu_update_curr_config(struct kbase_device * kbdev,struct kbase_hwcnt_curr_config * curr_config)327 static int kbasep_hwcnt_gpu_update_curr_config(struct kbase_device *kbdev,
328 struct kbase_hwcnt_curr_config *curr_config)
329 {
330 if (WARN_ON(!kbdev) || WARN_ON(!curr_config))
331 return -EINVAL;
332
333 lockdep_assert_held(&kbdev->hwaccess_lock);
334
335 curr_config->num_l2_slices = kbdev->gpu_props.curr_config.l2_slices;
336 curr_config->shader_present = kbdev->gpu_props.curr_config.shader_present;
337 return 0;
338 }
339
340 /* JM backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */
kbasep_hwcnt_backend_jm_timestamp_ns(struct kbase_hwcnt_backend * backend)341 static u64 kbasep_hwcnt_backend_jm_timestamp_ns(struct kbase_hwcnt_backend *backend)
342 {
343 (void)backend;
344 return ktime_get_raw_ns();
345 }
346
347 /* JM backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */
348 static int
kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend * backend,const struct kbase_hwcnt_enable_map * enable_map)349 kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend,
350 const struct kbase_hwcnt_enable_map *enable_map)
351 {
352 int errcode;
353 struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend;
354 struct kbase_context *kctx;
355 struct kbase_device *kbdev;
356 struct kbase_hwcnt_physical_enable_map phys_enable_map;
357 enum kbase_hwcnt_physical_set phys_counter_set;
358 struct kbase_instr_hwcnt_enable enable;
359 u64 timestamp_ns;
360
361 if (!backend_jm || !enable_map || backend_jm->enabled ||
362 (enable_map->metadata != backend_jm->info->metadata))
363 return -EINVAL;
364
365 kctx = backend_jm->kctx;
366 kbdev = backend_jm->kctx->kbdev;
367
368 lockdep_assert_held(&kbdev->hwaccess_lock);
369
370 kbase_hwcnt_gpu_enable_map_to_physical(&phys_enable_map, enable_map);
371
372 kbase_hwcnt_gpu_set_to_physical(&phys_counter_set, backend_jm->info->counter_set);
373
374 enable.fe_bm = phys_enable_map.fe_bm;
375 enable.shader_bm = phys_enable_map.shader_bm;
376 enable.tiler_bm = phys_enable_map.tiler_bm;
377 enable.mmu_l2_bm = phys_enable_map.mmu_l2_bm;
378 enable.counter_set = phys_counter_set;
379 #if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
380 /* The dummy model needs the CPU mapping. */
381 enable.dump_buffer = (uintptr_t)backend_jm->cpu_dump_va;
382 #else
383 enable.dump_buffer = backend_jm->gpu_dump_va;
384 #endif /* CONFIG_MALI_BIFROST_NO_MALI */
385 enable.dump_buffer_bytes = backend_jm->info->dump_bytes;
386
387 timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend);
388
389 /* Update the current configuration information. */
390 errcode = kbasep_hwcnt_gpu_update_curr_config(kbdev, &backend_jm->curr_config);
391 if (errcode)
392 goto error;
393
394 errcode = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &enable);
395 if (errcode)
396 goto error;
397
398 backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev);
399
400 backend_jm->enabled = true;
401
402 kbasep_hwcnt_backend_jm_cc_enable(backend_jm, enable_map, timestamp_ns);
403
404 return 0;
405 error:
406 return errcode;
407 }
408
409 /* JM backend implementation of kbase_hwcnt_backend_dump_enable_fn */
kbasep_hwcnt_backend_jm_dump_enable(struct kbase_hwcnt_backend * backend,const struct kbase_hwcnt_enable_map * enable_map)410 static int kbasep_hwcnt_backend_jm_dump_enable(struct kbase_hwcnt_backend *backend,
411 const struct kbase_hwcnt_enable_map *enable_map)
412 {
413 unsigned long flags;
414 int errcode;
415 struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend;
416 struct kbase_device *kbdev;
417
418 if (!backend_jm)
419 return -EINVAL;
420
421 kbdev = backend_jm->kctx->kbdev;
422
423 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
424
425 errcode = kbasep_hwcnt_backend_jm_dump_enable_nolock(backend, enable_map);
426
427 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
428
429 return errcode;
430 }
431
432 /* JM backend implementation of kbase_hwcnt_backend_dump_disable_fn */
kbasep_hwcnt_backend_jm_dump_disable(struct kbase_hwcnt_backend * backend)433 static void kbasep_hwcnt_backend_jm_dump_disable(struct kbase_hwcnt_backend *backend)
434 {
435 int errcode;
436 struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend;
437
438 if (WARN_ON(!backend_jm) || !backend_jm->enabled)
439 return;
440
441 kbasep_hwcnt_backend_jm_cc_disable(backend_jm);
442
443 errcode = kbase_instr_hwcnt_disable_internal(backend_jm->kctx);
444 WARN_ON(errcode);
445
446 backend_jm->enabled = false;
447 }
448
449 /* JM backend implementation of kbase_hwcnt_backend_dump_clear_fn */
kbasep_hwcnt_backend_jm_dump_clear(struct kbase_hwcnt_backend * backend)450 static int kbasep_hwcnt_backend_jm_dump_clear(struct kbase_hwcnt_backend *backend)
451 {
452 struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend;
453
454 if (!backend_jm || !backend_jm->enabled)
455 return -EINVAL;
456
457 return kbase_instr_hwcnt_clear(backend_jm->kctx);
458 }
459
460 /* JM backend implementation of kbase_hwcnt_backend_dump_request_fn */
kbasep_hwcnt_backend_jm_dump_request(struct kbase_hwcnt_backend * backend,u64 * dump_time_ns)461 static int kbasep_hwcnt_backend_jm_dump_request(struct kbase_hwcnt_backend *backend,
462 u64 *dump_time_ns)
463 {
464 struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend;
465 struct kbase_device *kbdev;
466 const struct kbase_hwcnt_metadata *metadata;
467 u64 current_cycle_count;
468 size_t clk;
469 int ret;
470
471 if (!backend_jm || !backend_jm->enabled || !dump_time_ns)
472 return -EINVAL;
473
474 kbdev = backend_jm->kctx->kbdev;
475 metadata = backend_jm->info->metadata;
476
477 /* Disable pre-emption, to make the timestamp as accurate as possible */
478 preempt_disable();
479 {
480 *dump_time_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend);
481 ret = kbase_instr_hwcnt_request_dump(backend_jm->kctx);
482
483 kbase_hwcnt_metadata_for_each_clock(metadata, clk)
484 {
485 if (!kbase_hwcnt_clk_enable_map_enabled(backend_jm->clk_enable_map, clk))
486 continue;
487
488 if (clk == KBASE_CLOCK_DOMAIN_TOP) {
489 /* Read cycle count for top clock domain. */
490 kbase_backend_get_gpu_time_norequest(kbdev, ¤t_cycle_count,
491 NULL, NULL);
492 } else {
493 /*
494 * Estimate cycle count for non-top clock
495 * domain.
496 */
497 current_cycle_count = kbase_ccswe_cycle_at(
498 &backend_jm->ccswe_shader_cores, *dump_time_ns);
499 }
500 backend_jm->cycle_count_elapsed[clk] =
501 current_cycle_count - backend_jm->prev_cycle_count[clk];
502
503 /*
504 * Keep the current cycle count for later calculation.
505 */
506 backend_jm->prev_cycle_count[clk] = current_cycle_count;
507 }
508 }
509 preempt_enable();
510
511 return ret;
512 }
513
514 /* JM backend implementation of kbase_hwcnt_backend_dump_wait_fn */
kbasep_hwcnt_backend_jm_dump_wait(struct kbase_hwcnt_backend * backend)515 static int kbasep_hwcnt_backend_jm_dump_wait(struct kbase_hwcnt_backend *backend)
516 {
517 struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend;
518
519 if (!backend_jm || !backend_jm->enabled)
520 return -EINVAL;
521
522 return kbase_instr_hwcnt_wait_for_dump(backend_jm->kctx);
523 }
524
525 /* JM backend implementation of kbase_hwcnt_backend_dump_get_fn */
kbasep_hwcnt_backend_jm_dump_get(struct kbase_hwcnt_backend * backend,struct kbase_hwcnt_dump_buffer * dst,const struct kbase_hwcnt_enable_map * dst_enable_map,bool accumulate)526 static int kbasep_hwcnt_backend_jm_dump_get(struct kbase_hwcnt_backend *backend,
527 struct kbase_hwcnt_dump_buffer *dst,
528 const struct kbase_hwcnt_enable_map *dst_enable_map,
529 bool accumulate)
530 {
531 struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend;
532 size_t clk;
533 #if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
534 struct kbase_device *kbdev;
535 unsigned long flags;
536 int errcode;
537 #endif /* CONFIG_MALI_BIFROST_NO_MALI */
538
539 if (!backend_jm || !dst || !dst_enable_map ||
540 (backend_jm->info->metadata != dst->metadata) ||
541 (dst_enable_map->metadata != dst->metadata))
542 return -EINVAL;
543
544 /* Invalidate the kernel buffer before reading from it. */
545 kbase_sync_mem_regions(backend_jm->kctx, backend_jm->vmap, KBASE_SYNC_TO_CPU);
546
547 /* Dump sample to the internal 64-bit user buffer. */
548 kbasep_hwcnt_backend_jm_dump_sample(backend_jm);
549
550 /* Extract elapsed cycle count for each clock domain if enabled. */
551 kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk)
552 {
553 if (!kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk))
554 continue;
555
556 /* Reset the counter to zero if accumulation is off. */
557 if (!accumulate)
558 dst->clk_cnt_buf[clk] = 0;
559 dst->clk_cnt_buf[clk] += backend_jm->cycle_count_elapsed[clk];
560 }
561
562 #if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
563 kbdev = backend_jm->kctx->kbdev;
564
565 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
566
567 /* Update the current configuration information. */
568 errcode = kbasep_hwcnt_gpu_update_curr_config(kbdev, &backend_jm->curr_config);
569
570 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
571
572 if (errcode)
573 return errcode;
574 #endif /* CONFIG_MALI_BIFROST_NO_MALI */
575 return kbase_hwcnt_jm_dump_get(dst, backend_jm->to_user_buf, dst_enable_map,
576 backend_jm->pm_core_mask, &backend_jm->curr_config,
577 accumulate);
578 }
579
580 /**
581 * kbasep_hwcnt_backend_jm_dump_alloc() - Allocate a GPU dump buffer.
582 * @info: Non-NULL pointer to JM backend info.
583 * @kctx: Non-NULL pointer to kbase context.
584 * @gpu_dump_va: Non-NULL pointer to where GPU dump buffer virtual address
585 * is stored on success.
586 *
587 * Return: 0 on success, else error code.
588 */
kbasep_hwcnt_backend_jm_dump_alloc(const struct kbase_hwcnt_backend_jm_info * info,struct kbase_context * kctx,u64 * gpu_dump_va)589 static int kbasep_hwcnt_backend_jm_dump_alloc(const struct kbase_hwcnt_backend_jm_info *info,
590 struct kbase_context *kctx, u64 *gpu_dump_va)
591 {
592 struct kbase_va_region *reg;
593 u64 flags;
594 u64 nr_pages;
595
596 /* Calls to this function are inherently asynchronous, with respect to
597 * MMU operations.
598 */
599 const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC;
600
601 WARN_ON(!info);
602 WARN_ON(!kctx);
603 WARN_ON(!gpu_dump_va);
604
605 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR | BASEP_MEM_PERMANENT_KERNEL_MAPPING |
606 BASE_MEM_CACHED_CPU | BASE_MEM_UNCACHED_GPU;
607
608 nr_pages = PFN_UP(info->dump_bytes);
609
610 reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, gpu_dump_va, mmu_sync_info);
611
612 if (!reg)
613 return -ENOMEM;
614
615 return 0;
616 }
617
618 /**
619 * kbasep_hwcnt_backend_jm_dump_free() - Free an allocated GPU dump buffer.
620 * @kctx: Non-NULL pointer to kbase context.
621 * @gpu_dump_va: GPU dump buffer virtual address.
622 */
kbasep_hwcnt_backend_jm_dump_free(struct kbase_context * kctx,u64 gpu_dump_va)623 static void kbasep_hwcnt_backend_jm_dump_free(struct kbase_context *kctx, u64 gpu_dump_va)
624 {
625 WARN_ON(!kctx);
626 if (gpu_dump_va)
627 kbase_mem_free(kctx, gpu_dump_va);
628 }
629
630 /**
631 * kbasep_hwcnt_backend_jm_destroy() - Destroy a JM backend.
632 * @backend: Pointer to JM backend to destroy.
633 *
634 * Can be safely called on a backend in any state of partial construction.
635 */
kbasep_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_jm * backend)636 static void kbasep_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_jm *backend)
637 {
638 if (!backend)
639 return;
640
641 if (backend->kctx) {
642 struct kbase_context *kctx = backend->kctx;
643 struct kbase_device *kbdev = kctx->kbdev;
644
645 if (backend->cpu_dump_va)
646 kbase_phy_alloc_mapping_put(kctx, backend->vmap);
647
648 if (backend->gpu_dump_va)
649 kbasep_hwcnt_backend_jm_dump_free(kctx, backend->gpu_dump_va);
650
651 kbasep_js_release_privileged_ctx(kbdev, kctx);
652 kbase_destroy_context(kctx);
653 }
654
655 kfree(backend->to_user_buf);
656
657 kfree(backend);
658 }
659
660 /**
661 * kbasep_hwcnt_backend_jm_create() - Create a JM backend.
662 * @info: Non-NULL pointer to backend info.
663 * @out_backend: Non-NULL pointer to where backend is stored on success.
664 *
665 * Return: 0 on success, else error code.
666 */
kbasep_hwcnt_backend_jm_create(const struct kbase_hwcnt_backend_jm_info * info,struct kbase_hwcnt_backend_jm ** out_backend)667 static int kbasep_hwcnt_backend_jm_create(const struct kbase_hwcnt_backend_jm_info *info,
668 struct kbase_hwcnt_backend_jm **out_backend)
669 {
670 int errcode;
671 struct kbase_device *kbdev;
672 struct kbase_hwcnt_backend_jm *backend = NULL;
673
674 WARN_ON(!info);
675 WARN_ON(!out_backend);
676
677 kbdev = info->kbdev;
678
679 backend = kzalloc(sizeof(*backend), GFP_KERNEL);
680 if (!backend)
681 goto alloc_error;
682
683 backend->info = info;
684 kbasep_hwcnt_backend_jm_init_layout(&info->hwcnt_gpu_info, &backend->phys_layout);
685
686 backend->kctx = kbase_create_context(kbdev, true,
687 BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED, 0, NULL);
688 if (!backend->kctx)
689 goto alloc_error;
690
691 kbasep_js_schedule_privileged_ctx(kbdev, backend->kctx);
692
693 errcode = kbasep_hwcnt_backend_jm_dump_alloc(info, backend->kctx, &backend->gpu_dump_va);
694 if (errcode)
695 goto error;
696
697 backend->cpu_dump_va =
698 kbase_phy_alloc_mapping_get(backend->kctx, backend->gpu_dump_va, &backend->vmap);
699 if (!backend->cpu_dump_va || !backend->vmap)
700 goto alloc_error;
701
702 backend->to_user_buf = kzalloc(info->metadata->dump_buf_bytes, GFP_KERNEL);
703 if (!backend->to_user_buf)
704 goto alloc_error;
705
706 kbase_ccswe_init(&backend->ccswe_shader_cores);
707 backend->rate_listener.notify = kbasep_hwcnt_backend_jm_on_freq_change;
708
709 *out_backend = backend;
710 return 0;
711
712 alloc_error:
713 errcode = -ENOMEM;
714 error:
715 kbasep_hwcnt_backend_jm_destroy(backend);
716 return errcode;
717 }
718
719 /* JM backend implementation of kbase_hwcnt_backend_metadata_fn */
720 static const struct kbase_hwcnt_metadata *
kbasep_hwcnt_backend_jm_metadata(const struct kbase_hwcnt_backend_info * info)721 kbasep_hwcnt_backend_jm_metadata(const struct kbase_hwcnt_backend_info *info)
722 {
723 if (!info)
724 return NULL;
725
726 return ((const struct kbase_hwcnt_backend_jm_info *)info)->metadata;
727 }
728
729 /* JM backend implementation of kbase_hwcnt_backend_init_fn */
kbasep_hwcnt_backend_jm_init(const struct kbase_hwcnt_backend_info * info,struct kbase_hwcnt_backend ** out_backend)730 static int kbasep_hwcnt_backend_jm_init(const struct kbase_hwcnt_backend_info *info,
731 struct kbase_hwcnt_backend **out_backend)
732 {
733 int errcode;
734 struct kbase_hwcnt_backend_jm *backend = NULL;
735
736 if (!info || !out_backend)
737 return -EINVAL;
738
739 errcode = kbasep_hwcnt_backend_jm_create((const struct kbase_hwcnt_backend_jm_info *)info,
740 &backend);
741 if (errcode)
742 return errcode;
743
744 *out_backend = (struct kbase_hwcnt_backend *)backend;
745
746 return 0;
747 }
748
749 /* JM backend implementation of kbase_hwcnt_backend_term_fn */
kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend * backend)750 static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend)
751 {
752 if (!backend)
753 return;
754
755 kbasep_hwcnt_backend_jm_dump_disable(backend);
756 kbasep_hwcnt_backend_jm_destroy((struct kbase_hwcnt_backend_jm *)backend);
757 }
758
759 /**
760 * kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info.
761 * @info: Pointer to info to destroy.
762 *
763 * Can be safely called on a backend info in any state of partial construction.
764 */
kbasep_hwcnt_backend_jm_info_destroy(const struct kbase_hwcnt_backend_jm_info * info)765 static void kbasep_hwcnt_backend_jm_info_destroy(const struct kbase_hwcnt_backend_jm_info *info)
766 {
767 if (!info)
768 return;
769
770 kbase_hwcnt_jm_metadata_destroy(info->metadata);
771 kfree(info);
772 }
773
774 /**
775 * kbasep_hwcnt_backend_jm_info_create() - Create a JM backend info.
776 * @kbdev: Non_NULL pointer to kbase device.
777 * @out_info: Non-NULL pointer to where info is stored on success.
778 *
779 * Return: 0 on success, else error code.
780 */
kbasep_hwcnt_backend_jm_info_create(struct kbase_device * kbdev,const struct kbase_hwcnt_backend_jm_info ** out_info)781 static int kbasep_hwcnt_backend_jm_info_create(struct kbase_device *kbdev,
782 const struct kbase_hwcnt_backend_jm_info **out_info)
783 {
784 int errcode = -ENOMEM;
785 struct kbase_hwcnt_backend_jm_info *info = NULL;
786
787 WARN_ON(!kbdev);
788 WARN_ON(!out_info);
789
790 info = kzalloc(sizeof(*info), GFP_KERNEL);
791 if (!info)
792 return errcode;
793
794 info->kbdev = kbdev;
795
796 #ifdef CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY
797 info->counter_set = KBASE_HWCNT_SET_SECONDARY;
798 #elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY)
799 info->counter_set = KBASE_HWCNT_SET_TERTIARY;
800 #else
801 /* Default to primary */
802 info->counter_set = KBASE_HWCNT_SET_PRIMARY;
803 #endif
804
805 errcode = kbasep_hwcnt_backend_jm_gpu_info_init(kbdev, &info->hwcnt_gpu_info);
806 if (errcode)
807 goto error;
808
809 errcode = kbase_hwcnt_jm_metadata_create(&info->hwcnt_gpu_info, info->counter_set,
810 &info->metadata, &info->dump_bytes);
811 if (errcode)
812 goto error;
813
814 *out_info = info;
815
816 return 0;
817 error:
818 kbasep_hwcnt_backend_jm_info_destroy(info);
819 return errcode;
820 }
821
kbase_hwcnt_backend_jm_create(struct kbase_device * kbdev,struct kbase_hwcnt_backend_interface * iface)822 int kbase_hwcnt_backend_jm_create(struct kbase_device *kbdev,
823 struct kbase_hwcnt_backend_interface *iface)
824 {
825 int errcode;
826 const struct kbase_hwcnt_backend_jm_info *info = NULL;
827
828 if (!kbdev || !iface)
829 return -EINVAL;
830
831 errcode = kbasep_hwcnt_backend_jm_info_create(kbdev, &info);
832
833 if (errcode)
834 return errcode;
835
836 iface->info = (struct kbase_hwcnt_backend_info *)info;
837 iface->metadata = kbasep_hwcnt_backend_jm_metadata;
838 iface->init = kbasep_hwcnt_backend_jm_init;
839 iface->term = kbasep_hwcnt_backend_jm_term;
840 iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns;
841 iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable;
842 iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock;
843 iface->dump_disable = kbasep_hwcnt_backend_jm_dump_disable;
844 iface->dump_clear = kbasep_hwcnt_backend_jm_dump_clear;
845 iface->dump_request = kbasep_hwcnt_backend_jm_dump_request;
846 iface->dump_wait = kbasep_hwcnt_backend_jm_dump_wait;
847 iface->dump_get = kbasep_hwcnt_backend_jm_dump_get;
848
849 return 0;
850 }
851
kbase_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_interface * iface)852 void kbase_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_interface *iface)
853 {
854 if (!iface)
855 return;
856
857 kbasep_hwcnt_backend_jm_info_destroy(
858 (const struct kbase_hwcnt_backend_jm_info *)iface->info);
859 memset(iface, 0, sizeof(*iface));
860 }
861