1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 #include "mali_kbase_csf_tl_reader.h"
23
24 #include "mali_kbase_csf_trace_buffer.h"
25 #include "mali_kbase_reset_gpu.h"
26
27 #include "tl/mali_kbase_tlstream.h"
28 #include "tl/mali_kbase_tl_serialize.h"
29 #include "tl/mali_kbase_tracepoints.h"
30
31 #include "mali_kbase_pm.h"
32 #include "mali_kbase_hwaccess_time.h"
33
34 #include <linux/math64.h>
35
36 #if IS_ENABLED(CONFIG_DEBUG_FS)
37 #include "tl/mali_kbase_timeline_priv.h"
38 #include <linux/debugfs.h>
39 #endif
40
41 /* Name of the CSFFW timeline tracebuffer. */
42 #define KBASE_CSFFW_TRACEBUFFER_NAME "timeline"
43 /* Name of the timeline header metatadata */
44 #define KBASE_CSFFW_TIMELINE_HEADER_NAME "timeline_header"
45
46 /**
47 * struct kbase_csffw_tl_message - CSFFW timeline message.
48 *
49 * @msg_id: Message ID.
50 * @timestamp: Timestamp of the event.
51 * @cycle_counter: Cycle number of the event.
52 *
53 * Contain fields that are common for all CSFFW timeline messages.
54 */
55 struct kbase_csffw_tl_message {
56 u32 msg_id;
57 u64 timestamp;
58 u64 cycle_counter;
59 } __packed __aligned(4);
60
61 #if IS_ENABLED(CONFIG_DEBUG_FS)
kbase_csf_tl_debugfs_poll_interval_read(void * data,u64 * val)62 static int kbase_csf_tl_debugfs_poll_interval_read(void *data, u64 *val)
63 {
64 struct kbase_device *kbdev = (struct kbase_device *)data;
65 struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader;
66
67 *val = self->timer_interval;
68
69 return 0;
70 }
71
kbase_csf_tl_debugfs_poll_interval_write(void * data,u64 val)72 static int kbase_csf_tl_debugfs_poll_interval_write(void *data, u64 val)
73 {
74 struct kbase_device *kbdev = (struct kbase_device *)data;
75 struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader;
76
77 if (val > KBASE_CSF_TL_READ_INTERVAL_MAX || val < KBASE_CSF_TL_READ_INTERVAL_MIN)
78 return -EINVAL;
79
80 self->timer_interval = (u32)val;
81
82 return 0;
83 }
84
85 DEFINE_DEBUGFS_ATTRIBUTE(kbase_csf_tl_poll_interval_fops,
86 kbase_csf_tl_debugfs_poll_interval_read,
87 kbase_csf_tl_debugfs_poll_interval_write, "%llu\n");
88
kbase_csf_tl_reader_debugfs_init(struct kbase_device * kbdev)89 void kbase_csf_tl_reader_debugfs_init(struct kbase_device *kbdev)
90 {
91 debugfs_create_file("csf_tl_poll_interval_in_ms", 0644,
92 kbdev->debugfs_instr_directory, kbdev,
93 &kbase_csf_tl_poll_interval_fops);
94 }
95 #endif
96
97 /**
98 * tl_reader_overflow_notify() - Emit stream overflow tracepoint.
99 *
100 * @self: CSFFW TL Reader instance.
101 * @msg_buf_start: Start of the message.
102 * @msg_buf_end: End of the message buffer.
103 */
tl_reader_overflow_notify(const struct kbase_csf_tl_reader * self,u8 * const msg_buf_start,u8 * const msg_buf_end)104 static void tl_reader_overflow_notify(
105 const struct kbase_csf_tl_reader *self,
106 u8 *const msg_buf_start,
107 u8 *const msg_buf_end)
108 {
109 struct kbase_device *kbdev = self->kbdev;
110 struct kbase_csffw_tl_message message = {0};
111
112 /* Reuse the timestamp and cycle count from current event if possible */
113 if (msg_buf_start + sizeof(message) <= msg_buf_end)
114 memcpy(&message, msg_buf_start, sizeof(message));
115
116 KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW(
117 kbdev, message.timestamp, message.cycle_counter);
118 }
119
120 /**
121 * tl_reader_overflow_check() - Check if an overflow has happened
122 *
123 * @self: CSFFW TL Reader instance.
124 * @event_id: Incoming event id.
125 *
126 * Return: True, if an overflow has happened, False otherwise.
127 */
tl_reader_overflow_check(struct kbase_csf_tl_reader * self,u16 event_id)128 static bool tl_reader_overflow_check(
129 struct kbase_csf_tl_reader *self,
130 u16 event_id)
131 {
132 struct kbase_device *kbdev = self->kbdev;
133 bool has_overflow = false;
134
135 /* 0 is a special event_id and reserved for the very first tracepoint
136 * after reset, we should skip overflow check when reset happened.
137 */
138 if (event_id != 0) {
139 has_overflow = self->got_first_event
140 && self->expected_event_id != event_id;
141
142 if (has_overflow)
143 dev_warn(kbdev->dev,
144 "CSFFW overflow, event_id: %u, expected: %u.",
145 event_id, self->expected_event_id);
146 }
147
148 self->got_first_event = true;
149 self->expected_event_id = event_id + 1;
150 /* When event_id reaches its max value, it skips 0 and wraps to 1. */
151 if (self->expected_event_id == 0)
152 self->expected_event_id++;
153
154 return has_overflow;
155 }
156
157 /**
158 * tl_reader_reset() - Reset timeline tracebuffer reader state machine.
159 *
160 * @self: CSFFW TL Reader instance.
161 *
162 * Reset the reader to the default state, i.e. set all the
163 * mutable fields to zero.
164 */
tl_reader_reset(struct kbase_csf_tl_reader * self)165 static void tl_reader_reset(struct kbase_csf_tl_reader *self)
166 {
167 self->got_first_event = false;
168 self->is_active = false;
169 self->expected_event_id = 0;
170 self->tl_header.btc = 0;
171 }
172
kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader * self)173 int kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self)
174 {
175 int ret = 0;
176 struct kbase_device *kbdev = self->kbdev;
177 struct kbase_tlstream *stream = self->stream;
178
179 u8 *read_buffer = self->read_buffer;
180 const size_t read_buffer_size = sizeof(self->read_buffer);
181
182 u32 bytes_read;
183 u8 *csffw_data_begin;
184 u8 *csffw_data_end;
185 u8 *csffw_data_it;
186
187 unsigned long flags;
188
189 spin_lock_irqsave(&self->read_lock, flags);
190
191 /* If not running, early exit. */
192 if (!self->is_active) {
193 spin_unlock_irqrestore(&self->read_lock, flags);
194 return -EBUSY;
195 }
196
197 /* Copying the whole buffer in a single shot. We assume
198 * that the buffer will not contain partially written messages.
199 */
200 bytes_read = kbase_csf_firmware_trace_buffer_read_data(
201 self->trace_buffer, read_buffer, read_buffer_size);
202 csffw_data_begin = read_buffer;
203 csffw_data_end = read_buffer + bytes_read;
204
205 for (csffw_data_it = csffw_data_begin;
206 csffw_data_it < csffw_data_end;) {
207 u32 event_header;
208 u16 event_id;
209 u16 event_size;
210 unsigned long acq_flags;
211 char *buffer;
212
213 /* Can we safely read event_id? */
214 if (csffw_data_it + sizeof(event_header) > csffw_data_end) {
215 dev_warn(
216 kbdev->dev,
217 "Unable to parse CSFFW tracebuffer event header.");
218 ret = -EBUSY;
219 break;
220 }
221
222 /* Read and parse the event header. */
223 memcpy(&event_header, csffw_data_it, sizeof(event_header));
224 event_id = (event_header >> 0) & 0xFFFF;
225 event_size = (event_header >> 16) & 0xFFFF;
226 csffw_data_it += sizeof(event_header);
227
228 /* Detect if an overflow has happened. */
229 if (tl_reader_overflow_check(self, event_id))
230 tl_reader_overflow_notify(self,
231 csffw_data_it,
232 csffw_data_end);
233
234 /* Can we safely read the message body? */
235 if (csffw_data_it + event_size > csffw_data_end) {
236 dev_warn(kbdev->dev,
237 "event_id: %u, can't read with event_size: %u.",
238 event_id, event_size);
239 ret = -EBUSY;
240 break;
241 }
242
243 /* Convert GPU timestamp to CPU timestamp. */
244 {
245 struct kbase_csffw_tl_message *msg =
246 (struct kbase_csffw_tl_message *) csffw_data_it;
247 msg->timestamp =
248 kbase_backend_time_convert_gpu_to_cpu(kbdev, msg->timestamp);
249 }
250
251 /* Copy the message out to the tl_stream. */
252 buffer = kbase_tlstream_msgbuf_acquire(
253 stream, event_size, &acq_flags);
254 kbasep_serialize_bytes(buffer, 0, csffw_data_it, event_size);
255 kbase_tlstream_msgbuf_release(stream, acq_flags);
256 csffw_data_it += event_size;
257 }
258
259 spin_unlock_irqrestore(&self->read_lock, flags);
260 return ret;
261 }
262
kbasep_csf_tl_reader_read_callback(struct timer_list * timer)263 static void kbasep_csf_tl_reader_read_callback(struct timer_list *timer)
264 {
265 struct kbase_csf_tl_reader *self =
266 container_of(timer, struct kbase_csf_tl_reader, read_timer);
267
268 int rcode;
269
270 kbase_csf_tl_reader_flush_buffer(self);
271
272 rcode = mod_timer(&self->read_timer,
273 jiffies + msecs_to_jiffies(self->timer_interval));
274
275 CSTD_UNUSED(rcode);
276 }
277
278 /**
279 * tl_reader_init_late() - Late CSFFW TL Reader initialization.
280 *
281 * @self: CSFFW TL Reader instance.
282 * @kbdev: Kbase device.
283 *
284 * Late initialization is done once at kbase_csf_tl_reader_start() time.
285 * This is because the firmware image is not parsed
286 * by the kbase_csf_tl_reader_init() time.
287 *
288 * Return: Zero on success, -1 otherwise.
289 */
tl_reader_init_late(struct kbase_csf_tl_reader * self,struct kbase_device * kbdev)290 static int tl_reader_init_late(
291 struct kbase_csf_tl_reader *self,
292 struct kbase_device *kbdev)
293 {
294 struct firmware_trace_buffer *tb;
295 size_t hdr_size = 0;
296 const char *hdr = NULL;
297
298 if (self->kbdev)
299 return 0;
300
301 tb = kbase_csf_firmware_get_trace_buffer(
302 kbdev, KBASE_CSFFW_TRACEBUFFER_NAME);
303 hdr = kbase_csf_firmware_get_timeline_metadata(
304 kbdev, KBASE_CSFFW_TIMELINE_HEADER_NAME, &hdr_size);
305
306 if (!tb) {
307 dev_warn(
308 kbdev->dev,
309 "'%s' tracebuffer is not present in the firmware image.",
310 KBASE_CSFFW_TRACEBUFFER_NAME);
311 return -1;
312 }
313
314 if (!hdr) {
315 dev_warn(
316 kbdev->dev,
317 "'%s' timeline metadata is not present in the firmware image.",
318 KBASE_CSFFW_TIMELINE_HEADER_NAME);
319 return -1;
320 }
321
322 self->kbdev = kbdev;
323 self->trace_buffer = tb;
324 self->tl_header.data = hdr;
325 self->tl_header.size = hdr_size;
326
327 return 0;
328 }
329
330 /**
331 * tl_reader_update_enable_bit() - Update the first bit of a CSFFW tracebuffer.
332 *
333 * @self: CSFFW TL Reader instance.
334 * @value: The value to set.
335 *
336 * Update the first bit of a CSFFW tracebufer and then reset the GPU.
337 * This is to make these changes visible to the MCU.
338 *
339 * Return: 0 on success, or negative error code for failure.
340 */
tl_reader_update_enable_bit(struct kbase_csf_tl_reader * self,bool value)341 static int tl_reader_update_enable_bit(
342 struct kbase_csf_tl_reader *self,
343 bool value)
344 {
345 int err = 0;
346
347 err = kbase_csf_firmware_trace_buffer_update_trace_enable_bit(
348 self->trace_buffer, 0, value);
349
350 return err;
351 }
352
kbase_csf_tl_reader_init(struct kbase_csf_tl_reader * self,struct kbase_tlstream * stream)353 void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self,
354 struct kbase_tlstream *stream)
355 {
356 self->timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT;
357
358 kbase_timer_setup(&self->read_timer,
359 kbasep_csf_tl_reader_read_callback);
360
361 self->stream = stream;
362
363 /* This will be initialized by tl_reader_init_late() */
364 self->kbdev = NULL;
365 self->trace_buffer = NULL;
366 self->tl_header.data = NULL;
367 self->tl_header.size = 0;
368
369 spin_lock_init(&self->read_lock);
370
371 tl_reader_reset(self);
372 }
373
kbase_csf_tl_reader_term(struct kbase_csf_tl_reader * self)374 void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self)
375 {
376 del_timer_sync(&self->read_timer);
377 }
378
kbase_csf_tl_reader_start(struct kbase_csf_tl_reader * self,struct kbase_device * kbdev)379 int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self,
380 struct kbase_device *kbdev)
381 {
382 int rcode;
383
384 /* If already running, early exit. */
385 if (self->is_active)
386 return 0;
387
388 if (tl_reader_init_late(self, kbdev)) {
389 #if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
390 dev_warn(
391 kbdev->dev,
392 "CSFFW timeline is not available for MALI_BIFROST_NO_MALI builds!");
393 return 0;
394 #else
395 return -EINVAL;
396 #endif
397 }
398
399 tl_reader_reset(self);
400
401 self->is_active = true;
402 /* Set bytes to copy to the header size. This is to trigger copying
403 * of the header to the user space.
404 */
405 self->tl_header.btc = self->tl_header.size;
406
407 /* Enable the tracebuffer on the CSFFW side. */
408 rcode = tl_reader_update_enable_bit(self, true);
409 if (rcode != 0)
410 return rcode;
411
412 rcode = mod_timer(&self->read_timer,
413 jiffies + msecs_to_jiffies(self->timer_interval));
414
415 return 0;
416 }
417
kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader * self)418 void kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader *self)
419 {
420 unsigned long flags;
421
422 /* If is not running, early exit. */
423 if (!self->is_active)
424 return;
425
426 /* Disable the tracebuffer on the CSFFW side. */
427 tl_reader_update_enable_bit(self, false);
428
429 del_timer_sync(&self->read_timer);
430
431 spin_lock_irqsave(&self->read_lock, flags);
432
433 tl_reader_reset(self);
434
435 spin_unlock_irqrestore(&self->read_lock, flags);
436 }
437
kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader * self)438 void kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader *self)
439 {
440 kbase_csf_tl_reader_flush_buffer(self);
441 }
442