1 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2 /*
3 *
4 * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 #ifndef _KBASE_CSF_H_
23 #define _KBASE_CSF_H_
24
25 #include "mali_kbase_csf_kcpu.h"
26 #include "mali_kbase_csf_scheduler.h"
27 #include "mali_kbase_csf_firmware.h"
28 #include "mali_kbase_csf_protected_memory.h"
29 #include "mali_kbase_hwaccess_time.h"
30
31 /* Indicate invalid CS h/w interface
32 */
33 #define KBASEP_IF_NR_INVALID ((s8)-1)
34
35 /* Indicate invalid CSG number for a GPU command queue group
36 */
37 #define KBASEP_CSG_NR_INVALID ((s8)-1)
38
39 /* Indicate invalid user doorbell number for a GPU command queue
40 */
41 #define KBASEP_USER_DB_NR_INVALID ((s8)-1)
42
43 /* Number of pages used for GPU command queue's User input & output data */
44 #define KBASEP_NUM_CS_USER_IO_PAGES (2)
45
46 /* Indicates an invalid value for the scan out sequence number, used to
47 * signify there is no group that has protected mode execution pending.
48 */
49 #define KBASEP_TICK_PROTM_PEND_SCAN_SEQ_NR_INVALID (U32_MAX)
50
51 #define FIRMWARE_IDLE_HYSTERESIS_TIME_USEC (10000) /* Default 10 milliseconds */
52
53 /* Idle hysteresis time can be scaled down when GPU sleep feature is used */
54 #define FIRMWARE_IDLE_HYSTERESIS_GPU_SLEEP_SCALER (5)
55
56 /**
57 * kbase_csf_ctx_init - Initialize the CSF interface for a GPU address space.
58 *
59 * @kctx: Pointer to the kbase context which is being initialized.
60 *
61 * Return: 0 if successful or a negative error code on failure.
62 */
63 int kbase_csf_ctx_init(struct kbase_context *kctx);
64
65 /**
66 * kbase_csf_ctx_handle_fault - Terminate queue groups & notify fault upon
67 * GPU bus fault, MMU page fault or similar.
68 *
69 * @kctx: Pointer to faulty kbase context.
70 * @fault: Pointer to the fault.
71 *
72 * This function terminates all GPU command queue groups in the context and
73 * notifies the event notification thread of the fault.
74 */
75 void kbase_csf_ctx_handle_fault(struct kbase_context *kctx,
76 struct kbase_fault *fault);
77
78 /**
79 * kbase_csf_ctx_term - Terminate the CSF interface for a GPU address space.
80 *
81 * @kctx: Pointer to the kbase context which is being terminated.
82 *
83 * This function terminates any remaining CSGs and CSs which weren't destroyed
84 * before context termination.
85 */
86 void kbase_csf_ctx_term(struct kbase_context *kctx);
87
88 /**
89 * kbase_csf_queue_register - Register a GPU command queue.
90 *
91 * @kctx: Pointer to the kbase context within which the
92 * queue is to be registered.
93 * @reg: Pointer to the structure which contains details of the
94 * queue to be registered within the provided
95 * context.
96 *
97 * Return: 0 on success, or negative on failure.
98 */
99 int kbase_csf_queue_register(struct kbase_context *kctx,
100 struct kbase_ioctl_cs_queue_register *reg);
101
102 /**
103 * kbase_csf_queue_register_ex - Register a GPU command queue with
104 * extended format.
105 *
106 * @kctx: Pointer to the kbase context within which the
107 * queue is to be registered.
108 * @reg: Pointer to the structure which contains details of the
109 * queue to be registered within the provided
110 * context, together with the extended parameter fields
111 * for supporting cs trace command.
112 *
113 * Return: 0 on success, or negative on failure.
114 */
115 int kbase_csf_queue_register_ex(struct kbase_context *kctx,
116 struct kbase_ioctl_cs_queue_register_ex *reg);
117
118 /**
119 * kbase_csf_queue_terminate - Terminate a GPU command queue.
120 *
121 * @kctx: Pointer to the kbase context within which the
122 * queue is to be terminated.
123 * @term: Pointer to the structure which identifies which
124 * queue is to be terminated.
125 */
126 void kbase_csf_queue_terminate(struct kbase_context *kctx,
127 struct kbase_ioctl_cs_queue_terminate *term);
128
129 /**
130 * kbase_csf_free_command_stream_user_pages() - Free the resources allocated
131 * for a queue at the time of bind.
132 *
133 * @kctx: Address of the kbase context within which the queue was created.
134 * @queue: Pointer to the queue to be unlinked.
135 *
136 * This function will free the pair of physical pages allocated for a GPU
137 * command queue, and also release the hardware doorbell page, that were mapped
138 * into the process address space to enable direct submission of commands to
139 * the hardware. Also releases the reference taken on the queue when the mapping
140 * was created.
141 *
142 * If an explicit or implicit unbind was missed by the userspace then the
143 * mapping will persist. On process exit kernel itself will remove the mapping.
144 */
145 void kbase_csf_free_command_stream_user_pages(struct kbase_context *kctx,
146 struct kbase_queue *queue);
147
148 /**
149 * kbase_csf_alloc_command_stream_user_pages - Allocate resources for a
150 * GPU command queue.
151 *
152 * @kctx: Pointer to the kbase context within which the resources
153 * for the queue are being allocated.
154 * @queue: Pointer to the queue for which to allocate resources.
155 *
156 * This function allocates a pair of User mode input/output pages for a
157 * GPU command queue and maps them in the shared interface segment of MCU
158 * firmware address space. Also reserves a hardware doorbell page for the queue.
159 *
160 * Return: 0 on success, or negative on failure.
161 */
162 int kbase_csf_alloc_command_stream_user_pages(struct kbase_context *kctx,
163 struct kbase_queue *queue);
164
165 /**
166 * kbase_csf_queue_bind - Bind a GPU command queue to a queue group.
167 *
168 * @kctx: The kbase context.
169 * @bind: Pointer to the union which specifies a queue group and a
170 * queue to be bound to that group.
171 *
172 * Return: 0 on success, or negative on failure.
173 */
174 int kbase_csf_queue_bind(struct kbase_context *kctx,
175 union kbase_ioctl_cs_queue_bind *bind);
176
177 /**
178 * kbase_csf_queue_unbind - Unbind a GPU command queue from a queue group
179 * to which it has been bound and free
180 * resources allocated for this queue if there
181 * are any.
182 *
183 * @queue: Pointer to queue to be unbound.
184 * @process_exit: Flag to indicate if process exit is happening.
185 */
186 void kbase_csf_queue_unbind(struct kbase_queue *queue, bool process_exit);
187
188 /**
189 * kbase_csf_queue_unbind_stopped - Unbind a GPU command queue in the case
190 * where it was never started.
191 * @queue: Pointer to queue to be unbound.
192 *
193 * Variant of kbase_csf_queue_unbind() for use on error paths for cleaning up
194 * queues that failed to fully bind.
195 */
196 void kbase_csf_queue_unbind_stopped(struct kbase_queue *queue);
197
198 /**
199 * kbase_csf_queue_kick - Schedule a GPU command queue on the firmware
200 *
201 * @kctx: The kbase context.
202 * @kick: Pointer to the struct which specifies the queue
203 * that needs to be scheduled.
204 *
205 * Return: 0 on success, or negative on failure.
206 */
207 int kbase_csf_queue_kick(struct kbase_context *kctx,
208 struct kbase_ioctl_cs_queue_kick *kick);
209
210 /**
211 * kbase_csf_queue_group_handle_is_valid - Find the queue group corresponding
212 * to the indicated handle.
213 *
214 * @kctx: The kbase context under which the queue group exists.
215 * @group_handle: Handle for the group which uniquely identifies it within
216 * the context with which it was created.
217 *
218 * This function is used to find the queue group when passed a handle.
219 *
220 * Return: Pointer to a queue group on success, NULL on failure
221 */
222 struct kbase_queue_group *kbase_csf_find_queue_group(struct kbase_context *kctx, u8 group_handle);
223
224 /**
225 * kbase_csf_queue_group_handle_is_valid - Find if the given queue group handle
226 * is valid.
227 *
228 * @kctx: The kbase context under which the queue group exists.
229 * @group_handle: Handle for the group which uniquely identifies it within
230 * the context with which it was created.
231 *
232 * This function is used to determine if the queue group handle is valid.
233 *
234 * Return: 0 on success, or negative on failure.
235 */
236 int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx,
237 u8 group_handle);
238
239 /**
240 * kbase_csf_queue_group_create - Create a GPU command queue group.
241 *
242 * @kctx: Pointer to the kbase context within which the
243 * queue group is to be created.
244 * @create: Pointer to the structure which contains details of the
245 * queue group which is to be created within the
246 * provided kbase context.
247 *
248 * Return: 0 on success, or negative on failure.
249 */
250 int kbase_csf_queue_group_create(struct kbase_context *kctx,
251 union kbase_ioctl_cs_queue_group_create *create);
252
253 /**
254 * kbase_csf_queue_group_terminate - Terminate a GPU command queue group.
255 *
256 * @kctx: Pointer to the kbase context within which the
257 * queue group is to be terminated.
258 * @group_handle: Pointer to the structure which identifies the queue
259 * group which is to be terminated.
260 */
261 void kbase_csf_queue_group_terminate(struct kbase_context *kctx,
262 u8 group_handle);
263
264 /**
265 * kbase_csf_term_descheduled_queue_group - Terminate a GPU command queue
266 * group that is not operational
267 * inside the scheduler.
268 *
269 * @group: Pointer to the structure which identifies the queue
270 * group to be terminated. The function assumes that the caller
271 * is sure that the given group is not operational inside the
272 * scheduler. If in doubt, use its alternative:
273 * @ref kbase_csf_queue_group_terminate().
274 */
275 void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group);
276
277 #if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
278 /**
279 * kbase_csf_queue_group_suspend - Suspend a GPU command queue group
280 *
281 * @kctx: The kbase context for which the queue group is to be
282 * suspended.
283 * @sus_buf: Pointer to the structure which contains details of the
284 * user buffer and its kernel pinned pages.
285 * @group_handle: Handle for the group which uniquely identifies it within
286 * the context within which it was created.
287 *
288 * This function is used to suspend a queue group and copy the suspend buffer.
289 *
290 * Return: 0 on success or negative value if failed to suspend
291 * queue group and copy suspend buffer contents.
292 */
293 int kbase_csf_queue_group_suspend(struct kbase_context *kctx,
294 struct kbase_suspend_copy_buffer *sus_buf, u8 group_handle);
295 #endif
296
297 /**
298 * kbase_csf_add_group_fatal_error - Report a fatal group error to userspace
299 *
300 * @group: GPU command queue group.
301 * @err_payload: Error payload to report.
302 */
303 void kbase_csf_add_group_fatal_error(
304 struct kbase_queue_group *const group,
305 struct base_gpu_queue_group_error const *const err_payload);
306
307 /**
308 * kbase_csf_interrupt - Handle interrupts issued by CSF firmware.
309 *
310 * @kbdev: The kbase device to handle an IRQ for
311 * @val: The value of JOB IRQ status register which triggered the interrupt
312 */
313 void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val);
314
315 /**
316 * kbase_csf_doorbell_mapping_init - Initialize the fields that facilitates
317 * the update of userspace mapping of HW
318 * doorbell page.
319 *
320 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
321 *
322 * The function creates a file and allocates a dummy page to facilitate the
323 * update of userspace mapping to point to the dummy page instead of the real
324 * HW doorbell page after the suspend of queue group.
325 *
326 * Return: 0 on success, or negative on failure.
327 */
328 int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev);
329
330 /**
331 * kbase_csf_doorbell_mapping_term - Free the dummy page & close the file used
332 * to update the userspace mapping of HW doorbell page
333 *
334 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
335 */
336 void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev);
337
338 /**
339 * kbase_csf_setup_dummy_user_reg_page - Setup the dummy page that is accessed
340 * instead of the User register page after
341 * the GPU power down.
342 *
343 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
344 *
345 * The function allocates a dummy page which is used to replace the User
346 * register page in the userspace mapping after the power down of GPU.
347 * On the power up of GPU, the mapping is updated to point to the real
348 * User register page. The mapping is used to allow access to LATEST_FLUSH
349 * register from userspace.
350 *
351 * Return: 0 on success, or negative on failure.
352 */
353 int kbase_csf_setup_dummy_user_reg_page(struct kbase_device *kbdev);
354
355 /**
356 * kbase_csf_free_dummy_user_reg_page - Free the dummy page that was used
357 * to replace the User register page
358 *
359 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
360 */
361 void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev);
362
363 /**
364 * kbase_csf_ring_csg_doorbell - ring the doorbell for a CSG interface.
365 *
366 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
367 * @slot: Index of CSG interface for ringing the door-bell.
368 *
369 * The function kicks a notification on the CSG interface to firmware.
370 */
371 void kbase_csf_ring_csg_doorbell(struct kbase_device *kbdev, int slot);
372
373 /**
374 * kbase_csf_ring_csg_slots_doorbell - ring the doorbell for a set of CSG
375 * interfaces.
376 *
377 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
378 * @slot_bitmap: bitmap for the given slots, slot-0 on bit-0, etc.
379 *
380 * The function kicks a notification on a set of CSG interfaces to firmware.
381 */
382 void kbase_csf_ring_csg_slots_doorbell(struct kbase_device *kbdev,
383 u32 slot_bitmap);
384
385 /**
386 * kbase_csf_ring_cs_kernel_doorbell - ring the kernel doorbell for a CSI
387 * assigned to a GPU queue
388 *
389 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
390 * @csi_index: ID of the CSI assigned to the GPU queue.
391 * @csg_nr: Index of the CSG slot assigned to the queue
392 * group to which the GPU queue is bound.
393 * @ring_csg_doorbell: Flag to indicate if the CSG doorbell needs to be rung
394 * after updating the CSG_DB_REQ. So if this flag is false
395 * the doorbell interrupt will not be sent to FW.
396 * The flag is supposed be false only when the input page
397 * for bound GPU queues is programmed at the time of
398 * starting/resuming the group on a CSG slot.
399 *
400 * The function sends a doorbell interrupt notification to the firmware for
401 * a CSI assigned to a GPU queue.
402 */
403 void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev,
404 int csi_index, int csg_nr,
405 bool ring_csg_doorbell);
406
407 /**
408 * kbase_csf_ring_cs_user_doorbell - ring the user doorbell allocated for a
409 * queue.
410 *
411 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
412 * @queue: Pointer to the queue for ringing the door-bell.
413 *
414 * The function kicks a notification to the firmware on the doorbell assigned
415 * to the queue.
416 */
417 void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev,
418 struct kbase_queue *queue);
419
420 /**
421 * kbase_csf_active_queue_groups_reset - Reset the state of all active GPU
422 * command queue groups associated with the context.
423 *
424 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
425 * @kctx: The kbase context.
426 *
427 * This function will iterate through all the active/scheduled GPU command
428 * queue groups associated with the context, deschedule and mark them as
429 * terminated (which will then lead to unbinding of all the queues bound to
430 * them) and also no more work would be allowed to execute for them.
431 *
432 * This is similar to the action taken in response to an unexpected OoM event.
433 */
434 void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev,
435 struct kbase_context *kctx);
436
437 /**
438 * kbase_csf_priority_check - Check the priority requested
439 *
440 * @kbdev: Device pointer
441 * @req_priority: Requested priority
442 *
443 * This will determine whether the requested priority can be satisfied.
444 *
445 * Return: The same or lower priority than requested.
446 */
447 u8 kbase_csf_priority_check(struct kbase_device *kbdev, u8 req_priority);
448
449 extern const u8 kbasep_csf_queue_group_priority_to_relative[BASE_QUEUE_GROUP_PRIORITY_COUNT];
450 extern const u8 kbasep_csf_relative_to_queue_group_priority[KBASE_QUEUE_GROUP_PRIORITY_COUNT];
451
452 /**
453 * kbase_csf_priority_relative_to_queue_group_priority - Convert relative to base priority
454 *
455 * @priority: kbase relative priority
456 *
457 * This will convert the monotonically increasing realtive priority to the
458 * fixed base priority list.
459 *
460 * Return: base_queue_group_priority priority.
461 */
kbase_csf_priority_relative_to_queue_group_priority(u8 priority)462 static inline u8 kbase_csf_priority_relative_to_queue_group_priority(u8 priority)
463 {
464 if (priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT)
465 priority = KBASE_QUEUE_GROUP_PRIORITY_LOW;
466 return kbasep_csf_relative_to_queue_group_priority[priority];
467 }
468
469 /**
470 * kbase_csf_priority_queue_group_priority_to_relative - Convert base priority to relative
471 *
472 * @priority: base_queue_group_priority priority
473 *
474 * This will convert the fixed base priority list to monotonically increasing realtive priority.
475 *
476 * Return: kbase relative priority.
477 */
kbase_csf_priority_queue_group_priority_to_relative(u8 priority)478 static inline u8 kbase_csf_priority_queue_group_priority_to_relative(u8 priority)
479 {
480 /* Apply low priority in case of invalid priority */
481 if (priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)
482 priority = BASE_QUEUE_GROUP_PRIORITY_LOW;
483 return kbasep_csf_queue_group_priority_to_relative[priority];
484 }
485
486 /**
487 * kbase_csf_ktrace_gpu_cycle_cnt - Wrapper to retreive the GPU cycle counter
488 * value for Ktrace purpose.
489 *
490 * @kbdev: Instance of a GPU platform device that implements a CSF interface.
491 *
492 * This function is just a wrapper to retreive the GPU cycle counter value, to
493 * avoid any overhead on Release builds where Ktrace is disabled by default.
494 *
495 * Return: Snapshot of the GPU cycle count register.
496 */
kbase_csf_ktrace_gpu_cycle_cnt(struct kbase_device * kbdev)497 static inline u64 kbase_csf_ktrace_gpu_cycle_cnt(struct kbase_device *kbdev)
498 {
499 #if KBASE_KTRACE_ENABLE
500 return kbase_backend_get_cycle_cnt(kbdev);
501 #else
502 return 0;
503 #endif
504 }
505
506 #endif /* _KBASE_CSF_H_ */
507