1 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ 2 /* 3 * 4 * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. 5 * 6 * This program is free software and is provided to you under the terms of the 7 * GNU General Public License version 2 as published by the Free Software 8 * Foundation, and any use by you of this program is subject to the terms 9 * of such GNU license. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, you can access it online at 18 * http://www.gnu.org/licenses/gpl-2.0.html. 19 * 20 */ 21 22 #ifndef _KBASE_CSF_KCPU_H_ 23 #define _KBASE_CSF_KCPU_H_ 24 25 #include <mali_kbase_fence.h> 26 #include <mali_kbase_sync.h> 27 28 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) 29 #include <linux/fence.h> 30 #else 31 #include <linux/dma-fence.h> 32 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ 33 34 /* The maximum number of KCPU commands in flight, enqueueing more commands 35 * than this value shall block. 36 */ 37 #define KBASEP_KCPU_QUEUE_SIZE ((size_t)256) 38 39 /** 40 * struct kbase_kcpu_command_import_info - Structure which holds information 41 * about the buffer to be imported 42 * 43 * @gpu_va: Address of the buffer to be imported. 44 */ 45 struct kbase_kcpu_command_import_info { 46 u64 gpu_va; 47 }; 48 49 /** 50 * struct kbase_kcpu_command_fence_info - Structure which holds information about the 51 * fence object enqueued in the kcpu command queue 52 * 53 * @fence_cb: Fence callback 54 * @fence: Fence 55 * @kcpu_queue: kcpu command queue 56 */ 57 struct kbase_kcpu_command_fence_info { 58 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) 59 struct fence_cb fence_cb; 60 struct fence *fence; 61 #else 62 struct dma_fence_cb fence_cb; 63 struct dma_fence *fence; 64 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ 65 struct kbase_kcpu_command_queue *kcpu_queue; 66 }; 67 68 /** 69 * struct kbase_kcpu_command_cqs_set_info - Structure which holds information 70 * about CQS objects for the kcpu CQS set command 71 * 72 * @objs: Array of structures which define CQS objects to be used by 73 * the kcpu command. 74 * @nr_objs: Number of CQS objects in the array. 75 */ 76 struct kbase_kcpu_command_cqs_set_info { 77 struct base_cqs_set *objs; 78 unsigned int nr_objs; 79 }; 80 81 /** 82 * struct kbase_kcpu_command_cqs_wait_info - Structure which holds information 83 * about CQS objects for the kcpu CQS wait command 84 * 85 * @objs: Array of structures which define CQS objects to be used by 86 * the kcpu command. 87 * @signaled: Bit array used to report the status of the CQS wait objects. 88 * 1 is signaled, 0 otherwise. 89 * @nr_objs: Number of CQS objects in the array. 90 * @inherit_err_flags: Bit-pattern for the CQSs in the array who's error field 91 * to be served as the source for importing into the 92 * queue's error-state. 93 */ 94 struct kbase_kcpu_command_cqs_wait_info { 95 struct base_cqs_wait_info *objs; 96 unsigned long *signaled; 97 unsigned int nr_objs; 98 u32 inherit_err_flags; 99 }; 100 101 /** 102 * struct kbase_kcpu_command_cqs_set_operation_info - Structure which holds information 103 * about CQS objects for the kcpu CQS timeline set command 104 * 105 * @objs: Array of structures which define CQS timeline objects to be used by 106 * the kcpu command. 107 * @nr_objs: Number of CQS objects in the array. 108 */ 109 struct kbase_kcpu_command_cqs_set_operation_info { 110 struct base_cqs_set_operation_info *objs; 111 unsigned int nr_objs; 112 }; 113 114 /** 115 * struct kbase_kcpu_command_cqs_wait_operation_info - Structure which holds information 116 * about CQS objects for the kcpu CQS timeline wait command 117 * 118 * @objs: Array of structures which define CQS timeline objects to be used by 119 * the kcpu command. 120 * @signaled: Bit array used to report the status of the CQS wait objects. 121 * 1 is signaled, 0 otherwise. 122 * @nr_objs: Number of CQS objects in the array. 123 * @inherit_err_flags: Bit-pattern for CQSs in the array who's error field is to 124 * be used as the source to import into the queue's error-state 125 */ 126 struct kbase_kcpu_command_cqs_wait_operation_info { 127 struct base_cqs_wait_operation_info *objs; 128 unsigned long *signaled; 129 unsigned int nr_objs; 130 u32 inherit_err_flags; 131 }; 132 133 /** 134 * struct kbase_kcpu_command_jit_alloc_info - Structure which holds information 135 * needed for the kcpu command for jit allocations 136 * 137 * @node: Used to keep track of all JIT free/alloc commands in submission 138 * order. This must be located in the front of this struct to 139 * match that of kbase_kcpu_command_jit_free_info. 140 * @info: Array of objects of the struct base_jit_alloc_info type which 141 * specify jit allocations to be made by the kcpu command. 142 * @count: Number of jit alloc objects in the array. 143 * @blocked: Whether this allocation has been put into the pending list to 144 * be retried later. 145 */ 146 struct kbase_kcpu_command_jit_alloc_info { 147 struct list_head node; 148 struct base_jit_alloc_info *info; 149 u8 count; 150 bool blocked; 151 }; 152 153 /** 154 * struct kbase_kcpu_command_jit_free_info - Structure which holds information 155 * needed for the kcpu jit free command 156 * 157 * @node: Used to keep track of all JIT free/alloc commands in submission 158 * order. This must be located in the front of this struct to 159 * match that of kbase_kcpu_command_jit_alloc_info. 160 * @ids: Array of identifiers of jit allocations which are to be freed 161 * by the kcpu command. 162 * @count: Number of elements in the array. 163 */ 164 struct kbase_kcpu_command_jit_free_info { 165 struct list_head node; 166 u8 *ids; 167 u8 count; 168 }; 169 170 /** 171 * struct kbase_suspend_copy_buffer - information about the suspend buffer 172 * to be copied. 173 * 174 * @size: size of the suspend buffer in bytes. 175 * @pages: pointer to an array of pointers to the pages which contain 176 * the user buffer. 177 * @nr_pages: number of pages. 178 * @offset: offset into the pages 179 * @cpu_alloc: Reference to physical pages of suspend buffer allocation. 180 */ 181 struct kbase_suspend_copy_buffer { 182 size_t size; 183 struct page **pages; 184 int nr_pages; 185 size_t offset; 186 struct kbase_mem_phy_alloc *cpu_alloc; 187 }; 188 189 #if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST 190 /** 191 * struct kbase_kcpu_command_group_suspend_info - structure which contains 192 * suspend buffer data captured for a suspended queue group. 193 * 194 * @sus_buf: Pointer to the structure which contains details of the 195 * user buffer and its kernel pinned pages. 196 * @group_handle: Handle to the mapping of CSG. 197 */ 198 struct kbase_kcpu_command_group_suspend_info { 199 struct kbase_suspend_copy_buffer *sus_buf; 200 u8 group_handle; 201 }; 202 #endif 203 204 205 /** 206 * struct kbase_kcpu_command - Command which is to be part of the kernel 207 * command queue 208 * 209 * @type: Type of the command. 210 * @enqueue_ts: Denotes the relative time of enqueueing, a smaller value 211 * indicates that it has been enqueued earlier. 212 * @info: Structure which holds information about the command 213 * dependent on the command type. 214 * @info.fence: Fence 215 * @info.cqs_wait: CQS wait 216 * @info.cqs_set: CQS set 217 * @info.cqs_wait_operation: CQS wait operation 218 * @info.cqs_set_operation: CQS set operation 219 * @info.import: import 220 * @info.jit_alloc: JIT allocation 221 * @info.jit_free: JIT deallocation 222 * @info.suspend_buf_copy: suspend buffer copy 223 * @info.sample_time: sample time 224 */ 225 struct kbase_kcpu_command { 226 enum base_kcpu_command_type type; 227 u64 enqueue_ts; 228 union { 229 struct kbase_kcpu_command_fence_info fence; 230 struct kbase_kcpu_command_cqs_wait_info cqs_wait; 231 struct kbase_kcpu_command_cqs_set_info cqs_set; 232 struct kbase_kcpu_command_cqs_wait_operation_info cqs_wait_operation; 233 struct kbase_kcpu_command_cqs_set_operation_info cqs_set_operation; 234 struct kbase_kcpu_command_import_info import; 235 struct kbase_kcpu_command_jit_alloc_info jit_alloc; 236 struct kbase_kcpu_command_jit_free_info jit_free; 237 #if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST 238 struct kbase_kcpu_command_group_suspend_info suspend_buf_copy; 239 #endif 240 } info; 241 }; 242 243 /** 244 * struct kbase_kcpu_command_queue - a command queue executed by the kernel 245 * 246 * @lock: Lock to protect accesses to this queue. 247 * @kctx: The context to which this command queue belongs. 248 * @commands: Array of commands which have been successfully 249 * enqueued to this command queue. 250 * @wq: Dedicated workqueue for processing commands. 251 * @work: struct work_struct which contains a pointer to 252 * the function which handles processing of kcpu 253 * commands enqueued into a kcpu command queue; 254 * part of kernel API for processing workqueues 255 * @start_offset: Index of the command to be executed next 256 * @id: KCPU command queue ID. 257 * @num_pending_cmds: The number of commands enqueued but not yet 258 * executed or pending 259 * @cqs_wait_count: Tracks the number of CQS wait commands enqueued 260 * @fence_context: The dma-buf fence context number for this kcpu 261 * queue. A unique context number is allocated for 262 * each kcpu queue. 263 * @fence_seqno: The dma-buf fence sequence number for the fence 264 * that is returned on the enqueue of fence signal 265 * command. This is increased every time the 266 * fence signal command is queued. 267 * @fence_wait_processed: Used to avoid reprocessing of the fence wait 268 * command which has blocked the processing of 269 * commands that follow it. 270 * @enqueue_failed: Indicates that no space has become available in 271 * the buffer since an enqueue operation failed 272 * because of insufficient free space. 273 * @command_started: Indicates that the command at the front of the 274 * queue has been started in a previous queue 275 * process, but was not completed due to some 276 * unmet dependencies. Ensures that instrumentation 277 * of the execution start of these commands is only 278 * fired exactly once. 279 * @has_error: Indicates that the kcpu queue is in error mode 280 * or without errors since last cleaned. 281 * @jit_blocked: Used to keep track of command queues blocked 282 * by a pending JIT allocation command. 283 * @fence_timeout: Timer used to detect the fence wait timeout. 284 * @metadata: Metadata structure containing basic information about 285 * this queue for any fence objects associated with this queue. 286 */ 287 struct kbase_kcpu_command_queue { 288 struct mutex lock; 289 struct kbase_context *kctx; 290 struct kbase_kcpu_command commands[KBASEP_KCPU_QUEUE_SIZE]; 291 struct workqueue_struct *wq; 292 struct work_struct work; 293 u8 start_offset; 294 u8 id; 295 u16 num_pending_cmds; 296 u32 cqs_wait_count; 297 u64 fence_context; 298 unsigned int fence_seqno; 299 bool fence_wait_processed; 300 bool enqueue_failed; 301 bool command_started; 302 struct list_head jit_blocked; 303 bool has_error; 304 #ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG 305 struct timer_list fence_timeout; 306 #endif /* CONFIG_MALI_BIFROST_FENCE_DEBUG */ 307 #if IS_ENABLED(CONFIG_SYNC_FILE) 308 struct kbase_kcpu_dma_fence_meta *metadata; 309 #endif /* CONFIG_SYNC_FILE */ 310 }; 311 312 /** 313 * kbase_csf_kcpu_queue_new - Create new KCPU command queue. 314 * 315 * @kctx: Pointer to the kbase context within which the KCPU command 316 * queue will be created. 317 * @newq: Pointer to the structure which contains information about 318 * the new KCPU command queue to be created. 319 * 320 * Return: 0 if successful or a negative error code on failure. 321 */ 322 int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, 323 struct kbase_ioctl_kcpu_queue_new *newq); 324 325 /** 326 * kbase_csf_kcpu_queue_delete - Delete KCPU command queue. 327 * 328 * @kctx: Pointer to the kbase context from which the KCPU command 329 * queue is to be deleted. 330 * @del: Pointer to the structure which specifies the KCPU command 331 * queue to be deleted. 332 * 333 * Return: 0 if successful or a negative error code on failure. 334 */ 335 int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, 336 struct kbase_ioctl_kcpu_queue_delete *del); 337 338 /** 339 * kbase_csf_kcpu_queue_enqueue - Enqueue a KCPU command into a KCPU command 340 * queue. 341 * 342 * @kctx: Pointer to the kbase context within which the KCPU command 343 * is to be enqueued into the KCPU command queue. 344 * @enq: Pointer to the structure which specifies the KCPU command 345 * as well as the KCPU command queue into which the command 346 * is to be enqueued. 347 * 348 * Return: 0 if successful or a negative error code on failure. 349 */ 350 int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, 351 struct kbase_ioctl_kcpu_queue_enqueue *enq); 352 353 /** 354 * kbase_csf_kcpu_queue_context_init - Initialize the kernel CPU queues context 355 * for a GPU address space 356 * 357 * @kctx: Pointer to the kbase context being initialized. 358 * 359 * Return: 0 if successful or a negative error code on failure. 360 */ 361 int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx); 362 363 /** 364 * kbase_csf_kcpu_queue_context_term - Terminate the kernel CPU queues context 365 * for a GPU address space 366 * @kctx: Pointer to the kbase context being terminated. 367 * 368 * This function deletes any kernel CPU queues that weren't deleted before 369 * context termination. 370 * 371 */ 372 void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx); 373 374 #if IS_ENABLED(CONFIG_SYNC_FILE) 375 /* Test wrappers for dma fence operations. */ 376 int kbase_kcpu_fence_signal_process(struct kbase_kcpu_command_queue *kcpu_queue, 377 struct kbase_kcpu_command_fence_info *fence_info); 378 379 int kbase_kcpu_fence_signal_init(struct kbase_kcpu_command_queue *kcpu_queue, 380 struct kbase_kcpu_command *current_command, 381 struct base_fence *fence, struct sync_file **sync_file, int *fd); 382 #endif /* CONFIG_SYNC_FILE */ 383 384 #endif /* _KBASE_CSF_KCPU_H_ */ 385