1 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2 /*
3 *
4 * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /*
23 * Definitions (types, defines, etcs) specific to Job Manager Kbase.
24 * They are placed here to allow the hierarchy of header files to work.
25 */
26
27 #ifndef _KBASE_JM_DEFS_H_
28 #define _KBASE_JM_DEFS_H_
29
30 #include "mali_kbase_js_defs.h"
31
32 /* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */
33 #define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1
34
35 /*
36 * Number of milliseconds before resetting the GPU when a job cannot be "zapped"
37 * from the hardware. Note that the time is actually
38 * ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and
39 * the GPU actually being reset to give other contexts time for their jobs
40 * to be soft-stopped and removed from the hardware before resetting.
41 */
42 #define ZAP_TIMEOUT 1000
43
44 /*
45 * Prevent soft-stops from occurring in scheduling situations
46 *
47 * This is not due to HW issues, but when scheduling is desired to be more
48 * predictable.
49 *
50 * Therefore, soft stop may still be disabled due to HW issues.
51 *
52 * Soft stop will still be used for non-scheduling purposes e.g. when
53 * terminating a context.
54 *
55 * if not in use, define this value to 0 instead of being undefined.
56 */
57 #define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0
58
59 /*
60 * Prevent hard-stops from occurring in scheduling situations
61 *
62 * This is not due to HW issues, but when scheduling is desired to be more
63 * predictable.
64 *
65 * Hard stop will still be used for non-scheduling purposes e.g. when
66 * terminating a context.
67 *
68 * if not in use, define this value to 0 instead of being undefined.
69 */
70 #define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0
71
72 /* Atom has been previously soft-stopped */
73 #define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1<<1)
74 /* Atom has been previously retried to execute */
75 #define KBASE_KATOM_FLAGS_RERUN (1<<2)
76 /* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps
77 * to disambiguate short-running job chains during soft/hard stopping of jobs
78 */
79 #define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3)
80 /* Atom has been previously hard-stopped. */
81 #define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4)
82 /* Atom has caused us to enter disjoint state */
83 #define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5)
84 /* Atom blocked on cross-slot dependency */
85 #define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7)
86 /* Atom has fail dependency on cross-slot dependency */
87 #define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8)
88 /* Atom is currently in the list of atoms blocked on cross-slot dependencies */
89 #define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9)
90 /* Atom requires GPU to be in protected mode */
91 #define KBASE_KATOM_FLAG_PROTECTED (1<<11)
92 /* Atom has been stored in runnable_tree */
93 #define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12)
94 /* Atom is waiting for L2 caches to power up in order to enter protected mode */
95 #define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1<<13)
96
97 /* SW related flags about types of JS_COMMAND action
98 * NOTE: These must be masked off by JS_COMMAND_MASK
99 */
100
101 /* This command causes a disjoint event */
102 #define JS_COMMAND_SW_CAUSES_DISJOINT 0x100
103
104 /* Bitmask of all SW related flags */
105 #define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT)
106
107 #if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK)
108 #error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \
109 "Must update JS_COMMAND_SW_<..> bitmasks"
110 #endif
111
112 /* Soft-stop command that causes a Disjoint event. This of course isn't
113 * entirely masked off by JS_COMMAND_MASK
114 */
115 #define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \
116 (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP)
117
118 #define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT
119
120 /* Serialize atoms within a slot (ie only one atom per job slot) */
121 #define KBASE_SERIALIZE_INTRA_SLOT (1 << 0)
122 /* Serialize atoms between slots (ie only one job slot running at any time) */
123 #define KBASE_SERIALIZE_INTER_SLOT (1 << 1)
124 /* Reset the GPU after each atom completion */
125 #define KBASE_SERIALIZE_RESET (1 << 2)
126
127 /**
128 * enum kbase_timeout_selector - The choice of which timeout to get scaled
129 * using the lowest GPU frequency.
130 * @MMU_AS_INACTIVE_WAIT_TIMEOUT: Maximum waiting time in ms for the completion
131 * of a MMU operation
132 * @JM_DEFAULT_JS_FREE_TIMEOUT: Maximum timeout to wait for JS_COMMAND_NEXT
133 * to be updated on HW side so a Job Slot is
134 * considered free.
135 * @KBASE_TIMEOUT_SELECTOR_COUNT: Number of timeout selectors. Must be last in
136 * the enum.
137 */
138 enum kbase_timeout_selector {
139 MMU_AS_INACTIVE_WAIT_TIMEOUT,
140 JM_DEFAULT_JS_FREE_TIMEOUT,
141
142 /* Must be the last in the enum */
143 KBASE_TIMEOUT_SELECTOR_COUNT
144 };
145
146 #if IS_ENABLED(CONFIG_DEBUG_FS)
147 /**
148 * struct base_job_fault_event - keeps track of the atom which faulted or which
149 * completed after the faulty atom but before the
150 * debug data for faulty atom was dumped.
151 *
152 * @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for
153 * the atom which faulted.
154 * @katom: pointer to the atom for which job fault occurred or which
155 * completed after the faulty atom.
156 * @job_fault_work: work item, queued only for the faulty atom, which waits for
157 * the dumping to get completed and then does the bottom half
158 * of job done for the atoms which followed the faulty atom.
159 * @head: List head used to store the atom in the global list of
160 * faulty atoms or context specific list of atoms which got
161 * completed during the dump.
162 * @reg_offset: offset of the register to be dumped next, only applicable
163 * for the faulty atom.
164 */
165 struct base_job_fault_event {
166
167 u32 event_code;
168 struct kbase_jd_atom *katom;
169 struct work_struct job_fault_work;
170 struct list_head head;
171 int reg_offset;
172 };
173 #endif
174
175 /**
176 * struct kbase_jd_atom_dependency - Contains the dependency info for an atom.
177 * @atom: pointer to the dependee atom.
178 * @dep_type: type of dependency on the dependee @atom, i.e. order or data
179 * dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency.
180 */
181 struct kbase_jd_atom_dependency {
182 struct kbase_jd_atom *atom;
183 u8 dep_type;
184 };
185
186 /**
187 * kbase_jd_katom_dep_atom - Retrieves a read-only reference to the
188 * dependee atom.
189 * @dep: pointer to the dependency info structure.
190 *
191 * Return: readonly reference to dependee atom.
192 */
193 static inline const struct kbase_jd_atom *
kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency * dep)194 kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep)
195 {
196 return (const struct kbase_jd_atom *)(dep->atom);
197 }
198
199 /**
200 * kbase_jd_katom_dep_type - Retrieves the dependency type info
201 *
202 * @dep: pointer to the dependency info structure.
203 *
204 * Return: the type of dependency there is on the dependee atom.
205 */
kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency * dep)206 static inline u8 kbase_jd_katom_dep_type(
207 const struct kbase_jd_atom_dependency *dep)
208 {
209 return dep->dep_type;
210 }
211
212 /**
213 * kbase_jd_katom_dep_set - sets up the dependency info structure
214 * as per the values passed.
215 * @const_dep: pointer to the dependency info structure to be setup.
216 * @a: pointer to the dependee atom.
217 * @type: type of dependency there is on the dependee atom.
218 */
kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency * const_dep,struct kbase_jd_atom * a,u8 type)219 static inline void kbase_jd_katom_dep_set(
220 const struct kbase_jd_atom_dependency *const_dep,
221 struct kbase_jd_atom *a, u8 type)
222 {
223 struct kbase_jd_atom_dependency *dep;
224
225 dep = (struct kbase_jd_atom_dependency *)const_dep;
226
227 dep->atom = a;
228 dep->dep_type = type;
229 }
230
231 /**
232 * kbase_jd_katom_dep_clear - resets the dependency info structure
233 *
234 * @const_dep: pointer to the dependency info structure to be setup.
235 */
kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency * const_dep)236 static inline void kbase_jd_katom_dep_clear(
237 const struct kbase_jd_atom_dependency *const_dep)
238 {
239 struct kbase_jd_atom_dependency *dep;
240
241 dep = (struct kbase_jd_atom_dependency *)const_dep;
242
243 dep->atom = NULL;
244 dep->dep_type = BASE_JD_DEP_TYPE_INVALID;
245 }
246
247 /**
248 * enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it
249 * becomes runnable, with respect to job slot
250 * ringbuffer/fifo.
251 * @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo,
252 * which implies that either atom has not become
253 * runnable due to dependency or has completed
254 * the execution on GPU.
255 * @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is
256 * blocked due to cross slot dependency,
257 * can't be submitted to GPU.
258 * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot
259 * fifo but is waiting for the completion of
260 * previously added atoms in current & other
261 * slots, as their protected mode requirements
262 * do not match with the current atom.
263 * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo
264 * and is waiting for completion of protected
265 * mode transition, needed before the atom is
266 * submitted to GPU.
267 * @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is
268 * waiting for the cores, which are needed to
269 * execute the job chain represented by the atom,
270 * to become available
271 * @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to
272 * GPU.
273 * @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted
274 * to GPU.
275 * @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some
276 * failure, but only after the previously added
277 * atoms in fifo have completed or have also
278 * been returned to JS.
279 */
280 enum kbase_atom_gpu_rb_state {
281 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB,
282 KBASE_ATOM_GPU_RB_WAITING_BLOCKED,
283 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV,
284 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION,
285 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE,
286 KBASE_ATOM_GPU_RB_READY,
287 KBASE_ATOM_GPU_RB_SUBMITTED,
288 KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1
289 };
290
291 /**
292 * enum kbase_atom_enter_protected_state - The state of an atom with respect to
293 * the preparation for GPU's entry into protected mode,
294 * becomes pertinent only after atom's state with respect
295 * to slot ringbuffer is
296 * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
297 * @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any
298 * atoms currently submitted to GPU and protected mode
299 * transition is not already in progress.
300 * @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to
301 * become disabled before entry into protected mode.
302 * @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in
303 * preparation for the coherency change. L2 shall be
304 * powered down and GPU shall come out of fully
305 * coherent mode before entering protected mode.
306 * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change;
307 * for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on
308 * so that coherency register contains correct value when
309 * GPU enters protected mode.
310 * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for
311 * BASE_HW_ISSUE_TGOX_R1_1234 check
312 * that L2 is powered up and switch GPU to protected mode.
313 */
314 enum kbase_atom_enter_protected_state {
315 /*
316 * NOTE: The integer value of this must match
317 * KBASE_ATOM_EXIT_PROTECTED_CHECK.
318 */
319 KBASE_ATOM_ENTER_PROTECTED_CHECK = 0,
320 KBASE_ATOM_ENTER_PROTECTED_HWCNT,
321 KBASE_ATOM_ENTER_PROTECTED_IDLE_L2,
322 KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY,
323 KBASE_ATOM_ENTER_PROTECTED_FINISHED,
324 };
325
326 /**
327 * enum kbase_atom_exit_protected_state - The state of an atom with respect to
328 * the preparation for GPU's exit from protected mode,
329 * becomes pertinent only after atom's state with respect
330 * to slot ngbuffer is
331 * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
332 * @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any
333 * atoms currently submitted to GPU and protected mode
334 * transition is not already in progress.
335 * @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in
336 * preparation for the reset, as exiting protected mode
337 * requires a reset.
338 * @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from
339 * protected mode
340 * @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to
341 * complete
342 */
343 enum kbase_atom_exit_protected_state {
344 /*
345 * NOTE: The integer value of this must match
346 * KBASE_ATOM_ENTER_PROTECTED_CHECK.
347 */
348 KBASE_ATOM_EXIT_PROTECTED_CHECK = 0,
349 KBASE_ATOM_EXIT_PROTECTED_IDLE_L2,
350 KBASE_ATOM_EXIT_PROTECTED_RESET,
351 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT,
352 };
353
354 /**
355 * struct kbase_jd_atom - object representing the atom, containing the complete
356 * state and attributes of an atom.
357 * @work: work item for the bottom half processing of the atom,
358 * by JD or JS, after it got executed on GPU or the
359 * input fence got signaled
360 * @start_timestamp: time at which the atom was submitted to the GPU, by
361 * updating the JS_HEAD_NEXTn register.
362 * @udata: copy of the user data sent for the atom in
363 * base_jd_submit.
364 * @kctx: Pointer to the base context with which the atom is
365 * associated.
366 * @dep_head: Array of 2 list heads, pointing to the two list of
367 * atoms
368 * which are blocked due to dependency on this atom.
369 * @dep_item: Array of 2 list heads, used to store the atom in the
370 * list of other atoms depending on the same dependee
371 * atom.
372 * @dep: Array containing the dependency info for the 2 atoms
373 * on which the atom depends upon.
374 * @jd_item: List head used during job dispatch job_done
375 * processing - as dependencies may not be entirely
376 * resolved at this point,
377 * we need to use a separate list head.
378 * @in_jd_list: flag set to true if atom's @jd_item is currently on
379 * a list, prevents atom being processed twice.
380 * @jit_ids: Zero-terminated array of IDs of just-in-time memory
381 * allocations written to by the atom. When the atom
382 * completes, the value stored at the
383 * &struct_base_jit_alloc_info.heap_info_gpu_addr of
384 * each allocation is read in order to enforce an
385 * overall physical memory usage limit.
386 * @nr_extres: number of external resources referenced by the atom.
387 * @extres: Pointer to @nr_extres VA regions containing the external
388 * resource allocation and other information.
389 * @nr_extres external resources referenced by the atom.
390 * @device_nr: indicates the coregroup with which the atom is
391 * associated, when
392 * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified.
393 * @jc: GPU address of the job-chain.
394 * @softjob_data: Copy of data read from the user space buffer that @jc
395 * points to.
396 * @fence: Stores either an input or output sync fence,
397 * depending on soft-job type
398 * @sync_waiter: Pointer to the sync fence waiter structure passed to
399 * the callback function on signaling of the input
400 * fence.
401 * @dma_fence: object containing pointers to both input & output
402 * fences and other related members used for explicit
403 * sync through soft jobs and for the implicit
404 * synchronization required on access to external
405 * resources.
406 * @dma_fence.fence_in: Points to the dma-buf input fence for this atom.
407 * The atom would complete only after the fence is
408 * signaled.
409 * @dma_fence.fence: Points to the dma-buf output fence for this atom.
410 * @dma_fence.fence_cb: The object that is passed at the time of adding the
411 * callback that gets invoked when @dma_fence.fence_in
412 * is signaled.
413 * @dma_fence.fence_cb_added: Flag to keep a track if the callback was successfully
414 * added for @dma_fence.fence_in, which is supposed to be
415 * invoked on the signaling of fence.
416 * @dma_fence.context: The dma-buf fence context number for this atom. A
417 * unique context number is allocated to each katom in
418 * the context on context creation.
419 * @dma_fence.seqno: The dma-buf fence sequence number for this atom. This
420 * is increased every time this katom uses dma-buf fence
421 * @event_code: Event code for the job chain represented by the atom,
422 * both HW and low-level SW events are represented by
423 * event codes.
424 * @core_req: bitmask of BASE_JD_REQ_* flags specifying either
425 * Hw or Sw requirements for the job chain represented
426 * by the atom.
427 * @ticks: Number of scheduling ticks for which atom has been
428 * running on the GPU.
429 * @sched_priority: Priority of the atom for Job scheduling, as per the
430 * KBASE_JS_ATOM_SCHED_PRIO_*.
431 * @completed: Wait queue to wait upon for the completion of atom.
432 * @status: Indicates at high level at what stage the atom is in,
433 * as per KBASE_JD_ATOM_STATE_*, that whether it is not
434 * in use or its queued in JD or given to JS or
435 * submitted to Hw or it completed the execution on Hw.
436 * @work_id: used for GPU tracepoints, its a snapshot of the
437 * 'work_id' counter in kbase_jd_context which is
438 * incremented on every call to base_jd_submit.
439 * @slot_nr: Job slot chosen for the atom.
440 * @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the
441 * excat low level state of the atom.
442 * @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely
443 * tracking atom's state after it has entered
444 * Job scheduler on becoming runnable. Atom
445 * could be blocked due to cross slot dependency
446 * or waiting for the shader cores to become available
447 * or waiting for protected mode transitions to
448 * complete.
449 * @need_cache_flush_cores_retained: flag indicating that manual flush of GPU
450 * cache is needed for the atom and the shader cores
451 * used for atom have been kept on.
452 * @blocked: flag indicating that atom's resubmission to GPU is
453 * blocked till the work item is scheduled to return the
454 * atom to JS.
455 * @seq_nr: user-space sequence number, to order atoms in some
456 * temporal order
457 * @pre_dep: Pointer to atom that this atom has same-slot
458 * dependency on
459 * @post_dep: Pointer to atom that has same-slot dependency on
460 * this atom
461 * @x_pre_dep: Pointer to atom that this atom has cross-slot
462 * dependency on
463 * @x_post_dep: Pointer to atom that has cross-slot dependency on
464 * this atom
465 * @flush_id: The GPU's flush count recorded at the time of
466 * submission,
467 * used for the cache flush optimization
468 * @fault_event: Info for dumping the debug data on Job fault.
469 * @queue: List head used for 4 different purposes :
470 * Adds atom to the list of dma-buf fence waiting atoms.
471 * Adds atom to the list of atoms blocked due to cross
472 * slot dependency.
473 * Adds atom to the list of softjob atoms for which JIT
474 * allocation has been deferred
475 * Adds atom to the list of softjob atoms waiting for
476 * the signaling of fence.
477 * @jit_node: Used to keep track of all JIT free/alloc jobs in
478 * submission order
479 * @jit_blocked: Flag indicating that JIT allocation requested through
480 * softjob atom will be reattempted after the impending
481 * free of other active JIT allocations.
482 * @will_fail_event_code: If non-zero, this indicates that the atom will fail
483 * with the set event_code when the atom is processed.
484 * Used for special handling of atoms, which have a data
485 * dependency on the failed atoms.
486 * @protected_state: State of the atom, as per
487 * KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*,
488 * when transitioning into or out of protected mode.
489 * Atom will be either entering or exiting the
490 * protected mode.
491 * @protected_state.enter: entering the protected mode.
492 * @protected_state.exit: exiting the protected mode.
493 * @runnable_tree_node: The node added to context's job slot specific rb tree
494 * when the atom becomes runnable.
495 * @age: Age of atom relative to other atoms in the context,
496 * is snapshot of the age_count counter in kbase
497 * context.
498 * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified.
499 * @renderpass_id:Renderpass identifier used to associate an atom that has
500 * BASE_JD_REQ_START_RENDERPASS set in its core requirements
501 * with an atom that has BASE_JD_REQ_END_RENDERPASS set.
502 * @jc_fragment: Set of GPU fragment job chains
503 */
504 struct kbase_jd_atom {
505 struct work_struct work;
506 ktime_t start_timestamp;
507
508 struct base_jd_udata udata;
509 struct kbase_context *kctx;
510
511 struct list_head dep_head[2];
512 struct list_head dep_item[2];
513 const struct kbase_jd_atom_dependency dep[2];
514 struct list_head jd_item;
515 bool in_jd_list;
516
517 #if MALI_JIT_PRESSURE_LIMIT_BASE
518 u8 jit_ids[2];
519 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
520
521 u16 nr_extres;
522 struct kbase_va_region **extres;
523
524 u32 device_nr;
525 u64 jc;
526 void *softjob_data;
527 #if IS_ENABLED(CONFIG_SYNC_FILE)
528 struct {
529 /* Use the functions/API defined in mali_kbase_fence.h to
530 * when working with this sub struct
531 */
532 #if IS_ENABLED(CONFIG_SYNC_FILE)
533 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
534 struct fence *fence_in;
535 #else
536 struct dma_fence *fence_in;
537 #endif
538 #endif
539 /* This points to the dma-buf output fence for this atom. If
540 * this is NULL then there is no fence for this atom and the
541 * following fields related to dma_fence may have invalid data.
542 *
543 * The context and seqno fields contain the details for this
544 * fence.
545 *
546 * This fence is signaled when the katom is completed,
547 * regardless of the event_code of the katom (signal also on
548 * failure).
549 */
550 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
551 struct fence *fence;
552 #else
553 struct dma_fence *fence;
554 #endif
555
556 /* This is the callback object that is registered for the fence_in.
557 * The callback is invoked when the fence_in is signaled.
558 */
559 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
560 struct fence_cb fence_cb;
561 #else
562 struct dma_fence_cb fence_cb;
563 #endif
564 bool fence_cb_added;
565
566 unsigned int context;
567 atomic_t seqno;
568 } dma_fence;
569 #endif /* CONFIG_SYNC_FILE */
570
571 /* Note: refer to kbasep_js_atom_retained_state, which will take a copy
572 * of some of the following members
573 */
574 enum base_jd_event_code event_code;
575 base_jd_core_req core_req;
576 u8 jobslot;
577 u8 renderpass_id;
578 struct base_jd_fragment jc_fragment;
579
580 u32 ticks;
581 int sched_priority;
582
583 wait_queue_head_t completed;
584 enum kbase_jd_atom_state status;
585 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
586 int work_id;
587 #endif
588 unsigned int slot_nr;
589
590 u32 atom_flags;
591
592 enum kbase_atom_gpu_rb_state gpu_rb_state;
593
594 bool need_cache_flush_cores_retained;
595
596 atomic_t blocked;
597
598 u64 seq_nr;
599
600 struct kbase_jd_atom *pre_dep;
601 struct kbase_jd_atom *post_dep;
602
603 struct kbase_jd_atom *x_pre_dep;
604 struct kbase_jd_atom *x_post_dep;
605
606 u32 flush_id;
607
608 #if IS_ENABLED(CONFIG_DEBUG_FS)
609 struct base_job_fault_event fault_event;
610 #endif
611 struct list_head queue;
612
613 struct list_head jit_node;
614 bool jit_blocked;
615
616 enum base_jd_event_code will_fail_event_code;
617
618 union {
619 enum kbase_atom_enter_protected_state enter;
620 enum kbase_atom_exit_protected_state exit;
621 } protected_state;
622
623 struct rb_node runnable_tree_node;
624
625 u32 age;
626 };
627
kbase_jd_katom_is_protected(const struct kbase_jd_atom * katom)628 static inline bool kbase_jd_katom_is_protected(
629 const struct kbase_jd_atom *katom)
630 {
631 return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED);
632 }
633
634 /**
635 * kbase_jd_atom_is_younger - query if one atom is younger by age than another
636 *
637 * @katom_a: the first atom
638 * @katom_b: the second atom
639 *
640 * Return: true if the first atom is strictly younger than the second,
641 * false otherwise.
642 */
kbase_jd_atom_is_younger(const struct kbase_jd_atom * katom_a,const struct kbase_jd_atom * katom_b)643 static inline bool kbase_jd_atom_is_younger(const struct kbase_jd_atom *katom_a,
644 const struct kbase_jd_atom *katom_b)
645 {
646 return ((s32)(katom_a->age - katom_b->age) < 0);
647 }
648
649 /**
650 * kbase_jd_atom_is_earlier - Check whether the first atom has been submitted
651 * earlier than the second one
652 *
653 * @katom_a: the first atom
654 * @katom_b: the second atom
655 *
656 * Return: true if the first atom has been submitted earlier than the
657 * second atom. It is used to understand if an atom that is ready has been
658 * submitted earlier than the currently running atom, so that the currently
659 * running atom should be preempted to allow the ready atom to run.
660 */
kbase_jd_atom_is_earlier(const struct kbase_jd_atom * katom_a,const struct kbase_jd_atom * katom_b)661 static inline bool kbase_jd_atom_is_earlier(const struct kbase_jd_atom *katom_a,
662 const struct kbase_jd_atom *katom_b)
663 {
664 /* No seq_nr set? */
665 if (!katom_a->seq_nr || !katom_b->seq_nr)
666 return false;
667
668 /* Efficiently handle the unlikely case of wrapping.
669 * The following code assumes that the delta between the sequence number
670 * of the two atoms is less than INT64_MAX.
671 * In the extremely unlikely case where the delta is higher, the comparison
672 * defaults for no preemption.
673 * The code also assumes that the conversion from unsigned to signed types
674 * works because the signed integers are 2's complement.
675 */
676 return (s64)(katom_a->seq_nr - katom_b->seq_nr) < 0;
677 }
678
679 /*
680 * Theory of operations:
681 *
682 * Atom objects are statically allocated within the context structure.
683 *
684 * Each atom is the head of two lists, one for the "left" set of dependencies,
685 * one for the "right" set.
686 */
687
688 #define KBASE_JD_DEP_QUEUE_SIZE 256
689
690 /**
691 * enum kbase_jd_renderpass_state - State of a renderpass
692 * @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to
693 * START.
694 * @KBASE_JD_RP_START: Renderpass making a first attempt at tiling.
695 * Can transition to PEND_OOM or COMPLETE.
696 * @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much
697 * memory and has a soft-stop pending. Can transition to
698 * OOM or COMPLETE.
699 * @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much
700 * memory and therefore switched to incremental
701 * rendering. The fragment job chain is forced to run.
702 * Can only transition to RETRY.
703 * @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at
704 * tiling. Can transition to RETRY_PEND_OOM or COMPLETE.
705 * @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at
706 * tiling used too much memory again and has a
707 * soft-stop pending. Can transition to RETRY_OOM
708 * or COMPLETE.
709 * @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at
710 * tiling used too much memory again. The fragment job
711 * chain is forced to run. Can only transition to RETRY.
712 *
713 * A state machine is used to control incremental rendering.
714 */
715 enum kbase_jd_renderpass_state {
716 KBASE_JD_RP_COMPLETE, /* COMPLETE => START */
717 KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */
718 KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */
719 KBASE_JD_RP_OOM, /* OOM => RETRY */
720 KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or COMPLETE */
721 KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or COMPLETE */
722 KBASE_JD_RP_RETRY_OOM /* RETRY_OOM => RETRY */
723 };
724
725 /**
726 * struct kbase_jd_renderpass - Data for a renderpass
727 * @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then
728 * all other members are invalid.
729 * Both the job dispatcher context and hwaccess_lock must be
730 * locked to modify this so that it can be read with either
731 * (or both) locked.
732 * @start_katom: Address of the atom that is the start of a renderpass.
733 * Both the job dispatcher context and hwaccess_lock must be
734 * locked to modify this so that it can be read with either
735 * (or both) locked.
736 * @end_katom: Address of the atom that is the end of a renderpass, or NULL
737 * if that atom hasn't been added to the job scheduler yet.
738 * The job dispatcher context and hwaccess_lock must be
739 * locked to modify this so that it can be read with either
740 * (or both) locked.
741 * @oom_reg_list: A list of region structures which triggered out-of-memory.
742 * The hwaccess_lock must be locked to access this.
743 *
744 * Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS
745 * are associated with an object of this type, which is created and maintained
746 * by kbase to keep track of each renderpass.
747 */
748 struct kbase_jd_renderpass {
749 enum kbase_jd_renderpass_state state;
750 struct kbase_jd_atom *start_katom;
751 struct kbase_jd_atom *end_katom;
752 struct list_head oom_reg_list;
753 };
754
755 /**
756 * struct kbase_jd_context - per context object encapsulating all the
757 * Job dispatcher related state.
758 * @lock: lock to serialize the updates made to the
759 * Job dispatcher state and kbase_jd_atom objects.
760 * @sched_info: Structure encapsulating all the Job scheduling
761 * info.
762 * @atoms: Array of the objects representing atoms,
763 * containing the complete state and attributes
764 * of an atom.
765 * @renderpasses: Array of renderpass state for incremental
766 * rendering, indexed by user-specified renderpass
767 * ID.
768 * @job_nr: Tracks the number of atoms being processed by the
769 * kbase. This includes atoms that are not tracked by
770 * scheduler: 'not ready to run' & 'dependency-only'
771 * jobs.
772 * @zero_jobs_wait: Waitq that reflects whether there are no jobs
773 * (including SW-only dependency jobs). This is set
774 * when no jobs are present on the ctx, and clear
775 * when there are jobs.
776 * This must be updated atomically with @job_nr.
777 * note: Job Dispatcher knows about more jobs than
778 * the Job Scheduler as it is unaware of jobs that
779 * are blocked on dependencies and SW-only dependency
780 * jobs. This waitq can be waited upon to find out
781 * when the context jobs are all done/cancelled
782 * (including those that might've been blocked
783 * on dependencies) - and so, whether it can be
784 * terminated. However, it should only be terminated
785 * once it is not present in the run-pool.
786 * Since the waitq is only set under @lock,
787 * the waiter should also briefly obtain and drop
788 * @lock to guarantee that the setter has completed
789 * its work on the kbase_context
790 * @job_done_wq: Workqueue to which the per atom work item is
791 * queued for bottom half processing when the
792 * atom completes
793 * execution on GPU or the input fence get signaled.
794 * @tb_lock: Lock to serialize the write access made to @tb to
795 * store the register access trace messages.
796 * @tb: Pointer to the Userspace accessible buffer storing
797 * the trace messages for register read/write
798 * accesses made by the Kbase. The buffer is filled
799 * in circular fashion.
800 * @tb_wrap_offset: Offset to the end location in the trace buffer,
801 * the write pointer is moved to the beginning on
802 * reaching this offset.
803 * @work_id: atomic variable used for GPU tracepoints,
804 * incremented on every call to base_jd_submit.
805 * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both
806 * allocate & free, in submission order, protected
807 * by kbase_jd_context.lock.
808 * @jit_pending_alloc: A list of just-in-time memory allocation
809 * soft-jobs which will be reattempted after the
810 * impending free of other active allocations.
811 * @max_priority: Max priority level allowed for this context.
812 */
813 struct kbase_jd_context {
814 struct mutex lock;
815 struct kbasep_js_kctx_info sched_info;
816 struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT];
817 struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT];
818 struct workqueue_struct *job_done_wq;
819
820 wait_queue_head_t zero_jobs_wait;
821 spinlock_t tb_lock;
822 u32 *tb;
823 u32 job_nr;
824 size_t tb_wrap_offset;
825
826 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
827 atomic_t work_id;
828 #endif
829
830 struct list_head jit_atoms_head;
831 struct list_head jit_pending_alloc;
832 int max_priority;
833 };
834
835 /**
836 * struct jsctx_queue - JS context atom queue
837 * @runnable_tree: Root of RB-tree containing currently runnable atoms on this
838 * job slot.
839 * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot
840 * dependencies. Atoms on this list will be moved to the
841 * runnable_tree when the blocking atom completes.
842 *
843 * hwaccess_lock must be held when accessing this structure.
844 */
845 struct jsctx_queue {
846 struct rb_root runnable_tree;
847 struct list_head x_dep_head;
848 };
849
850 /**
851 * struct kbase_as - Object representing an address space of GPU.
852 * @number: Index at which this address space structure is present
853 * in an array of address space structures embedded inside
854 * the &struct kbase_device.
855 * @pf_wq: Workqueue for processing work items related to
856 * Page fault and Bus fault handling.
857 * @work_pagefault: Work item for the Page fault handling.
858 * @work_busfault: Work item for the Bus fault handling.
859 * @pf_data: Data relating to Page fault.
860 * @bf_data: Data relating to Bus fault.
861 * @current_setup: Stores the MMU configuration for this address space.
862 * @is_unresponsive: Flag to indicate MMU is not responding.
863 * Set if a MMU command isn't completed within
864 * &kbase_device:mmu_as_inactive_wait_time_ms.
865 * Clear by kbase_ctx_sched_restore_all_as() after GPU reset completes.
866 */
867 struct kbase_as {
868 int number;
869 struct workqueue_struct *pf_wq;
870 struct work_struct work_pagefault;
871 struct work_struct work_busfault;
872 struct kbase_fault pf_data;
873 struct kbase_fault bf_data;
874 struct kbase_mmu_setup current_setup;
875 bool is_unresponsive;
876 };
877
878 #endif /* _KBASE_JM_DEFS_H_ */
879