1 /*
2 * Copyright (C) 2013-2018 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10 #include <linux/file.h>
11 #include "mali_timeline.h"
12 #include "mali_kernel_common.h"
13 #include "mali_scheduler.h"
14 #include "mali_soft_job.h"
15 #include "mali_timeline_fence_wait.h"
16 #include "mali_timeline_sync_fence.h"
17 #include "mali_executor.h"
18 #include "mali_pp_job.h"
19
20 #define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid()))
21
22 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
23 _mali_osk_wq_work_t *sync_fence_callback_work_t = NULL;
24 _mali_osk_spinlock_irq_t *sync_fence_callback_list_lock = NULL;
25 static _MALI_OSK_LIST_HEAD_STATIC_INIT(sync_fence_callback_queue);
26 #endif
27
28 /*
29 * Following three elements are used to record how many
30 * gp, physical pp or virtual pp jobs are delayed in the whole
31 * timeline system, we can use these three value to decide
32 * if need to deactivate idle group.
33 */
34 _mali_osk_atomic_t gp_tracker_count;
35 _mali_osk_atomic_t phy_pp_tracker_count;
36 _mali_osk_atomic_t virt_pp_tracker_count;
37
38 static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
39 struct mali_timeline_waiter *waiter);
40
41 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
42 #include <linux/version.h>
43 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
44 #include <linux/list.h>
45 #include <linux/workqueue.h>
46 #include <linux/spinlock.h>
47
48 struct mali_deferred_fence_put_entry {
49 struct hlist_node list;
50 struct sync_fence *fence;
51 };
52
53 static HLIST_HEAD(mali_timeline_sync_fence_to_free_list);
54 static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock);
55
put_sync_fences(struct work_struct * ignore)56 static void put_sync_fences(struct work_struct *ignore)
57 {
58 struct hlist_head list;
59 struct hlist_node *tmp, *pos;
60 unsigned long flags;
61 struct mali_deferred_fence_put_entry *o;
62
63 spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
64 hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list);
65 spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
66
67 hlist_for_each_entry_safe(o, pos, tmp, &list, list) {
68 sync_fence_put(o->fence);
69 kfree(o);
70 }
71 }
72
73 static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences);
74 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */
75
76 /* Callback that is called when a sync fence a tracker is waiting on is signaled. */
77 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
mali_timeline_sync_fence_callback(struct sync_fence * sync_fence,struct sync_fence_waiter * sync_fence_waiter)78 static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter)
79 #else
80 static void mali_timeline_sync_fence_callback(struct mali_internal_sync_fence *sync_fence, struct mali_internal_sync_fence_waiter *sync_fence_waiter)
81 #endif
82 {
83 struct mali_timeline_tracker *tracker;
84
85 MALI_IGNORE(sync_fence);
86 MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter);
87
88 tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter);
89 MALI_DEBUG_ASSERT_POINTER(tracker);
90
91 _mali_osk_spinlock_irq_lock(sync_fence_callback_list_lock);
92 _mali_osk_list_addtail(&tracker->sync_fence_signal_list, &sync_fence_callback_queue);
93 _mali_osk_spinlock_irq_unlock(sync_fence_callback_list_lock);
94
95 _mali_osk_wq_schedule_work(sync_fence_callback_work_t);
96 }
97 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
98
mali_timeline_tracker_time_out(struct mali_timeline_tracker * tracker)99 static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker)
100 {
101 MALI_DEBUG_ASSERT_POINTER(tracker);
102 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type);
103
104 return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job);
105 }
106
mali_timeline_timer_callback(void * data)107 static void mali_timeline_timer_callback(void *data)
108 {
109 struct mali_timeline_system *system;
110 struct mali_timeline_tracker *tracker;
111 struct mali_timeline *timeline;
112 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
113 u32 tid = _mali_osk_get_tid();
114
115 timeline = (struct mali_timeline *) data;
116 MALI_DEBUG_ASSERT_POINTER(timeline);
117
118 system = timeline->system;
119 MALI_DEBUG_ASSERT_POINTER(system);
120
121 mali_spinlock_reentrant_wait(system->spinlock, tid);
122
123 if (!system->timer_enabled) {
124 mali_spinlock_reentrant_signal(system->spinlock, tid);
125 return;
126 }
127
128 tracker = timeline->tracker_tail;
129 timeline->timer_active = MALI_FALSE;
130
131 if (NULL != tracker && MALI_TRUE == tracker->timer_active) {
132 /* This is likely the delayed work that has been schedule out before cancelled. */
133 if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) {
134 mali_spinlock_reentrant_signal(system->spinlock, tid);
135 return;
136 }
137
138 schedule_mask = mali_timeline_tracker_time_out(tracker);
139 tracker->timer_active = MALI_FALSE;
140 } else {
141 MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n"));
142 }
143
144 mali_spinlock_reentrant_signal(system->spinlock, tid);
145
146 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
147 }
148
mali_timeline_system_stop_timer(struct mali_timeline_system * system)149 void mali_timeline_system_stop_timer(struct mali_timeline_system *system)
150 {
151 u32 i;
152 u32 tid = _mali_osk_get_tid();
153
154 MALI_DEBUG_ASSERT_POINTER(system);
155
156 mali_spinlock_reentrant_wait(system->spinlock, tid);
157 system->timer_enabled = MALI_FALSE;
158 mali_spinlock_reentrant_signal(system->spinlock, tid);
159
160 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
161 struct mali_timeline *timeline = system->timelines[i];
162
163 MALI_DEBUG_ASSERT_POINTER(timeline);
164
165 if (NULL != timeline->delayed_work) {
166 _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
167 timeline->timer_active = MALI_FALSE;
168 }
169 }
170 }
171
mali_timeline_destroy(struct mali_timeline * timeline)172 static void mali_timeline_destroy(struct mali_timeline *timeline)
173 {
174 MALI_DEBUG_ASSERT_POINTER(timeline);
175 if (NULL != timeline) {
176 /* Assert that the timeline object has been properly cleaned up before destroying it. */
177 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
178 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
179 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
180 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
181 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
182 MALI_DEBUG_ASSERT(NULL != timeline->system);
183 MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id);
184
185 if (NULL != timeline->delayed_work) {
186 _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
187 _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work);
188 }
189
190 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
191 if (NULL != timeline->sync_tl) {
192 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
193 sync_timeline_destroy(timeline->sync_tl);
194 #else
195 mali_internal_sync_timeline_destroy(timeline->sync_tl);
196 #endif
197 }
198 #else
199 _mali_osk_free(timeline);
200 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
201 }
202 }
203
mali_timeline_create(struct mali_timeline_system * system,enum mali_timeline_id id)204 static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id)
205 {
206 struct mali_timeline *timeline;
207
208 MALI_DEBUG_ASSERT_POINTER(system);
209 MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX);
210
211 timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline));
212 if (NULL == timeline) {
213 return NULL;
214 }
215
216 /* Initially the timeline is empty. */
217 #if defined(MALI_TIMELINE_DEBUG_START_POINT)
218 /* Start the timeline a bit before wrapping when debugging. */
219 timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128;
220 #else
221 timeline->point_next = 1;
222 #endif
223 timeline->point_oldest = timeline->point_next;
224
225 /* The tracker and waiter lists will initially be empty. */
226
227 timeline->system = system;
228 timeline->id = id;
229
230 timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline);
231 if (NULL == timeline->delayed_work) {
232 mali_timeline_destroy(timeline);
233 return NULL;
234 }
235
236 timeline->timer_active = MALI_FALSE;
237
238 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
239 {
240 char timeline_name[32];
241
242 switch (id) {
243 case MALI_TIMELINE_GP:
244 _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid());
245 break;
246 case MALI_TIMELINE_PP:
247 _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid());
248 break;
249 case MALI_TIMELINE_SOFT:
250 _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid());
251 break;
252 default:
253 MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id));
254 mali_timeline_destroy(timeline);
255 return NULL;
256 }
257
258 timeline->destroyed = MALI_FALSE;
259
260 timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name);
261 if (NULL == timeline->sync_tl) {
262 mali_timeline_destroy(timeline);
263 return NULL;
264 }
265
266 timeline->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
267 if (NULL == timeline->spinlock) {
268 mali_timeline_destroy(timeline);
269 return NULL;
270 }
271 }
272 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
273
274 return timeline;
275 }
276
mali_timeline_insert_tracker(struct mali_timeline * timeline,struct mali_timeline_tracker * tracker)277 static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker)
278 {
279 MALI_DEBUG_ASSERT_POINTER(timeline);
280 MALI_DEBUG_ASSERT_POINTER(tracker);
281
282 if (mali_timeline_is_full(timeline)) {
283 /* Don't add tracker if timeline is full. */
284 tracker->point = MALI_TIMELINE_NO_POINT;
285 return;
286 }
287
288 tracker->timeline = timeline;
289 tracker->point = timeline->point_next;
290
291 /* Find next available point. */
292 timeline->point_next++;
293 if (MALI_TIMELINE_NO_POINT == timeline->point_next) {
294 timeline->point_next++;
295 }
296
297 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
298
299 if (MALI_TIMELINE_TRACKER_GP == tracker->type) {
300 _mali_osk_atomic_inc(&gp_tracker_count);
301 } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) {
302 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
303 _mali_osk_atomic_inc(&virt_pp_tracker_count);
304 } else {
305 _mali_osk_atomic_inc(&phy_pp_tracker_count);
306 }
307 }
308
309 /* Add tracker as new head on timeline's tracker list. */
310 if (NULL == timeline->tracker_head) {
311 /* Tracker list is empty. */
312 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
313
314 timeline->tracker_tail = tracker;
315
316 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
317 MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev);
318 } else {
319 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
320
321 tracker->timeline_prev = timeline->tracker_head;
322 timeline->tracker_head->timeline_next = tracker;
323
324 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
325 }
326 timeline->tracker_head = tracker;
327
328 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
329 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev);
330 }
331
332 /* Inserting the waiter object into the given timeline */
mali_timeline_insert_waiter(struct mali_timeline * timeline,struct mali_timeline_waiter * waiter_new)333 static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new)
334 {
335 struct mali_timeline_waiter *waiter_prev;
336 struct mali_timeline_waiter *waiter_next;
337
338 /* Waiter time must be between timeline head and tail, and there must
339 * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */
340 MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN);
341 MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN);
342
343 /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/
344 waiter_prev = timeline->waiter_head; /* Insert new after waiter_prev */
345 waiter_next = NULL; /* Insert new before waiter_next */
346
347 /* Iterating backwards from head (newest) to tail (oldest) until we
348 * find the correct spot to insert the new waiter */
349 while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) {
350 waiter_next = waiter_prev;
351 waiter_prev = waiter_prev->timeline_prev;
352 }
353
354 if (NULL == waiter_prev && NULL == waiter_next) {
355 /* list is empty */
356 timeline->waiter_head = waiter_new;
357 timeline->waiter_tail = waiter_new;
358 } else if (NULL == waiter_next) {
359 /* insert at head */
360 waiter_new->timeline_prev = timeline->waiter_head;
361 timeline->waiter_head->timeline_next = waiter_new;
362 timeline->waiter_head = waiter_new;
363 } else if (NULL == waiter_prev) {
364 /* insert at tail */
365 waiter_new->timeline_next = timeline->waiter_tail;
366 timeline->waiter_tail->timeline_prev = waiter_new;
367 timeline->waiter_tail = waiter_new;
368 } else {
369 /* insert between */
370 waiter_new->timeline_next = waiter_next;
371 waiter_new->timeline_prev = waiter_prev;
372 waiter_next->timeline_prev = waiter_new;
373 waiter_prev->timeline_next = waiter_new;
374 }
375 }
376
mali_timeline_update_delayed_work(struct mali_timeline * timeline)377 static void mali_timeline_update_delayed_work(struct mali_timeline *timeline)
378 {
379 struct mali_timeline_system *system;
380 struct mali_timeline_tracker *oldest_tracker;
381
382 MALI_DEBUG_ASSERT_POINTER(timeline);
383 MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
384
385 system = timeline->system;
386 MALI_DEBUG_ASSERT_POINTER(system);
387
388 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
389
390 /* Timer is disabled, early out. */
391 if (!system->timer_enabled) return;
392
393 oldest_tracker = timeline->tracker_tail;
394 if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) {
395 if (MALI_FALSE == oldest_tracker->timer_active) {
396 if (MALI_TRUE == timeline->timer_active) {
397 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
398 }
399 _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ);
400 oldest_tracker->timer_active = MALI_TRUE;
401 timeline->timer_active = MALI_TRUE;
402 }
403 } else if (MALI_TRUE == timeline->timer_active) {
404 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
405 timeline->timer_active = MALI_FALSE;
406 }
407 }
408
mali_timeline_update_oldest_point(struct mali_timeline * timeline)409 static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline)
410 {
411 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
412
413 MALI_DEBUG_ASSERT_POINTER(timeline);
414
415 MALI_DEBUG_CODE({
416 struct mali_timeline_system *system = timeline->system;
417 MALI_DEBUG_ASSERT_POINTER(system);
418
419 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
420 });
421
422 if (NULL != timeline->tracker_tail) {
423 /* Set oldest point to oldest tracker's point */
424 timeline->point_oldest = timeline->tracker_tail->point;
425 } else {
426 /* No trackers, mark point list as empty */
427 timeline->point_oldest = timeline->point_next;
428 }
429
430 /* Release all waiters no longer on the timeline's point list.
431 * Releasing a waiter can trigger this function to be called again, so
432 * we do not store any pointers on stack. */
433 while (NULL != timeline->waiter_tail) {
434 u32 waiter_time_relative;
435 u32 time_head_relative;
436 struct mali_timeline_waiter *waiter = timeline->waiter_tail;
437
438 time_head_relative = timeline->point_next - timeline->point_oldest;
439 waiter_time_relative = waiter->point - timeline->point_oldest;
440
441 if (waiter_time_relative < time_head_relative) {
442 /* This and all following waiters are on the point list, so we are done. */
443 break;
444 }
445
446 /* Remove waiter from timeline's waiter list. */
447 if (NULL != waiter->timeline_next) {
448 waiter->timeline_next->timeline_prev = NULL;
449 } else {
450 /* This was the last waiter */
451 timeline->waiter_head = NULL;
452 }
453 timeline->waiter_tail = waiter->timeline_next;
454
455 /* Release waiter. This could activate a tracker, if this was
456 * the last waiter for the tracker. */
457 schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
458 }
459
460 return schedule_mask;
461 }
462
mali_timeline_release_with_depended_point(struct mali_timeline_tracker * tracker)463 static mali_scheduler_mask mali_timeline_release_with_depended_point(struct mali_timeline_tracker *tracker)
464 {
465 struct mali_timeline *timeline;
466 struct mali_timeline_waiter *waiter;
467 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
468
469 timeline = tracker->timeline;
470 MALI_DEBUG_ASSERT_POINTER(timeline);
471 MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
472
473 MALI_DEBUG_CODE({
474 struct mali_timeline_system *system = timeline->system;
475 MALI_DEBUG_ASSERT_POINTER(system);
476
477 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
478 });
479
480 /* Only release the waiter that wait for the tracker. */
481 waiter = timeline->waiter_tail;
482 while (NULL != waiter) {
483 if (waiter->point == tracker->point) {
484
485 struct mali_timeline_waiter *waiter_next;
486 struct mali_timeline_waiter *waiter_prev;
487
488 waiter_next = waiter->timeline_next;
489 waiter_prev = waiter->timeline_prev;
490 waiter->timeline_next = NULL;
491 waiter->timeline_prev = NULL;
492
493 if (NULL != waiter_prev) {
494 waiter_prev->timeline_next = waiter_next;
495 }
496
497 if (NULL != waiter_next) {
498 waiter_next->timeline_prev = waiter_prev;
499 }
500
501 if (waiter == timeline->waiter_tail)
502 timeline->waiter_tail = waiter_next;
503
504 if (waiter == timeline->waiter_head)
505 timeline->waiter_head = NULL;
506
507 schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
508 waiter = waiter_next;
509 }else {
510
511 waiter = waiter->timeline_next;
512 }
513 }
514
515 return schedule_mask;
516 }
517
mali_timeline_tracker_init(struct mali_timeline_tracker * tracker,mali_timeline_tracker_type type,struct mali_timeline_fence * fence,void * job)518 void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker,
519 mali_timeline_tracker_type type,
520 struct mali_timeline_fence *fence,
521 void *job)
522 {
523 MALI_DEBUG_ASSERT_POINTER(tracker);
524 MALI_DEBUG_ASSERT_POINTER(job);
525
526 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type);
527
528 /* Zero out all tracker members. */
529 _mali_osk_memset(tracker, 0, sizeof(*tracker));
530
531 tracker->type = type;
532 tracker->job = job;
533 tracker->trigger_ref_count = 1; /* Prevents any callback from trigging while adding it */
534 tracker->os_tick_create = _mali_osk_time_tickcount();
535 MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC);
536
537 tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE;
538
539 /* Copy fence. */
540 if (NULL != fence) {
541 _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence));
542 }
543 }
544
mali_timeline_tracker_release(struct mali_timeline_tracker * tracker)545 mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker)
546 {
547 struct mali_timeline *timeline;
548 struct mali_timeline_system *system;
549 struct mali_timeline_tracker *tracker_next, *tracker_prev;
550 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
551 u32 tid = _mali_osk_get_tid();
552
553 /* Upon entry a group lock will be held, but not a scheduler lock. */
554 MALI_DEBUG_ASSERT_POINTER(tracker);
555 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
556
557 /* Tracker should have been triggered */
558 MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count);
559
560 /* All waiters should have been released at this point */
561 MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
562 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
563
564 MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job));
565
566 timeline = tracker->timeline;
567 if (NULL == timeline) {
568 /* Tracker was not on a timeline, there is nothing to release. */
569 return MALI_SCHEDULER_MASK_EMPTY;
570 }
571
572 system = timeline->system;
573 MALI_DEBUG_ASSERT_POINTER(system);
574
575 mali_spinlock_reentrant_wait(system->spinlock, tid);
576
577 /* Tracker should still be on timeline */
578 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
579 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point));
580
581 /* Tracker is no longer valid. */
582 MALI_DEBUG_CODE(tracker->magic = 0);
583
584 tracker_next = tracker->timeline_next;
585 tracker_prev = tracker->timeline_prev;
586 tracker->timeline_next = NULL;
587 tracker->timeline_prev = NULL;
588
589 /* Removing tracker from timeline's tracker list */
590 if (NULL == tracker_next) {
591 /* This tracker was the head */
592 timeline->tracker_head = tracker_prev;
593 } else {
594 tracker_next->timeline_prev = tracker_prev;
595 }
596
597 if (NULL == tracker_prev) {
598 /* This tracker was the tail */
599 timeline->tracker_tail = tracker_next;
600 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
601 /* Update the timeline's oldest time and release any waiters */
602 schedule_mask |= mali_timeline_update_oldest_point(timeline);
603 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
604 } else {
605 tracker_prev->timeline_next = tracker_next;
606 if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
607 /* Use the signaled soft tracker to release the depended soft waiter */
608 schedule_mask |= mali_timeline_release_with_depended_point(tracker);
609 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
610 }
611 }
612
613 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
614
615 /* Update delayed work only when it is the soft job timeline */
616 if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
617 mali_timeline_update_delayed_work(tracker->timeline);
618 }
619
620 mali_spinlock_reentrant_signal(system->spinlock, tid);
621
622 return schedule_mask;
623 }
624
mali_timeline_system_release_waiter_list(struct mali_timeline_system * system,struct mali_timeline_waiter * tail,struct mali_timeline_waiter * head)625 void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system,
626 struct mali_timeline_waiter *tail,
627 struct mali_timeline_waiter *head)
628 {
629 MALI_DEBUG_ASSERT_POINTER(system);
630 MALI_DEBUG_ASSERT_POINTER(head);
631 MALI_DEBUG_ASSERT_POINTER(tail);
632 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
633
634 head->tracker_next = system->waiter_empty_list;
635 system->waiter_empty_list = tail;
636 }
637
mali_timeline_tracker_activate(struct mali_timeline_tracker * tracker)638 static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker)
639 {
640 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
641 struct mali_timeline_system *system;
642 struct mali_timeline *timeline;
643 u32 tid = _mali_osk_get_tid();
644
645 MALI_DEBUG_ASSERT_POINTER(tracker);
646 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
647
648 system = tracker->system;
649 MALI_DEBUG_ASSERT_POINTER(system);
650 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
651
652 tracker->os_tick_activate = _mali_osk_time_tickcount();
653
654 if (NULL != tracker->waiter_head) {
655 mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head);
656 tracker->waiter_head = NULL;
657 tracker->waiter_tail = NULL;
658 }
659
660 switch (tracker->type) {
661 case MALI_TIMELINE_TRACKER_GP:
662 schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job);
663
664 _mali_osk_atomic_dec(&gp_tracker_count);
665 break;
666 case MALI_TIMELINE_TRACKER_PP:
667 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
668 _mali_osk_atomic_dec(&virt_pp_tracker_count);
669 } else {
670 _mali_osk_atomic_dec(&phy_pp_tracker_count);
671 }
672 schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job);
673 break;
674 case MALI_TIMELINE_TRACKER_SOFT:
675 timeline = tracker->timeline;
676 MALI_DEBUG_ASSERT_POINTER(timeline);
677
678 schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job);
679
680 /* Start a soft timer to make sure the soft job be released in a limited time */
681 mali_spinlock_reentrant_wait(system->spinlock, tid);
682 mali_timeline_update_delayed_work(timeline);
683 mali_spinlock_reentrant_signal(system->spinlock, tid);
684 break;
685 case MALI_TIMELINE_TRACKER_WAIT:
686 mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job);
687 break;
688 case MALI_TIMELINE_TRACKER_SYNC:
689 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
690 mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job);
691 #else
692 MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type));
693 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
694 break;
695 default:
696 MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type));
697 break;
698 }
699
700 return schedule_mask;
701 }
702
mali_timeline_system_tracker_get(struct mali_timeline_system * system,struct mali_timeline_tracker * tracker)703 void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker)
704 {
705 u32 tid = _mali_osk_get_tid();
706
707 MALI_DEBUG_ASSERT_POINTER(tracker);
708 MALI_DEBUG_ASSERT_POINTER(system);
709
710 mali_spinlock_reentrant_wait(system->spinlock, tid);
711
712 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
713 tracker->trigger_ref_count++;
714
715 mali_spinlock_reentrant_signal(system->spinlock, tid);
716 }
717
mali_timeline_system_tracker_put(struct mali_timeline_system * system,struct mali_timeline_tracker * tracker,mali_timeline_activation_error activation_error)718 mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error)
719 {
720 u32 tid = _mali_osk_get_tid();
721 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
722
723 MALI_DEBUG_ASSERT_POINTER(tracker);
724 MALI_DEBUG_ASSERT_POINTER(system);
725
726 mali_spinlock_reentrant_wait(system->spinlock, tid);
727
728 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
729 tracker->trigger_ref_count--;
730
731 tracker->activation_error |= activation_error;
732
733 if (0 == tracker->trigger_ref_count) {
734 schedule_mask |= mali_timeline_tracker_activate(tracker);
735 tracker = NULL;
736 }
737
738 mali_spinlock_reentrant_signal(system->spinlock, tid);
739
740 return schedule_mask;
741 }
742
mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence * fence,_mali_uk_fence_t * uk_fence)743 void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence)
744 {
745 u32 i;
746
747 MALI_DEBUG_ASSERT_POINTER(fence);
748 MALI_DEBUG_ASSERT_POINTER(uk_fence);
749
750 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
751 fence->points[i] = uk_fence->points[i];
752 }
753
754 fence->sync_fd = uk_fence->sync_fd;
755 }
756
mali_timeline_system_create(struct mali_session_data * session)757 struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session)
758 {
759 u32 i;
760 struct mali_timeline_system *system;
761
762 MALI_DEBUG_ASSERT_POINTER(session);
763 MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n"));
764
765 system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system));
766 if (NULL == system) {
767 return NULL;
768 }
769
770 system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
771 if (NULL == system->spinlock) {
772 mali_timeline_system_destroy(system);
773 return NULL;
774 }
775
776 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
777 system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i);
778 if (NULL == system->timelines[i]) {
779 mali_timeline_system_destroy(system);
780 return NULL;
781 }
782 }
783
784 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
785 system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled");
786 if (NULL == system->signaled_sync_tl) {
787 mali_timeline_system_destroy(system);
788 return NULL;
789 }
790 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
791
792 system->waiter_empty_list = NULL;
793 system->session = session;
794 system->timer_enabled = MALI_TRUE;
795
796 system->wait_queue = _mali_osk_wait_queue_init();
797 if (NULL == system->wait_queue) {
798 mali_timeline_system_destroy(system);
799 return NULL;
800 }
801
802 return system;
803 }
804
805 #if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC) ||defined(CONFIG_SYNC_FILE)
806 /**
807 * Check if there are any trackers left on timeline.
808 *
809 * Used as a wait queue conditional.
810 *
811 * @param data Timeline.
812 * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not.
813 */
mali_timeline_has_no_trackers(void * data)814 static mali_bool mali_timeline_has_no_trackers(void *data)
815 {
816 struct mali_timeline *timeline = (struct mali_timeline *) data;
817
818 MALI_DEBUG_ASSERT_POINTER(timeline);
819
820 return mali_timeline_is_empty(timeline);
821 }
822 #if defined(CONFIG_SYNC) ||defined(CONFIG_SYNC_FILE)
823 /**
824 * Cancel sync fence waiters waited upon by trackers on all timelines.
825 *
826 * Will return after all timelines have no trackers left.
827 *
828 * @param system Timeline system.
829 */
mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system * system)830 static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system)
831 {
832 u32 i;
833 u32 tid = _mali_osk_get_tid();
834 struct mali_timeline_tracker *tracker, *tracker_next;
835 _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list);
836
837 MALI_DEBUG_ASSERT_POINTER(system);
838 MALI_DEBUG_ASSERT_POINTER(system->session);
839 MALI_DEBUG_ASSERT(system->session->is_aborting);
840
841 mali_spinlock_reentrant_wait(system->spinlock, tid);
842
843 /* Cancel sync fence waiters. */
844 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
845 struct mali_timeline *timeline = system->timelines[i];
846
847 MALI_DEBUG_ASSERT_POINTER(timeline);
848
849 tracker_next = timeline->tracker_tail;
850 while (NULL != tracker_next) {
851 tracker = tracker_next;
852 tracker_next = tracker->timeline_next;
853
854 if (NULL == tracker->sync_fence) continue;
855
856 MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker));
857
858 /* Cancel sync fence waiter. */
859 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
860 if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
861 #else
862 if (0 == mali_internal_sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
863 #endif
864 /* Callback was not called, move tracker to local list. */
865 _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list);
866 }
867 }
868 }
869
870 mali_spinlock_reentrant_signal(system->spinlock, tid);
871
872 /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */
873 _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) {
874 mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter);
875 }
876
877 /* Sleep until all sync fence callbacks are done and all timelines are empty. */
878 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
879 struct mali_timeline *timeline = system->timelines[i];
880
881 MALI_DEBUG_ASSERT_POINTER(timeline);
882
883 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
884 }
885 }
886
887 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
888
889 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
890 static void mali_timeline_cancel_dma_fence_waiters(struct mali_timeline_system *system)
891 {
892 u32 i, j;
893 u32 tid = _mali_osk_get_tid();
894 struct mali_pp_job *pp_job = NULL;
895 struct mali_pp_job *next_pp_job = NULL;
896 struct mali_timeline *timeline = NULL;
897 struct mali_timeline_tracker *tracker, *tracker_next;
898 _MALI_OSK_LIST_HEAD_STATIC_INIT(pp_job_list);
899
900 MALI_DEBUG_ASSERT_POINTER(system);
901 MALI_DEBUG_ASSERT_POINTER(system->session);
902 MALI_DEBUG_ASSERT(system->session->is_aborting);
903
904 mali_spinlock_reentrant_wait(system->spinlock, tid);
905
906 /* Cancel dma fence waiters. */
907 timeline = system->timelines[MALI_TIMELINE_PP];
908 MALI_DEBUG_ASSERT_POINTER(timeline);
909
910 tracker_next = timeline->tracker_tail;
911 while (NULL != tracker_next) {
912 mali_bool fence_is_signaled = MALI_TRUE;
913 tracker = tracker_next;
914 tracker_next = tracker->timeline_next;
915
916 if (NULL == tracker->waiter_dma_fence) continue;
917 pp_job = (struct mali_pp_job *)tracker->job;
918 MALI_DEBUG_ASSERT_POINTER(pp_job);
919 MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling dma fence waiter for tracker 0x%08X.\n", tracker));
920
921 for (j = 0; j < pp_job->dma_fence_context.num_dma_fence_waiter; j++) {
922 if (pp_job->dma_fence_context.mali_dma_fence_waiters[j]) {
923 /* Cancel a previously callback from the fence.
924 * This function returns true if the callback is successfully removed,
925 * or false if the fence has already been signaled.
926 */
927 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
928 bool ret = dma_fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence,
929 &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base);
930
931 #else
932 bool ret = fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence,
933 &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base);
934 #endif
935 if (ret) {
936 fence_is_signaled = MALI_FALSE;
937 }
938 }
939 }
940
941 /* Callbacks were not called, move pp job to local list. */
942 if (MALI_FALSE == fence_is_signaled)
943 _mali_osk_list_add(&pp_job->list, &pp_job_list);
944 }
945
946 mali_spinlock_reentrant_signal(system->spinlock, tid);
947
948 /* Manually call dma fence callback in order to release waiter and trigger activation of tracker. */
949 _MALI_OSK_LIST_FOREACHENTRY(pp_job, next_pp_job, &pp_job_list, struct mali_pp_job, list) {
950 mali_timeline_dma_fence_callback((void *)pp_job);
951 }
952
953 /* Sleep until all dma fence callbacks are done and all timelines are empty. */
954 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
955 struct mali_timeline *timeline = system->timelines[i];
956 MALI_DEBUG_ASSERT_POINTER(timeline);
957 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
958 }
959 }
960 #endif
961 #endif
962 void mali_timeline_system_abort(struct mali_timeline_system *system)
963 {
964 MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid(););
965
966 MALI_DEBUG_ASSERT_POINTER(system);
967 MALI_DEBUG_ASSERT_POINTER(system->session);
968 MALI_DEBUG_ASSERT(system->session->is_aborting);
969
970 MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session));
971
972 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
973 mali_timeline_cancel_sync_fence_waiters(system);
974 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
975
976 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
977 mali_timeline_cancel_dma_fence_waiters(system);
978 #endif
979
980 /* Should not be any waiters or trackers left at this point. */
981 MALI_DEBUG_CODE({
982 u32 i;
983 mali_spinlock_reentrant_wait(system->spinlock, tid);
984 for (i = 0; i < MALI_TIMELINE_MAX; ++i)
985 {
986 struct mali_timeline *timeline = system->timelines[i];
987 MALI_DEBUG_ASSERT_POINTER(timeline);
988 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
989 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
990 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
991 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
992 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
993 }
994 mali_spinlock_reentrant_signal(system->spinlock, tid);
995 });
996 }
997
998 void mali_timeline_system_destroy(struct mali_timeline_system *system)
999 {
1000 u32 i;
1001 struct mali_timeline_waiter *waiter, *next;
1002 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1003 u32 tid = _mali_osk_get_tid();
1004 #endif
1005
1006 MALI_DEBUG_ASSERT_POINTER(system);
1007 MALI_DEBUG_ASSERT_POINTER(system->session);
1008
1009 MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n"));
1010
1011 if (NULL != system) {
1012
1013 /* There should be no waiters left on this queue. */
1014 if (NULL != system->wait_queue) {
1015 _mali_osk_wait_queue_term(system->wait_queue);
1016 system->wait_queue = NULL;
1017 }
1018
1019 /* Free all waiters in empty list */
1020 waiter = system->waiter_empty_list;
1021 while (NULL != waiter) {
1022 next = waiter->tracker_next;
1023 _mali_osk_free(waiter);
1024 waiter = next;
1025 }
1026
1027 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1028 if (NULL != system->signaled_sync_tl) {
1029 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1030 sync_timeline_destroy(system->signaled_sync_tl);
1031 #else
1032 mali_internal_sync_timeline_destroy(system->signaled_sync_tl);
1033 #endif
1034 }
1035
1036 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1037 if ((NULL != system->timelines[i]) && (NULL != system->timelines[i]->spinlock)) {
1038 mali_spinlock_reentrant_wait(system->timelines[i]->spinlock, tid);
1039 system->timelines[i]->destroyed = MALI_TRUE;
1040 mali_spinlock_reentrant_signal(system->timelines[i]->spinlock, tid);
1041 }
1042 }
1043 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
1044
1045 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1046 if (NULL != system->timelines[i]) {
1047 mali_timeline_destroy(system->timelines[i]);
1048 }
1049 }
1050
1051 if (NULL != system->spinlock) {
1052 mali_spinlock_reentrant_term(system->spinlock);
1053 }
1054
1055 _mali_osk_free(system);
1056 }
1057 }
1058
1059 /**
1060 * Find how many waiters are needed for a given fence.
1061 *
1062 * @param fence The fence to check.
1063 * @return Number of waiters needed for fence.
1064 */
1065 static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence)
1066 {
1067 u32 i, num_waiters = 0;
1068
1069 MALI_DEBUG_ASSERT_POINTER(fence);
1070
1071 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1072 if (MALI_TIMELINE_NO_POINT != fence->points[i]) {
1073 ++num_waiters;
1074 }
1075 }
1076
1077 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1078 if (-1 != fence->sync_fd) ++num_waiters;
1079 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
1080
1081 return num_waiters;
1082 }
1083
1084 static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system)
1085 {
1086 struct mali_timeline_waiter *waiter;
1087
1088 MALI_DEBUG_ASSERT_POINTER(system);
1089 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1090
1091 waiter = system->waiter_empty_list;
1092 if (NULL != waiter) {
1093 /* Remove waiter from empty list and zero it */
1094 system->waiter_empty_list = waiter->tracker_next;
1095 _mali_osk_memset(waiter, 0, sizeof(*waiter));
1096 }
1097
1098 /* Return NULL if list was empty. */
1099 return waiter;
1100 }
1101
1102 static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system,
1103 struct mali_timeline_waiter **tail,
1104 struct mali_timeline_waiter **head,
1105 int max_num_waiters)
1106 {
1107 u32 i, tid = _mali_osk_get_tid();
1108 mali_bool do_alloc;
1109 struct mali_timeline_waiter *waiter;
1110
1111 MALI_DEBUG_ASSERT_POINTER(system);
1112 MALI_DEBUG_ASSERT_POINTER(tail);
1113 MALI_DEBUG_ASSERT_POINTER(head);
1114
1115 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1116
1117 *head = *tail = NULL;
1118 do_alloc = MALI_FALSE;
1119 i = 0;
1120 while (i < max_num_waiters) {
1121 if (MALI_FALSE == do_alloc) {
1122 waiter = mali_timeline_system_get_zeroed_waiter(system);
1123 if (NULL == waiter) {
1124 do_alloc = MALI_TRUE;
1125 mali_spinlock_reentrant_signal(system->spinlock, tid);
1126 continue;
1127 }
1128 } else {
1129 waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter));
1130 if (NULL == waiter) break;
1131 }
1132 ++i;
1133 if (NULL == *tail) {
1134 *tail = waiter;
1135 *head = waiter;
1136 } else {
1137 (*head)->tracker_next = waiter;
1138 *head = waiter;
1139 }
1140 }
1141 if (MALI_TRUE == do_alloc) {
1142 mali_spinlock_reentrant_wait(system->spinlock, tid);
1143 }
1144 }
1145
1146 /**
1147 * Create waiters for the given tracker. The tracker is activated when all waiters are release.
1148 *
1149 * @note Tracker can potentially be activated before this function returns.
1150 *
1151 * @param system Timeline system.
1152 * @param tracker Tracker we will create waiters for.
1153 * @param waiter_tail List of pre-allocated waiters.
1154 * @param waiter_head List of pre-allocated waiters.
1155 */
1156 static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system,
1157 struct mali_timeline_tracker *tracker,
1158 struct mali_timeline_waiter *waiter_tail,
1159 struct mali_timeline_waiter *waiter_head)
1160 {
1161 int i;
1162 u32 tid = _mali_osk_get_tid();
1163 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1164 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1165 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1166 struct sync_fence *sync_fence = NULL;
1167 #else
1168 struct mali_internal_sync_fence *sync_fence = NULL;
1169 #endif
1170 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
1171
1172 MALI_DEBUG_ASSERT_POINTER(system);
1173 MALI_DEBUG_ASSERT_POINTER(tracker);
1174
1175 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1176
1177 MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
1178 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1179 MALI_DEBUG_ASSERT(NULL != tracker->job);
1180
1181 /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter
1182 * into the timelines sorted list of waiters */
1183 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1184 mali_timeline_point point;
1185 struct mali_timeline *timeline;
1186 struct mali_timeline_waiter *waiter;
1187
1188 /* Get point on current timeline from tracker's fence. */
1189 point = tracker->fence.points[i];
1190
1191 if (likely(MALI_TIMELINE_NO_POINT == point)) {
1192 /* Fence contains no point on this timeline so we don't need a waiter. */
1193 continue;
1194 }
1195
1196 timeline = system->timelines[i];
1197 MALI_DEBUG_ASSERT_POINTER(timeline);
1198
1199 if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
1200 MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n",
1201 point, timeline->point_oldest, timeline->point_next));
1202 continue;
1203 }
1204
1205 if (likely(mali_timeline_is_point_released(timeline, point))) {
1206 /* Tracker representing the point has been released so we don't need a
1207 * waiter. */
1208 continue;
1209 }
1210
1211 if ((MALI_TIMELINE_SOFT == timeline->id) && mali_timeline_is_tracker_released(timeline, point)) {
1212 /* The tracker that the point related to has already been released, so no need to a waiter. */
1213 continue;
1214 }
1215
1216 /* The point is on timeline. */
1217 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point));
1218
1219 /* Get a new zeroed waiter object. */
1220 if (likely(NULL != waiter_tail)) {
1221 waiter = waiter_tail;
1222 waiter_tail = waiter_tail->tracker_next;
1223 } else {
1224 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1225 continue;
1226 }
1227
1228 /* Yanking the trigger ref count of the tracker. */
1229 tracker->trigger_ref_count++;
1230
1231 waiter->point = point;
1232 waiter->tracker = tracker;
1233
1234 /* Insert waiter on tracker's singly-linked waiter list. */
1235 if (NULL == tracker->waiter_head) {
1236 /* list is empty */
1237 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1238 tracker->waiter_tail = waiter;
1239 } else {
1240 tracker->waiter_head->tracker_next = waiter;
1241 }
1242 tracker->waiter_head = waiter;
1243
1244 /* Add waiter to timeline. */
1245 mali_timeline_insert_waiter(timeline, waiter);
1246 }
1247 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1248 if (-1 != tracker->fence.sync_fd) {
1249 int ret;
1250 struct mali_timeline_waiter *waiter;
1251 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1252 sync_fence = sync_fence_fdget(tracker->fence.sync_fd);
1253 #else
1254 sync_fence = mali_internal_sync_fence_fdget(tracker->fence.sync_fd);
1255 #endif
1256 if (unlikely(NULL == sync_fence)) {
1257 MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd));
1258 goto exit;
1259 }
1260
1261 /* Check if we have a zeroed waiter object available. */
1262 if (unlikely(NULL == waiter_tail)) {
1263 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1264 goto exit;
1265 }
1266
1267 /* Start asynchronous wait that will release waiter when the fence is signaled. */
1268 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1269 sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
1270 ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
1271 #else
1272 mali_internal_sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
1273 ret = mali_internal_sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
1274 #endif
1275 if (1 == ret) {
1276 /* Fence already signaled, no waiter needed. */
1277 tracker->fence.sync_fd = -1;
1278 goto exit;
1279 } else if (0 != ret) {
1280 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret));
1281 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
1282 goto exit;
1283 }
1284
1285 /* Grab new zeroed waiter object. */
1286 waiter = waiter_tail;
1287 waiter_tail = waiter_tail->tracker_next;
1288
1289 /* Increase the trigger ref count of the tracker. */
1290 tracker->trigger_ref_count++;
1291
1292 waiter->point = MALI_TIMELINE_NO_POINT;
1293 waiter->tracker = tracker;
1294
1295 /* Insert waiter on tracker's singly-linked waiter list. */
1296 if (NULL == tracker->waiter_head) {
1297 /* list is empty */
1298 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1299 tracker->waiter_tail = waiter;
1300 } else {
1301 tracker->waiter_head->tracker_next = waiter;
1302 }
1303 tracker->waiter_head = waiter;
1304
1305 /* Also store waiter in separate field for easy access by sync callback. */
1306 tracker->waiter_sync = waiter;
1307
1308 /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */
1309 tracker->sync_fence = sync_fence;
1310
1311 sync_fence = NULL;
1312 }
1313 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)*/
1314 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
1315 if ((NULL != tracker->timeline) && (MALI_TIMELINE_PP == tracker->timeline->id)) {
1316
1317 struct mali_pp_job *job = (struct mali_pp_job *)tracker->job;
1318
1319 if (0 < job->dma_fence_context.num_dma_fence_waiter) {
1320 struct mali_timeline_waiter *waiter;
1321 /* Check if we have a zeroed waiter object available. */
1322 if (unlikely(NULL == waiter_tail)) {
1323 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1324 goto exit;
1325 }
1326
1327 /* Grab new zeroed waiter object. */
1328 waiter = waiter_tail;
1329 waiter_tail = waiter_tail->tracker_next;
1330
1331 /* Increase the trigger ref count of the tracker. */
1332 tracker->trigger_ref_count++;
1333
1334 waiter->point = MALI_TIMELINE_NO_POINT;
1335 waiter->tracker = tracker;
1336
1337 /* Insert waiter on tracker's singly-linked waiter list. */
1338 if (NULL == tracker->waiter_head) {
1339 /* list is empty */
1340 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1341 tracker->waiter_tail = waiter;
1342 } else {
1343 tracker->waiter_head->tracker_next = waiter;
1344 }
1345 tracker->waiter_head = waiter;
1346
1347 /* Also store waiter in separate field for easy access by sync callback. */
1348 tracker->waiter_dma_fence = waiter;
1349 }
1350 }
1351 #endif /* defined(CONFIG_MALI_DMA_BUF_FENCE)*/
1352
1353 #if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1354 exit:
1355 #endif /* defined(CONFIG_MALI_DMA_BUF_FENCE) || defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
1356
1357 if (NULL != waiter_tail) {
1358 mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head);
1359 }
1360
1361 /* Release the initial trigger ref count. */
1362 tracker->trigger_ref_count--;
1363
1364 /* If there were no waiters added to this tracker we activate immediately. */
1365 if (0 == tracker->trigger_ref_count) {
1366 schedule_mask |= mali_timeline_tracker_activate(tracker);
1367 }
1368
1369 mali_spinlock_reentrant_signal(system->spinlock, tid);
1370
1371 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1372 if (NULL != sync_fence) {
1373 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1374 sync_fence_put(sync_fence);
1375 #else
1376 fput(sync_fence->file);
1377 #endif
1378 }
1379 #endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */
1380
1381 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1382 }
1383
1384 mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system,
1385 struct mali_timeline_tracker *tracker,
1386 enum mali_timeline_id timeline_id)
1387 {
1388 int num_waiters = 0;
1389 struct mali_timeline_waiter *waiter_tail, *waiter_head;
1390 u32 tid = _mali_osk_get_tid();
1391
1392 mali_timeline_point point = MALI_TIMELINE_NO_POINT;
1393
1394 MALI_DEBUG_ASSERT_POINTER(system);
1395 MALI_DEBUG_ASSERT_POINTER(system->session);
1396 MALI_DEBUG_ASSERT_POINTER(tracker);
1397
1398 MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting);
1399 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type);
1400 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
1401
1402 MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id));
1403
1404 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
1405 tracker->system = system;
1406
1407 mali_spinlock_reentrant_wait(system->spinlock, tid);
1408
1409 num_waiters = mali_timeline_fence_num_waiters(&tracker->fence);
1410
1411 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
1412 if (MALI_TIMELINE_PP == timeline_id) {
1413 struct mali_pp_job *job = (struct mali_pp_job *)tracker->job;
1414 if (0 < job->dma_fence_context.num_dma_fence_waiter)
1415 num_waiters++;
1416 }
1417 #endif
1418
1419 /* Allocate waiters. */
1420 mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters);
1421 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1422
1423 /* Add tracker to timeline. This will allocate a point for the tracker on the timeline. If
1424 * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the
1425 * point will be MALI_TIMELINE_NO_POINT.
1426 *
1427 * NOTE: the tracker can fail to be added if the timeline is full. If this happens, the
1428 * point will be MALI_TIMELINE_NO_POINT. */
1429 MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE);
1430 if (likely(timeline_id < MALI_TIMELINE_MAX)) {
1431 struct mali_timeline *timeline = system->timelines[timeline_id];
1432 mali_timeline_insert_tracker(timeline, tracker);
1433 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
1434 }
1435
1436 point = tracker->point;
1437
1438 /* Create waiters for tracker based on supplied fence. Each waiter will increase the
1439 * trigger ref count. */
1440 mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head);
1441 tracker = NULL;
1442
1443 /* At this point the tracker object might have been freed so we should no longer
1444 * access it. */
1445
1446
1447 /* The tracker will always be activated after calling add_tracker, even if NO_POINT is
1448 * returned. */
1449 return point;
1450 }
1451
1452 static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
1453 struct mali_timeline_waiter *waiter)
1454 {
1455 struct mali_timeline_tracker *tracker;
1456 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1457
1458 MALI_DEBUG_ASSERT_POINTER(system);
1459 MALI_DEBUG_ASSERT_POINTER(waiter);
1460
1461 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1462
1463 tracker = waiter->tracker;
1464 MALI_DEBUG_ASSERT_POINTER(tracker);
1465
1466 /* At this point the waiter has been removed from the timeline's waiter list, but it is
1467 * still on the tracker's waiter list. All of the tracker's waiters will be released when
1468 * the tracker is activated. */
1469
1470 waiter->point = MALI_TIMELINE_NO_POINT;
1471 waiter->tracker = NULL;
1472
1473 tracker->trigger_ref_count--;
1474 if (0 == tracker->trigger_ref_count) {
1475 /* This was the last waiter; activate tracker */
1476 schedule_mask |= mali_timeline_tracker_activate(tracker);
1477 tracker = NULL;
1478 }
1479
1480 return schedule_mask;
1481 }
1482
1483 mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system,
1484 enum mali_timeline_id timeline_id)
1485 {
1486 mali_timeline_point point;
1487 struct mali_timeline *timeline;
1488 u32 tid = _mali_osk_get_tid();
1489
1490 MALI_DEBUG_ASSERT_POINTER(system);
1491
1492 if (MALI_TIMELINE_MAX <= timeline_id) {
1493 return MALI_TIMELINE_NO_POINT;
1494 }
1495
1496 mali_spinlock_reentrant_wait(system->spinlock, tid);
1497
1498 timeline = system->timelines[timeline_id];
1499 MALI_DEBUG_ASSERT_POINTER(timeline);
1500
1501 point = MALI_TIMELINE_NO_POINT;
1502 if (timeline->point_oldest != timeline->point_next) {
1503 point = timeline->point_next - 1;
1504 if (MALI_TIMELINE_NO_POINT == point) point--;
1505 }
1506
1507 mali_spinlock_reentrant_signal(system->spinlock, tid);
1508
1509 return point;
1510 }
1511
1512 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1513 static void mali_timeline_do_sync_fence_callback(void *arg)
1514 {
1515 _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1516 struct mali_timeline_tracker *tracker;
1517 struct mali_timeline_tracker *tmp_tracker;
1518 u32 tid = _mali_osk_get_tid();
1519
1520 MALI_IGNORE(arg);
1521
1522 /*
1523 * Quickly "unhook" the jobs pending to be deleted, so we can release
1524 * the lock before we start deleting the job objects
1525 * (without any locks held)
1526 */
1527 _mali_osk_spinlock_irq_lock(sync_fence_callback_list_lock);
1528 _mali_osk_list_move_list(&sync_fence_callback_queue, &list);
1529 _mali_osk_spinlock_irq_unlock(sync_fence_callback_list_lock);
1530
1531 _MALI_OSK_LIST_FOREACHENTRY(tracker, tmp_tracker, &list,
1532 struct mali_timeline_tracker, sync_fence_signal_list) {
1533 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1534 mali_bool is_aborting = MALI_FALSE;
1535 int fence_status = 0;
1536 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1537 struct sync_fence *sync_fence = NULL;
1538 #else
1539 struct mali_internal_sync_fence *sync_fence = NULL;
1540 #endif
1541 struct mali_timeline_system *system = NULL;
1542 struct mali_timeline_waiter *waiter = NULL;
1543
1544 _mali_osk_list_delinit(&tracker->sync_fence_signal_list);
1545
1546 sync_fence = tracker->sync_fence;
1547 MALI_DEBUG_ASSERT_POINTER(sync_fence);
1548
1549 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
1550 fence_status = sync_fence->status;
1551 #elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0)
1552 fence_status = atomic_read(&sync_fence->status);
1553 #else
1554 fence_status = sync_fence->fence->ops->signaled(sync_fence->fence);
1555 #endif
1556
1557 system = tracker->system;
1558 MALI_DEBUG_ASSERT_POINTER(system);
1559 MALI_DEBUG_ASSERT_POINTER(system->session);
1560
1561 mali_spinlock_reentrant_wait(system->spinlock, tid);
1562
1563 is_aborting = system->session->is_aborting;
1564 if (!is_aborting && (0 > fence_status)) {
1565 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status));
1566 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
1567 }
1568
1569 waiter = tracker->waiter_sync;
1570 MALI_DEBUG_ASSERT_POINTER(waiter);
1571
1572 tracker->sync_fence = NULL;
1573 tracker->fence.sync_fd = -1;
1574
1575 schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
1576
1577 /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */
1578 if (is_aborting) {
1579 _mali_osk_wait_queue_wake_up(system->wait_queue);
1580 }
1581
1582 mali_spinlock_reentrant_signal(system->spinlock, tid);
1583
1584 /*
1585 * Older versions of Linux, before 3.5, doesn't support fput() in interrupt
1586 * context. For those older kernels, allocate a list object and put the
1587 * fence object on that and defer the call to sync_fence_put() to a workqueue.
1588 */
1589 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
1590 {
1591 struct mali_deferred_fence_put_entry *obj;
1592
1593 obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC);
1594 if (obj) {
1595 unsigned long flags;
1596 mali_bool schedule = MALI_FALSE;
1597
1598 obj->fence = sync_fence;
1599
1600 spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
1601 if (hlist_empty(&mali_timeline_sync_fence_to_free_list))
1602 schedule = MALI_TRUE;
1603 hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list);
1604 spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
1605
1606 if (schedule)
1607 schedule_delayed_work(&delayed_sync_fence_put, 0);
1608 }
1609 }
1610 #else
1611 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1612 sync_fence_put(sync_fence);
1613 #else
1614 fput(sync_fence->file);
1615 #endif
1616 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */
1617
1618 if (!is_aborting) {
1619 mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE);
1620 }
1621 }
1622 }
1623 #endif
1624 _mali_osk_errcode_t mali_timeline_initialize(void)
1625 {
1626 _mali_osk_atomic_init(&gp_tracker_count, 0);
1627 _mali_osk_atomic_init(&phy_pp_tracker_count, 0);
1628 _mali_osk_atomic_init(&virt_pp_tracker_count, 0);
1629
1630 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1631 sync_fence_callback_list_lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST);
1632 if (NULL == sync_fence_callback_list_lock) {
1633 return _MALI_OSK_ERR_NOMEM;
1634 }
1635
1636 sync_fence_callback_work_t = _mali_osk_wq_create_work(
1637 mali_timeline_do_sync_fence_callback, NULL);
1638
1639 if (NULL == sync_fence_callback_work_t) {
1640 return _MALI_OSK_ERR_FAULT;
1641 }
1642 #endif
1643 return _MALI_OSK_ERR_OK;
1644 }
1645
1646
1647 void mali_timeline_terminate(void)
1648 {
1649 _mali_osk_atomic_term(&gp_tracker_count);
1650 _mali_osk_atomic_term(&phy_pp_tracker_count);
1651 _mali_osk_atomic_term(&virt_pp_tracker_count);
1652
1653 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1654 if (NULL != sync_fence_callback_list_lock) {
1655 _mali_osk_spinlock_irq_term(sync_fence_callback_list_lock);
1656 sync_fence_callback_list_lock = NULL;
1657 }
1658
1659 if (NULL != sync_fence_callback_work_t) {
1660 _mali_osk_wq_delete_work(sync_fence_callback_work_t);
1661 sync_fence_callback_work_t = NULL;
1662 }
1663 #endif
1664 }
1665
1666 #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
1667
1668 static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id)
1669 {
1670 struct mali_timeline *timeline;
1671 struct mali_timeline_system *system;
1672
1673 MALI_DEBUG_ASSERT_POINTER(tracker);
1674
1675 MALI_DEBUG_ASSERT_POINTER(tracker->timeline);
1676 timeline = tracker->timeline;
1677
1678 MALI_DEBUG_ASSERT_POINTER(timeline->system);
1679 system = timeline->system;
1680
1681 if (MALI_TIMELINE_MAX > id) {
1682 if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) {
1683 return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]);
1684 } else {
1685 return MALI_FALSE;
1686 }
1687 } else {
1688 MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id);
1689 return MALI_FALSE;
1690 }
1691 }
1692
1693 static const char *timeline_id_to_string(enum mali_timeline_id id)
1694 {
1695 switch (id) {
1696 case MALI_TIMELINE_GP:
1697 return "GP";
1698 case MALI_TIMELINE_PP:
1699 return "PP";
1700 case MALI_TIMELINE_SOFT:
1701 return "SOFT";
1702 default:
1703 return "NONE";
1704 }
1705 }
1706
1707 static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type)
1708 {
1709 switch (type) {
1710 case MALI_TIMELINE_TRACKER_GP:
1711 return "GP";
1712 case MALI_TIMELINE_TRACKER_PP:
1713 return "PP";
1714 case MALI_TIMELINE_TRACKER_SOFT:
1715 return "SOFT";
1716 case MALI_TIMELINE_TRACKER_WAIT:
1717 return "WAIT";
1718 case MALI_TIMELINE_TRACKER_SYNC:
1719 return "SYNC";
1720 default:
1721 return "INVALID";
1722 }
1723 }
1724
1725 mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker)
1726 {
1727 struct mali_timeline *timeline = NULL;
1728
1729 MALI_DEBUG_ASSERT_POINTER(tracker);
1730 timeline = tracker->timeline;
1731
1732 if (0 != tracker->trigger_ref_count) {
1733 return MALI_TIMELINE_TS_WAITING;
1734 }
1735
1736 if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) {
1737 return MALI_TIMELINE_TS_ACTIVE;
1738 }
1739
1740 if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) {
1741 return MALI_TIMELINE_TS_INIT;
1742 }
1743
1744 return MALI_TIMELINE_TS_FINISH;
1745 }
1746
1747 void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx)
1748 {
1749 const char *tracker_state = "IWAF";
1750 char state_char = 'I';
1751 char tracker_type[32] = {0};
1752
1753 MALI_DEBUG_ASSERT_POINTER(tracker);
1754
1755 state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1756 _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1757
1758 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1759 if (0 != tracker->trigger_ref_count) {
1760 if (print_ctx)
1761 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n",
1762 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1763 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1764 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1765 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1766 tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job));
1767 else
1768 MALI_DEBUG_PRINT(2, ("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n",
1769 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1770 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1771 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1772 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1773 tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job)));
1774 } else {
1775 if (print_ctx)
1776 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n",
1777 tracker_type, tracker->point, state_char,
1778 tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job));
1779 else
1780 MALI_DEBUG_PRINT(2, ("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n",
1781 tracker_type, tracker->point, state_char,
1782 tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job)));
1783
1784 }
1785 #else
1786 if (0 != tracker->trigger_ref_count) {
1787 if (print_ctx)
1788 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n",
1789 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1790 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1791 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1792 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1793 (unsigned int)(uintptr_t)(tracker->job));
1794 else
1795 MALI_DEBUG_PRINT(2, ("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n",
1796 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1797 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1798 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1799 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1800 (unsigned int)(uintptr_t)(tracker->job)));
1801 } else {
1802 if (print_ctx)
1803 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c job:(0x%08X)\n",
1804 tracker_type, tracker->point, state_char,
1805 (unsigned int)(uintptr_t)(tracker->job));
1806 else
1807 MALI_DEBUG_PRINT(2, ("TL: %s %u %c job:(0x%08X)\n",
1808 tracker_type, tracker->point, state_char,
1809 (unsigned int)(uintptr_t)(tracker->job)));
1810
1811 }
1812 #endif
1813 }
1814
1815 void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx)
1816 {
1817 struct mali_timeline_tracker *tracker = NULL;
1818
1819 MALI_DEBUG_ASSERT_POINTER(timeline);
1820
1821 tracker = timeline->tracker_tail;
1822 while (NULL != tracker) {
1823 mali_timeline_debug_print_tracker(tracker, print_ctx);
1824 tracker = tracker->timeline_next;
1825 }
1826 }
1827
1828 #if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0))
1829 void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker)
1830 {
1831 const char *tracker_state = "IWAF";
1832 char state_char = 'I';
1833 char tracker_type[32] = {0};
1834
1835 MALI_DEBUG_ASSERT_POINTER(tracker);
1836
1837 state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1838 _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1839
1840 #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1841 if (0 != tracker->trigger_ref_count) {
1842 MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n",
1843 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1844 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1845 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1846 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1847 tracker->fence.sync_fd, tracker->sync_fence, tracker->job));
1848 } else {
1849 MALI_PRINT(("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n",
1850 tracker_type, tracker->point, state_char,
1851 tracker->fence.sync_fd, tracker->sync_fence, tracker->job));
1852 }
1853 #else
1854 if (0 != tracker->trigger_ref_count) {
1855 MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n",
1856 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1857 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1858 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1859 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1860 tracker->job));
1861 } else {
1862 MALI_PRINT(("TL: %s %u %c job:(0x%08X)\n",
1863 tracker_type, tracker->point, state_char,
1864 tracker->job));
1865 }
1866 #endif
1867 }
1868
1869 void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline)
1870 {
1871 struct mali_timeline_tracker *tracker = NULL;
1872
1873 MALI_DEBUG_ASSERT_POINTER(timeline);
1874
1875 tracker = timeline->tracker_tail;
1876 while (NULL != tracker) {
1877 mali_timeline_debug_direct_print_tracker(tracker);
1878 tracker = tracker->timeline_next;
1879 }
1880 }
1881
1882 #endif
1883
1884 void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx)
1885 {
1886 int i;
1887 int num_printed = 0;
1888 u32 tid = _mali_osk_get_tid();
1889
1890 MALI_DEBUG_ASSERT_POINTER(system);
1891
1892 mali_spinlock_reentrant_wait(system->spinlock, tid);
1893
1894 /* Print all timelines */
1895 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1896 struct mali_timeline *timeline = system->timelines[i];
1897
1898 MALI_DEBUG_ASSERT_POINTER(timeline);
1899
1900 if (NULL == timeline->tracker_head) continue;
1901 if (print_ctx)
1902 _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n",
1903 timeline_id_to_string((enum mali_timeline_id)i));
1904 else
1905 MALI_DEBUG_PRINT(2, ("TL: Timeline %s: oldest (%u) next(%u)\n",
1906 timeline_id_to_string((enum mali_timeline_id)i), timeline->point_oldest, timeline->point_next));
1907
1908 mali_timeline_debug_print_timeline(timeline, print_ctx);
1909 num_printed++;
1910 }
1911
1912 if (0 == num_printed) {
1913 if (print_ctx)
1914 _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n");
1915 else
1916 MALI_DEBUG_PRINT(2, ("TL: All timelines empty\n"));
1917 }
1918
1919 mali_spinlock_reentrant_signal(system->spinlock, tid);
1920 }
1921
1922 #endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */
1923
1924 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
1925 void mali_timeline_dma_fence_callback(void *pp_job_ptr)
1926 {
1927 struct mali_timeline_system *system;
1928 struct mali_timeline_waiter *waiter;
1929 struct mali_timeline_tracker *tracker;
1930 struct mali_pp_job *pp_job = (struct mali_pp_job *)pp_job_ptr;
1931 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1932 u32 tid = _mali_osk_get_tid();
1933 mali_bool is_aborting = MALI_FALSE;
1934
1935 MALI_DEBUG_ASSERT_POINTER(pp_job);
1936
1937 tracker = &pp_job->tracker;
1938 MALI_DEBUG_ASSERT_POINTER(tracker);
1939
1940 system = tracker->system;
1941 MALI_DEBUG_ASSERT_POINTER(system);
1942 MALI_DEBUG_ASSERT_POINTER(system->session);
1943
1944 mali_spinlock_reentrant_wait(system->spinlock, tid);
1945
1946 waiter = tracker->waiter_dma_fence;
1947 MALI_DEBUG_ASSERT_POINTER(waiter);
1948
1949 schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
1950
1951 is_aborting = system->session->is_aborting;
1952
1953 /* If aborting, wake up sleepers that are waiting for dma fence callbacks to complete. */
1954 if (is_aborting) {
1955 _mali_osk_wait_queue_wake_up(system->wait_queue);
1956 }
1957
1958 mali_spinlock_reentrant_signal(system->spinlock, tid);
1959
1960 if (!is_aborting) {
1961 mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE);
1962 }
1963 }
1964 #endif
1965